From 66c268a564806a6760973b3e763a5458f939130d Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Fri, 20 Feb 2026 13:31:27 +0000 Subject: [PATCH] Add static and stochastic Docker test harnesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move examples/docker-network/ to testing/static/ and add testing/chaos/ as a new stochastic simulation harness. testing/static/ — Static 5-node test harness: - Fixed mesh, chain, and mesh-public topologies with docker compose - Manual test scripts (ping, iperf, netem) - Build script, config generation, key derivation testing/chaos/ — Stochastic network simulation: - Python orchestrator generating N-node FIPS meshes with dynamic network conditions, driven by reproducible YAML scenarios - Topology generation: random geometric, Erdos-Renyi, or chain graphs with BFS connectivity guarantee - Per-link netem: HTB classful qdiscs with u32 filters for per-peer impairment (delay, loss, jitter), stochastic mutation across configurable policy profiles - Per-link bandwidth pacing: HTB rate limiting with configurable tiers (1/10/100/1000 mbps) randomly assigned per edge - Link flaps: tc netem 100% loss with graph connectivity protection - Node churn: docker stop/start with netem re-application on restart, shared down_nodes tracking across all managers - Traffic generation: random iperf3 sessions between node pairs - Down-node guards: all docker exec callers check container liveness, auto-detect crashed containers via is_container_running() safety net - Log collection and post-run analysis (panics, errors, sessions, MMP metrics, tree reconvergence) - chaos.sh wrapper with --seed, --duration, --verbose, --list options - Four scenarios: smoke-10, chaos-10, churn-10, churn-20 --- examples/docker-network/.gitignore | 2 - testing/README.md | 22 ++ testing/chaos/.gitignore | 4 + .../chaos}/Dockerfile | 0 testing/chaos/README.md | 136 ++++++++ .../chaos}/configs/node.template.yaml | 0 .../chaos}/resolv.conf | 0 testing/chaos/scenarios/chaos-10.yaml | 53 +++ testing/chaos/scenarios/churn-10.yaml | 56 ++++ testing/chaos/scenarios/churn-20.yaml | 60 ++++ testing/chaos/scenarios/smoke-10.yaml | 27 ++ testing/chaos/scripts/build.sh | 48 +++ testing/chaos/scripts/chaos.sh | 144 +++++++++ .../chaos}/scripts/derive-keys.py | 0 testing/chaos/sim/__init__.py | 1 + testing/chaos/sim/__main__.py | 61 ++++ testing/chaos/sim/compose.py | 76 +++++ testing/chaos/sim/config_gen.py | 75 +++++ testing/chaos/sim/docker_exec.py | 71 ++++ testing/chaos/sim/keys.py | 14 + testing/chaos/sim/links.py | 210 ++++++++++++ testing/chaos/sim/logs.py | 167 ++++++++++ testing/chaos/sim/netem.py | 306 ++++++++++++++++++ testing/chaos/sim/nodes.py | 193 +++++++++++ testing/chaos/sim/runner.py | 272 ++++++++++++++++ testing/chaos/sim/scenario.py | 270 ++++++++++++++++ testing/chaos/sim/topology.py | 181 +++++++++++ testing/chaos/sim/traffic.py | 127 ++++++++ .../docker-network => testing/static}/.env | 0 testing/static/.gitignore | 2 + testing/static/Dockerfile | 34 ++ .../static}/README.md | 138 ++++---- testing/static/configs/node.template.yaml | 30 ++ .../static}/configs/topologies/chain.yaml | 0 .../configs/topologies/mesh-public.yaml | 0 .../static}/configs/topologies/mesh.yaml | 0 .../static}/docker-chain-topology.svg | 0 .../static}/docker-compose.yml | 0 .../static}/docker-mesh-topology.svg | 0 testing/static/resolv.conf | 1 + .../static}/scripts/build.sh | 6 +- testing/static/scripts/derive-keys.py | 111 +++++++ .../static}/scripts/generate-configs.sh | 0 .../static}/scripts/iperf-test.sh | 0 .../static}/scripts/netem.sh | 0 .../static}/scripts/ping-test.sh | 0 46 files changed, 2827 insertions(+), 71 deletions(-) delete mode 100644 examples/docker-network/.gitignore create mode 100644 testing/README.md create mode 100644 testing/chaos/.gitignore rename {examples/docker-network => testing/chaos}/Dockerfile (100%) create mode 100644 testing/chaos/README.md rename {examples/docker-network => testing/chaos}/configs/node.template.yaml (100%) rename {examples/docker-network => testing/chaos}/resolv.conf (100%) create mode 100644 testing/chaos/scenarios/chaos-10.yaml create mode 100644 testing/chaos/scenarios/churn-10.yaml create mode 100644 testing/chaos/scenarios/churn-20.yaml create mode 100644 testing/chaos/scenarios/smoke-10.yaml create mode 100755 testing/chaos/scripts/build.sh create mode 100755 testing/chaos/scripts/chaos.sh rename {examples/docker-network => testing/chaos}/scripts/derive-keys.py (100%) create mode 100644 testing/chaos/sim/__init__.py create mode 100644 testing/chaos/sim/__main__.py create mode 100644 testing/chaos/sim/compose.py create mode 100644 testing/chaos/sim/config_gen.py create mode 100644 testing/chaos/sim/docker_exec.py create mode 100644 testing/chaos/sim/keys.py create mode 100644 testing/chaos/sim/links.py create mode 100644 testing/chaos/sim/logs.py create mode 100644 testing/chaos/sim/netem.py create mode 100644 testing/chaos/sim/nodes.py create mode 100644 testing/chaos/sim/runner.py create mode 100644 testing/chaos/sim/scenario.py create mode 100644 testing/chaos/sim/topology.py create mode 100644 testing/chaos/sim/traffic.py rename {examples/docker-network => testing/static}/.env (100%) create mode 100644 testing/static/.gitignore create mode 100644 testing/static/Dockerfile rename {examples/docker-network => testing/static}/README.md (65%) create mode 100644 testing/static/configs/node.template.yaml rename {examples/docker-network => testing/static}/configs/topologies/chain.yaml (100%) rename {examples/docker-network => testing/static}/configs/topologies/mesh-public.yaml (100%) rename {examples/docker-network => testing/static}/configs/topologies/mesh.yaml (100%) rename {examples/docker-network => testing/static}/docker-chain-topology.svg (100%) rename {examples/docker-network => testing/static}/docker-compose.yml (100%) rename {examples/docker-network => testing/static}/docker-mesh-topology.svg (100%) create mode 100644 testing/static/resolv.conf rename {examples/docker-network => testing/static}/scripts/build.sh (90%) create mode 100755 testing/static/scripts/derive-keys.py rename {examples/docker-network => testing/static}/scripts/generate-configs.sh (100%) rename {examples/docker-network => testing/static}/scripts/iperf-test.sh (100%) rename {examples/docker-network => testing/static}/scripts/netem.sh (100%) rename {examples/docker-network => testing/static}/scripts/ping-test.sh (100%) diff --git a/examples/docker-network/.gitignore b/examples/docker-network/.gitignore deleted file mode 100644 index 0a254e6..0000000 --- a/examples/docker-network/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -fips -generated-configs \ No newline at end of file diff --git a/testing/README.md b/testing/README.md new file mode 100644 index 0000000..3ffe2fd --- /dev/null +++ b/testing/README.md @@ -0,0 +1,22 @@ +# FIPS Testing + +Integration and simulation test harnesses for FIPS, using Docker +containers running the full protocol stack. + +## Test Harnesses + +### [static/](static/) -- Static Docker Network + +Fixed topologies (mesh, chain) with 5 nodes. Manual scripts for +building, config generation, connectivity tests (ping, iperf), and +network impairment (netem). Useful for deterministic debugging and +validating specific topology configurations. + +### [chaos/](chaos/) -- Stochastic Simulation + +Automated randomized testing with configurable node counts, topology +algorithms (random geometric, Erdos-Renyi, chain), and fault +injection (netem mutation, link flaps, traffic generation, node +churn). Scenarios are defined in YAML and executed via a Python +harness that manages the full lifecycle: topology generation, Docker +orchestration, fault scheduling, log collection, and analysis. diff --git a/testing/chaos/.gitignore b/testing/chaos/.gitignore new file mode 100644 index 0000000..148f94d --- /dev/null +++ b/testing/chaos/.gitignore @@ -0,0 +1,4 @@ +fips +generated-configs +__pycache__ +sim-results diff --git a/examples/docker-network/Dockerfile b/testing/chaos/Dockerfile similarity index 100% rename from examples/docker-network/Dockerfile rename to testing/chaos/Dockerfile diff --git a/testing/chaos/README.md b/testing/chaos/README.md new file mode 100644 index 0000000..c77d320 --- /dev/null +++ b/testing/chaos/README.md @@ -0,0 +1,136 @@ +# Stochastic Network Simulation + +Automated stochastic network testing for FIPS. Generates random +topologies, spins up Docker containers, and applies configurable +stressors (network impairment, link flaps, traffic generation, node +churn) over a timed simulation run. Logs are collected and analyzed +automatically. + +## Prerequisites + +- Docker with the compose plugin +- Rust toolchain (for building the FIPS binary) +- Python 3 with `pyyaml` and `jinja2` packages + +## Quick Start + +```bash +./testing/chaos/scripts/build.sh +./testing/chaos/scripts/chaos.sh smoke-10 +``` + +## Available Scenarios + +| Scenario | Nodes | Topology | Duration | Netem | Link Flaps | Traffic | Node Churn | Bandwidth | +| -------- | ----- | ---------------- | -------- | ----- | ---------- | ------- | ---------- | --------- | +| smoke-10 | 10 | random_geometric | 60s | -- | -- | -- | -- | -- | +| chaos-10 | 10 | random_geometric | 120s | yes | yes | yes | -- | -- | +| churn-10 | 10 | random_geometric | 600s | yes | yes | yes | yes | -- | +| churn-20 | 20 | erdos_renyi | 600s | yes | yes | yes | yes | yes | + +## CLI Options + +| Option | Description | +| ----------------- | ------------------------------------ | +| `-v`, `--verbose` | Enable debug logging | +| `--seed N` | Override the scenario's random seed | +| `--duration secs` | Override the scenario's duration | +| `--list` | List available scenarios | + +The scenario argument accepts either a name (`churn-10`) or a file +path (`scenarios/churn-10.yaml`). + +## Scenario YAML Format + +Annotated example based on `churn-10.yaml`: + +```yaml +scenario: + name: "churn-10" + seed: 42 # deterministic RNG seed + duration_secs: 600 # total simulation time + +topology: + num_nodes: 10 + algorithm: random_geometric # or erdos_renyi, chain + params: + radius: 0.5 # algorithm-specific parameter + ensure_connected: true # retry until graph is connected + subnet: "172.20.0.0/24" + ip_start: 10 # first node gets .10 + +netem: + enabled: true + default_policy: + delay_ms: { min: 5, max: 50 } + jitter_ms: { min: 1, max: 10 } + loss_pct: { min: 0, max: 2 } + mutation: + interval_secs: { min: 20, max: 45 } # re-roll interval + fraction: 0.3 # fraction of links mutated + policies: # named policy profiles + normal: + delay_ms: [5, 20] + loss_pct: [0, 1] + degraded: + delay_ms: [50, 100] + jitter_ms: [10, 30] + loss_pct: [3, 8] + +link_flaps: + enabled: true + interval_secs: { min: 30, max: 60 } + max_down_links: 2 + down_duration_secs: { min: 10, max: 30 } + protect_connectivity: true # never partition the graph + +traffic: + enabled: true + max_concurrent: 3 + interval_secs: { min: 10, max: 30 } + duration_secs: { min: 5, max: 15 } + parallel_streams: 4 + +node_churn: + enabled: true + interval_secs: { min: 60, max: 180 } + max_down_nodes: 1 + down_duration_secs: { min: 30, max: 90 } + protect_connectivity: true # never kill the last path + +bandwidth: + enabled: false # per-link HTB rate limiting + tiers_mbps: [1, 10, 100, 1000] # each link randomly assigned a tier + +logging: + rust_log: "debug" + output_dir: "./sim-results" +``` + +## Topology Algorithms + +| Algorithm | Parameters | Description | +| ---------------- | ------------------- | --------------------------------------------------------- | +| random_geometric | radius (default 0.5)| Place nodes in unit square, connect pairs within radius | +| erdos_renyi | p (default 0.3) | Include each edge independently with probability p | +| chain | -- | Linear chain: n01--n02--...--nN | + +When `ensure_connected` is true (default), the generator retries up to +50 times to produce a connected graph. + +## Output + +Results written to `sim-results/` (configurable via +`logging.output_dir`): + +- `analysis.txt` -- Summary: panics, errors, sessions, metrics +- `metadata.txt` -- Seed, node count, edges, adjacency list +- `fips-node-nXX.log` -- Per-node log output + +Exit code 0 on success, 2 if panics detected. + +## Creating Custom Scenarios + +1. Copy an existing scenario from `scenarios/`. +2. Adjust topology size, algorithm, and stressor parameters. +3. Run with `./testing/chaos/scripts/chaos.sh path/to/custom.yaml`. diff --git a/examples/docker-network/configs/node.template.yaml b/testing/chaos/configs/node.template.yaml similarity index 100% rename from examples/docker-network/configs/node.template.yaml rename to testing/chaos/configs/node.template.yaml diff --git a/examples/docker-network/resolv.conf b/testing/chaos/resolv.conf similarity index 100% rename from examples/docker-network/resolv.conf rename to testing/chaos/resolv.conf diff --git a/testing/chaos/scenarios/chaos-10.yaml b/testing/chaos/scenarios/chaos-10.yaml new file mode 100644 index 0000000..27f8ce5 --- /dev/null +++ b/testing/chaos/scenarios/chaos-10.yaml @@ -0,0 +1,53 @@ +scenario: + name: "chaos-10" + seed: 42 + duration_secs: 120 + +topology: + num_nodes: 10 + algorithm: random_geometric + params: + radius: 0.5 + ensure_connected: true + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + delay_ms: { min: 5, max: 50 } + jitter_ms: { min: 1, max: 10 } + loss_pct: { min: 0, max: 2 } + mutation: + interval_secs: { min: 15, max: 30 } + fraction: 0.3 + policies: + normal: + delay_ms: [5, 20] + loss_pct: [0, 1] + degraded: + delay_ms: [50, 100] + jitter_ms: [10, 30] + loss_pct: [3, 8] + terrible: + delay_ms: [100, 200] + jitter_ms: [30, 60] + loss_pct: [10, 20] + +link_flaps: + enabled: true + interval_secs: { min: 20, max: 60 } + max_down_links: 2 + down_duration_secs: { min: 10, max: 30 } + protect_connectivity: true + +traffic: + enabled: true + max_concurrent: 3 + interval_secs: { min: 10, max: 30 } + duration_secs: { min: 5, max: 15 } + parallel_streams: 4 + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/churn-10.yaml b/testing/chaos/scenarios/churn-10.yaml new file mode 100644 index 0000000..1313138 --- /dev/null +++ b/testing/chaos/scenarios/churn-10.yaml @@ -0,0 +1,56 @@ +scenario: + name: "churn-10" + seed: 42 + duration_secs: 600 + +topology: + num_nodes: 10 + algorithm: random_geometric + params: + radius: 0.5 + ensure_connected: true + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + delay_ms: { min: 5, max: 50 } + jitter_ms: { min: 1, max: 10 } + loss_pct: { min: 0, max: 2 } + mutation: + interval_secs: { min: 20, max: 45 } + fraction: 0.3 + policies: + normal: + delay_ms: [5, 20] + loss_pct: [0, 1] + degraded: + delay_ms: [50, 100] + jitter_ms: [10, 30] + loss_pct: [3, 8] + +link_flaps: + enabled: true + interval_secs: { min: 30, max: 60 } + max_down_links: 2 + down_duration_secs: { min: 10, max: 30 } + protect_connectivity: true + +traffic: + enabled: true + max_concurrent: 3 + interval_secs: { min: 10, max: 30 } + duration_secs: { min: 5, max: 15 } + parallel_streams: 4 + +node_churn: + enabled: true + interval_secs: { min: 60, max: 180 } + max_down_nodes: 1 + down_duration_secs: { min: 30, max: 90 } + protect_connectivity: true + +logging: + rust_log: "debug" + output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/churn-20.yaml b/testing/chaos/scenarios/churn-20.yaml new file mode 100644 index 0000000..c112d3a --- /dev/null +++ b/testing/chaos/scenarios/churn-20.yaml @@ -0,0 +1,60 @@ +scenario: + name: "churn-20" + seed: 42 + duration_secs: 600 + +topology: + num_nodes: 20 + algorithm: erdos_renyi + params: + p: 0.3 + ensure_connected: true + subnet: "172.20.0.0/23" + ip_start: 10 + +netem: + enabled: true + default_policy: + delay_ms: { min: 5, max: 50 } + jitter_ms: { min: 1, max: 10 } + loss_pct: { min: 0, max: 2 } + mutation: + interval_secs: { min: 20, max: 45 } + fraction: 0.3 + policies: + normal: + delay_ms: [5, 20] + loss_pct: [0, 1] + degraded: + delay_ms: [50, 100] + jitter_ms: [10, 30] + loss_pct: [3, 8] + +link_flaps: + enabled: true + interval_secs: { min: 30, max: 60 } + max_down_links: 3 + down_duration_secs: { min: 10, max: 30 } + protect_connectivity: true + +traffic: + enabled: true + max_concurrent: 5 + interval_secs: { min: 10, max: 30 } + duration_secs: { min: 5, max: 15 } + parallel_streams: 4 + +node_churn: + enabled: true + interval_secs: { min: 60, max: 90 } + max_down_nodes: 3 + down_duration_secs: { min: 30, max: 90 } + protect_connectivity: false + +bandwidth: + enabled: true + tiers_mbps: [1, 10, 100, 1000] + +logging: + rust_log: "debug" + output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/smoke-10.yaml b/testing/chaos/scenarios/smoke-10.yaml new file mode 100644 index 0000000..3c1fde8 --- /dev/null +++ b/testing/chaos/scenarios/smoke-10.yaml @@ -0,0 +1,27 @@ +scenario: + name: "smoke-10" + seed: 42 + duration_secs: 60 + +topology: + num_nodes: 10 + algorithm: random_geometric + params: + radius: 0.5 + ensure_connected: true + subnet: "172.20.0.0/24" + ip_start: 10 + +# Phase 2+ features (disabled for MVP) +netem: + enabled: false + +link_flaps: + enabled: false + +traffic: + enabled: false + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/scripts/build.sh b/testing/chaos/scripts/build.sh new file mode 100755 index 0000000..95b309f --- /dev/null +++ b/testing/chaos/scripts/build.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Build the FIPS binary for the chaos simulation Docker image. +# Usage: ./scripts/build.sh +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CHAOS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Find project root (directory containing Cargo.toml) +PROJECT_ROOT="$(cd "$CHAOS_DIR/../.." && pwd)" +if [ ! -f "$PROJECT_ROOT/Cargo.toml" ]; then + echo "Error: Cannot find Cargo.toml at $PROJECT_ROOT" >&2 + echo "Expected layout: /testing/chaos/scripts/build.sh" >&2 + exit 1 +fi + +# Detect host OS +UNAME_S=$(uname -s) +CARGO_TARGET="x86_64-unknown-linux-musl" + +if [ "$UNAME_S" = "Darwin" ]; then + echo "Detected macOS host - using cross-compilation for Linux..." + + if ! command -v cargo-zigbuild &> /dev/null; then + echo "Error: cargo-zigbuild not found." >&2 + echo "Please install it: cargo install cargo-zigbuild" >&2 + exit 1 + fi + + if ! rustup target list --installed | grep -q "$CARGO_TARGET"; then + echo "Installing Rust target $CARGO_TARGET..." + rustup target add "$CARGO_TARGET" + fi + + echo "Building FIPS for Linux (release) using cargo-zigbuild..." + cargo zigbuild --release --target "$CARGO_TARGET" --manifest-path="$PROJECT_ROOT/Cargo.toml" + + echo "Copying binary to docker context..." + cp "$PROJECT_ROOT/target/$CARGO_TARGET/release/fips" "$CHAOS_DIR/fips" +else + echo "Building FIPS (release)..." + cargo build --release --manifest-path="$PROJECT_ROOT/Cargo.toml" + + echo "Copying binary to docker context..." + cp "$PROJECT_ROOT/target/release/fips" "$CHAOS_DIR/fips" +fi + +echo "Done. Binary at $CHAOS_DIR/fips" diff --git a/testing/chaos/scripts/chaos.sh b/testing/chaos/scripts/chaos.sh new file mode 100755 index 0000000..803b2b3 --- /dev/null +++ b/testing/chaos/scripts/chaos.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Run a FIPS stochastic network simulation. +# +# Usage: ./scripts/chaos.sh [options] +# scenario: path to YAML file, or scenario name (e.g., "churn-10") +# +# Options: +# -v, --verbose Enable debug logging +# --seed Override scenario seed +# --duration Override scenario duration +# --list List available scenarios +# +# Examples: +# ./scripts/chaos.sh churn-10 +# ./scripts/chaos.sh churn-10 --seed 123 --verbose +# ./scripts/chaos.sh scenarios/churn-10.yaml --duration 300 +# ./scripts/chaos.sh --list +set -e + +trap 'echo ""; echo "Simulation interrupted"; exit 130' INT + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CHAOS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SCENARIO_DIR="$CHAOS_DIR/scenarios" + +usage() { + echo "Usage: $0 [options]" + echo "" + echo "Arguments:" + echo " scenario Path to YAML file, or scenario name (e.g., churn-10)" + echo "" + echo "Options:" + echo " -v, --verbose Enable debug logging" + echo " --seed Override scenario seed" + echo " --duration Override scenario duration" + echo " --list List available scenarios" + exit 1 +} + +list_scenarios() { + echo "=== Available Scenarios ===" + echo "" + for f in "$SCENARIO_DIR"/*.yaml; do + [ -f "$f" ] || continue + echo " $(basename "$f" .yaml)" + done + exit 0 +} + +# Handle --list before requiring a positional arg +for arg in "$@"; do + case "$arg" in + --list) list_scenarios ;; + esac +done + +# Require at least one argument (the scenario) +[ $# -lt 1 ] && usage + +# Parse arguments +SCENARIO_ARG="" +VERBOSE="" +SEED="" +DURATION="" + +while [ $# -gt 0 ]; do + case "$1" in + -v|--verbose) VERBOSE="--verbose"; shift ;; + --seed) SEED="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --list) list_scenarios ;; + -*) echo "Error: Unknown option '$1'" >&2; usage ;; + *) + if [ -z "$SCENARIO_ARG" ]; then + SCENARIO_ARG="$1" + else + echo "Error: Unexpected argument '$1'" >&2 + usage + fi + shift + ;; + esac +done + +[ -z "$SCENARIO_ARG" ] && usage + +# Resolve scenario path +if [ -f "$SCENARIO_ARG" ]; then + SCENARIO_FILE="$SCENARIO_ARG" +elif [ -f "$SCENARIO_DIR/$SCENARIO_ARG.yaml" ]; then + SCENARIO_FILE="$SCENARIO_DIR/$SCENARIO_ARG.yaml" +else + echo "Error: Scenario not found: $SCENARIO_ARG" >&2 + echo "Tried:" >&2 + echo " $SCENARIO_ARG" >&2 + echo " $SCENARIO_DIR/$SCENARIO_ARG.yaml" >&2 + echo "" >&2 + echo "Available scenarios:" >&2 + for f in "$SCENARIO_DIR"/*.yaml; do + [ -f "$f" ] || continue + echo " $(basename "$f" .yaml)" >&2 + done + exit 1 +fi + +# Check prerequisites +if ! command -v python3 &> /dev/null; then + echo "Error: python3 not found" >&2 + exit 1 +fi + +if ! command -v docker &> /dev/null; then + echo "Error: docker not found" >&2 + exit 1 +fi + +if ! docker info &> /dev/null; then + echo "Error: Docker is not running" >&2 + exit 1 +fi + +if [ ! -f "$CHAOS_DIR/fips" ]; then + echo "Error: FIPS binary not found at $CHAOS_DIR/fips" >&2 + echo "Run testing/chaos/scripts/build.sh first" >&2 + exit 1 +fi + +# Build python args +PYTHON_ARGS=("$SCENARIO_FILE") +[ -n "$VERBOSE" ] && PYTHON_ARGS+=("$VERBOSE") +[ -n "$SEED" ] && PYTHON_ARGS+=("--seed" "$SEED") +[ -n "$DURATION" ] && PYTHON_ARGS+=("--duration" "$DURATION") + +echo "=== FIPS Stochastic Simulation ===" +echo "" +echo " Scenario: $(basename "$SCENARIO_FILE" .yaml)" +echo " File: $SCENARIO_FILE" +[ -n "$SEED" ] && echo " Seed: $SEED (override)" +[ -n "$DURATION" ] && echo " Duration: ${DURATION}s (override)" +echo "" + +# Run from testing/chaos directory (sim expects relative paths) +cd "$CHAOS_DIR" +python3 -m sim "${PYTHON_ARGS[@]}" diff --git a/examples/docker-network/scripts/derive-keys.py b/testing/chaos/scripts/derive-keys.py similarity index 100% rename from examples/docker-network/scripts/derive-keys.py rename to testing/chaos/scripts/derive-keys.py diff --git a/testing/chaos/sim/__init__.py b/testing/chaos/sim/__init__.py new file mode 100644 index 0000000..d1550b8 --- /dev/null +++ b/testing/chaos/sim/__init__.py @@ -0,0 +1 @@ +"""FIPS stochastic network simulation tool.""" diff --git a/testing/chaos/sim/__main__.py b/testing/chaos/sim/__main__.py new file mode 100644 index 0000000..f88bcd8 --- /dev/null +++ b/testing/chaos/sim/__main__.py @@ -0,0 +1,61 @@ +"""CLI entry point: python -m sim """ + +import argparse +import logging +import sys + +from .runner import SimRunner +from .scenario import load_scenario + + +def main(): + parser = argparse.ArgumentParser( + prog="sim", + description="FIPS stochastic network simulation", + ) + parser.add_argument("scenario", help="Path to scenario YAML file") + parser.add_argument( + "-v", "--verbose", action="store_true", help="Enable debug logging" + ) + parser.add_argument( + "--seed", type=int, default=None, + help="Override scenario seed", + ) + parser.add_argument( + "--duration", type=int, default=None, + help="Override scenario duration in seconds", + ) + args = parser.parse_args() + + level = logging.DEBUG if args.verbose else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)-5s %(name)s: %(message)s", + datefmt="%H:%M:%S", + ) + + try: + scenario = load_scenario(args.scenario) + except (FileNotFoundError, ValueError) as e: + print(f"Error loading scenario: {e}", file=sys.stderr) + sys.exit(1) + + # Apply CLI overrides + if args.seed is not None: + scenario.seed = args.seed + if args.duration is not None: + if args.duration < 1: + print("Error: --duration must be >= 1", file=sys.stderr) + sys.exit(1) + scenario.duration_secs = args.duration + + runner = SimRunner(scenario) + result = runner.run() + + if result and result.panics: + sys.exit(2) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/testing/chaos/sim/compose.py b/testing/chaos/sim/compose.py new file mode 100644 index 0000000..1f21b6b --- /dev/null +++ b/testing/chaos/sim/compose.py @@ -0,0 +1,76 @@ +"""Generate docker-compose.yml for a simulation topology.""" + +from __future__ import annotations + +import os + +from jinja2 import Template + +from .scenario import Scenario +from .topology import SimTopology + +# Jinja2 template for the compose file. +# build context points back to the testing/chaos root where the Dockerfile lives. +_COMPOSE_TEMPLATE = Template( + """\ +networks: + fips-net: + driver: bridge + ipam: + config: + - subnet: {{ subnet }} + +x-fips-common: &fips-common + build: + context: ../.. + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + sysctls: + - net.ipv6.conf.all.disable_ipv6=0 + restart: "no" + env_file: + - ./npubs.env + environment: + - RUST_LOG={{ rust_log }} + - RUST_BACKTRACE=1 + +services: +{% for node in nodes %} + {{ node.node_id }}: + <<: *fips-common + container_name: fips-node-{{ node.node_id }} + hostname: {{ node.node_id }} + volumes: + - ../../resolv.conf:/etc/resolv.conf:ro + - ./{{ node.node_id }}.yaml:/etc/fips/fips.yaml:ro + networks: + fips-net: + ipv4_address: {{ node.docker_ip }} +{% endfor %} +""" +) + + +def generate_compose( + topology: SimTopology, + scenario: Scenario, + output_dir: str, +) -> str: + """Render docker-compose.yml and write to output_dir. Returns the file path.""" + os.makedirs(output_dir, exist_ok=True) + + nodes = [topology.nodes[nid] for nid in sorted(topology.nodes)] + + content = _COMPOSE_TEMPLATE.render( + subnet=scenario.topology.subnet, + rust_log=scenario.logging.rust_log, + nodes=nodes, + ) + + path = os.path.join(output_dir, "docker-compose.yml") + with open(path, "w") as f: + f.write(content) + + return path diff --git a/testing/chaos/sim/config_gen.py b/testing/chaos/sim/config_gen.py new file mode 100644 index 0000000..6381bed --- /dev/null +++ b/testing/chaos/sim/config_gen.py @@ -0,0 +1,75 @@ +"""FIPS node config generation from template + topology.""" + +from __future__ import annotations + +import os + +from .topology import SimTopology + +# Path to the shared node config template +_TEMPLATE_PATH = os.path.join( + os.path.dirname(__file__), "..", "configs", "node.template.yaml" +) + + +def _load_template() -> str: + with open(_TEMPLATE_PATH) as f: + return f.read() + + +def generate_peers_block(topology: SimTopology, node_id: str) -> str: + """Generate the YAML peers block for a node.""" + peers = topology.nodes[node_id].peers + if not peers: + return " []" + + lines = [] + for peer_id in sorted(peers): + peer = topology.nodes[peer_id] + lines.append(f' - npub: "{peer.npub}"') + lines.append(f' alias: "{peer_id}"') + lines.append(f" addresses:") + lines.append(f" - transport: udp") + lines.append(f' addr: "{peer.docker_ip}:4000"') + lines.append(f" connect_policy: auto_connect") + return "\n".join(lines) + + +def generate_node_config(topology: SimTopology, node_id: str) -> str: + """Generate a complete FIPS config YAML for one node.""" + template = _load_template() + node = topology.nodes[node_id] + peers_yaml = generate_peers_block(topology, node_id) + + config = template + config = config.replace("{{NODE_NAME}}", node_id.upper()) + config = config.replace("{{TOPOLOGY}}", "sim") + config = config.replace("{{NPUB}}", node.npub) + config = config.replace("{{NSEC}}", node.nsec) + config = config.replace("{{PEERS}}", peers_yaml) + return config + + +def generate_npubs_env(topology: SimTopology) -> str: + """Generate npubs.env content mapping NPUB_= for all nodes.""" + lines = [] + for node_id in sorted(topology.nodes): + node = topology.nodes[node_id] + env_name = f"NPUB_{node_id.upper()}" + lines.append(f"{env_name}={node.npub}") + return "\n".join(lines) + "\n" + + +def write_configs(topology: SimTopology, output_dir: str): + """Write all node configs and npubs.env to the output directory.""" + os.makedirs(output_dir, exist_ok=True) + + for node_id in topology.nodes: + config = generate_node_config(topology, node_id) + path = os.path.join(output_dir, f"{node_id}.yaml") + with open(path, "w") as f: + f.write(config) + + env_path = os.path.join(output_dir, "npubs.env") + with open(env_path, "w") as f: + f.write(generate_npubs_env(topology)) diff --git a/testing/chaos/sim/docker_exec.py b/testing/chaos/sim/docker_exec.py new file mode 100644 index 0000000..558dd8f --- /dev/null +++ b/testing/chaos/sim/docker_exec.py @@ -0,0 +1,71 @@ +"""Thin wrapper around subprocess for docker exec calls.""" + +import subprocess +import logging + +log = logging.getLogger(__name__) + + +class DockerExecError(Exception): + def __init__(self, container, cmd, returncode, stderr): + self.container = container + self.cmd = cmd + self.returncode = returncode + self.stderr = stderr + super().__init__( + f"docker exec {container}: {cmd!r} returned {returncode}: {stderr}" + ) + + +def docker_exec(container: str, cmd: str, timeout: int = 30) -> str: + """Execute a command in a running container, return stdout.""" + result = subprocess.run( + ["docker", "exec", container, "/bin/bash", "-c", cmd], + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != 0: + raise DockerExecError(container, cmd, result.returncode, result.stderr) + return result.stdout + + +def docker_exec_quiet(container: str, cmd: str, timeout: int = 30) -> str | None: + """Execute a command, return stdout on success or None on failure (logged).""" + try: + return docker_exec(container, cmd, timeout) + except (DockerExecError, subprocess.TimeoutExpired) as e: + log.warning("docker exec failed on %s: %s", container, e) + return None + + +def docker_compose( + compose_file: str, + args: list[str], + timeout: int = 300, + check: bool = True, +) -> subprocess.CompletedProcess: + """Run a docker compose command with the given compose file.""" + cmd = ["docker", "compose", "-f", compose_file] + args + log.info("Running: %s", " ".join(cmd)) + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + check=check, + ) + + +def is_container_running(container: str) -> bool: + """Check if a container is running.""" + try: + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container], + capture_output=True, + text=True, + timeout=10, + ) + return result.returncode == 0 and result.stdout.strip() == "true" + except subprocess.TimeoutExpired: + return False diff --git a/testing/chaos/sim/keys.py b/testing/chaos/sim/keys.py new file mode 100644 index 0000000..6fd3ed4 --- /dev/null +++ b/testing/chaos/sim/keys.py @@ -0,0 +1,14 @@ +"""Key derivation wrapper, reusing logic from scripts/derive-keys.py.""" + +import importlib.util +import os + +# Import derive() from scripts/derive-keys.py +_SCRIPTS_DIR = os.path.join(os.path.dirname(__file__), "..", "scripts") +_DERIVE_KEYS_PATH = os.path.join(_SCRIPTS_DIR, "derive-keys.py") + +_spec = importlib.util.spec_from_file_location("derive_keys", _DERIVE_KEYS_PATH) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) + +derive = _mod.derive # derive(mesh_name, node_name) -> (nsec_hex, npub_bech32) diff --git a/testing/chaos/sim/links.py b/testing/chaos/sim/links.py new file mode 100644 index 0000000..7417533 --- /dev/null +++ b/testing/chaos/sim/links.py @@ -0,0 +1,210 @@ +"""Link up/down simulation via netem 100% loss. + +Simulates link failures by setting netem to 100% packet loss on the +specific tc class for that peer. Requires the NetemManager to have +already set up per-link classful qdiscs. Includes connectivity +protection to prevent graph partitioning. +""" + +from __future__ import annotations + +import logging +import random +import time +from dataclasses import dataclass, field + +from .docker_exec import docker_exec_quiet, is_container_running +from .scenario import LinkFlapsConfig +from .topology import SimTopology + +log = logging.getLogger(__name__) + +IFACE = "eth0" + + +@dataclass +class LinkState: + edge: tuple[str, str] # (node_a, node_b) — canonical sorted order + is_down: bool = False + down_since: float | None = None + restore_at: float | None = None + # Saved netem params to restore when link comes back up + saved_params_a: str | None = None # tc args for a->b direction + saved_params_b: str | None = None # tc args for b->a direction + + +class LinkManager: + """Manages link up/down state using tc netem 100% loss.""" + + def __init__( + self, + topology: SimTopology, + config: LinkFlapsConfig, + rng: random.Random, + netem_mgr=None, + ): + self.topology = topology + self.config = config + self.rng = rng + self.netem_mgr = netem_mgr # Optional: for coordinated tc manipulation + self.link_states: dict[tuple[str, str], LinkState] = { + edge: LinkState(edge=edge) for edge in topology.edges + } + + @property + def down_count(self) -> int: + return sum(1 for ls in self.link_states.values() if ls.is_down) + + def maybe_flap(self): + """Attempt to bring down a random link.""" + if self.down_count >= self.config.max_down_links: + log.debug("At max_down_links (%d), skipping flap", self.config.max_down_links) + return + + # Pick a random up link whose endpoints are both running + down = self.netem_mgr.down_nodes if self.netem_mgr else set() + up_links = [ + e for e, ls in self.link_states.items() + if not ls.is_down and e[0] not in down and e[1] not in down + ] + if not up_links: + return + + self.rng.shuffle(up_links) + + for edge in up_links: + # Connectivity protection + if self.config.protect_connectivity and self._would_disconnect(edge): + log.debug("Skipping %s-%s (would disconnect graph)", edge[0], edge[1]) + continue + + # Bring it down + down_duration = self.rng.uniform( + self.config.down_duration_secs.min, + self.config.down_duration_secs.max, + ) + self._link_down(edge, down_duration) + return + + log.debug("No safe link to flap (all would disconnect)") + + def restore_expired(self): + """Restore links whose down duration has expired.""" + now = time.time() + for edge, state in self.link_states.items(): + if state.is_down and state.restore_at and now >= state.restore_at: + self._link_up(edge) + + def restore_all(self): + """Restore all downed links (for teardown).""" + for edge, state in list(self.link_states.items()): + if state.is_down: + self._link_up(edge) + + def _link_down(self, edge: tuple[str, str], duration: float): + """Simulate link failure by setting netem to 100% loss on both directions.""" + a, b = edge + state = self.link_states[edge] + + # Save current netem params and apply 100% loss + state.saved_params_a = self._set_loss(a, b, "loss 100%") + state.saved_params_b = self._set_loss(b, a, "loss 100%") + + now = time.time() + state.is_down = True + state.down_since = now + state.restore_at = now + duration + + log.info("Link DOWN: %s -- %s (restore in %.0fs)", a, b, duration) + + def _link_up(self, edge: tuple[str, str]): + """Restore link by reverting netem to saved params.""" + a, b = edge + state = self.link_states[edge] + + # Restore previous netem params + if state.saved_params_a: + self._set_loss(a, b, state.saved_params_a) + if state.saved_params_b: + self._set_loss(b, a, state.saved_params_b) + + down_for = time.time() - state.down_since if state.down_since else 0 + state.is_down = False + state.down_since = None + state.restore_at = None + state.saved_params_a = None + state.saved_params_b = None + + log.info("Link UP: %s -- %s (was down %.0fs)", a, b, down_for) + + def _set_loss(self, src_node: str, dst_node: str, netem_args: str) -> str | None: + """Set netem args on the tc class for src->dst. Returns the previous netem args.""" + if not self.netem_mgr: + return None + + # Skip if the node's container is down + if src_node in self.netem_mgr.down_nodes: + return None + + container = self.topology.container_name(src_node) + + # Safety net: detect containers that crashed outside of NodeManager + if not is_container_running(container): + log.debug( + "Container %s not running (unexpected), marking %s as down", + container, + src_node, + ) + self.netem_mgr.down_nodes.add(src_node) + return None + + dest_ip = self.topology.nodes[dst_node].docker_ip + + states = self.netem_mgr.states.get(container, {}) + link_state = states.get(dest_ip) + if link_state is None: + log.warning("No netem state for %s -> %s", src_node, dst_node) + return None + + # Save current params + prev_args = link_state.params.to_tc_args() + + # Apply new netem + cmd = ( + f"tc qdisc replace dev {IFACE} parent {link_state.class_id} " + f"handle {link_state.netem_handle} netem {netem_args}" + ) + docker_exec_quiet(container, cmd) + + return prev_args + + def _would_disconnect(self, edge: tuple[str, str]) -> bool: + """Check if removing this edge (plus currently-down edges) disconnects the graph.""" + # Build set of active edges (excluding already-down links and the candidate) + active_edges = set() + for e, state in self.link_states.items(): + if not state.is_down and e != edge: + active_edges.add(e) + + # BFS on active edges + if not self.topology.nodes: + return True + + adj: dict[str, list[str]] = {nid: [] for nid in self.topology.nodes} + for a, b in active_edges: + adj[a].append(b) + adj[b].append(a) + + start = next(iter(self.topology.nodes)) + visited = set() + queue = [start] + while queue: + node = queue.pop() + if node in visited: + continue + visited.add(node) + for neighbor in adj[node]: + if neighbor not in visited: + queue.append(neighbor) + + return len(visited) < len(self.topology.nodes) diff --git a/testing/chaos/sim/logs.py b/testing/chaos/sim/logs.py new file mode 100644 index 0000000..30cec94 --- /dev/null +++ b/testing/chaos/sim/logs.py @@ -0,0 +1,167 @@ +"""Log collection and post-run analysis.""" + +from __future__ import annotations + +import os +import re +import subprocess +import logging +from dataclasses import dataclass, field + +log = logging.getLogger(__name__) + +# Regex to strip ANSI escape codes from tracing output +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +@dataclass +class AnalysisResult: + errors: list[tuple[str, str]] = field(default_factory=list) + warnings: list[tuple[str, str]] = field(default_factory=list) + sessions_established: list[tuple[str, str]] = field(default_factory=list) + peers_promoted: list[tuple[str, str]] = field(default_factory=list) + peer_removals: list[tuple[str, str]] = field(default_factory=list) + parent_switches: list[tuple[str, str]] = field(default_factory=list) + mmp_link_metrics: list[tuple[str, str]] = field(default_factory=list) + mmp_session_metrics: list[tuple[str, str]] = field(default_factory=list) + handshake_timeouts: list[tuple[str, str]] = field(default_factory=list) + panics: list[tuple[str, str]] = field(default_factory=list) + + def summary(self) -> str: + lines = [ + "=== Simulation Analysis ===", + "", + f"Panics: {len(self.panics)}", + f"Errors: {len(self.errors)}", + f"Warnings: {len(self.warnings)}", + f"Sessions established: {len(self.sessions_established)}", + f"Peers promoted: {len(self.peers_promoted)}", + f"Peer removals: {len(self.peer_removals)}", + f"Parent switches: {len(self.parent_switches)}", + f"Handshake timeouts: {len(self.handshake_timeouts)}", + f"MMP link samples: {len(self.mmp_link_metrics)}", + f"MMP session samples: {len(self.mmp_session_metrics)}", + ] + + if self.panics: + lines.append("") + lines.append("--- PANICS ---") + for container, line in self.panics[:10]: + lines.append(f" [{container}] {line.strip()}") + + if self.errors: + lines.append("") + lines.append("--- ERRORS (first 20) ---") + for container, line in self.errors[:20]: + lines.append(f" [{container}] {line.strip()}") + + if self.handshake_timeouts: + lines.append("") + lines.append("--- HANDSHAKE TIMEOUTS (first 10) ---") + for container, line in self.handshake_timeouts[:10]: + lines.append(f" [{container}] {line.strip()}") + + lines.append("") + return "\n".join(lines) + + +def collect_logs(container_names: list[str], output_dir: str) -> dict[str, str]: + """Collect all output (stdout + stderr) from all containers.""" + os.makedirs(output_dir, exist_ok=True) + logs = {} + + for name in container_names: + try: + result = subprocess.run( + ["docker", "logs", name], + capture_output=True, + text=True, + timeout=30, + ) + # Combine stdout and stderr — tracing may go to either + # depending on the subscriber configuration. + # Strip ANSI escape codes for clean log files. + raw = result.stdout + result.stderr + log_text = _ANSI_RE.sub("", raw) + logs[name] = log_text + + path = os.path.join(output_dir, f"{name}.log") + with open(path, "w") as f: + f.write(log_text) + + except (subprocess.TimeoutExpired, Exception) as e: + log.warning("Failed to collect logs from %s: %s", name, e) + logs[name] = "" + + return logs + + +def analyze_logs(logs: dict[str, str]) -> AnalysisResult: + """Parse structured tracing output and categorize events.""" + result = AnalysisResult() + + for container, log_text in logs.items(): + for raw_line in log_text.splitlines(): + # Strip ANSI escape codes for reliable matching + line = _ANSI_RE.sub("", raw_line) + + # Panics + if "panicked" in line or "PANIC" in line: + result.panics.append((container, line)) + # Errors and warnings + elif " ERROR " in line: + result.errors.append((container, line)) + elif " WARN " in line: + result.warnings.append((container, line)) + + # Session establishment + if "Session established" in line: + result.sessions_established.append((container, line)) + # Peer promotion + if "Inbound peer promoted" in line or "Outbound handshake completed" in line: + result.peers_promoted.append((container, line)) + # Peer removal + if "Peer removed" in line: + result.peer_removals.append((container, line)) + # Parent switches + if "Parent switched" in line: + result.parent_switches.append((container, line)) + # Handshake timeouts + if "timed out" in line and ("handshake" in line.lower() or "Handshake" in line): + result.handshake_timeouts.append((container, line)) + # MMP metrics + if "MMP link metrics" in line: + result.mmp_link_metrics.append((container, line)) + if "MMP session metrics" in line: + result.mmp_session_metrics.append((container, line)) + + return result + + +def write_sim_metadata( + output_dir: str, + scenario_name: str, + seed: int, + num_nodes: int, + num_edges: int, + duration_secs: int, + topology=None, +): + """Write simulation metadata for reproducibility.""" + path = os.path.join(output_dir, "metadata.txt") + with open(path, "w") as f: + f.write(f"scenario: {scenario_name}\n") + f.write(f"seed: {seed}\n") + f.write(f"nodes: {num_nodes}\n") + f.write(f"edges: {num_edges}\n") + f.write(f"duration_secs: {duration_secs}\n") + + if topology: + f.write("\nadjacency:\n") + for nid in sorted(topology.nodes): + node = topology.nodes[nid] + peers = sorted(node.peers) + f.write(f" {nid} ({node.docker_ip}): {', '.join(peers)}\n") + f.write("\nedges:\n") + for a, b in sorted(topology.edges): + f.write(f" {a} -- {b}\n") diff --git a/testing/chaos/sim/netem.py b/testing/chaos/sim/netem.py new file mode 100644 index 0000000..c73a209 --- /dev/null +++ b/testing/chaos/sim/netem.py @@ -0,0 +1,306 @@ +"""Per-link network impairment via tc HTB + netem + u32 filters. + +Each container gets an HTB root qdisc on eth0 with one class per peer. +Each class has a netem leaf qdisc for that specific link's impairment. +u32 filters match destination IP to direct traffic to the right class. + + eth0 root (HTB 1:) + ├── class 1:1 → netem 11: (peer 1) ← u32 filter: dst= + ├── class 1:2 → netem 12: (peer 2) ← u32 filter: dst= + └── class 1:99 → pfifo (default, no impairment) +""" + +from __future__ import annotations + +import logging +import random +from dataclasses import dataclass, field + +from .docker_exec import docker_exec_quiet, is_container_running +from .scenario import BandwidthConfig, NetemConfig, NetemPolicy +from .topology import SimTopology + +log = logging.getLogger(__name__) + +IFACE = "eth0" + + +@dataclass +class NetemParams: + """Concrete netem parameters for one link direction.""" + + delay_ms: int = 0 + jitter_ms: int = 0 + loss_pct: float = 0.0 + duplicate_pct: float = 0.0 + reorder_pct: float = 0.0 + corrupt_pct: float = 0.0 + + def to_tc_args(self) -> str: + """Build the netem arguments string for tc.""" + parts = [] + if self.delay_ms > 0: + if self.jitter_ms > 0: + parts.append(f"delay {self.delay_ms}ms {self.jitter_ms}ms") + else: + parts.append(f"delay {self.delay_ms}ms") + if self.loss_pct > 0: + parts.append(f"loss {self.loss_pct:.1f}%") + if self.duplicate_pct > 0: + parts.append(f"duplicate {self.duplicate_pct:.1f}%") + if self.reorder_pct > 0 and self.delay_ms > 0: + parts.append(f"reorder {self.reorder_pct:.1f}%") + if self.corrupt_pct > 0: + parts.append(f"corrupt {self.corrupt_pct:.1f}%") + return " ".join(parts) if parts else "delay 0ms" + + +@dataclass +class LinkNetemState: + """Tracks the netem state for a single link direction (one container's view).""" + + container: str + dest_ip: str + class_id: str # e.g., "1:1" + netem_handle: str # e.g., "11:" + params: NetemParams = field(default_factory=NetemParams) + rate_mbit: int = 0 # 0 = unlimited (1gbit default) + + +class NetemManager: + """Manages per-link netem impairment across all containers.""" + + def __init__( + self, + topology: SimTopology, + config: NetemConfig, + rng: random.Random, + bandwidth: BandwidthConfig | None = None, + ): + self.topology = topology + self.config = config + self.rng = rng + # Per-container, per-dest-ip netem state + self.states: dict[str, dict[str, LinkNetemState]] = {} + # Nodes currently down (updated by NodeManager) — skip tc ops on these + self.down_nodes: set[str] = set() + # Per-edge bandwidth: (node_a, node_b) -> rate in mbit + self._edge_rates: dict[tuple[str, str], int] = {} + if bandwidth and bandwidth.enabled: + for a, b in topology.edges: + rate = rng.choice(bandwidth.tiers_mbps) + self._edge_rates[(a, b)] = rate + self._edge_rates[(b, a)] = rate + + def _htb_rate(self, node_id: str, peer_id: str) -> str: + """Return the HTB rate string for a link direction.""" + rate = self._edge_rates.get((node_id, peer_id), 0) + return f"{rate}mbit" if rate > 0 else "1gbit" + + def setup_initial(self): + """Set up HTB qdiscs and initial netem on all containers.""" + if self._edge_rates: + log.info("Bandwidth pacing enabled (%d edges with rate limits)", + len(self._edge_rates) // 2) + + for node_id in sorted(self.topology.nodes): + node = self.topology.nodes[node_id] + container = self.topology.container_name(node_id) + peer_ips = {} + for peer_id in sorted(node.peers): + peer_ips[peer_id] = self.topology.nodes[peer_id].docker_ip + + if not peer_ips: + continue + + # Build all tc commands for this container + cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"] + cmds.append( + f"tc qdisc add dev {IFACE} root handle 1: htb default 99" + ) + cmds.append( + f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit" + ) + + container_states = {} + + for idx, (peer_id, dest_ip) in enumerate(peer_ips.items(), start=1): + class_id = f"1:{idx}" + netem_handle = f"{idx + 10}:" + + # Sample initial params from default policy + params = self._sample_policy(self.config.default_policy) + + rate = self._htb_rate(node_id, peer_id) + rate_mbit = self._edge_rates.get((node_id, peer_id), 0) + cmds.append( + f"tc class add dev {IFACE} parent 1: classid {class_id} htb rate {rate}" + ) + cmds.append( + f"tc qdisc add dev {IFACE} parent {class_id} " + f"handle {netem_handle} netem {params.to_tc_args()}" + ) + cmds.append( + f"tc filter add dev {IFACE} parent 1: protocol ip " + f"prio {idx} u32 match ip dst {dest_ip}/32 flowid {class_id}" + ) + + state = LinkNetemState( + container=container, + dest_ip=dest_ip, + class_id=class_id, + netem_handle=netem_handle, + params=params, + rate_mbit=rate_mbit, + ) + container_states[dest_ip] = state + + # Execute all commands in one docker exec + full_cmd = " && ".join(cmds) + result = docker_exec_quiet(container, full_cmd, timeout=30) + if result is not None: + log.info( + "Configured per-link netem on %s (%d peers)", + container, + len(peer_ips), + ) + else: + log.warning("Failed to configure netem on %s", container) + + self.states[container] = container_states + + def setup_node(self, node_id: str): + """Re-apply HTB/netem/filters for a single node (after container restart). + + Uses the saved state from the initial setup so the node gets the same + class IDs and current netem params it had before going down. + """ + container = self.topology.container_name(node_id) + container_states = self.states.get(container) + if not container_states: + log.warning("No netem state for %s, skipping setup", container) + return + + cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"] + cmds.append( + f"tc qdisc add dev {IFACE} root handle 1: htb default 99" + ) + cmds.append( + f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit" + ) + + for dest_ip, state in container_states.items(): + rate = f"{state.rate_mbit}mbit" if state.rate_mbit > 0 else "1gbit" + cmds.append( + f"tc class add dev {IFACE} parent 1: classid {state.class_id} htb rate {rate}" + ) + cmds.append( + f"tc qdisc add dev {IFACE} parent {state.class_id} " + f"handle {state.netem_handle} netem {state.params.to_tc_args()}" + ) + # Extract the priority from class_id (e.g., "1:3" -> 3) + prio = state.class_id.split(":")[1] + cmds.append( + f"tc filter add dev {IFACE} parent 1: protocol ip " + f"prio {prio} u32 match ip dst {dest_ip}/32 flowid {state.class_id}" + ) + + full_cmd = " && ".join(cmds) + result = docker_exec_quiet(container, full_cmd, timeout=30) + if result is not None: + log.info( + "Re-applied netem on %s (%d peers)", + container, + len(container_states), + ) + else: + log.warning("Failed to re-apply netem on %s", container) + + def mutate(self): + """Randomly mutate netem params on a fraction of links.""" + if not self.config.mutation.policies: + return + + # Only consider edges where both endpoints are up + live_edges = [ + (a, b) for a, b in self.topology.edges + if a not in self.down_nodes and b not in self.down_nodes + ] + if not live_edges: + return + + num_to_mutate = max(1, int(len(live_edges) * self.config.mutation.fraction)) + edges_to_mutate = self.rng.sample( + live_edges, min(num_to_mutate, len(live_edges)) + ) + + # Pick a random policy for this mutation round + policy_name = self.rng.choice(list(self.config.mutation.policies.keys())) + policy = self.config.mutation.policies[policy_name] + + log.info( + "Netem mutation: %d links -> '%s' policy", + len(edges_to_mutate), + policy_name, + ) + + for a, b in edges_to_mutate: + params = self._sample_policy(policy) + self._update_link(a, b, params) + + def _update_link(self, node_a: str, node_b: str, params: NetemParams): + """Update netem on both directions of a link.""" + for src, dst in [(node_a, node_b), (node_b, node_a)]: + if src in self.down_nodes: + continue + container = self.topology.container_name(src) + + # Safety net: detect containers that crashed outside of NodeManager + if not is_container_running(container): + log.debug( + "Container %s not running (unexpected), marking %s as down", + container, + src, + ) + self.down_nodes.add(src) + continue + + dest_ip = self.topology.nodes[dst].docker_ip + + states = self.states.get(container, {}) + state = states.get(dest_ip) + if state is None: + continue + + cmd = ( + f"tc qdisc replace dev {IFACE} parent {state.class_id} " + f"handle {state.netem_handle} netem {params.to_tc_args()}" + ) + result = docker_exec_quiet(container, cmd) + if result is not None: + state.params = params + log.debug( + "Updated netem %s -> %s: %s", + src, + dst, + params.to_tc_args(), + ) + + def _sample_policy(self, policy: NetemPolicy) -> NetemParams: + """Sample concrete params from a policy's ranges.""" + return NetemParams( + delay_ms=int(self.rng.uniform(policy.delay_ms[0], policy.delay_ms[1])), + jitter_ms=int(self.rng.uniform(policy.jitter_ms[0], policy.jitter_ms[1])), + loss_pct=round( + self.rng.uniform(policy.loss_pct[0], policy.loss_pct[1]), 1 + ), + duplicate_pct=round( + self.rng.uniform(policy.duplicate_pct[0], policy.duplicate_pct[1]), 1 + ), + reorder_pct=round( + self.rng.uniform(policy.reorder_pct[0], policy.reorder_pct[1]), 1 + ), + corrupt_pct=round( + self.rng.uniform(policy.corrupt_pct[0], policy.corrupt_pct[1]), 1 + ), + ) diff --git a/testing/chaos/sim/nodes.py b/testing/chaos/sim/nodes.py new file mode 100644 index 0000000..4ca6972 --- /dev/null +++ b/testing/chaos/sim/nodes.py @@ -0,0 +1,193 @@ +"""Node churn simulation via docker stop/start. + +Simulates node loss and recovery by stopping and restarting containers. +When a node restarts, its FIPS process starts fresh — all peer +connections, sessions, and routing state are lost. Peers detect the +loss via handshake/link timeouts and reconverge the spanning tree. + +After restart, netem rules must be re-applied since tc state is lost +when the container stops. +""" + +from __future__ import annotations + +import logging +import random +import time +from collections import deque +from dataclasses import dataclass, field + +from .docker_exec import docker_exec_quiet, is_container_running +from .scenario import NodeChurnConfig +from .topology import SimTopology + +log = logging.getLogger(__name__) + + +@dataclass +class NodeState: + node_id: str + is_down: bool = False + down_since: float | None = None + restore_at: float | None = None + + +class NodeManager: + """Manages node stop/start lifecycle.""" + + def __init__( + self, + topology: SimTopology, + config: NodeChurnConfig, + rng: random.Random, + netem_mgr=None, + down_nodes: set[str] | None = None, + ): + self.topology = topology + self.config = config + self.rng = rng + self.netem_mgr = netem_mgr + self.down_nodes = down_nodes or set() + self.node_states: dict[str, NodeState] = { + nid: NodeState(node_id=nid) for nid in topology.nodes + } + + @property + def down_count(self) -> int: + return sum(1 for ns in self.node_states.values() if ns.is_down) + + def maybe_kill(self): + """Attempt to stop a random node.""" + if self.down_count >= self.config.max_down_nodes: + log.debug( + "At max_down_nodes (%d), skipping churn", + self.config.max_down_nodes, + ) + return + + # Candidate: any node that's currently up + up_nodes = [nid for nid, ns in self.node_states.items() if not ns.is_down] + if not up_nodes: + return + + self.rng.shuffle(up_nodes) + + for node_id in up_nodes: + if self.config.protect_connectivity and self._would_disconnect(node_id): + log.debug("Skipping %s (would disconnect graph)", node_id) + continue + + down_duration = self.rng.uniform( + self.config.down_duration_secs.min, + self.config.down_duration_secs.max, + ) + self._stop_node(node_id, down_duration) + return + + log.debug("No safe node to kill (all would disconnect)") + + def restore_expired(self): + """Restart nodes whose down duration has expired.""" + now = time.time() + for nid, state in self.node_states.items(): + if state.is_down and state.restore_at and now >= state.restore_at: + self._start_node(nid) + + def restore_all(self): + """Restart all stopped nodes (for teardown — needed for log collection).""" + for nid, state in list(self.node_states.items()): + if state.is_down: + self._start_node(nid) + + def _stop_node(self, node_id: str, duration: float): + """Stop a container.""" + container = self.topology.container_name(node_id) + docker_exec_quiet(container, "kill 1", timeout=5) # SIGTERM to PID 1 + # Use docker stop with a short grace period + import subprocess + + subprocess.run( + ["docker", "stop", "-t", "2", container], + capture_output=True, + timeout=15, + ) + + now = time.time() + state = self.node_states[node_id] + state.is_down = True + state.down_since = now + state.restore_at = now + duration + + # Tell other managers to skip this node + self.down_nodes.add(node_id) + + log.info("Node STOPPED: %s (restore in %.0fs)", node_id, duration) + + def _start_node(self, node_id: str): + """Start a stopped container and re-apply netem.""" + container = self.topology.container_name(node_id) + import subprocess + + result = subprocess.run( + ["docker", "start", container], + capture_output=True, + text=True, + timeout=15, + ) + if result.returncode != 0: + log.warning("Failed to start %s: %s", container, result.stderr) + return + + state = self.node_states[node_id] + down_for = time.time() - state.down_since if state.down_since else 0 + state.is_down = False + state.down_since = None + state.restore_at = None + + # Clear the down-node flag before re-applying netem + self.down_nodes.discard(node_id) + + log.info("Node STARTED: %s (was down %.0fs)", node_id, down_for) + + # Re-apply netem after a brief delay for the container to initialize + if self.netem_mgr: + # Small delay for container networking to be ready + time.sleep(1) + self.netem_mgr.setup_node(node_id) + + def _would_disconnect(self, node_id: str) -> bool: + """Check if removing this node (plus currently-down nodes) disconnects the graph. + + Builds the subgraph of currently-up nodes (excluding the candidate) + and checks connectivity via BFS. + """ + # Active nodes: up and not the candidate + active_nodes = set() + for nid, state in self.node_states.items(): + if not state.is_down and nid != node_id: + active_nodes.add(nid) + + if len(active_nodes) <= 1: + return True # Can't remove from a 1-node graph + + # Build adjacency for active nodes only + adj: dict[str, list[str]] = {nid: [] for nid in active_nodes} + for a, b in self.topology.edges: + if a in active_nodes and b in active_nodes: + adj[a].append(b) + adj[b].append(a) + + # BFS + start = next(iter(active_nodes)) + visited = set() + queue = deque([start]) + while queue: + node = queue.popleft() + if node in visited: + continue + visited.add(node) + for neighbor in adj[node]: + if neighbor not in visited: + queue.append(neighbor) + + return len(visited) < len(active_nodes) diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py new file mode 100644 index 0000000..9e74787 --- /dev/null +++ b/testing/chaos/sim/runner.py @@ -0,0 +1,272 @@ +"""Main simulation orchestration.""" + +from __future__ import annotations + +import logging +import os +import random +import signal +import sys +import time + +from .compose import generate_compose +from .config_gen import write_configs +from .docker_exec import docker_compose +from .links import LinkManager +from .logs import AnalysisResult, analyze_logs, collect_logs, write_sim_metadata +from .netem import NetemManager +from .nodes import NodeManager +from .scenario import Scenario +from .topology import SimTopology, generate_topology +from .traffic import TrafficManager + +log = logging.getLogger(__name__) + + +class SimRunner: + def __init__(self, scenario: Scenario): + self.scenario = scenario + self.rng = random.Random(scenario.seed) + self.topology: SimTopology | None = None + self.compose_file: str | None = None + self.output_dir: str = scenario.logging.output_dir + self._interrupted = False + + # Shared set of currently-down node IDs (updated by NodeManager, + # read by NetemManager, LinkManager, TrafficManager) + self._down_nodes: set[str] = set() + + # Managers (initialized during setup) + self.netem_mgr: NetemManager | None = None + self.link_mgr: LinkManager | None = None + self.traffic_mgr: TrafficManager | None = None + self.node_mgr: NodeManager | None = None + + def run(self) -> AnalysisResult | None: + """Run the full simulation lifecycle.""" + signal.signal(signal.SIGINT, self._handle_sigint) + signal.signal(signal.SIGTERM, self._handle_sigint) + + result = None + try: + self._setup() + self._warmup() + self._simulation_loop() + except Exception: + log.exception("Simulation failed") + finally: + result = self._teardown() + + return result + + def _handle_sigint(self, signum, frame): + if self._interrupted: + log.warning("Force exit") + sys.exit(1) + log.info("Interrupt received, shutting down gracefully...") + self._interrupted = True + + def _setup(self): + """Generate topology, configs, compose file. Start containers.""" + s = self.scenario + mesh_name = f"sim-{s.name}-{s.seed}" + + # 1. Generate topology + log.info( + "Generating %d-node %s topology (seed=%d)...", + s.topology.num_nodes, + s.topology.algorithm, + s.seed, + ) + self.topology = generate_topology(s.topology, self.rng, mesh_name) + log.info( + "Topology: %d nodes, %d edges", + len(self.topology.nodes), + len(self.topology.edges), + ) + + # Log adjacency summary + for nid in sorted(self.topology.nodes): + peers = sorted(self.topology.nodes[nid].peers) + log.info(" %s: peers=%s", nid, ",".join(peers)) + + # 2. Generate configs + docker_network_dir = os.path.join(os.path.dirname(__file__), "..") + config_dir = os.path.normpath( + os.path.join(docker_network_dir, "generated-configs", "sim") + ) + write_configs(self.topology, config_dir) + log.info("Wrote node configs to %s", config_dir) + + # 3. Generate docker-compose.yml + self.compose_file = generate_compose(self.topology, self.scenario, config_dir) + log.info("Wrote %s", self.compose_file) + + # 4. Build images (reuses Docker cache) + log.info("Building Docker images...") + docker_compose(self.compose_file, ["build"]) + + # 5. Start containers + log.info("Starting %d containers...", len(self.topology.nodes)) + docker_compose(self.compose_file, ["up", "-d"]) + + # 6. Initialize managers + if s.netem.enabled: + bw = s.bandwidth if s.bandwidth.enabled else None + self.netem_mgr = NetemManager(self.topology, s.netem, self.rng, bandwidth=bw) + self.netem_mgr.down_nodes = self._down_nodes + log.info("Applying initial per-link netem...") + self.netem_mgr.setup_initial() + + if s.link_flaps.enabled: + self.link_mgr = LinkManager( + self.topology, s.link_flaps, self.rng, netem_mgr=self.netem_mgr + ) + + if s.traffic.enabled: + self.traffic_mgr = TrafficManager( + self.topology, s.traffic, self.rng, down_nodes=self._down_nodes + ) + + if s.node_churn.enabled: + self.node_mgr = NodeManager( + self.topology, s.node_churn, self.rng, + netem_mgr=self.netem_mgr, down_nodes=self._down_nodes, + ) + + def _warmup(self): + """Wait for mesh convergence.""" + n = len(self.topology.nodes) + wait = max(10, n) # Heuristic: ~1s per node, minimum 10s + log.info("Waiting %ds for mesh convergence...", wait) + self._sleep(wait) + + def _simulation_loop(self): + """Main event loop driving stochastic behavior.""" + start = time.time() + s = self.scenario + duration = s.duration_secs + log.info("Simulation running for %ds...", duration) + + # Schedule first events + next_netem = self._schedule_next(start, s.netem.mutation.interval_secs) if self.netem_mgr else float("inf") + next_flap = self._schedule_next(start, s.link_flaps.interval_secs) if self.link_mgr else float("inf") + next_traffic = self._schedule_next(start, s.traffic.interval_secs) if self.traffic_mgr else float("inf") + next_churn = self._schedule_next(start, s.node_churn.interval_secs) if self.node_mgr else float("inf") + + while not self._interrupted: + now = time.time() + elapsed = now - start + if elapsed >= duration: + break + + # Netem mutation + if self.netem_mgr and now >= next_netem: + self.netem_mgr.mutate() + next_netem = self._schedule_next(now, s.netem.mutation.interval_secs) + + # Link flaps + if self.link_mgr: + if now >= next_flap: + self.link_mgr.maybe_flap() + next_flap = self._schedule_next(now, s.link_flaps.interval_secs) + self.link_mgr.restore_expired() + + # Traffic generation + if self.traffic_mgr: + if now >= next_traffic: + self.traffic_mgr.maybe_spawn() + next_traffic = self._schedule_next(now, s.traffic.interval_secs) + self.traffic_mgr.cleanup_expired() + + # Node churn + if self.node_mgr: + if now >= next_churn: + self.node_mgr.maybe_kill() + next_churn = self._schedule_next(now, s.node_churn.interval_secs) + self.node_mgr.restore_expired() + + # Status line + down_links = self.link_mgr.down_count if self.link_mgr else 0 + down_nodes = self.node_mgr.down_count if self.node_mgr else 0 + active = self.traffic_mgr.active_count if self.traffic_mgr else 0 + print( + f"\r [{elapsed:.0f}s/{duration}s] " + f"nodes={len(self.topology.nodes)} " + f"edges={len(self.topology.edges)} " + f"links_down={down_links} " + f"nodes_down={down_nodes} " + f"traffic={active} ", + end="", + flush=True, + ) + + self._sleep(1) + + print() # Clear status line + + def _teardown(self) -> AnalysisResult | None: + """Stop dynamic elements, collect logs, analyze, stop containers.""" + result = None + + if self.topology and self.compose_file: + # Stop traffic + if self.traffic_mgr: + log.info("Stopping traffic sessions...") + self.traffic_mgr.stop_all() + + # Restore links + if self.link_mgr: + log.info("Restoring downed links...") + self.link_mgr.restore_all() + + # Restore stopped nodes (needed for log collection) + if self.node_mgr: + log.info("Restoring stopped nodes...") + self.node_mgr.restore_all() + + # Collect logs before stopping containers + os.makedirs(self.output_dir, exist_ok=True) + container_names = [ + self.topology.container_name(nid) for nid in sorted(self.topology.nodes) + ] + log.info("Collecting logs from %d containers...", len(container_names)) + logs = collect_logs(container_names, self.output_dir) + + # Analyze + result = analyze_logs(logs) + analysis_path = os.path.join(self.output_dir, "analysis.txt") + with open(analysis_path, "w") as f: + f.write(result.summary()) + print(result.summary()) + + # Write metadata + write_sim_metadata( + self.output_dir, + scenario_name=self.scenario.name, + seed=self.scenario.seed, + num_nodes=len(self.topology.nodes), + num_edges=len(self.topology.edges), + duration_secs=self.scenario.duration_secs, + topology=self.topology, + ) + + # Stop containers + log.info("Stopping containers...") + docker_compose( + self.compose_file, + ["down"], + check=False, + ) + + return result + + def _schedule_next(self, now: float, interval) -> float: + """Schedule the next event using a Range interval.""" + return now + self.rng.uniform(interval.min, interval.max) + + def _sleep(self, seconds: float): + """Sleep in small increments so SIGINT can break out.""" + end = time.time() + seconds + while time.time() < end and not self._interrupted: + time.sleep(min(0.5, end - time.time())) diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py new file mode 100644 index 0000000..46113d5 --- /dev/null +++ b/testing/chaos/sim/scenario.py @@ -0,0 +1,270 @@ +"""Scenario YAML loading and validation.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field + +import yaml + + +@dataclass +class Range: + """A min/max range for stochastic parameters.""" + + min: float + max: float + + def validate(self, name: str): + if self.min > self.max: + raise ValueError(f"{name}: min ({self.min}) > max ({self.max})") + if self.min < 0: + raise ValueError(f"{name}: min ({self.min}) must be >= 0") + + +@dataclass +class TopologyConfig: + num_nodes: int = 10 + algorithm: str = "random_geometric" + params: dict = field(default_factory=dict) + ensure_connected: bool = True + subnet: str = "172.20.0.0/24" + ip_start: int = 10 + + +@dataclass +class NetemPolicy: + delay_ms: tuple[float, float] = (0, 0) + jitter_ms: tuple[float, float] = (0, 0) + loss_pct: tuple[float, float] = (0, 0) + duplicate_pct: tuple[float, float] = (0, 0) + reorder_pct: tuple[float, float] = (0, 0) + corrupt_pct: tuple[float, float] = (0, 0) + + +@dataclass +class NetemMutationConfig: + interval_secs: Range = field(default_factory=lambda: Range(15, 30)) + fraction: float = 0.3 + policies: dict[str, NetemPolicy] = field(default_factory=dict) + + +@dataclass +class NetemConfig: + enabled: bool = False + default_policy: NetemPolicy = field(default_factory=NetemPolicy) + mutation: NetemMutationConfig = field(default_factory=NetemMutationConfig) + + +@dataclass +class LinkFlapsConfig: + enabled: bool = False + interval_secs: Range = field(default_factory=lambda: Range(20, 60)) + max_down_links: int = 2 + down_duration_secs: Range = field(default_factory=lambda: Range(10, 30)) + protect_connectivity: bool = True + + +@dataclass +class TrafficConfig: + enabled: bool = False + max_concurrent: int = 3 + interval_secs: Range = field(default_factory=lambda: Range(10, 30)) + duration_secs: Range = field(default_factory=lambda: Range(5, 15)) + parallel_streams: int = 4 + + +@dataclass +class NodeChurnConfig: + enabled: bool = False + interval_secs: Range = field(default_factory=lambda: Range(60, 180)) + max_down_nodes: int = 1 + down_duration_secs: Range = field(default_factory=lambda: Range(30, 90)) + protect_connectivity: bool = True + + +@dataclass +class BandwidthConfig: + """Per-link bandwidth pacing via HTB rate limiting. + + When enabled, each link is randomly assigned a rate from the tiers list. + # TODO: Add structured bandwidth assignment (e.g., per-node roles, + # asymmetric uplink/downlink, tiered by topology distance, or + # time-varying bandwidth mutations similar to netem mutation). + """ + + enabled: bool = False + tiers_mbps: list[int] = field(default_factory=lambda: [1, 10, 100, 1000]) + + +@dataclass +class LoggingConfig: + rust_log: str = "info" + output_dir: str = "./sim-results" + + +@dataclass +class Scenario: + name: str = "unnamed" + seed: int = 42 + duration_secs: int = 120 + topology: TopologyConfig = field(default_factory=TopologyConfig) + netem: NetemConfig = field(default_factory=NetemConfig) + link_flaps: LinkFlapsConfig = field(default_factory=LinkFlapsConfig) + traffic: TrafficConfig = field(default_factory=TrafficConfig) + node_churn: NodeChurnConfig = field(default_factory=NodeChurnConfig) + bandwidth: BandwidthConfig = field(default_factory=BandwidthConfig) + logging: LoggingConfig = field(default_factory=LoggingConfig) + + +def _parse_range(data, name: str) -> Range: + """Parse a {min, max} dict into a Range.""" + if isinstance(data, dict): + return Range(min=float(data["min"]), max=float(data["max"])) + raise ValueError(f"{name}: expected {{min, max}} dict, got {type(data).__name__}") + + +def _parse_netem_policy(data: dict) -> NetemPolicy: + """Parse a netem policy from a dict with [min, max] lists or {min, max} dicts.""" + policy = NetemPolicy() + for attr in ( + "delay_ms", + "jitter_ms", + "loss_pct", + "duplicate_pct", + "reorder_pct", + "corrupt_pct", + ): + if attr in data: + val = data[attr] + if isinstance(val, list) and len(val) == 2: + setattr(policy, attr, (float(val[0]), float(val[1]))) + elif isinstance(val, dict): + setattr(policy, attr, (float(val["min"]), float(val["max"]))) + else: + raise ValueError(f"netem policy {attr}: expected [min, max] or {{min, max}}") + return policy + + +def load_scenario(path: str) -> Scenario: + """Load and validate a scenario from a YAML file.""" + with open(path) as f: + raw = yaml.safe_load(f) + + s = Scenario() + + # Scenario section + sc = raw.get("scenario", {}) + s.name = sc.get("name", os.path.splitext(os.path.basename(path))[0]) + s.seed = int(sc.get("seed", 42)) + s.duration_secs = int(sc.get("duration_secs", 120)) + + # Topology section + tc = raw.get("topology", {}) + s.topology.num_nodes = int(tc.get("num_nodes", 10)) + s.topology.algorithm = tc.get("algorithm", "random_geometric") + s.topology.params = tc.get("params", {}) + s.topology.ensure_connected = tc.get("ensure_connected", True) + s.topology.subnet = tc.get("subnet", "172.20.0.0/24") + s.topology.ip_start = int(tc.get("ip_start", 10)) + + # Netem section + nc = raw.get("netem", {}) + s.netem.enabled = nc.get("enabled", False) + if "default_policy" in nc: + s.netem.default_policy = _parse_netem_policy(nc["default_policy"]) + if "mutation" in nc: + mc = nc["mutation"] + s.netem.mutation.interval_secs = _parse_range( + mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs" + ) + s.netem.mutation.fraction = float(mc.get("fraction", 0.3)) + if "policies" in mc: + s.netem.mutation.policies = { + name: _parse_netem_policy(pdata) + for name, pdata in mc["policies"].items() + } + + # Link flaps section + lf = raw.get("link_flaps", {}) + s.link_flaps.enabled = lf.get("enabled", False) + if "interval_secs" in lf: + s.link_flaps.interval_secs = _parse_range(lf["interval_secs"], "link_flaps.interval_secs") + s.link_flaps.max_down_links = int(lf.get("max_down_links", 2)) + if "down_duration_secs" in lf: + s.link_flaps.down_duration_secs = _parse_range( + lf["down_duration_secs"], "link_flaps.down_duration_secs" + ) + s.link_flaps.protect_connectivity = lf.get("protect_connectivity", True) + + # Traffic section + tf = raw.get("traffic", {}) + s.traffic.enabled = tf.get("enabled", False) + s.traffic.max_concurrent = int(tf.get("max_concurrent", 3)) + if "interval_secs" in tf: + s.traffic.interval_secs = _parse_range(tf["interval_secs"], "traffic.interval_secs") + if "duration_secs" in tf: + s.traffic.duration_secs = _parse_range(tf["duration_secs"], "traffic.duration_secs") + s.traffic.parallel_streams = int(tf.get("parallel_streams", 4)) + + # Node churn section + nc2 = raw.get("node_churn", {}) + s.node_churn.enabled = nc2.get("enabled", False) + if "interval_secs" in nc2: + s.node_churn.interval_secs = _parse_range(nc2["interval_secs"], "node_churn.interval_secs") + s.node_churn.max_down_nodes = int(nc2.get("max_down_nodes", 1)) + if "down_duration_secs" in nc2: + s.node_churn.down_duration_secs = _parse_range( + nc2["down_duration_secs"], "node_churn.down_duration_secs" + ) + s.node_churn.protect_connectivity = nc2.get("protect_connectivity", True) + + # Bandwidth section + bw = raw.get("bandwidth", {}) + s.bandwidth.enabled = bw.get("enabled", False) + if "tiers_mbps" in bw: + tiers = bw["tiers_mbps"] + if not isinstance(tiers, list) or not tiers: + raise ValueError("bandwidth.tiers_mbps must be a non-empty list") + s.bandwidth.tiers_mbps = [int(t) for t in tiers] + + # Logging section + lg = raw.get("logging", {}) + s.logging.rust_log = lg.get("rust_log", "info") + s.logging.output_dir = lg.get("output_dir", "./sim-results") + + # Validation + _validate(s) + + return s + + +def _validate(s: Scenario): + """Validate scenario constraints.""" + if s.topology.num_nodes < 2: + raise ValueError("topology.num_nodes must be >= 2") + if s.topology.num_nodes > 250: + raise ValueError("topology.num_nodes must be <= 250 (subnet limit)") + if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain"): + raise ValueError(f"Unknown topology algorithm: {s.topology.algorithm}") + if s.duration_secs < 1: + raise ValueError("duration_secs must be >= 1") + + # Validate ranges + if s.netem.enabled and s.netem.mutation.policies: + s.netem.mutation.interval_secs.validate("netem.mutation.interval_secs") + if s.link_flaps.enabled: + s.link_flaps.interval_secs.validate("link_flaps.interval_secs") + s.link_flaps.down_duration_secs.validate("link_flaps.down_duration_secs") + if s.traffic.enabled: + s.traffic.interval_secs.validate("traffic.interval_secs") + s.traffic.duration_secs.validate("traffic.duration_secs") + if s.node_churn.enabled: + s.node_churn.interval_secs.validate("node_churn.interval_secs") + s.node_churn.down_duration_secs.validate("node_churn.down_duration_secs") + if s.node_churn.max_down_nodes >= s.topology.num_nodes: + raise ValueError("node_churn.max_down_nodes must be < topology.num_nodes") + if s.bandwidth.enabled: + for tier in s.bandwidth.tiers_mbps: + if tier <= 0: + raise ValueError(f"bandwidth.tiers_mbps: all values must be > 0, got {tier}") diff --git a/testing/chaos/sim/topology.py b/testing/chaos/sim/topology.py new file mode 100644 index 0000000..c510890 --- /dev/null +++ b/testing/chaos/sim/topology.py @@ -0,0 +1,181 @@ +"""Topology generation: random graphs with connectivity guarantees.""" + +from __future__ import annotations + +import math +import random +from collections import deque +from dataclasses import dataclass, field + +from .keys import derive +from .scenario import TopologyConfig + + +@dataclass +class SimNode: + node_id: str # "n01", "n02", ... + docker_ip: str # "172.20.0.10", ... + nsec: str # 64-char hex + npub: str # bech32 npub1... + peers: list[str] = field(default_factory=list) + + +@dataclass +class SimTopology: + nodes: dict[str, SimNode] = field(default_factory=dict) + edges: set[tuple[str, str]] = field(default_factory=set) + + def is_connected(self) -> bool: + """BFS connectivity check.""" + if len(self.nodes) <= 1: + return True + start = next(iter(self.nodes)) + visited = set() + queue = deque([start]) + while queue: + node = queue.popleft() + if node in visited: + continue + visited.add(node) + for peer in self.nodes[node].peers: + if peer not in visited: + queue.append(peer) + return len(visited) == len(self.nodes) + + def neighbors(self, node_id: str) -> list[str]: + return self.nodes[node_id].peers + + def would_disconnect(self, edge: tuple[str, str]) -> bool: + """Check if removing this edge would disconnect the graph.""" + a, b = edge + # Temporarily remove edge + self.nodes[a].peers.remove(b) + self.nodes[b].peers.remove(a) + connected = self.is_connected() + # Restore + self.nodes[a].peers.append(b) + self.nodes[b].peers.append(a) + return not connected + + def container_name(self, node_id: str) -> str: + return f"fips-node-{node_id}" + + +def generate_topology( + config: TopologyConfig, + rng: random.Random, + mesh_name: str, +) -> SimTopology: + """Generate a topology according to the config.""" + n = config.num_nodes + subnet_base = config.subnet.rsplit(".", 1)[0] # "172.20.0" + + # Create nodes with IPs and keys + nodes: dict[str, SimNode] = {} + for i in range(n): + node_id = f"n{i + 1:02d}" + docker_ip = f"{subnet_base}.{config.ip_start + i}" + nsec, npub = derive(mesh_name, node_id) + nodes[node_id] = SimNode( + node_id=node_id, + docker_ip=docker_ip, + nsec=nsec, + npub=npub, + ) + + node_ids = sorted(nodes.keys()) + + # Generate edges + if config.algorithm == "chain": + edges = _generate_chain(node_ids) + elif config.algorithm == "random_geometric": + radius = config.params.get("radius", 0.5) + edges = _generate_random_geometric(node_ids, radius, rng) + elif config.algorithm == "erdos_renyi": + p = config.params.get("p", 0.3) + edges = _generate_erdos_renyi(node_ids, p, rng) + else: + raise ValueError(f"Unknown algorithm: {config.algorithm}") + + # Build peer lists from edges + for a, b in edges: + nodes[a].peers.append(b) + nodes[b].peers.append(a) + + topo = SimTopology(nodes=nodes, edges=edges) + + # Connectivity check with retry + if config.ensure_connected: + max_retries = 50 + attempt = 0 + while not topo.is_connected() and attempt < max_retries: + attempt += 1 + # Clear and regenerate + for node in nodes.values(): + node.peers.clear() + + if config.algorithm == "random_geometric": + edges = _generate_random_geometric(node_ids, radius, rng) + elif config.algorithm == "erdos_renyi": + edges = _generate_erdos_renyi(node_ids, p, rng) + else: + break # chain is always connected + + for a, b in edges: + nodes[a].peers.append(b) + nodes[b].peers.append(a) + + topo.edges = edges + + if not topo.is_connected(): + raise RuntimeError( + f"Failed to generate connected topology after {max_retries} attempts" + ) + + return topo + + +def _generate_chain(node_ids: list[str]) -> set[tuple[str, str]]: + """Linear topology: n01-n02-n03-...""" + edges = set() + for i in range(len(node_ids) - 1): + edge = _make_edge(node_ids[i], node_ids[i + 1]) + edges.add(edge) + return edges + + +def _generate_random_geometric( + node_ids: list[str], + radius: float, + rng: random.Random, +) -> set[tuple[str, str]]: + """Place nodes randomly in [0,1]^2, connect if distance < radius.""" + positions = {nid: (rng.random(), rng.random()) for nid in node_ids} + edges = set() + for i, a in enumerate(node_ids): + for b in node_ids[i + 1 :]: + ax, ay = positions[a] + bx, by = positions[b] + dist = math.sqrt((ax - bx) ** 2 + (ay - by) ** 2) + if dist < radius: + edges.add(_make_edge(a, b)) + return edges + + +def _generate_erdos_renyi( + node_ids: list[str], + p: float, + rng: random.Random, +) -> set[tuple[str, str]]: + """Include each edge with probability p.""" + edges = set() + for i, a in enumerate(node_ids): + for b in node_ids[i + 1 :]: + if rng.random() < p: + edges.add(_make_edge(a, b)) + return edges + + +def _make_edge(a: str, b: str) -> tuple[str, str]: + """Canonical edge representation (sorted).""" + return (min(a, b), max(a, b)) diff --git a/testing/chaos/sim/traffic.py b/testing/chaos/sim/traffic.py new file mode 100644 index 0000000..12ea717 --- /dev/null +++ b/testing/chaos/sim/traffic.py @@ -0,0 +1,127 @@ +"""Random iperf3 traffic generation between node pairs. + +Spawns iperf3 clients as background processes in containers. The iperf3 +server is already running in each container (started by the Dockerfile +entrypoint). +""" + +from __future__ import annotations + +import logging +import random +import time +from dataclasses import dataclass, field + +from .docker_exec import docker_exec_quiet +from .scenario import TrafficConfig +from .topology import SimTopology + +log = logging.getLogger(__name__) + + +@dataclass +class TrafficSession: + client_node: str + server_node: str + started_at: float + duration_secs: int + container: str + + +class TrafficManager: + """Manages random iperf3 sessions across the mesh.""" + + def __init__( + self, + topology: SimTopology, + config: TrafficConfig, + rng: random.Random, + down_nodes: set[str] | None = None, + ): + self.topology = topology + self.config = config + self.rng = rng + self.down_nodes = down_nodes or set() + self.active_sessions: list[TrafficSession] = [] + + @property + def active_count(self) -> int: + return len(self.active_sessions) + + def maybe_spawn(self): + """Spawn a new iperf3 session if under the concurrency limit.""" + if self.active_count >= self.config.max_concurrent: + log.debug( + "At max_concurrent (%d), skipping traffic spawn", + self.config.max_concurrent, + ) + return + + node_ids = [nid for nid in self.topology.nodes if nid not in self.down_nodes] + if len(node_ids) < 2: + return + + # Pick random client and server (different nodes, both up) + client, server = self.rng.sample(node_ids, 2) + server_npub = self.topology.nodes[server].npub + container = self.topology.container_name(client) + + duration = int( + self.rng.uniform( + self.config.duration_secs.min, + self.config.duration_secs.max, + ) + ) + streams = self.config.parallel_streams + + # Start iperf3 in background (nohup, stdout to /dev/null) + cmd = ( + f"nohup iperf3 -c {server_npub}.fips -t {duration} " + f"-P {streams} > /dev/null 2>&1 &" + ) + result = docker_exec_quiet(container, cmd) + if result is not None: + session = TrafficSession( + client_node=client, + server_node=server, + started_at=time.time(), + duration_secs=duration, + container=container, + ) + self.active_sessions.append(session) + log.info( + "Traffic: %s -> %s (%ds, %d streams)", + client, + server, + duration, + streams, + ) + else: + log.warning("Failed to start iperf3 on %s", container) + + def cleanup_expired(self): + """Remove sessions that have completed (based on time).""" + now = time.time() + grace = 5 # seconds after expected completion + before = len(self.active_sessions) + self.active_sessions = [ + s + for s in self.active_sessions + if now - s.started_at < s.duration_secs + grace + ] + removed = before - len(self.active_sessions) + if removed > 0: + log.debug("Cleaned up %d expired traffic sessions", removed) + + def stop_all(self): + """Kill all iperf3 client processes in running containers.""" + seen = set() + for session in self.active_sessions: + if session.container not in seen: + if session.client_node not in self.down_nodes: + docker_exec_quiet( + session.container, + "killall iperf3 2>/dev/null; true", + ) + seen.add(session.container) + self.active_sessions.clear() diff --git a/examples/docker-network/.env b/testing/static/.env similarity index 100% rename from examples/docker-network/.env rename to testing/static/.env diff --git a/testing/static/.gitignore b/testing/static/.gitignore new file mode 100644 index 0000000..151aa8e --- /dev/null +++ b/testing/static/.gitignore @@ -0,0 +1,2 @@ +fips +generated-configs diff --git a/testing/static/Dockerfile b/testing/static/Dockerfile new file mode 100644 index 0000000..7be38c6 --- /dev/null +++ b/testing/static/Dockerfile @@ -0,0 +1,34 @@ +FROM debian:bookworm-slim + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + iproute2 iputils-ping dnsutils openssh-client openssh-server iperf3 \ + dnsmasq curl python3 rsync && \ + rm -rf /var/lib/apt/lists/* + +# Setup SSH server with no authentication (test only!) +RUN mkdir -p /var/run/sshd && \ + ssh-keygen -A && \ + sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + sed -i 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/' /etc/ssh/sshd_config && \ + sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config && \ + passwd -d root + +# dnsmasq: forward .fips to FIPS daemon, everything else to Docker DNS +RUN printf '%s\n' \ + 'port=53' \ + 'listen-address=127.0.0.1' \ + 'bind-interfaces' \ + 'server=/fips/127.0.0.1#5354' \ + 'server=127.0.0.11' \ + 'no-resolv' \ + >> /etc/dnsmasq.conf + +COPY fips /usr/local/bin/ +RUN chmod +x /usr/local/bin/fips + +# Static web page served via Python HTTP server +RUN printf 'Fuck IPs!\n' > /root/index.html + +# Start dnsmasq, SSH server, iperf3, and HTTP server in background, then run FIPS +ENTRYPOINT ["/bin/bash", "-c", "dnsmasq && /usr/sbin/sshd && iperf3 -s -D && python3 -m http.server 8000 -d /root -b :: &>/dev/null & exec fips --config /etc/fips/fips.yaml"] diff --git a/examples/docker-network/README.md b/testing/static/README.md similarity index 65% rename from examples/docker-network/README.md rename to testing/static/README.md index b4945c9..57fd655 100644 --- a/examples/docker-network/README.md +++ b/testing/static/README.md @@ -1,10 +1,11 @@ -# Docker Network Test Harness +# Static Docker Network Test Harness -Multi-node integration test for FIPS using Docker containers. Multiple -topologies are provided: a sparse mesh (5 nodes, 6 links), a linear chain -(5 nodes, 4 links), and a mesh with a public external node. All exercise the -full FIPS stack including TUN devices, DNS resolution, peer link encryption, -spanning tree construction, and discovery-driven multi-hop routing. +Multi-node integration test for FIPS using Docker containers with fixed +topologies. Multiple topologies are provided: a sparse mesh (5 nodes, 6 +links), a linear chain (5 nodes, 4 links), and a mesh with a public external +node. All exercise the full FIPS stack including TUN devices, DNS resolution, +peer link encryption, spanning tree construction, and discovery-driven +multi-hop routing. ## Prerequisites @@ -17,25 +18,25 @@ spanning tree construction, and discovery-driven multi-hop routing. Build the binary and generate configs: ```bash -./scripts/build.sh +./testing/static/scripts/build.sh ``` Start the mesh (default topology): ```bash -docker compose up -d -./scripts/ping-test.sh mesh # 20/20 expected -./scripts/iperf-test.sh mesh # bandwidth test -docker compose down +docker compose -f testing/static/docker-compose.yml up -d +./testing/static/scripts/ping-test.sh mesh # 20/20 expected +./testing/static/scripts/iperf-test.sh mesh # bandwidth test +docker compose -f testing/static/docker-compose.yml down ``` The mesh profile is activated by default via `.env`. To use a different topology, specify the profile explicitly: ```bash -docker compose --profile chain up -d -./scripts/ping-test.sh chain -docker compose --profile chain down +docker compose -f testing/static/docker-compose.yml --profile chain up -d +./testing/static/scripts/ping-test.sh chain +docker compose -f testing/static/docker-compose.yml --profile chain down ``` ## Topologies @@ -45,7 +46,7 @@ docker compose --profile chain down ![Mesh Topology](docker-mesh-topology.svg) Five nodes with 6 bidirectional UDP links forming a sparse, fully connected -graph. Not all nodes are direct peers — non-adjacent pairs require +graph. Not all nodes are direct peers -- non-adjacent pairs require discovery-driven multi-hop routing to establish end-to-end sessions. The spanning tree is rooted at node A, which has the lexicographically @@ -57,26 +58,26 @@ covering both direct-peer and multi-hop paths. | Link | Type | |------|------| -| A — D | tree edge (D's parent is A) | -| A — E | tree edge (E's parent is A) | -| C — D | tree edge (C's parent is D) | -| B — C | tree edge (B's parent is C) | -| D — E | non-tree link | -| C — E | non-tree link | +| A -- D | tree edge (D's parent is A) | +| A -- E | tree edge (E's parent is A) | +| C -- D | tree edge (C's parent is D) | +| B -- C | tree edge (B's parent is C) | +| D -- E | non-tree link | +| C -- E | non-tree link | ### Chain ![Chain Topology](docker-chain-topology.svg) -Five nodes in a linear chain: A — B — C — D — E. Each node peers only with +Five nodes in a linear chain: A -- B -- C -- D -- E. Each node peers only with its immediate neighbors. Multi-hop communication (e.g., A to E) requires the discovery protocol to find routes through intermediate nodes. The ping test covers: -- Adjacent hops: A→B, B→C (1 hop each) -- Multi-hop: A→C (2 hops), A→D (3 hops), A→E (4 hops) -- Reverse: E→A (4 hops) +- Adjacent hops: A->B, B->C (1 hop each) +- Multi-hop: A->C (2 hops), A->D (3 hops), A->E (4 hops) +- Reverse: E->A (4 hops) ### Mesh-Public @@ -84,7 +85,7 @@ Same five Docker nodes as the mesh topology, plus an external public node (`pub`) at a remote IP. Nodes A, B, and C peer with the public node. This topology is for testing mixed local/remote mesh operation. -External nodes are not managed by Docker — only their identity and address +External nodes are not managed by Docker -- only their identity and address appear in the topology file so that Docker nodes can peer with them. ## Configuration Management @@ -92,29 +93,34 @@ appear in the topology file so that Docker nodes can peer with them. ### File Structure ```text -configs/ -├── node.template.yaml # Template for all node configs -└── topologies/ - ├── mesh.yaml # Mesh topology definition - ├── chain.yaml # Chain topology definition - └── mesh-public.yaml # Mesh + external public node - -generated-configs/ # Auto-generated (gitignored) -├── npubs.env # NPUB_A=..., NPUB_B=..., etc. -├── mesh/ -│ ├── node-a.yaml ... node-e.yaml -├── mesh-public/ -│ ├── node-a.yaml ... node-e.yaml -└── chain/ - ├── node-a.yaml ... node-e.yaml - -scripts/ -├── build.sh # Build binary + generate configs -├── generate-configs.sh # Generate node configs from topology -├── derive-keys.py # Deterministic nsec/npub derivation -├── ping-test.sh # Connectivity test -├── iperf-test.sh # Bandwidth test -└── netem.sh # Network impairment +testing/static/ +├── Dockerfile # Container image definition +├── docker-compose.yml # Service definitions for all topologies +├── resolv.conf # DNS config pointing to FIPS resolver +├── .env # Default compose profile +├── configs/ +│ ├── node.template.yaml # Template for all node configs +│ └── topologies/ +│ ├── mesh.yaml # Mesh topology definition +│ ├── chain.yaml # Chain topology definition +│ └── mesh-public.yaml # Mesh + external public node +├── generated-configs/ # Auto-generated (gitignored) +│ ├── npubs.env # NPUB_A=..., NPUB_B=..., etc. +│ ├── mesh/ +│ │ ├── node-a.yaml ... node-e.yaml +│ ├── mesh-public/ +│ │ ├── node-a.yaml ... node-e.yaml +│ └── chain/ +│ ├── node-a.yaml ... node-e.yaml +├── scripts/ +│ ├── build.sh # Build binary + generate configs +│ ├── generate-configs.sh # Generate node configs from topology +│ ├── derive-keys.py # Deterministic nsec/npub derivation +│ ├── ping-test.sh # Connectivity test +│ ├── iperf-test.sh # Bandwidth test +│ └── netem.sh # Network impairment +├── docker-mesh-topology.svg # Mesh topology diagram +└── docker-chain-topology.svg # Chain topology diagram ``` ### Topology Files @@ -144,7 +150,7 @@ in peer blocks and the npubs environment file. ### Generating Configs ```bash -./scripts/generate-configs.sh [mesh-name] +./testing/static/scripts/generate-configs.sh [mesh-name] ``` This reads the topology definition and generates: @@ -164,7 +170,7 @@ automatically after compiling. `mesh.yaml` 2. Add corresponding service definitions to `docker-compose.yml` with `profiles: [""]` -3. Run `./scripts/generate-configs.sh ` to generate configs +3. Run `./testing/static/scripts/generate-configs.sh ` to generate configs ## Deterministic Mesh Identity Derivation @@ -174,11 +180,11 @@ each mesh needs unique node identities to avoid key conflicts. The optional ```bash # Build with derived identities -./scripts/build.sh mesh my-mesh-1 +./testing/static/scripts/build.sh mesh my-mesh-1 # Or generate configs directly -./scripts/generate-configs.sh mesh my-mesh-1 -./scripts/generate-configs.sh mesh-public my-mesh-1 +./testing/static/scripts/generate-configs.sh mesh my-mesh-1 +./testing/static/scripts/generate-configs.sh mesh-public my-mesh-1 ``` ### How It Works @@ -204,7 +210,7 @@ with no external dependencies (pure Python stdlib: hashlib for SHA-256, manual secp256k1 scalar multiplication, and BIP-173 bech32 encoding): ```bash -$ ./scripts/derive-keys.py my-mesh-1 a +$ ./testing/static/scripts/derive-keys.py my-mesh-1 a nsec=<64-char-hex> npub=npub1... ``` @@ -234,8 +240,8 @@ This file is: ## Performance Testing ```bash -./scripts/iperf-test.sh [mesh|chain] -./scripts/iperf-test.sh mesh --live # show live iperf3 output +./testing/static/scripts/iperf-test.sh [mesh|chain] +./testing/static/scripts/iperf-test.sh mesh --live # show live iperf3 output ``` Runs iperf3 with: @@ -250,7 +256,7 @@ The `netem.sh` script simulates adverse network conditions using `tc`/`netem` on all running containers: ```bash -./scripts/netem.sh [mesh|chain] [options] +./testing/static/scripts/netem.sh [mesh|chain] [options] ``` ### Options @@ -277,16 +283,16 @@ on all running containers: ```bash # Apply 50ms delay with 5% packet loss -./scripts/netem.sh mesh apply --delay 50 --loss 5 +./testing/static/scripts/netem.sh mesh apply --delay 50 --loss 5 # Use a preset -./scripts/netem.sh chain apply --preset congested +./testing/static/scripts/netem.sh chain apply --preset congested # Check current rules -./scripts/netem.sh mesh status +./testing/static/scripts/netem.sh mesh status # Remove all impairment -./scripts/netem.sh mesh remove +./testing/static/scripts/netem.sh mesh remove ``` Rules are applied to egress on each container's `eth0` interface. With all @@ -336,7 +342,7 @@ docker exec fips-node-b iperf3 -c $NPUB_A.fips Force a clean rebuild: ```bash -docker compose build --no-cache +docker compose -f testing/static/docker-compose.yml build --no-cache ``` **Check node logs**: @@ -356,7 +362,7 @@ docker exec fips-node-a dig AAAA .fips @127.0.0.1 the binary inside the container: ```bash -md5sum examples/docker-network/fips +md5sum testing/static/fips docker exec fips-node-a md5sum /usr/local/bin/fips ``` @@ -365,5 +371,5 @@ convergence wait in `ping-test.sh` may be insufficient. Edit the `sleep` value at the top of the script. **Missing npubs.env**: If test scripts fail with "npubs.env not found", run -`./scripts/generate-configs.sh mesh` (or your topology) first, or use -`./scripts/build.sh` which generates configs automatically. +`./testing/static/scripts/generate-configs.sh mesh` (or your topology) first, +or use `./testing/static/scripts/build.sh` which generates configs automatically. diff --git a/testing/static/configs/node.template.yaml b/testing/static/configs/node.template.yaml new file mode 100644 index 0000000..171a1fd --- /dev/null +++ b/testing/static/configs/node.template.yaml @@ -0,0 +1,30 @@ +# FIPS Node {{NODE_NAME}} configuration ({{TOPOLOGY}} topology) +# +# Identity: {{NPUB}} + +node: + identity: + nsec: "{{NSEC}}" + +tun: + enabled: true + name: fips0 + mtu: 1280 + +dns: + enabled: true + bind_addr: "127.0.0.1" + +transports: + udp: + bind_addr: "0.0.0.0:4000" + mtu: 1472 + # recv_buf_size: 2097152 # 2 MB (default) + # send_buf_size: 2097152 # 2 MB (default) + # Note: The kernel clamps socket buffers to net.core.rmem_max / wmem_max. + # Ensure these sysctls are >= the configured buffer sizes: + # sysctl -w net.core.rmem_max=2097152 + # sysctl -w net.core.wmem_max=2097152 + +peers: +{{PEERS}} \ No newline at end of file diff --git a/examples/docker-network/configs/topologies/chain.yaml b/testing/static/configs/topologies/chain.yaml similarity index 100% rename from examples/docker-network/configs/topologies/chain.yaml rename to testing/static/configs/topologies/chain.yaml diff --git a/examples/docker-network/configs/topologies/mesh-public.yaml b/testing/static/configs/topologies/mesh-public.yaml similarity index 100% rename from examples/docker-network/configs/topologies/mesh-public.yaml rename to testing/static/configs/topologies/mesh-public.yaml diff --git a/examples/docker-network/configs/topologies/mesh.yaml b/testing/static/configs/topologies/mesh.yaml similarity index 100% rename from examples/docker-network/configs/topologies/mesh.yaml rename to testing/static/configs/topologies/mesh.yaml diff --git a/examples/docker-network/docker-chain-topology.svg b/testing/static/docker-chain-topology.svg similarity index 100% rename from examples/docker-network/docker-chain-topology.svg rename to testing/static/docker-chain-topology.svg diff --git a/examples/docker-network/docker-compose.yml b/testing/static/docker-compose.yml similarity index 100% rename from examples/docker-network/docker-compose.yml rename to testing/static/docker-compose.yml diff --git a/examples/docker-network/docker-mesh-topology.svg b/testing/static/docker-mesh-topology.svg similarity index 100% rename from examples/docker-network/docker-mesh-topology.svg rename to testing/static/docker-mesh-topology.svg diff --git a/testing/static/resolv.conf b/testing/static/resolv.conf new file mode 100644 index 0000000..bbc8559 --- /dev/null +++ b/testing/static/resolv.conf @@ -0,0 +1 @@ +nameserver 127.0.0.1 diff --git a/examples/docker-network/scripts/build.sh b/testing/static/scripts/build.sh similarity index 90% rename from examples/docker-network/scripts/build.sh rename to testing/static/scripts/build.sh index 55e2246..8d53fed 100755 --- a/examples/docker-network/scripts/build.sh +++ b/testing/static/scripts/build.sh @@ -17,7 +17,7 @@ MESH_NAME="${2:-}" PROJECT_ROOT="$(cd "$DOCKER_DIR/../.." && pwd)" if [ ! -f "$PROJECT_ROOT/Cargo.toml" ]; then echo "Error: Cannot find Cargo.toml at $PROJECT_ROOT" >&2 - echo "Expected layout: /examples/docker-network/scripts/build.sh" >&2 + echo "Expected layout: /testing/static/scripts/build.sh" >&2 exit 1 fi @@ -66,6 +66,6 @@ echo "Generating node configurations from templates..." "$SCRIPT_DIR/generate-configs.sh" "$TOPOLOGY" $MESH_NAME echo "" echo "Building Docker images..." -docker compose --profile "$TOPOLOGY" build +docker compose -f "$DOCKER_DIR/docker-compose.yml" --profile "$TOPOLOGY" build echo "" -echo "Ready: docker compose --profile $TOPOLOGY up -d" +echo "Ready: docker compose -f testing/static/docker-compose.yml --profile $TOPOLOGY up -d" diff --git a/testing/static/scripts/derive-keys.py b/testing/static/scripts/derive-keys.py new file mode 100755 index 0000000..37495e5 --- /dev/null +++ b/testing/static/scripts/derive-keys.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Derive deterministic nostr nsec/npub from mesh-name and node-name. + +Usage: derive-keys.py +Output: nsec=\nnpub= + +Derivation: nsec = sha256(mesh_name + "|" + node_name) + npub = bech32("npub", secp256k1_pubkey_x(nsec)) + +Pure Python, no external dependencies. +""" + +import hashlib +import sys + +# --- secp256k1 --- + +P = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F +Gx = 0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798 +Gy = 0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8 + + +def _modinv(a, m): + return pow(a, m - 2, m) + + +def _point_add(p1, p2): + if p1 is None: + return p2 + if p2 is None: + return p1 + x1, y1 = p1 + x2, y2 = p2 + if x1 == x2 and y1 != y2: + return None + if x1 == x2: + lam = (3 * x1 * x1) * _modinv(2 * y1, P) % P + else: + lam = (y2 - y1) * _modinv(x2 - x1, P) % P + x3 = (lam * lam - x1 - x2) % P + y3 = (lam * (x1 - x3) - y1) % P + return (x3, y3) + + +def _scalar_mult(k, point): + result = None + addend = point + while k: + if k & 1: + result = _point_add(result, addend) + addend = _point_add(addend, addend) + k >>= 1 + return result + + +# --- bech32 (BIP-173) --- + +_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" + + +def _bech32_polymod(values): + gen = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3] + chk = 1 + for v in values: + b = chk >> 25 + chk = (chk & 0x1FFFFFF) << 5 ^ v + for i in range(5): + chk ^= gen[i] if ((b >> i) & 1) else 0 + return chk + + +def _bech32_encode(hrp, data_5bit): + hrp_expand = [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp] + polymod = _bech32_polymod(hrp_expand + data_5bit + [0] * 6) ^ 1 + checksum = [(polymod >> 5 * (5 - i)) & 31 for i in range(6)] + return hrp + "1" + "".join(_CHARSET[d] for d in data_5bit + checksum) + + +def _convertbits(data, frombits, tobits): + acc, bits, ret = 0, 0, [] + maxv = (1 << tobits) - 1 + for value in data: + acc = (acc << frombits) | value + bits += frombits + while bits >= tobits: + bits -= tobits + ret.append((acc >> bits) & maxv) + if bits: + ret.append((acc << (tobits - bits)) & maxv) + return ret + + +# --- public API --- + +def derive(mesh_name, node_name): + nsec_hex = hashlib.sha256(f"{mesh_name}|{node_name}".encode()).hexdigest() + k = int(nsec_hex, 16) + pub = _scalar_mult(k, (Gx, Gy)) + x_hex = format(pub[0], "064x") + data_5bit = _convertbits(list(bytes.fromhex(x_hex)), 8, 5) + npub = _bech32_encode("npub", data_5bit) + return nsec_hex, npub + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + nsec, npub = derive(sys.argv[1], sys.argv[2]) + print(f"nsec={nsec}") + print(f"npub={npub}") diff --git a/examples/docker-network/scripts/generate-configs.sh b/testing/static/scripts/generate-configs.sh similarity index 100% rename from examples/docker-network/scripts/generate-configs.sh rename to testing/static/scripts/generate-configs.sh diff --git a/examples/docker-network/scripts/iperf-test.sh b/testing/static/scripts/iperf-test.sh similarity index 100% rename from examples/docker-network/scripts/iperf-test.sh rename to testing/static/scripts/iperf-test.sh diff --git a/examples/docker-network/scripts/netem.sh b/testing/static/scripts/netem.sh similarity index 100% rename from examples/docker-network/scripts/netem.sh rename to testing/static/scripts/netem.sh diff --git a/examples/docker-network/scripts/ping-test.sh b/testing/static/scripts/ping-test.sh similarity index 100% rename from examples/docker-network/scripts/ping-test.sh rename to testing/static/scripts/ping-test.sh