From fed6cc6987cdd61b261f577cd39cd8778af85f7d Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Fri, 20 Mar 2026 16:14:10 +0000 Subject: [PATCH] Fix chaos sim DNS pipeline, ephemeral npub tracking, and analysis Three infrastructure bugs fixed: - DNS resolution never populated the identity cache because Docker overwrites /etc/resolv.conf at container start. Fix: bind-mount resolv.conf in the chaos compose template, matching the static and sidecar test configs. - Traffic generator used config-time npubs for iperf3 targets, but ephemeral nodes generate fresh keypairs at startup. Fix: share the peer churn manager's runtime npub cache with the traffic manager. - Log analysis had no discovery counters and used wrong patterns. Fix: add discovery section with correct log message matching. Also adds timestamped output directories (YYYYMMDD-HHMMSS-scenario), coord_ttl_secs override in maelstrom.yaml, FIPS_SIM_OUTPUT env var support, and maelstrom-sparse scenario for sparse topology testing. --- testing/chaos/scenarios/maelstrom-sparse.yaml | 82 +++++++++++++++++++ testing/chaos/scenarios/maelstrom.yaml | 5 ++ testing/chaos/sim/compose.py | 8 ++ testing/chaos/sim/runner.py | 21 ++++- testing/chaos/sim/traffic.py | 4 +- testing/lib/log_analysis.py | 39 +++++++++ 6 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 testing/chaos/scenarios/maelstrom-sparse.yaml diff --git a/testing/chaos/scenarios/maelstrom-sparse.yaml b/testing/chaos/scenarios/maelstrom-sparse.yaml new file mode 100644 index 0000000..3576720 --- /dev/null +++ b/testing/chaos/scenarios/maelstrom-sparse.yaml @@ -0,0 +1,82 @@ +# Maelstrom-sparse: sparse geometric topology for discovery stress testing +# +# Uses random geometric graph with low radius to produce sparse connectivity +# (~3-4 peers per node), forcing multi-hop routing and heavy discovery usage. +# The short coord cache TTL (10s) ensures transit-warmed entries expire +# quickly, requiring rediscovery. +# +# Default: 50 nodes. Override with --nodes flag. + +scenario: + name: "maelstrom-sparse" + seed: 42 + duration_secs: 600 + +topology: + num_nodes: 50 + algorithm: random_geometric + params: + radius: 0.20 + ensure_connected: true + subnet: "172.20.0.0/16" + ip_start: 10 + transport_mix: + udp: 0.8 + tcp: 0.2 + +netem: + enabled: true + default_policy: + delay_ms: { min: 5, max: 50 } + jitter_ms: { min: 1, max: 10 } + loss_pct: { min: 0, max: 2 } + mutation: + interval_secs: { min: 20, max: 45 } + fraction: 0.3 + policies: + normal: + delay_ms: [5, 20] + loss_pct: [0, 1] + degraded: + delay_ms: [50, 100] + jitter_ms: [10, 30] + loss_pct: [3, 8] + +link_flaps: + enabled: true + interval_secs: { min: 30, max: 60 } + max_down_links: 3 + down_duration_secs: { min: 10, max: 30 } + protect_connectivity: true + +traffic: + enabled: true + max_concurrent: 10 + interval_secs: { min: 5, max: 30 } + duration_secs: { min: 5, max: 60 } + parallel_streams: 4 + +node_churn: + enabled: true + interval_secs: { min: 60, max: 120 } + max_down_nodes: 5 + down_duration_secs: { min: 30, max: 90 } + protect_connectivity: false + +peer_churn: + enabled: true + interval_secs: { min: 8, max: 12 } + ephemeral_fraction: 0.5 + +bandwidth: + enabled: true + tiers_mbps: [1, 10, 100, 1000] + +logging: + rust_log: "debug" + output_dir: "./sim-results" + +fips_overrides: + node: + cache: + coord_ttl_secs: 10 diff --git a/testing/chaos/scenarios/maelstrom.yaml b/testing/chaos/scenarios/maelstrom.yaml index 1f5b1ca..66d4180 100644 --- a/testing/chaos/scenarios/maelstrom.yaml +++ b/testing/chaos/scenarios/maelstrom.yaml @@ -74,3 +74,8 @@ bandwidth: logging: rust_log: "debug" output_dir: "./sim-results" + +fips_overrides: + node: + cache: + coord_ttl_secs: 10 diff --git a/testing/chaos/sim/compose.py b/testing/chaos/sim/compose.py index 514ed67..1358ee1 100644 --- a/testing/chaos/sim/compose.py +++ b/testing/chaos/sim/compose.py @@ -49,6 +49,7 @@ services: hostname: {{ node.node_id }} volumes: - ./{{ node.node_id }}.yaml:/etc/fips/fips.yaml:ro + - {{ resolv_conf }}:/etc/resolv.conf:ro networks: fips-net: ipv4_address: {{ node.docker_ip }} @@ -67,11 +68,18 @@ def generate_compose( nodes = [topology.nodes[nid] for nid in sorted(topology.nodes)] + # Absolute path to the shared resolv.conf (bind-mounted into containers + # so Docker's runtime resolv.conf generation doesn't overwrite it). + resolv_conf = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..", "..", "docker", "resolv.conf") + ) + content = _COMPOSE_TEMPLATE.render( subnet=scenario.topology.subnet, rust_log=scenario.logging.rust_log, image=FIPS_SIM_IMAGE, nodes=nodes, + resolv_conf=resolv_conf, ) path = os.path.join(output_dir, "docker-compose.yml") diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index 1a07bb7..a259f9d 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -10,6 +10,7 @@ import signal import subprocess import sys import time +from datetime import datetime from .compose import generate_compose from .config_gen import write_configs @@ -34,7 +35,7 @@ class SimRunner: self.rng = random.Random(scenario.seed) self.topology: SimTopology | None = None self.compose_file: str | None = None - self.output_dir: str = scenario.logging.output_dir + self.output_dir: str = self._resolve_output_dir(scenario) self._interrupted = False # Shared set of currently-down node IDs (updated by NodeManager, @@ -49,6 +50,20 @@ class SimRunner: self.node_mgr: NodeManager | None = None self.peer_churn_mgr: PeerChurnManager | None = None + @staticmethod + def _resolve_output_dir(scenario: Scenario) -> str: + """Build a timestamped output directory path. + + Format: {base}/{scenario_name}-{YYYYMMDD-HHMMSS}/ + + The base path is determined by (in priority order): + 1. FIPS_SIM_OUTPUT environment variable + 2. The scenario YAML's logging.output_dir (default: ./sim-results) + """ + base = os.environ.get("FIPS_SIM_OUTPUT", scenario.logging.output_dir) + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + return os.path.join(base, f"{timestamp}-{scenario.name}") + def run(self) -> AnalysisResult | None: """Run the full simulation lifecycle.""" signal.signal(signal.SIGINT, self._handle_sigint) @@ -194,6 +209,10 @@ class SimRunner: down_nodes=self._down_nodes, ephemeral_nodes=self._ephemeral_nodes, ) + # Share npub cache with traffic manager so iperf3 targets + # use runtime npubs (critical for ephemeral identity nodes). + if self.traffic_mgr: + self.traffic_mgr.npub_cache = self.peer_churn_mgr.npub_cache def _warmup(self): """Wait for mesh convergence.""" diff --git a/testing/chaos/sim/traffic.py b/testing/chaos/sim/traffic.py index 76061be..8d9d1e5 100644 --- a/testing/chaos/sim/traffic.py +++ b/testing/chaos/sim/traffic.py @@ -39,11 +39,13 @@ class TrafficManager: config: TrafficConfig, rng: random.Random, down_nodes: set[str] | None = None, + npub_cache: dict[str, str] | None = None, ): self.topology = topology self.config = config self.rng = rng self.down_nodes = down_nodes or set() + self.npub_cache = npub_cache or {} self.active_sessions: list[TrafficSession] = [] self.completed_results: list[dict] = [] @@ -66,7 +68,7 @@ class TrafficManager: # Pick random client and server (different nodes, both up) client, server = self.rng.sample(node_ids, 2) - server_npub = self.topology.nodes[server].npub + server_npub = self.npub_cache.get(server, self.topology.nodes[server].npub) container = self.topology.container_name(client) duration = int( diff --git a/testing/lib/log_analysis.py b/testing/lib/log_analysis.py index e4c73c3..02fe66b 100644 --- a/testing/lib/log_analysis.py +++ b/testing/lib/log_analysis.py @@ -47,6 +47,15 @@ class AnalysisResult: congestion_detected: list[tuple[str, str]] = field(default_factory=list) kernel_drop_events: list[tuple[str, str]] = field(default_factory=list) rekey_cutovers: list[tuple[str, str]] = field(default_factory=list) + discovery_initiated: list[tuple[str, str]] = field(default_factory=list) + discovery_succeeded: list[tuple[str, str]] = field(default_factory=list) + discovery_bloom_miss: list[tuple[str, str]] = field(default_factory=list) + discovery_backoff: list[tuple[str, str]] = field(default_factory=list) + discovery_dedup: list[tuple[str, str]] = field(default_factory=list) + discovery_timeout: list[tuple[str, str]] = field(default_factory=list) + discovery_retry: list[tuple[str, str]] = field(default_factory=list) + discovery_no_tree_peer: list[tuple[str, str]] = field(default_factory=list) + discovery_trigger: list[tuple[str, str]] = field(default_factory=list) def summary(self) -> str: """Format a human-readable summary of the analysis.""" @@ -66,6 +75,17 @@ class AnalysisResult: f"Congestion events: {len(self.congestion_detected)}", f"Kernel drop events: {len(self.kernel_drop_events)}", f"Rekey cutovers: {len(self.rekey_cutovers)}", + "", + "--- Discovery ---", + f"Triggers: {len(self.discovery_trigger)}", + f"Initiated: {len(self.discovery_initiated)}", + f"Succeeded: {len(self.discovery_succeeded)}", + f"Retries: {len(self.discovery_retry)}", + f"Bloom miss: {len(self.discovery_bloom_miss)}", + f"Backoff suppressed: {len(self.discovery_backoff)}", + f"Deduplicated: {len(self.discovery_dedup)}", + f"No tree peer: {len(self.discovery_no_tree_peer)}", + f"Timed out: {len(self.discovery_timeout)}", ] if self.panics: @@ -151,6 +171,25 @@ def _analyze_lines(result: AnalysisResult, source: str, log_text: str): # Rekey cutovers if "Rekey cutover complete" in line or "FSP rekey cutover complete" in line: result.rekey_cutovers.append((source, line)) + # Discovery + if "Initiating LookupRequest" in line: + result.discovery_initiated.append((source, line)) + if "proof verified, caching route" in line: + result.discovery_succeeded.append((source, line)) + if "target not in any peer bloom filter" in line: + result.discovery_bloom_miss.append((source, line)) + if "suppressed by backoff" in line: + result.discovery_backoff.append((source, line)) + if "deduplicated, already pending" in line: + result.discovery_dedup.append((source, line)) + if "lookup timed out" in line: + result.discovery_timeout.append((source, line)) + if "Discovery retry sent" in line: + result.discovery_retry.append((source, line)) + if "no tree peers with bloom match" in line: + result.discovery_no_tree_peer.append((source, line)) + if "Failed to initiate session, trying discovery" in line: + result.discovery_trigger.append((source, line)) def collect_docker_logs(containers: list[str]) -> dict[str, str]: