Fix chaos sim DNS pipeline, ephemeral npub tracking, and analysis

Three infrastructure bugs fixed:

- DNS resolution never populated the identity cache because Docker
  overwrites /etc/resolv.conf at container start. Fix: bind-mount
  resolv.conf in the chaos compose template, matching the static and
  sidecar test configs.

- Traffic generator used config-time npubs for iperf3 targets, but
  ephemeral nodes generate fresh keypairs at startup. Fix: share the
  peer churn manager's runtime npub cache with the traffic manager.

- Log analysis had no discovery counters and used wrong patterns.
  Fix: add discovery section with correct log message matching.

Also adds timestamped output directories (YYYYMMDD-HHMMSS-scenario),
coord_ttl_secs override in maelstrom.yaml, FIPS_SIM_OUTPUT env var
support, and maelstrom-sparse scenario for sparse topology testing.

testing/chaos/scenarios/maelstrom-sparse.yaml

@@ -0,0 +1,82 @@
+# Maelstrom-sparse: sparse geometric topology for discovery stress testing
+#
+# Uses random geometric graph with low radius to produce sparse connectivity
+# (~3-4 peers per node), forcing multi-hop routing and heavy discovery usage.
+# The short coord cache TTL (10s) ensures transit-warmed entries expire
+# quickly, requiring rediscovery.
+#
+# Default: 50 nodes. Override with --nodes flag.
+
+scenario:
+  name: "maelstrom-sparse"
+  seed: 42
+  duration_secs: 600
+
+topology:
+  num_nodes: 50
+  algorithm: random_geometric
+  params:
+    radius: 0.20
+  ensure_connected: true
+  subnet: "172.20.0.0/16"
+  ip_start: 10
+  transport_mix:
+    udp: 0.8
+    tcp: 0.2
+
+netem:
+  enabled: true
+  default_policy:
+    delay_ms: { min: 5, max: 50 }
+    jitter_ms: { min: 1, max: 10 }
+    loss_pct: { min: 0, max: 2 }
+  mutation:
+    interval_secs: { min: 20, max: 45 }
+    fraction: 0.3
+    policies:
+      normal:
+        delay_ms: [5, 20]
+        loss_pct: [0, 1]
+      degraded:
+        delay_ms: [50, 100]
+        jitter_ms: [10, 30]
+        loss_pct: [3, 8]
+
+link_flaps:
+  enabled: true
+  interval_secs: { min: 30, max: 60 }
+  max_down_links: 3
+  down_duration_secs: { min: 10, max: 30 }
+  protect_connectivity: true
+
+traffic:
+  enabled: true
+  max_concurrent: 10
+  interval_secs: { min: 5, max: 30 }
+  duration_secs: { min: 5, max: 60 }
+  parallel_streams: 4
+
+node_churn:
+  enabled: true
+  interval_secs: { min: 60, max: 120 }
+  max_down_nodes: 5
+  down_duration_secs: { min: 30, max: 90 }
+  protect_connectivity: false
+
+peer_churn:
+  enabled: true
+  interval_secs: { min: 8, max: 12 }
+  ephemeral_fraction: 0.5
+
+bandwidth:
+  enabled: true
+  tiers_mbps: [1, 10, 100, 1000]
+
+logging:
+  rust_log: "debug"
+  output_dir: "./sim-results"
+
+fips_overrides:
+  node:
+    cache:
+      coord_ttl_secs: 10

testing/chaos/scenarios/maelstrom.yaml

@@ -74,3 +74,8 @@ bandwidth:
 logging:
   rust_log: "debug"
   output_dir: "./sim-results"
+
+fips_overrides:
+  node:
+    cache:
+      coord_ttl_secs: 10

testing/chaos/sim/compose.py

@@ -49,6 +49,7 @@ services:
     hostname: {{ node.node_id }}
     volumes:
       - ./{{ node.node_id }}.yaml:/etc/fips/fips.yaml:ro
+      - {{ resolv_conf }}:/etc/resolv.conf:ro
     networks:
       fips-net:
         ipv4_address: {{ node.docker_ip }}
@@ -67,11 +68,18 @@ def generate_compose(
 
     nodes = [topology.nodes[nid] for nid in sorted(topology.nodes)]
 
+    # Absolute path to the shared resolv.conf (bind-mounted into containers
+    # so Docker's runtime resolv.conf generation doesn't overwrite it).
+    resolv_conf = os.path.normpath(
+        os.path.join(os.path.dirname(__file__), "..", "..", "docker", "resolv.conf")
+    )
+
     content = _COMPOSE_TEMPLATE.render(
         subnet=scenario.topology.subnet,
         rust_log=scenario.logging.rust_log,
         image=FIPS_SIM_IMAGE,
         nodes=nodes,
+        resolv_conf=resolv_conf,
     )
 
     path = os.path.join(output_dir, "docker-compose.yml")

testing/chaos/sim/runner.py

@@ -10,6 +10,7 @@ import signal
 import subprocess
 import sys
 import time
+from datetime import datetime
 
 from .compose import generate_compose
 from .config_gen import write_configs
@@ -34,7 +35,7 @@ class SimRunner:
         self.rng = random.Random(scenario.seed)
         self.topology: SimTopology | None = None
         self.compose_file: str | None = None
-        self.output_dir: str = scenario.logging.output_dir
+        self.output_dir: str = self._resolve_output_dir(scenario)
         self._interrupted = False
 
         # Shared set of currently-down node IDs (updated by NodeManager,
@@ -49,6 +50,20 @@ class SimRunner:
         self.node_mgr: NodeManager | None = None
         self.peer_churn_mgr: PeerChurnManager | None = None
 
+    @staticmethod
+    def _resolve_output_dir(scenario: Scenario) -> str:
+        """Build a timestamped output directory path.
+
+        Format: {base}/{scenario_name}-{YYYYMMDD-HHMMSS}/
+
+        The base path is determined by (in priority order):
+        1. FIPS_SIM_OUTPUT environment variable
+        2. The scenario YAML's logging.output_dir (default: ./sim-results)
+        """
+        base = os.environ.get("FIPS_SIM_OUTPUT", scenario.logging.output_dir)
+        timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+        return os.path.join(base, f"{timestamp}-{scenario.name}")
+
     def run(self) -> AnalysisResult | None:
         """Run the full simulation lifecycle."""
         signal.signal(signal.SIGINT, self._handle_sigint)
@@ -194,6 +209,10 @@ class SimRunner:
                 down_nodes=self._down_nodes,
                 ephemeral_nodes=self._ephemeral_nodes,
             )
+            # Share npub cache with traffic manager so iperf3 targets
+            # use runtime npubs (critical for ephemeral identity nodes).
+            if self.traffic_mgr:
+                self.traffic_mgr.npub_cache = self.peer_churn_mgr.npub_cache
 
     def _warmup(self):
         """Wait for mesh convergence."""

testing/chaos/sim/traffic.py

@@ -39,11 +39,13 @@ class TrafficManager:
         config: TrafficConfig,
         rng: random.Random,
         down_nodes: set[str] | None = None,
+        npub_cache: dict[str, str] | None = None,
     ):
         self.topology = topology
         self.config = config
         self.rng = rng
         self.down_nodes = down_nodes or set()
+        self.npub_cache = npub_cache or {}
         self.active_sessions: list[TrafficSession] = []
         self.completed_results: list[dict] = []
 
@@ -66,7 +68,7 @@ class TrafficManager:
 
         # Pick random client and server (different nodes, both up)
         client, server = self.rng.sample(node_ids, 2)
-        server_npub = self.topology.nodes[server].npub
+        server_npub = self.npub_cache.get(server, self.topology.nodes[server].npub)
         container = self.topology.container_name(client)
 
         duration = int(

testing/lib/log_analysis.py

@@ -47,6 +47,15 @@ class AnalysisResult:
     congestion_detected: list[tuple[str, str]] = field(default_factory=list)
     kernel_drop_events: list[tuple[str, str]] = field(default_factory=list)
     rekey_cutovers: list[tuple[str, str]] = field(default_factory=list)
+    discovery_initiated: list[tuple[str, str]] = field(default_factory=list)
+    discovery_succeeded: list[tuple[str, str]] = field(default_factory=list)
+    discovery_bloom_miss: list[tuple[str, str]] = field(default_factory=list)
+    discovery_backoff: list[tuple[str, str]] = field(default_factory=list)
+    discovery_dedup: list[tuple[str, str]] = field(default_factory=list)
+    discovery_timeout: list[tuple[str, str]] = field(default_factory=list)
+    discovery_retry: list[tuple[str, str]] = field(default_factory=list)
+    discovery_no_tree_peer: list[tuple[str, str]] = field(default_factory=list)
+    discovery_trigger: list[tuple[str, str]] = field(default_factory=list)
 
     def summary(self) -> str:
         """Format a human-readable summary of the analysis."""
@@ -66,6 +75,17 @@ class AnalysisResult:
             f"Congestion events:     {len(self.congestion_detected)}",
             f"Kernel drop events:    {len(self.kernel_drop_events)}",
             f"Rekey cutovers:        {len(self.rekey_cutovers)}",
+            "",
+            "--- Discovery ---",
+            f"Triggers:              {len(self.discovery_trigger)}",
+            f"Initiated:             {len(self.discovery_initiated)}",
+            f"Succeeded:             {len(self.discovery_succeeded)}",
+            f"Retries:               {len(self.discovery_retry)}",
+            f"Bloom miss:            {len(self.discovery_bloom_miss)}",
+            f"Backoff suppressed:    {len(self.discovery_backoff)}",
+            f"Deduplicated:          {len(self.discovery_dedup)}",
+            f"No tree peer:          {len(self.discovery_no_tree_peer)}",
+            f"Timed out:             {len(self.discovery_timeout)}",
         ]
 
         if self.panics:
@@ -151,6 +171,25 @@ def _analyze_lines(result: AnalysisResult, source: str, log_text: str):
         # Rekey cutovers
         if "Rekey cutover complete" in line or "FSP rekey cutover complete" in line:
             result.rekey_cutovers.append((source, line))
+        # Discovery
+        if "Initiating LookupRequest" in line:
+            result.discovery_initiated.append((source, line))
+        if "proof verified, caching route" in line:
+            result.discovery_succeeded.append((source, line))
+        if "target not in any peer bloom filter" in line:
+            result.discovery_bloom_miss.append((source, line))
+        if "suppressed by backoff" in line:
+            result.discovery_backoff.append((source, line))
+        if "deduplicated, already pending" in line:
+            result.discovery_dedup.append((source, line))
+        if "lookup timed out" in line:
+            result.discovery_timeout.append((source, line))
+        if "Discovery retry sent" in line:
+            result.discovery_retry.append((source, line))
+        if "no tree peers with bloom match" in line:
+            result.discovery_no_tree_peer.append((source, line))
+        if "Failed to initiate session, trying discovery" in line:
+            result.discovery_trigger.append((source, line))
 
 
 def collect_docker_logs(containers: list[str]) -> dict[str, str]: