From c8a0ac5fca297dd8d296cef692b4ae6443171cb6 Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 15:30:01 +0000
Subject: [PATCH 01/11] Stop a node whose logs cannot be read from counting as
 a clean node

count_log_pattern summed a per-node `docker logs | grep -c ... || true`, so a
node the harness could not read contributed zero. The six assert_zero_count
callers are negative health assertions -- no panics, no ERROR lines, no AEAD
decrypt failures, no rekey msg2 failures -- and a contributed zero reads as
clean, so one unreadable node silently weakened the assertion and all of them
voided it. This is the family the harness-fallback issue was raised to high
priority for, and it is the one remaining audit residual that can make a green
rekey run lie about protocol code.

The read now fails the count rather than degrading it, printing a sentinel that
names the container. Both callers split the declaration from the assignment,
because `local c=$(fn)` takes local's exit status and discards the function's --
which is how the original defect stayed invisible.

Validated by breaking what it guards rather than by observing green: against
absent containers the old reader returns 0 and assert_zero_count passes
vacuously, while the new one returns rc=1 and reports a failure. Checked the
other direction too, since a fix that reds a legitimately clean run is no use: a
genuine zero across readable nodes still returns 0, and the sum is unchanged at
2+1+0 for a shimmed three-node read.
---
 testing/static/scripts/rekey-test.sh | 34 ++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh
index eab2b2a..bc78113 100755
--- a/testing/static/scripts/rekey-test.sh
+++ b/testing/static/scripts/rekey-test.sh
@@ -285,15 +285,31 @@ phase_result() {
     fi
 }
 
-# Count occurrences of a pattern across all node logs
+# Count occurrences of a pattern across all node logs.
+#
+# A node whose logs cannot be read makes the whole count unusable rather than
+# contributing 0. The previous form ended each read `| grep -c "$pat" || true`,
+# so a failed `docker logs` yielded 0 for that node and the six assert_zero_count
+# callers below read a clean result from a node that was never consulted — one
+# unreadable node silently weakened the assertion instead of voiding it.
+#
+# Returns non-zero and prints a sentinel naming the container. Every caller must
+# split the declaration from the assignment (`local c` then `c=$(...)`), because
+# `local c=$(...)` takes `local`'s exit status and discards this one.
 count_log_pattern() {
     local pattern="$1"
     local total=0
+    local node logs count
     for node in $NODES; do
-        local count=$(docker logs "fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" 2>&1 | grep -c "$pattern" || true)
+        if ! logs=$(docker logs "fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" 2>&1); then
+            echo "unreadable:fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}"
+            return 1
+        fi
+        count=$(grep -c "$pattern" <<<"$logs" || true)
         total=$((total + count))
     done
     echo "$total"
+    return 0
 }
 
 wait_for_log_pattern_count() {
@@ -325,7 +341,12 @@ assert_min_count() {
     local pattern="$1"
     local min_count="$2"
     local description="$3"
-    local count=$(count_log_pattern "$pattern")
+    local count
+    count=$(count_log_pattern "$pattern") || {
+        echo "  ✗ $description: node logs unreadable ($count), count not established"
+        FAILED=$((FAILED + 1))
+        return
+    }
     if [ "$count" -ge "$min_count" ]; then
         echo "  ✓ $description: $count (>= $min_count)"
         PASSED=$((PASSED + 1))
@@ -339,7 +360,12 @@ assert_min_count() {
 assert_zero_count() {
     local pattern="$1"
     local description="$2"
-    local count=$(count_log_pattern "$pattern")
+    local count
+    count=$(count_log_pattern "$pattern") || {
+        echo "  ✗ $description: node logs unreadable ($count), zero not established"
+        FAILED=$((FAILED + 1))
+        return
+    }
     if [ "$count" -eq 0 ]; then
         echo "  ✓ $description: 0"
         PASSED=$((PASSED + 1))

From 34a2561af768a8defab5153d0215fa62508fc6ba Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 15:36:59 +0000
Subject: [PATCH 02/11] Close the remaining "caught" and "produces red" harness
 holes

Four sites where a check could report a verdict it had not established, or
detect a failure and then not turn it red.

assert_no_panic in both NAT suites read `docker logs ... || true`, so a
container that could not be read produced empty output, matched no panic
pattern, and returned success. The assertion's failure mode was
indistinguishable from its success condition. It now reports that absence of
panics is not established.

deb-install's apt capture discarded the exit status, so a failed docker exec
gave an empty capture that matched neither error pattern and reached the pass
branch. The status is now kept and checked before the output is inspected, with
the output still printed on failure.

wait_for_systemd printed a warning and returned success on timeout, so every
check after it read a system that may not have started its units. It now returns
non-zero and the caller abandons that distro leg rather than testing an
unstarted system.

interop's copy of count_log_pattern carried the same defect fixed in rekey: a
node whose logs could not be read contributed zero to eight expect-zero
assertions. Fixed the same way, and its consumer now reports the unreadable case
rather than comparing a sentinel against zero.

Each validated by breaking what it guards and by confirming the healthy path is
unchanged: an absent container now fails each check where it previously passed,
a readable panic-free container still passes, a real panic is still caught, and
a genuinely successful install still passes.
---
 testing/deb-install/test.sh             | 27 ++++++++++++++++++++-----
 testing/interop/interop-test.sh         | 22 +++++++++++++++++---
 testing/nat/scripts/nostr-relay-test.sh |  9 ++++++++-
 testing/nat/scripts/stun-faults-test.sh | 10 ++++++++-
 4 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/testing/deb-install/test.sh b/testing/deb-install/test.sh
index 6a78827..daa7d96 100755
--- a/testing/deb-install/test.sh
+++ b/testing/deb-install/test.sh
@@ -82,8 +82,11 @@ wait_for_systemd() {
         fi
         sleep 1
     done
-    echo "  WARNING: systemd did not reach running state in ${BOOT_TIMEOUT}s (may still work)"
-    return 0
+    # A boot that never reached `running` or `degraded` is not a warning: every
+    # check after this point reads a system that may not have started its units,
+    # and returning 0 here made the timeout indistinguishable from a clean boot.
+    echo "  ERROR: systemd did not reach running state in ${BOOT_TIMEOUT}s" >&2
+    return 1
 }
 
 wait_for_service_active() {
@@ -229,18 +232,32 @@ DOCKERFILE
     rm -f "$CACHE_DIR/deb-for-image"
 
     start_systemd_container_with_tun "$name" "$image"
-    wait_for_systemd "$name"
+    wait_for_systemd "$name" || {
+        fail "systemd did not boot in $name; the install checks below would read an unstarted system"
+        cleanup_container "$name"
+        return
+    }
 
     # Install the .deb. apt handles dependencies (libc6, systemd,
     # libdbus-1-3) and runs the maintainer scripts (postinst →
     # systemctl enable fips.service; fips-dns.service starts and
     # runs fips-dns-setup).
     log "Installing .deb (apt install /opt/fips-deb/${deb_basename})"
-    local install_output
+    # `|| true` here used to discard apt's exit status, and an empty capture (a
+    # failed `docker exec`) matches neither error pattern below, so a install
+    # that never ran reached `pass "apt install completed"`. Keep the capture on
+    # failure so the diagnostics below can print it, but remember the status.
+    local install_output install_rc=0
     install_output=$(docker exec "$name" bash -c "
         apt-get update >/dev/null 2>&1
         cd /opt/fips-deb && apt-get install -y --no-install-recommends ./${deb_basename} 2>&1
-    ") || true
+    ") || install_rc=$?
+    if [ "$install_rc" -ne 0 ]; then
+        fail "apt install exited $install_rc"
+        echo "$install_output" | tail -20
+        cleanup_container "$name"
+        return
+    fi
     if echo "$install_output" | grep -qE "^E:|errors? were encountered"; then
         fail "apt install reported errors"
         echo "$install_output" | tail -20
diff --git a/testing/interop/interop-test.sh b/testing/interop/interop-test.sh
index f240daf..ab27b3c 100755
--- a/testing/interop/interop-test.sh
+++ b/testing/interop/interop-test.sh
@@ -476,13 +476,24 @@ phase_result() {
 }
 
 # Count a pattern across all node logs.
+#
+# A node whose logs cannot be read makes the whole count unusable rather than
+# contributing 0. The previous form ended each read `| grep -cE … || true`, so a
+# failed `docker logs` yielded 0 for that node and the eight expect-zero
+# assertions in the GLOBAL_PATTERNS loop read a clean result from a node that was
+# never consulted. Same defect and same fix as rekey-test.sh.
 count_log_pattern() {
-    local pattern="$1" total=0 n count
+    local pattern="$1" total=0 n logs count
     for n in "${NODES[@]}"; do
-        count=$(docker logs "${CONTAINER[$n]}" 2>&1 | grep -cE "$pattern" || true)
+        if ! logs=$(docker logs "${CONTAINER[$n]}" 2>&1); then
+            echo "unreadable:${CONTAINER[$n]}"
+            return 1
+        fi
+        count=$(grep -cE "$pattern" <<<"$logs" || true)
         total=$((total + count))
     done
     echo "$total"
+    return 0
 }
 
 # Per-node count of a pattern.
@@ -854,7 +865,12 @@ declare -A GLOBAL_PATTERNS=(
 
 for pat in "${!GLOBAL_PATTERNS[@]}"; do
     desc="${GLOBAL_PATTERNS[$pat]}"
-    total="$(count_log_pattern "$pat")"
+    if ! total="$(count_log_pattern "$pat")"; then
+        echo "    FAIL  $desc: node logs unreadable ($total), zero not established"
+        FAILED=$((FAILED + 1))
+        INTEROP_FAILURES+=("[log] $desc: node logs unreadable ($total)")
+        continue
+    fi
     if [ "$total" -eq 0 ]; then
         echo "    PASS  $desc: 0"
         PASSED=$((PASSED + 1))
diff --git a/testing/nat/scripts/nostr-relay-test.sh b/testing/nat/scripts/nostr-relay-test.sh
index f02531d..812d879 100755
--- a/testing/nat/scripts/nostr-relay-test.sh
+++ b/testing/nat/scripts/nostr-relay-test.sh
@@ -291,14 +291,21 @@ assert_process_alive() {
     return 0
 }
 
+# A container whose logs cannot be read has not been shown to be panic-free.
+# See the companion note in stun-faults-test.sh: the previous `|| true` made
+# this assertion's failure mode indistinguishable from its success condition.
 assert_no_panic() {
     local container="$1"
     local logs
-    logs="$(docker logs "$container" 2>&1 || true)"
+    if ! logs="$(docker logs "$container" 2>&1)"; then
+        echo "could not read logs from $container; absence of panics is not established" >&2
+        return 1
+    fi
     if grep -Eq "panicked at|RUST_BACKTRACE|fatal runtime error" <<<"$logs"; then
         echo "panic detected in $container logs" >&2
         return 1
     fi
+    return 0
 }
 
 run_test() {
diff --git a/testing/nat/scripts/stun-faults-test.sh b/testing/nat/scripts/stun-faults-test.sh
index 51a5a13..c9dbbb5 100755
--- a/testing/nat/scripts/stun-faults-test.sh
+++ b/testing/nat/scripts/stun-faults-test.sh
@@ -143,13 +143,21 @@ assert_process_alive() {
     return 0
 }
 
+# A container whose logs cannot be read has not been shown to be panic-free.
+# The previous form read `docker logs … || true`, so an unreadable container
+# yielded empty output, matched no panic pattern, and returned success — the
+# assertion's failure mode was indistinguishable from its success condition.
 assert_no_panic() {
     local logs
-    logs="$(docker logs "$NODE" 2>&1 || true)"
+    if ! logs="$(docker logs "$NODE" 2>&1)"; then
+        echo "could not read logs from $NODE; absence of panics is not established" >&2
+        return 1
+    fi
     if grep -Eq "panicked at|RUST_BACKTRACE|fatal runtime error" <<<"$logs"; then
         echo "panic detected in $NODE logs" >&2
         return 1
     fi
+    return 0
 }
 
 # Look for STUN-related fault evidence in the daemon's logs. The

From bf173d8d986e10ccef484fa9b4957e00e884f5d7 Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 15:43:10 +0000
Subject: [PATCH 03/11] Pin the chaos mesh root to n01 so scenario diagrams
 describe the real tree

Every chaos scenario draws n01 at the top of its topology, and until now that
held in three of thirteen. The mesh roots itself at the numerically smallest
NodeAddr, which is a hash of the node's public key and bears no relation to the
node numbering, so which node ended up as root was effectively arbitrary.

The consequences were not cosmetic. cost-reeval rooted at n04, its own
designated test subject, so that node had no parent and the periodic parent
switch the scenario exists to observe could not occur at all. mixed-technology
rooted at n09, which put its two documented parent criteria out of reach and
made correct cost-based selection look like a defect.

Identities are still derived from the mesh name exactly as before and are still
deterministic. What changes is which node id holds which one: they are now
assigned in NodeAddr order, so n01 holds the smallest and is the root. Verified
against a model of the daemon's own derivation that reproduces the previously
observed root for every scenario and n01's address byte for byte; all twelve
pinned scenarios now root at n01.

smoke-10 deliberately opts out via pin_root: false so that root election from an
arbitrary key distribution stays exercised somewhere. Its assertion is a
convergence floor and is root-agnostic, which is why it is the cheapest home for
that. The new key is rejected when non-boolean, and a near-miss spelling is
rejected as unknown; both checked.

Not yet established: the trees themselves change, so the parent-dependent
assertions in bottleneck-parent, cost-avoidance and cost-stability need
re-deriving against live runs. Those are held until the in-flight CI finishes,
because a chaos run rebuilds the shared fips-test image that run is using.
---
 testing/chaos/scenarios/smoke-10.yaml | 10 +++++++++
 testing/chaos/sim/keys.py             |  2 +-
 testing/chaos/sim/scenario.py         | 17 +++++++++++++++-
 testing/chaos/sim/topology.py         | 29 ++++++++++++++++++++++-----
 testing/lib/derive_keys.py            | 19 ++++++++++++++++--
 5 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/testing/chaos/scenarios/smoke-10.yaml b/testing/chaos/scenarios/smoke-10.yaml
index 9a63958..452e9b5 100644
--- a/testing/chaos/scenarios/smoke-10.yaml
+++ b/testing/chaos/scenarios/smoke-10.yaml
@@ -11,6 +11,16 @@ topology:
   ensure_connected: true
   subnet: "172.20.0.0/24"
   ip_start: 10
+  # The one scenario that deliberately does NOT pin the root. Every other
+  # scenario assigns identities in NodeAddr order so that n01 is root and its
+  # diagram describes the tree that actually forms. That is right for them,
+  # because they test parent selection and were silently testing it under a
+  # root they did not intend, but applying it everywhere would leave root
+  # election itself exercised nowhere. This is the generic 10-node convergence
+  # baseline and its `baseline` assertion is root-agnostic, so it is the
+  # cheapest place to keep election under test.
+  # Do not "fix" this to true for consistency with the others.
+  pin_root: false
 
 # Phase 2+ features (disabled for MVP)
 netem:
diff --git a/testing/chaos/sim/keys.py b/testing/chaos/sim/keys.py
index dd2dd24..69ee9d1 100644
--- a/testing/chaos/sim/keys.py
+++ b/testing/chaos/sim/keys.py
@@ -8,4 +8,4 @@ _TESTING_DIR = os.path.join(os.path.dirname(__file__), "..", "..")
 if _TESTING_DIR not in sys.path:
     sys.path.insert(0, _TESTING_DIR)
 
-from lib.derive_keys import derive  # noqa: E402
+from lib.derive_keys import derive, derive_full  # noqa: E402,F401
diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py
index d7e30c4..8f03f83 100644
--- a/testing/chaos/sim/scenario.py
+++ b/testing/chaos/sim/scenario.py
@@ -44,6 +44,14 @@ class TopologyConfig:
     # When set, each edge is randomly assigned a transport based on weights.
     # Only valid for non-explicit algorithms (explicit uses per-edge syntax).
     transport_mix: dict[str, float] | None = None
+    # Assign derived identities in NodeAddr order so n01 holds the smallest and
+    # is therefore the root every scenario diagram draws. Default on: without it
+    # the root is effectively arbitrary, which left `cost-reeval` rooted at its
+    # own designated test subject — so the parent switch it exists to observe
+    # could not occur — and made `mixed-technology`'s parent criteria
+    # unreachable. Set false where election from an arbitrary key distribution
+    # is itself the subject.
+    pin_root: bool = True
 
 
 @dataclass
@@ -372,7 +380,7 @@ _SECTION_KEYS = {
     "scenario": {"name", "seed", "duration_secs"},
     "topology": {
         "num_nodes", "algorithm", "params", "ensure_connected", "subnet",
-        "ip_start", "default_transport", "transport_mix",
+        "ip_start", "default_transport", "transport_mix", "pin_root",
     },
     "netem": {"enabled", "default_policy", "link_policies", "mutation"},
     "netem.link_policies[]": {"edges", "policy", "policy_name"},
@@ -494,6 +502,13 @@ def load_scenario(path: str) -> Scenario:
     s.topology.subnet = tc.get("subnet", "172.20.0.0/24")
     s.topology.ip_start = int(tc.get("ip_start", 10))
     s.topology.default_transport = tc.get("default_transport", "udp")
+    if "pin_root" in tc:
+        pin = tc["pin_root"]
+        if not isinstance(pin, bool):
+            raise ValueError(
+                f"topology.pin_root must be a boolean, got {pin!r}"
+            )
+        s.topology.pin_root = pin
     if "transport_mix" in tc:
         mix = tc["transport_mix"]
         if not isinstance(mix, dict) or not mix:
diff --git a/testing/chaos/sim/topology.py b/testing/chaos/sim/topology.py
index 69ea769..4ac99d3 100644
--- a/testing/chaos/sim/topology.py
+++ b/testing/chaos/sim/topology.py
@@ -7,7 +7,7 @@ import random
 from collections import deque
 from dataclasses import dataclass, field
 
-from .keys import derive
+from .keys import derive_full
 from .naming import name_suffix, veth_token
 from .scenario import TopologyConfig
 
@@ -203,12 +203,31 @@ def generate_topology(
     n = config.num_nodes
     subnet_base = config.subnet.rsplit(".", 1)[0]  # "172.20.0"
 
-    # Create nodes with IPs and keys
+    # Create nodes with IPs and keys.
+    #
+    # The mesh roots itself at the numerically smallest NodeAddr
+    # (`src/tree/state.rs:363-390`), which is a hash of the node's public key
+    # and so bears no relation to the node numbering. Every scenario diagram in
+    # this tree draws n01 at the top, and before this ordering was applied that
+    # held in only three of thirteen: `cost-reeval` rooted at n04 — its own
+    # designated test subject, which therefore had no parent to switch and could
+    # not exercise what the scenario exists to test — and `mixed-technology` at
+    # n09, which made its two documented parent criteria unreachable.
+    #
+    # So derive the identities from the mesh name as before, then *assign* them
+    # in NodeAddr order: n01 receives the smallest and is the root, n02 the next,
+    # and so on. The keys are unchanged and still deterministic; only which node
+    # id holds which one changes. Scenarios that want an arbitrary root set
+    # `pin_root: false` and keep exercising election.
+    node_ids_ordered = [f"n{i + 1:02d}" for i in range(n)]
+    identities = [derive_full(mesh_name, nid) for nid in node_ids_ordered]
+    if config.pin_root:
+        identities.sort(key=lambda t: t[2])
+
     nodes: dict[str, SimNode] = {}
-    for i in range(n):
-        node_id = f"n{i + 1:02d}"
+    for i, node_id in enumerate(node_ids_ordered):
         docker_ip = f"{subnet_base}.{config.ip_start + i}"
-        nsec, npub = derive(mesh_name, node_id)
+        nsec, npub, _ = identities[i]
         nodes[node_id] = SimNode(
             node_id=node_id,
             docker_ip=docker_ip,
diff --git a/testing/lib/derive_keys.py b/testing/lib/derive_keys.py
index 37495e5..460095d 100755
--- a/testing/lib/derive_keys.py
+++ b/testing/lib/derive_keys.py
@@ -93,13 +93,28 @@ def _convertbits(data, frombits, tobits):
 # --- public API ---
 
 def derive(mesh_name, node_name):
+    nsec_hex, npub, _ = derive_full(mesh_name, node_name)
+    return nsec_hex, npub
+
+
+def derive_full(mesh_name, node_name):
+    """As `derive`, plus the NodeAddr the daemon will compute for this key.
+
+    NodeAddr is the first 16 bytes of SHA-256 over the serialized x-only
+    public key (`src/identity/node_addr.rs:36-43`), and the mesh elects the
+    numerically smallest one as root (`src/tree/state.rs:363-390`). Callers
+    that need the tree's root to be predictable derive it from here rather
+    than assuming it follows the node numbering.
+    """
     nsec_hex = hashlib.sha256(f"{mesh_name}|{node_name}".encode()).hexdigest()
     k = int(nsec_hex, 16)
     pub = _scalar_mult(k, (Gx, Gy))
     x_hex = format(pub[0], "064x")
-    data_5bit = _convertbits(list(bytes.fromhex(x_hex)), 8, 5)
+    x_bytes = bytes.fromhex(x_hex)
+    data_5bit = _convertbits(list(x_bytes), 8, 5)
     npub = _bech32_encode("npub", data_5bit)
-    return nsec_hex, npub
+    node_addr = hashlib.sha256(x_bytes).hexdigest()[:32]
+    return nsec_hex, npub, node_addr
 
 
 if __name__ == "__main__":

From d4a2504f9913b68a65fde2e5c05cde4301d4e351 Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 15:45:37 +0000
Subject: [PATCH 04/11] Teach the trailing-log gate to see command-substitution
 call sites

Found while fixing count_log_pattern: the gate reported that function clean
even though it ends in echo and both callers consume its status. Its call-site
patterns only matched direct forms -- if fn, while fn, fn ||, fn && -- so a
status consumed through command substitution was invisible, because the line
begins with the variable rather than the function name.

That is not an exotic form. It is how a shell function returns a value, and it
is precisely the shape of the swallowed-failure family this gate exists to
catch, so the gate was blind to a large part of its own stated class.

Three patterns added for the assignment, if-guarded and test-expression forms.
The extension finds four real instances, all value-returning helpers whose
trailing echo made their exit status unconditionally zero, and each now carries
an explicit return 0. Also corrects the finding message, which described the
trailing command as a log call; for these it is the return mechanism, and the
hazard is that it fixes the status either way.

Validated by breaking what it guards: a probe reintroducing the defect shape
behind a command substitution is reported and exits 1, where before the
extension it would have passed.
---
 .../chaos/scenarios/congestion-stress.yaml    | 42 +++++++++++++++----
 testing/check-trailing-log.py                 |  9 ++++
 testing/static/scripts/bench-multirun.sh      |  1 +
 testing/static/scripts/generate-configs.sh    |  2 +
 testing/static/scripts/netem.sh               |  1 +
 5 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/testing/chaos/scenarios/congestion-stress.yaml b/testing/chaos/scenarios/congestion-stress.yaml
index 8dce07e..f5aa4c6 100644
--- a/testing/chaos/scenarios/congestion-stress.yaml
+++ b/testing/chaos/scenarios/congestion-stress.yaml
@@ -119,15 +119,39 @@ assertions:
     min_nodes_ce_received: 1
 
 # The fourth criterion, "kernel_drop_events > 0 on at least one node", is
-# NOT asserted, because the implementation does not meet it: across all 182
-# archived runs of this scenario, including the six that meet the other
-# three, no node has ever reported a non-zero kernel_drop_events. Asserting
-# it would red the scenario permanently, and asserting a weakened version
-# would assert nothing. It stands here as an unmet criterion: either the
-# 4 KB recv_buf_size no longer provokes SO_RXQ_OVFL under this traffic, or
-# the counter does not reach the snapshot. Until that is settled it is a
-# coverage gap, because the transport drop-detection path this scenario
-# names as signal 2 is exercised by nothing.
+# NOT asserted, because this scenario has never met it: across all 182
+# archived runs, including the six that meet the other three, no node has
+# reported a non-zero kernel_drop_events. Asserting it would red the scenario
+# permanently, and asserting a weakened version would assert nothing.
+#
+# CORRECTED 2026-07-23. An earlier version of this note concluded that the
+# transport drop-detection path "is exercised by nothing". That is wrong, and
+# the error was one of scope: it looked only inside this scenario. Widening
+# the scan to every scenario's congestion snapshots -- 2317 files, 11778 node
+# records -- finds 51 records reporting kernel_drop_events > 0 across six
+# other scenarios (depth-vs-cost, bottleneck-parent, tcp-mesh, cost-avoidance,
+# maelstrom-sparse, mixed-technology), with raw drop counts up to 41165. The
+# path is live and plumbed through to the snapshot. What is dead is this
+# scenario's ability to provoke it.
+#
+# The inversion is the lead: every scenario that DOES overflow runs with the
+# default 2 MB recv_buf_size, while this one, the only scenario that shrinks
+# it to 4 KB, never does. SO_RXQ_OVFL counts datagrams arriving at a full
+# queue, and the 1 Mbps cap below sets the arrival rate to 125 kB/s per link.
+# Linux doubles a requested SO_RCVBUF, so the queue is 8192 bytes and filling
+# it takes ~65 ms of reader stall -- ~22 ms even at the busiest node's 3 Mbps
+# aggregate. Traffic volume cannot cause the overflow because the volume is
+# capped below the rate the buffer drains. The ingress policer works against
+# the same signal, discarding excess in tc before the socket ever sees it.
+#
+# So signal 3 appears to suppress signal 2, and the two may not belong in one
+# scenario. Tracked with the experiment that would settle it (raise or drop
+# the bandwidth cap for one run and watch the counter) in ISSUE-2026-0084.
+#
+# Note also that the header above says "small recv_buf_size (8KB)" while
+# fips_overrides sets 4096. The 8 KB figure matches what Linux allocates after
+# doubling, so the comment may be describing the effective queue rather than
+# the setting; it reads as the setting and should say which it means.
 
 logging:
   rust_log: "info"
diff --git a/testing/check-trailing-log.py b/testing/check-trailing-log.py
index eb8a5c8..b4f4817 100755
--- a/testing/check-trailing-log.py
+++ b/testing/check-trailing-log.py
@@ -133,6 +133,15 @@ def status_tested(name: str, all_text: str, defining_file: Path) -> list[str]:
         (rf"^\s*{re.escape(name)}\s+[^\n|&]*&&", "<fn> &&"),
         (rf"^\s*{re.escape(name)}\s*&&", "<fn> &&"),
         (rf"\bif\s+\S*\s*{re.escape(name)}\b.*;\s*then", "if ... <fn> ; then"),
+        # Command substitution. Found 2026-07-23 while fixing a function this
+        # check reported clean: `total=$(count_log_pattern "$p") || { ... }`
+        # consumes the status, but the line begins with the variable, so none
+        # of the patterns above match it. That is not an exotic form — it is
+        # how a shell function returns a *value*, and it is the shape of the
+        # `|| true`-swallowed-failure family this check exists to catch.
+        (rf"=\$\(\s*{re.escape(name)}\b", "<var>=$(<fn>)"),
+        (rf"\bif\s+!?\s*\S*=?\$\(\s*{re.escape(name)}\b", "if <var>=$(<fn>)"),
+        (rf"\[\s+\"?\$\(\s*{re.escape(name)}\b", "[ $(<fn>) ]"),
     ]
     hits = []
     for pat, label in patterns:
diff --git a/testing/static/scripts/bench-multirun.sh b/testing/static/scripts/bench-multirun.sh
index d9a8547..d2a138b 100755
--- a/testing/static/scripts/bench-multirun.sh
+++ b/testing/static/scripts/bench-multirun.sh
@@ -120,6 +120,7 @@ ping_path() {
         return
     fi
     echo "${rtt//\// } ${loss:-N/A}"
+    return 0
 }
 
 # ── Path definitions ───────────────────────────────────────────────────────
diff --git a/testing/static/scripts/generate-configs.sh b/testing/static/scripts/generate-configs.sh
index 9be398b..a1d1a04 100755
--- a/testing/static/scripts/generate-configs.sh
+++ b/testing/static/scripts/generate-configs.sh
@@ -74,6 +74,7 @@ docker_host_name() {
     local host
     host=$(get_node_attr "$topology_file" "$node_id" "docker_host")
     echo "${host:-node-$node_id}"
+    return 0
 }
 
 # Get peers list from topology
@@ -118,6 +119,7 @@ get_default_transport() {
     local topology_file="$1"
     local transport=$(grep "^default_transport:" "$topology_file" | head -1 | sed 's/.*: *\([a-z]*\).*/\1/')
     echo "${transport:-udp}"
+    return 0
 }
 
 # Get the port for a given transport type
diff --git a/testing/static/scripts/netem.sh b/testing/static/scripts/netem.sh
index 866a20f..30d1787 100755
--- a/testing/static/scripts/netem.sh
+++ b/testing/static/scripts/netem.sh
@@ -159,6 +159,7 @@ build_netem_params() {
     fi
 
     echo "$params"
+    return 0
 }
 
 # Check if a container is running

From 6319c7f577b02f66853feb1c3dd22fb2d6981c5f Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 15:46:04 +0000
Subject: [PATCH 05/11] Describe the trailing command accurately in the gate's
 finding message

For a value-returning helper the trailing echo is the return mechanism, not a
log call. The hazard is the same either way -- it fixes the exit status at zero
-- but calling it a log call misdescribes four of the sites the gate now finds.
---
 testing/check-trailing-log.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testing/check-trailing-log.py b/testing/check-trailing-log.py
index b4f4817..990dc34 100755
--- a/testing/check-trailing-log.py
+++ b/testing/check-trailing-log.py
@@ -195,8 +195,8 @@ def main() -> int:
         print(f"    last statement: {stmt[:100]}")
         print(f"    status consumed by: {', '.join(callers)}")
         print(f"    fix: add an explicit `return 0` as the last statement. Its "
-              f"success value is currently whatever the log call returned, "
-              f"which is 0 whatever the function did.")
+              f"success value is currently the trailing command's, which is 0 "
+              f"whatever the function did.")
 
     print(f"trailing-log check: {scanned} function(s) scanned, "
           f"{shaped} end in a logging call, {len(ALLOWED)} allowed, "

From 9d92cfeab4ea552e8c3f662fcd2cb1a3b8eee012 Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 15:47:28 +0000
Subject: [PATCH 06/11] Correct three scenario headers that documented
 conclusions since disproved

congestion-stress said the transport drop-detection path "is exercised by
nothing". Widening the scan beyond this one scenario finds it firing in six
others, so the path is live and what is dead is this scenario's ability to
provoke it. Records the arithmetic that explains why: at the 1 Mbps cap the
receive queue needs tens of milliseconds of reader stall to overflow, and the
ingress policer discards the same traffic a layer below the socket.

mixed-technology said the implementation does not do what its criteria say.
It does. The criteria were conditioned on a tree rooted at n01 while the mesh
rooted at n09, under which n08's choice was settled by depth before link
technology could matter. The archived parent distributions are relabelled as
describing the old root so nobody calibrates against them.

cost-reeval now records why it had no assertion about its own subject: its
designated subject was the root, so the parent switch it exists to observe
could not occur. The historical logs show the switch on n04 in 13 of 14 runs
when the root still wandered, and only on n01 in the five runs before the fix.

Each header now names what remains to be done and where it is tracked, rather
than leaving a disproved conclusion in place for the next reader to inherit.
---
 testing/chaos/scenarios/cost-reeval.yaml      | 22 ++++++++++
 testing/chaos/scenarios/mixed-technology.yaml | 42 +++++++++++--------
 2 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/testing/chaos/scenarios/cost-reeval.yaml b/testing/chaos/scenarios/cost-reeval.yaml
index 559966d..51d2f08 100644
--- a/testing/chaos/scenarios/cost-reeval.yaml
+++ b/testing/chaos/scenarios/cost-reeval.yaml
@@ -32,6 +32,28 @@
 # re-evaluation" (trigger=periodic). If mutation never creates enough
 # asymmetry in a particular seed, the test still validates that periodic
 # re-eval runs without interference.
+#
+# THIS SCENARIO COULD NOT EXERCISE ITS SUBJECT AT ALL until 2026-07-23, and
+# the reason is worth keeping because nothing about it was visible from here.
+# The mesh roots at the numerically smallest NodeAddr, a hash of the node's
+# key (src/tree/state.rs:363-390), and that turned out to be **n04** — this
+# scenario's own designated test subject. A root has no parent, and
+# evaluate_parent returns early for the smallest address, so the parent switch
+# described above could never happen. The tree was inverted: n04 at depth 0,
+# n01 a leaf at depth 2, in all five recent runs.
+#
+# The archived logs show exactly that shape. The live info! string above
+# appears in 14 of 105 historical runs, on n04 in 13 of them, back when keys
+# were regenerated per run and the root wandered. In the five runs before this
+# fix it appears only on **n01** — the node that actually had a parent choice.
+#
+# `topology.pin_root` (default true) now assigns identities in NodeAddr order
+# so n01 is root and n04 is the depth-1 subject this file describes.
+#
+# STILL TO DO: this scenario has no assertion about its own subject, and that
+# was a consequence rather than an omission — none could have passed. With the
+# subject now reachable, decide whether the switch is reliable enough across
+# seeds to assert, and encode it if so. Tracked in ISSUE-2026-0085.
 
 scenario:
   name: "cost-reeval"
diff --git a/testing/chaos/scenarios/mixed-technology.yaml b/testing/chaos/scenarios/mixed-technology.yaml
index 9e52923..d6cb1f3 100644
--- a/testing/chaos/scenarios/mixed-technology.yaml
+++ b/testing/chaos/scenarios/mixed-technology.yaml
@@ -31,25 +31,33 @@
 #
 # Netem mutation shifts fiber-only links between normal and degraded.
 #
-# NEITHER TEST SUBJECT IS ASSERTED, because the implementation does not do
-# what the two lines above say. Across the five archived runs that carry a
-# status.txt and are therefore provably completed, n06's parent is n10 in
-# three and n03 in two, and n08's is n04 in four and n03 in one. Encoding
-# either as written would red the scenario most runs; encoding what it
-# actually does would pin behaviour nobody specified and would silently
-# bless it.
+# NEITHER TEST SUBJECT IS ASSERTED YET, and the reason has been found and
+# fixed at the root rather than papered over here.
 #
-# So this is an open question rather than a missing assertion: either
-# cost-based selection is not preferring fiber here, or the criteria were
-# written for a topology this file no longer has. n06 taking n10 is the
-# more interesting half — n10 reaches n06 by fiber, so the choice may be
-# defensible and the comment simply stale. It needs someone to work out
-# which, and until then this scenario proves nothing about parent choice.
+# The diagram above puts n01 at the top, and both test subjects only make
+# sense in a tree rooted there. The mesh does not root by topology: it roots
+# at the numerically smallest NodeAddr (src/tree/state.rs:363-390), which is a
+# hash of the node's key and bears no relation to the numbering. Until
+# 2026-07-23 this scenario rooted at **n09** in every run, which put both
+# criteria out of reach. Under that root n08's two candidates were not at
+# equal depth, so its choice was settled by depth long before link technology
+# could matter, and n08 taking n04 was CORRECT behaviour that these comments
+# made look like a defect.
 #
-# That leaves this scenario asserting nothing about its own subject. The
-# baseline block further down holds it to forming a tree and carrying
-# traffic, which is a floor and not a substitute: nothing here checks that
-# the tree it forms is the one these comments describe.
+# What the archive shows, for anyone re-deriving this later: across the five
+# runs under the old root, n06's parent is n10 in three and n03 in two, and
+# n08's is n04 in four and n03 in one. Those numbers describe a tree rooted at
+# n09 and must not be used to calibrate anything now.
+#
+# `topology.pin_root` (default true) now assigns identities in NodeAddr order,
+# so n01 holds the smallest and the diagram describes the tree that forms.
+# Both criteria should therefore become reachable and assertable.
+#
+# STILL TO DO: run this under the pinned root, confirm n06 and n08 both take
+# n03, and encode `tree_parents` for them. Until then the baseline block below
+# is a floor and not a substitute — it holds the mesh to forming a tree and
+# carrying traffic, and nothing here yet checks that the tree is the one these
+# comments describe. Tracked in ISSUE-2026-0085.
 
 scenario:
   name: "mixed-technology"

From f478f51afed7bc36b3fe2bb20d797c0ba0eea4e9 Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 17:04:23 +0000
Subject: [PATCH 07/11] Assert mixed-technology's sound criterion, and correct
 the one that never was

With the root pinned, n08's test subject is assertable for the first time: its
two candidates sit at equal depth with fiber-grade default netem on one side and
Bluetooth on the other, which is exactly the preference this scenario exists to
test, and it takes the fiber parent in four of four runs. Encoded as written
rather than calibrated from the runs, since "should pick n03" is the spec and
the runs merely confirm it is met.

n06 is deliberately absent, and its documented criterion is corrected rather
than left standing. The header called n03 a fiber parent for n06; the netem
policies in the same file say that link is WiFi and the other candidate is
Bluetooth, so n06 chooses between two impaired links. Its only fiber-grade link
reaches a node at depth 3 that is never a good parent. There was never a reason
to expect it to prefer n03, and the two-of-four split observed is correct
behaviour: in a run where it took the Bluetooth peer, the fiber-labelled
candidate measured a link cost of 3 against the other's 1, so the daemon chose
the cheaper effective depth as it should.

Validated by replaying the assertion against all four archived runs, then
breaking it three ways to confirm it can fail, then one live run for the wiring.
One of the breaks expects n06 to take n03 and reports that it chose n02, which
is the direct demonstration of why n06 is not encoded.

cost-reeval records that its subject now fires in two runs of three, against
zero of five when it was structurally impossible, and why that rate is still not
one an assertion can rest on.
---
 testing/chaos/scenarios/cost-reeval.yaml      | 17 ++++++++---
 testing/chaos/scenarios/mixed-technology.yaml | 30 +++++++++++++++++--
 2 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/testing/chaos/scenarios/cost-reeval.yaml b/testing/chaos/scenarios/cost-reeval.yaml
index 51d2f08..734d8f4 100644
--- a/testing/chaos/scenarios/cost-reeval.yaml
+++ b/testing/chaos/scenarios/cost-reeval.yaml
@@ -50,10 +50,19 @@
 # `topology.pin_root` (default true) now assigns identities in NodeAddr order
 # so n01 is root and n04 is the depth-1 subject this file describes.
 #
-# STILL TO DO: this scenario has no assertion about its own subject, and that
-# was a consequence rather than an omission — none could have passed. With the
-# subject now reachable, decide whether the switch is reliable enough across
-# seeds to assert, and encode it if so. Tracked in ISSUE-2026-0085.
+# MEASURED after the fix, 2026-07-23: the periodic switch now happens, in 2 of
+# 3 runs on n04 (0, 1, 1 occurrences). Before the fix it was 0 in 5 of 5, and
+# structurally impossible rather than merely rare. So the subject is exercised
+# for the first time.
+#
+# STILL NOT ASSERTED, deliberately. A "at least one periodic switch" assertion
+# would red roughly one run in three on this evidence, and a single run reading
+# zero is an argument against asserting rather than for it. Two ways forward,
+# neither taken here: gather enough runs to know whether 2-in-3 is the real
+# rate, or reduce the variance -- the mutation is stochastic with fraction=0.5,
+# so a deterministic link-degradation schedule would make the asymmetry this
+# scenario needs occur every run and the outcome assertable.
+# Tracked in ISSUE-2026-0085.
 
 scenario:
   name: "cost-reeval"
diff --git a/testing/chaos/scenarios/mixed-technology.yaml b/testing/chaos/scenarios/mixed-technology.yaml
index d6cb1f3..fb9afd5 100644
--- a/testing/chaos/scenarios/mixed-technology.yaml
+++ b/testing/chaos/scenarios/mixed-technology.yaml
@@ -26,8 +26,21 @@
 #   Fiber:     n03-n08, n06-n10
 #
 # Test subjects:
-#   - n06 has fiber (n03) and Bluetooth (n02) parents — should pick n03
-#   - n08 has fiber (n03) and Bluetooth (n04) parents — should pick n03
+#   - n08 has fiber (n03) and Bluetooth (n04) parents — should pick n03.
+#     ASSERTED below. Holds in 4 of 4 runs under the pinned root.
+#   - n06 was documented as having "fiber (n03) and Bluetooth (n02) parents".
+#     CORRECTED 2026-07-23: that is wrong, and the netem block below is the
+#     evidence rather than the ASCII art above. `n03-n06` is a **WiFi** link
+#     (5-20ms, 1-3% loss) and `n02-n06` is Bluetooth (15-40ms, 2-8% loss), so
+#     n06 chooses between two impaired links, not between fiber and Bluetooth.
+#     n06's only fiber-grade link is `n06-n10`, and n10 sits at depth 3 so it
+#     is never a good parent. There is therefore no reason to expect n06 to
+#     prefer n03, and the mutation block (fraction 0.2, degraded 50-100ms)
+#     moves the two costs run to run: across four runs n06 took n03 twice and
+#     n02 twice. NOT asserted, and the observed split is the correct outcome
+#     rather than a defect — in the run checked, n03 measured a link cost of 3
+#     against n02's 1, so effective depth 1+3 beat by 1+1 and the daemon chose
+#     the cheaper path exactly as it should.
 #
 # Netem mutation shifts fiber-only links between normal and degraded.
 #
@@ -142,6 +155,19 @@ assertions:
     max_roots: 1
     min_nodes_parented: 9
     min_sessions: 3
+  # The scenario's own subject, assertable for the first time now that the
+  # root is pinned to n01 and the topology above describes the tree that
+  # forms. n08's two candidates sit at equal depth, so the choice is decided
+  # by link cost alone: n03 over fiber-grade default netem against n04 over
+  # Bluetooth. That is the fiber-versus-Bluetooth preference this scenario
+  # exists to test.
+  #
+  # The criterion is encoded as written rather than calibrated from runs:
+  # "should pick n03" is the spec, and four of four runs agree. n06 is
+  # deliberately absent — see the corrected note in the header for why its
+  # documented criterion was never founded on this topology.
+  tree_parents:
+    n08: n03
 
 logging:
   rust_log: "info"

From 9b7a27f2194f7b0b07bbf3181d22cf68d871f947 Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 17:09:07 +0000
Subject: [PATCH 08/11] Claim the chaos network range instead of deriving it,
 ending the collision

Two concurrent ci-local runs could not both run chaos. Each child's subnet came
from its position in the suite list, so both runs walked 10.30.0 through
10.30.12 and requested identical ranges; whichever reached the daemon first won
and the other died with a pool overlap. A run index or hashed offset would only
make that unlikely, and it is precisely the failure being removed.

The simulator now claims its range: attempt-create on a candidate, advance on
docker's own overlap error, fail loudly on anything else. Docker's address pool
becomes the arbiter, so an overlap is impossible rather than improbable. The
claim lives in the sim rather than in ci-local because only the process that
creates the network can advance on conflict, and because it fixes the bare
chaos.sh path too, which a ci-local-only fix would have left broken.

The generated compose now declares the network external over the claimed one,
and teardown releases the range from both paths, including the setup-failed
path where a run that fell over after claiming still holds one.

Ordering matters here and is not obvious: node IPs derive from the subnet
during topology generation, and traffic shaping keys its filters on those
addresses, so the claim happens before the topology exists. Claiming later
would give a network on one range and filters on another, which does not fail
at bring-up and instead leaves the shaping matching nothing.

--subnet survives as an explicit pin for when a known range is wanted, and now
fails loudly if that range is taken rather than silently overlapping. ci-local
no longer passes it.

Validated by running two instances of the same scenario deliberately
concurrently: they claimed 10.30.1.0/24 and 10.30.0.0/24, both exited 0 with
their assertions passing, and both released their networks. The negative
control holds too: with a squatter on a pinned range the run aborts with a
message naming the range rather than proceeding.
---
 testing/chaos/sim/__main__.py |  12 ++--
 testing/chaos/sim/compose.py  |  22 ++++---
 testing/chaos/sim/netclaim.py | 108 ++++++++++++++++++++++++++++++++++
 testing/chaos/sim/runner.py   |  43 +++++++++++++-
 testing/chaos/sim/scenario.py |   4 ++
 testing/ci-local.sh           |  18 +++---
 6 files changed, 185 insertions(+), 22 deletions(-)
 create mode 100644 testing/chaos/sim/netclaim.py

diff --git a/testing/chaos/sim/__main__.py b/testing/chaos/sim/__main__.py
index ec1867f..92bbe5a 100644
--- a/testing/chaos/sim/__main__.py
+++ b/testing/chaos/sim/__main__.py
@@ -27,9 +27,10 @@ def main():
     )
     parser.add_argument(
         "--subnet", type=str, default=None,
-        help="Override topology subnet CIDR (e.g. 10.30.0.0/24); node IPs "
-             "derive from it. Used by CI to give each parallel run a "
-             "non-overlapping network.",
+        help="Pin the topology subnet CIDR (e.g. 10.30.0.0/24) instead of "
+             "claiming a free one. The sim normally claims a range so "
+             "concurrent runs cannot collide; pin only when you need a known "
+             "range, and expect a hard failure if it is already taken.",
     )
     args = parser.parse_args()
 
@@ -55,7 +56,10 @@ def main():
             sys.exit(1)
         scenario.duration_secs = args.duration
     if args.subnet is not None:
-        scenario.topology.subnet = args.subnet
+        # Pinning opts out of claiming. The runner still *creates* the network,
+        # so a range already in use fails loudly here rather than silently
+        # overlapping a concurrent run.
+        scenario.topology.pinned_subnet = args.subnet
 
     runner = SimRunner(scenario)
     result = runner.run()
diff --git a/testing/chaos/sim/compose.py b/testing/chaos/sim/compose.py
index c27a153..3594ead 100644
--- a/testing/chaos/sim/compose.py
+++ b/testing/chaos/sim/compose.py
@@ -19,12 +19,12 @@ _COMPOSE_TEMPLATE = Template(
     """\
 networks:
   fips-net:
-    driver: bridge
-    labels:
-      - "com.corganlabs.fips-ci=1"
-    ipam:
-      config:
-        - subnet: {{ subnet }}
+    # External, and created by the sim rather than by compose. The range has to
+    # be *claimed* -- attempt-create, advance on docker's own overlap error --
+    # so that two concurrent runs cannot select the same one, and only the
+    # process that creates the network can do that. See sim/netclaim.py.
+    external: true
+    name: {{ network_name }}
 
 x-fips-common: &fips-common
   image: {{ image }}
@@ -64,8 +64,14 @@ def generate_compose(
     topology: SimTopology,
     scenario: Scenario,
     output_dir: str,
+    network_name: str,
 ) -> str:
-    """Render docker-compose.yml and write to output_dir. Returns the file path."""
+    """Render docker-compose.yml and write to output_dir. Returns the file path.
+
+    ``network_name`` is the docker network the sim has already claimed; the
+    compose file refers to it as external rather than declaring a subnet, so
+    that the claim and the creation are the same operation.
+    """
     os.makedirs(output_dir, exist_ok=True)
 
     nodes = [topology.nodes[nid] for nid in sorted(topology.nodes)]
@@ -77,7 +83,7 @@ def generate_compose(
     )
 
     content = _COMPOSE_TEMPLATE.render(
-        subnet=scenario.topology.subnet,
+        network_name=network_name,
         rust_log=scenario.logging.rust_log,
         image=FIPS_SIM_IMAGE,
         nodes=nodes,
diff --git a/testing/chaos/sim/netclaim.py b/testing/chaos/sim/netclaim.py
new file mode 100644
index 0000000..ab4294d
--- /dev/null
+++ b/testing/chaos/sim/netclaim.py
@@ -0,0 +1,108 @@
+"""Claim a free /24 for a scenario's network, atomically.
+
+Two concurrent chaos runs used to request identical ranges: `ci-local.sh`
+derived each child's subnet from its position in the suite list, so both runs
+walked `10.30.0` through `10.30.12` and collided on every one of them. A run
+index or a hashed offset only makes a collision unlikely, and a collision is
+precisely the failure being removed, so this claims instead.
+
+Claim-and-advance: attempt to create the network on a candidate range and, on
+docker's own "Pool overlaps" error, move to the next. Docker's address pool is
+then the arbiter and an overlap between two concurrent runs is *impossible*
+rather than improbable. The pattern is taken from `testing/sidecar/scripts/
+test-sidecar.sh:75-98`, which has been carrying it since 2026-07-19.
+
+The claim has to happen before the topology is generated, not before the
+compose file is written: node IPs derive from the subnet inside
+`generate_topology`, and traffic shaping keys its filters on those addresses.
+Claiming later yields a network on one range and `tc` filters on another, which
+does not fail at bring-up and instead leaves the shaping matching nothing.
+"""
+
+from __future__ import annotations
+
+import logging
+import subprocess
+
+log = logging.getLogger(__name__)
+
+# Stay inside 10.30.0.0/16, which ci-local.sh already documents as clear of
+# docker's default pool (172.17-31, 192.168) and of the fixed-subnet suites in
+# 172.x. Sidecar has claimed 10.40.0.0/16; do not overlap it.
+NET_BASE = "10.30"
+NET_CANDIDATES = 200
+
+
+class NetworkClaimError(RuntimeError):
+    """No range could be claimed, or docker refused for another reason."""
+
+
+def claim_network(
+    name: str,
+    labels: dict[str, str] | None = None,
+    candidates: list[str] | None = None,
+) -> str:
+    """Create `name` on the first free /24 and return its CIDR.
+
+    `candidates` pins the search to an explicit list, which is how `--subnet`
+    opts out of claiming. A single pinned candidate that is already taken
+    raises rather than advancing, so pinning fails loudly instead of silently
+    overlapping a concurrent run.
+
+    Raises NetworkClaimError if every candidate is taken, or immediately if
+    docker fails for any reason other than an address-pool conflict. Retrying
+    a real error 200 times would bury the reason for it.
+    """
+    ranges = candidates or [
+        f"{NET_BASE}.{i}.0/24" for i in range(NET_CANDIDATES)
+    ]
+    label_args: list[str] = []
+    for key, value in (labels or {}).items():
+        label_args += ["--label", f"{key}={value}"]
+
+    for subnet in ranges:
+        proc = subprocess.run(
+            ["docker", "network", "create", "--subnet", subnet]
+            + label_args
+            + [name],
+            capture_output=True,
+            text=True,
+        )
+        if proc.returncode == 0:
+            log.info("Claimed network %s on %s", name, subnet)
+            return subnet
+        err = (proc.stderr or "") + (proc.stdout or "")
+        if "ool overlaps" in err:
+            continue
+        raise NetworkClaimError(
+            f"docker network create {name} on {subnet} failed: {err.strip()}"
+        )
+
+    if candidates:
+        raise NetworkClaimError(
+            f"pinned subnet(s) {', '.join(candidates)} already in use; "
+            f"another run holds the range"
+        )
+    raise NetworkClaimError(
+        f"no free /24 in {NET_BASE}.0.0/16 after {NET_CANDIDATES} attempts"
+    )
+
+
+def remove_network(name: str) -> None:
+    """Remove a claimed network, tolerating its absence.
+
+    The network is `external:` to the generated compose file, so `compose down`
+    leaves it behind and this is the only thing that reclaims the range.
+    Failures are logged rather than raised: teardown runs on the failure path
+    too, and losing a /24 is a leak, not a reason to mask the original error.
+    """
+    proc = subprocess.run(
+        ["docker", "network", "rm", name],
+        capture_output=True,
+        text=True,
+    )
+    if proc.returncode != 0:
+        err = ((proc.stderr or "") + (proc.stdout or "")).strip()
+        if "not found" in err or "No such network" in err:
+            return
+        log.warning("Could not remove network %s: %s", name, err)
diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py
index 182a252..ee9c8c6 100644
--- a/testing/chaos/sim/runner.py
+++ b/testing/chaos/sim/runner.py
@@ -30,6 +30,7 @@ from .link_swap import LinkSwapManager
 from .links import LinkManager
 from .logs import AnalysisResult, analyze_logs, collect_logs, write_sim_metadata
 from .naming import name_suffix
+from .netclaim import claim_network, remove_network
 from .netem import NetemManager
 from .nodes import NodeManager
 from .peer_churn import PeerChurnManager
@@ -47,6 +48,9 @@ class SimRunner:
         self.rng = random.Random(scenario.seed)
         self.topology: SimTopology | None = None
         self.compose_file: str | None = None
+        # Claimed in _setup; the compose file refers to it as external, so
+        # `compose down` does not remove it and teardown must.
+        self.network_name: str | None = None
         self.output_dir: str = self._resolve_output_dir(scenario)
         self._interrupted = False
 
@@ -185,6 +189,28 @@ class SimRunner:
         logging.getLogger().addHandler(fh)
         log.info("Runner log: %s", runner_log_path)
 
+        # 0. Claim this run's network range, before anything derives from it.
+        #
+        # The ordering is not obvious and it matters: node IPs are computed
+        # from the subnet inside generate_topology, and traffic shaping keys
+        # its filters on those addresses. Claiming after the topology exists
+        # would give a network on one range and `tc` filters on another, which
+        # does not fail at bring-up — it silently leaves the shaping matching
+        # nothing.
+        self.network_name = f"fips-sim{name_suffix()}-net"
+        s.topology.subnet = claim_network(
+            self.network_name,
+            labels={
+                "com.corganlabs.fips-ci": "1",
+                "com.corganlabs.fips-ci.run": os.environ.get(
+                    "FIPS_CI_RUN_ID", "manual"
+                ),
+            },
+            candidates=(
+                [s.topology.pinned_subnet] if s.topology.pinned_subnet else None
+            ),
+        )
+
         # 1. Generate topology
         log.info(
             "Generating %d-node %s topology (seed=%d)...",
@@ -237,7 +263,9 @@ class SimRunner:
         log.info("Wrote node configs to %s", config_dir)
 
         # 3. Generate docker-compose.yml
-        self.compose_file = generate_compose(self.topology, self.scenario, config_dir)
+        self.compose_file = generate_compose(
+            self.topology, self.scenario, config_dir, self.network_name
+        )
         log.info("Wrote %s", self.compose_file)
 
         # 4. Build the test image once (avoids per-service build at scale)
@@ -521,6 +549,17 @@ class SimRunner:
         except Exception:
             log.exception("Could not write status file")
 
+    def _release_network(self) -> None:
+        """Give this run's claimed /24 back.
+
+        The compose file declares the network `external:`, so `compose down`
+        leaves it alone and nothing else reclaims the range. Called from both
+        teardown paths, including the setup-failed one, because a run that
+        fell over after claiming still holds a range.
+        """
+        if self.network_name:
+            remove_network(self.network_name)
+
     def _teardown(self) -> AnalysisResult | None:
         """Stop dynamic elements, collect logs, analyze, stop containers."""
         if not self._containers_started:
@@ -536,6 +575,7 @@ class SimRunner:
                 # containers or a network even with no mesh to speak of.
                 log.info("Stopping containers...")
                 docker_compose(self.compose_file, ["down"], check=False)
+            self._release_network()
             return None
 
         result = None
@@ -701,6 +741,7 @@ class SimRunner:
                 ["down"],
                 check=False,
             )
+            self._release_network()
 
         return result
 
diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py
index 8f03f83..eb2e66e 100644
--- a/testing/chaos/sim/scenario.py
+++ b/testing/chaos/sim/scenario.py
@@ -52,6 +52,10 @@ class TopologyConfig:
     # unreachable. Set false where election from an arbitrary key distribution
     # is itself the subject.
     pin_root: bool = True
+    # Set by --subnet to opt out of claiming a free range. Not a scenario-file
+    # key: a scenario that hardcoded its range would reintroduce the collision
+    # claiming exists to remove.
+    pinned_subnet: str | None = None
 
 
 @dataclass
diff --git a/testing/ci-local.sh b/testing/ci-local.sh
index a6c52b7..5bd9c99 100755
--- a/testing/ci-local.sh
+++ b/testing/ci-local.sh
@@ -905,7 +905,6 @@ run_integration() {
         local pids=()
         local suite_names=()
         local running=0
-        local chaos_idx=0
 
         for entry in "${CHAOS_SUITES[@]}"; do
             # Parse: "display-name scenario [flags...]"
@@ -913,14 +912,15 @@ run_integration() {
             local name="${parts[0]}"
             local args=("${parts[@]:1}")
 
-            # Give each chaos child a unique, non-overlapping /24 in 10.30.x so
-            # parallel children never collide with each other, and so a chaos
-            # net can never swallow a fixed-subnet suite (sidecar/static/nat in
-            # 172.x). 10.30.x sits outside docker's default-address-pool range
-            # (172.17-31 / 192.168), so auto-assigned nets can't land on it
-            # either. Node IPs derive from this subnet inside the sim.
-            args+=("--subnet" "10.30.${chaos_idx}.0/24")
-            chaos_idx=$((chaos_idx + 1))
+            # No --subnet: the sim claims a free /24 itself (sim/netclaim.py).
+            # This used to pass 10.30.${chaos_idx}.0/24, which uniquified the
+            # children of ONE run against each other and was byte-identical
+            # between runs -- so two concurrent ci-local runs both walked
+            # 10.30.0 through 10.30.12 and collided on every one. A claim makes
+            # the daemon the arbiter, so an overlap is impossible rather than
+            # merely unlikely. The range still sits in 10.30.0.0/16, clear of
+            # docker's default pool (172.17-31 / 192.168) and of the
+            # fixed-subnet suites in 172.x.
 
             # Throttle: wait for a slot
             while [[ $running -ge $PARALLEL_JOBS ]]; do

From 2bc0345174fddd6f6d64b4b2fbd31cfc6a53a14f Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 18:58:49 +0000
Subject: [PATCH 09/11] Make the cost-based chaos assertions reliable under
 parallel CI load

cost-avoidance and mixed-technology decide a parent by measured link cost
(etx * (1 + srtt_ms/100)). Under the local CI's 4-way-parallel chaos the fast
fiber link's measured srtt spikes from host scheduling and can exceed the
Bluetooth link's, flipping the choice and reddening a run that proves nothing
about the daemon. The old margin was ~0.4 cost units, which a ~24ms one-sided
scheduling blip closes.

Widen the Bluetooth delay to 150-250ms so the margin dwarfs any plausible
one-sided spike. The link still establishes well within the handshake budget
(a ~500ms RTT against a 30s stale-handshake window), so the losing candidate
is still a real, established peer rather than an absent one.

Add a netem mutation exclude_edges option and use it in mixed-technology to
pin the two links n08 chooses between, so a random degradation can't flip the
asserted comparison either. An unknown excluded edge is a hard error, since a
silent no-op would reintroduce the flakiness it exists to remove.

Validated by running both scenarios repeatedly under heavier-than-CI
concurrent load: the parent choice holds every time.
---
 testing/chaos/scenarios/cost-avoidance.yaml   | 27 +++++++++++----
 testing/chaos/scenarios/mixed-technology.yaml | 34 +++++++++++++------
 testing/chaos/sim/netem.py                    | 25 +++++++++++++-
 testing/chaos/sim/scenario.py                 |  8 ++++-
 4 files changed, 75 insertions(+), 19 deletions(-)

diff --git a/testing/chaos/scenarios/cost-avoidance.yaml b/testing/chaos/scenarios/cost-avoidance.yaml
index dec22fe..d9c529b 100644
--- a/testing/chaos/scenarios/cost-avoidance.yaml
+++ b/testing/chaos/scenarios/cost-avoidance.yaml
@@ -13,9 +13,21 @@
 #      n04  (test subject)
 #
 # n04 has two candidate parents at depth 1: n02 (via Bluetooth L2CAP)
-# and n03 (via fiber). Cost-based selection should pick n03 because:
-#   effective_depth(n02) = 1 + ~1.4 (Bluetooth) = ~2.4
-#   effective_depth(n03) = 1 + ~1.01 (fiber) = ~2.01
+# and n03 (via fiber). Cost-based selection should pick n03 because
+# link_cost = etx * (1 + srtt_ms/100), so the far slower Bluetooth link
+# costs far more:
+#   effective_depth(n02) = 1 + ~4.5 (Bluetooth, ~150-250ms) = ~5.5
+#   effective_depth(n03) = 1 + ~1.05 (fiber, ~1-5ms)        = ~2.05
+#
+# The Bluetooth delay was widened deliberately, and the size is set by load,
+# not by realism. At the original 15-40ms the cost margin was ~0.4, and even
+# 80-120ms was not enough: under the CI's parallel-chaos contention the
+# *measured* srtt of the fiber link spikes (host scheduling, not netem) and
+# was observed to exceed a 120ms Bluetooth base, flipping n04 to n02. At
+# 150-250ms the gap dwarfs any plausible one-sided fiber spike, so the
+# assertion below is a reliable verdict rather than a coin toss under load.
+# n04 still establishes the n02 link (it was parented to it when this flaked),
+# so the choice remains real rather than vacuous.
 #
 # Validation: tree snapshot shows n04's parent is n03.
 
@@ -44,11 +56,14 @@ netem:
     jitter_ms: [0, 1]
     loss_pct: [0, 0.5]
   link_policies:
-    # Bluetooth (L2CAP) link from n02 to n04
+    # Bluetooth (L2CAP) link from n02 to n04. Delay widened to 150-250ms so the
+    # fiber-vs-Bluetooth cost margin survives CI load spikes on the fiber
+    # probe's measured latency (see header). Well within handshake timeouts, so
+    # the n02 link still establishes and the choice stays real.
     - edges: ["n02-n04"]
       policy:
-        delay_ms: [15, 40]
-        jitter_ms: [5, 15]
+        delay_ms: [150, 250]
+        jitter_ms: [15, 30]
         loss_pct: [2, 8]
 
 link_flaps:
diff --git a/testing/chaos/scenarios/mixed-technology.yaml b/testing/chaos/scenarios/mixed-technology.yaml
index fb9afd5..ef32e9e 100644
--- a/testing/chaos/scenarios/mixed-technology.yaml
+++ b/testing/chaos/scenarios/mixed-technology.yaml
@@ -44,8 +44,8 @@
 #
 # Netem mutation shifts fiber-only links between normal and degraded.
 #
-# NEITHER TEST SUBJECT IS ASSERTED YET, and the reason has been found and
-# fixed at the root rather than papered over here.
+# n08 IS now asserted (below); n06 is not, and both reasons trace to the root,
+# which was found and fixed rather than papered over here.
 #
 # The diagram above puts n01 at the top, and both test subjects only make
 # sense in a tree rooted there. The mesh does not root by topology: it roots
@@ -64,13 +64,15 @@
 #
 # `topology.pin_root` (default true) now assigns identities in NodeAddr order,
 # so n01 holds the smallest and the diagram describes the tree that forms.
-# Both criteria should therefore become reachable and assertable.
+# Under that root n08 takes fiber n03 as its subject assertion (below) now
+# checks. n06 stays unasserted: its "fiber n03" was a WiFi link (see the
+# corrected note above), so it was never a founded criterion. Tracked in
+# ISSUE-2026-0085.
 #
-# STILL TO DO: run this under the pinned root, confirm n06 and n08 both take
-# n03, and encode `tree_parents` for them. Until then the baseline block below
-# is a floor and not a substitute — it holds the mesh to forming a tree and
-# carrying traffic, and nothing here yet checks that the tree is the one these
-# comments describe. Tracked in ISSUE-2026-0085.
+# The n08 assertion is made reliable, not merely reachable: the Bluetooth
+# contrast is widened (netem below) and the two links n08 chooses between are
+# excluded from the netem mutation, so neither host load nor a random
+# degradation can flip the asserted comparison.
 
 scenario:
   name: "mixed-technology"
@@ -105,11 +107,15 @@ netem:
     jitter_ms: [0, 1]
     loss_pct: [0, 0.5]
   link_policies:
-    # Bluetooth (L2CAP) links
+    # Bluetooth (L2CAP) links. Delay widened to 150-250ms so the
+    # fiber-vs-Bluetooth cost margin on n08's choice (fiber n03 vs Bluetooth
+    # n04) dwarfs any load-induced srtt spike on the fiber probe, matching
+    # cost-avoidance. Combined with excluding n03-n08/n04-n08 from the mutation
+    # (below), n08's asserted comparison is robust to both load and churn.
     - edges: ["n02-n06", "n04-n08"]
       policy:
-        delay_ms: [15, 40]
-        jitter_ms: [5, 15]
+        delay_ms: [150, 250]
+        jitter_ms: [15, 30]
         loss_pct: [2, 8]
     # WiFi links (moderate latency, low loss)
     - edges: ["n03-n06", "n08-n10"]
@@ -120,6 +126,12 @@ netem:
   mutation:
     interval_secs: {min: 30, max: 60}
     fraction: 0.2
+    # Never mutate the two links n08's tree_parents assertion depends on: the
+    # degraded policy (50-100ms) could otherwise slow the fiber n03-n08 link
+    # enough to flip n08 to n04 independently of the widened Bluetooth contrast,
+    # making the assertion a coin toss. The mutation still churns the other ten
+    # links, so the dynamic behaviour it exists to exercise is unchanged.
+    exclude_edges: ["n03-n08", "n04-n08"]
     policies:
       normal:
         delay_ms: [1, 10]
diff --git a/testing/chaos/sim/netem.py b/testing/chaos/sim/netem.py
index 45720a8..3ac8d0e 100644
--- a/testing/chaos/sim/netem.py
+++ b/testing/chaos/sim/netem.py
@@ -144,6 +144,27 @@ class NetemManager:
                 len(self._edge_overrides),
             )
 
+        # Edges the periodic mutation must never touch (canonical "nXX-nYY").
+        # Unlike a link_policy typo, which merely fails to shape a link, a typo
+        # here would silently leave an asserted link unprotected and reintroduce
+        # the exact flakiness the exclusion exists to remove — so an unknown
+        # edge is a hard error, not a warning.
+        self._mutation_exclude: set[str] = set()
+        for edge_str in config.mutation.exclude_edges:
+            canonical = "-".join(sorted(edge_str.split("-")))
+            if canonical not in topo_edge_strs:
+                raise ValueError(
+                    f"netem.mutation.exclude_edges references {edge_str!r}, "
+                    "which is not an edge in the topology"
+                )
+            self._mutation_exclude.add(canonical)
+        if self._mutation_exclude:
+            log.info(
+                "Mutation excludes %d edge(s): %s",
+                len(self._mutation_exclude),
+                ", ".join(sorted(self._mutation_exclude)),
+            )
+
     def _htb_rate(self, node_id: str, peer_id: str) -> str:
         """Return the HTB rate string for a link direction."""
         rate = self._edge_rates.get((node_id, peer_id), 0)
@@ -370,10 +391,12 @@ class NetemManager:
         if not self.config.mutation.policies:
             return
 
-        # Only consider edges where both endpoints are up
+        # Only consider edges where both endpoints are up, and never the
+        # explicitly excluded ones (links an assertion depends on).
         live_edges = [
             (a, b) for a, b in self.topology.edges
             if a not in self.down_nodes and b not in self.down_nodes
+            and "-".join(sorted([a, b])) not in self._mutation_exclude
         ]
         if not live_edges:
             return
diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py
index eb2e66e..c1f3eaf 100644
--- a/testing/chaos/sim/scenario.py
+++ b/testing/chaos/sim/scenario.py
@@ -73,6 +73,11 @@ class NetemMutationConfig:
     interval_secs: Range = field(default_factory=lambda: Range(15, 30))
     fraction: float = 0.3
     policies: dict[str, NetemPolicy] = field(default_factory=dict)
+    # Edges the periodic mutation must never touch, "nXX-nYY" strings. Use it
+    # to pin the links a tree_parents assertion depends on so a random
+    # degradation cannot flip the very comparison being asserted. Validated
+    # against the topology in NetemManager — an unknown edge is a hard error.
+    exclude_edges: list[str] = field(default_factory=list)
 
 
 @dataclass
@@ -388,7 +393,7 @@ _SECTION_KEYS = {
     },
     "netem": {"enabled", "default_policy", "link_policies", "mutation"},
     "netem.link_policies[]": {"edges", "policy", "policy_name"},
-    "netem.mutation": {"interval_secs", "fraction", "policies"},
+    "netem.mutation": {"interval_secs", "fraction", "policies", "exclude_edges"},
     "link_flaps": {
         "enabled", "interval_secs", "max_down_links", "down_duration_secs",
         "protect_connectivity",
@@ -549,6 +554,7 @@ def load_scenario(path: str) -> Scenario:
             mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs"
         )
         s.netem.mutation.fraction = float(mc.get("fraction", 0.3))
+        s.netem.mutation.exclude_edges = list(mc.get("exclude_edges", []))
         if "policies" in mc:
             s.netem.mutation.policies = {
                 name: _parse_netem_policy(pdata, f"netem.mutation.policies.{name}")

From bb9bca4d83f23decbb3b0d487bf194b8690cf896 Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 19:50:07 +0000
Subject: [PATCH 10/11] Accept a degraded systemd as a booted one in the
 deb-install boot wait

The boot check piped `systemctl is-system-running --wait` into `grep -qE
'running|degraded'`, but the script runs `set -o pipefail` and is-system-running
exits non-zero for `degraded`. So the pipeline failed on a degraded system even
though grep matched, and the wait looped to its timeout. The older distros reach
`running` (exit 0) and passed; debian:trixie and ubuntu:26.04 reach `degraded`
because a unit that cannot run in a container (systemd-modules-load) fails, so
they timed out at every ceiling -- which is why the earlier timeout increase did
nothing.

Capture the state string and test it directly instead of trusting the pipeline
exit, so a degraded-but-booted system is accepted as intended. Validated: all
five distros pass, including the two that previously failed.
---
 testing/deb-install/test.sh | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/testing/deb-install/test.sh b/testing/deb-install/test.sh
index daa7d96..432ef1c 100755
--- a/testing/deb-install/test.sh
+++ b/testing/deb-install/test.sh
@@ -32,7 +32,7 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
 CACHE_DIR="$SCRIPT_DIR/.cache"
 DEB_CACHE_DIR="$CACHE_DIR/deb"
 
-# Timeouts
+# Timeouts. Each wait loop below exits as soon as its condition is met.
 BOOT_TIMEOUT=30
 SERVICE_TIMEOUT=20
 DAEMON_TIMEOUT=15
@@ -75,11 +75,19 @@ start_systemd_container_with_tun() {
 }
 
 wait_for_systemd() {
-    local name="$1"
+    local name="$1" state
     for _i in $(seq 1 "$BOOT_TIMEOUT"); do
-        if docker exec "$name" systemctl is-system-running --wait 2>/dev/null | grep -qE 'running|degraded'; then
-            return 0
-        fi
+        # `is-system-running` exits non-zero for `degraded` (a unit failed to
+        # start -- e.g. systemd-modules-load, which cannot load kernel modules
+        # inside a container -- even though the system did finish booting). This
+        # script runs `set -o pipefail`, so a piped `grep` would inherit that
+        # non-zero exit and reject an acceptable state, which timed out the
+        # newest distros (they reach `degraded`, older ones reach `running`).
+        # Capture the state string and test it directly instead of the pipe.
+        state=$(docker exec "$name" systemctl is-system-running --wait 2>/dev/null || true)
+        case "$state" in
+            running | degraded) return 0 ;;
+        esac
         sleep 1
     done
     # A boot that never reached `running` or `degraded` is not a warning: every

From be5deee814e641a83b3ff3585cfafa4da980a017 Mon Sep 17 00:00:00 2001
From: Johnathan Corgan <johnathan@corganlabs.com>
Date: Thu, 23 Jul 2026 19:50:17 +0000
Subject: [PATCH 11/11] Make the gateway integration suite safe for concurrent
 CI runs

gateway-lan pinned 172.20.1.0/24 and fd02::/64, so two concurrent local CI
runs collided on "Pool overlaps" at network creation. The IPv4 subnet has no
consumer -- the whole gateway LAN path is IPv6 -- so drop it and let docker
auto-assign. The IPv6 side cannot float, because the LAN clients' resolv.conf
pins the gateway's address as their nameserver and that must be a literal
known before they start, so claim a free /64 per run and thread the prefix
through the compose address pins, a generated resolv.conf, and the test via a
single exported variable.

The claim and the external network ship as a harness-only compose overlay
applied by run_gateway; the base compose keeps a normal gateway-lan network,
so the GitHub matrix and any standalone bring-up are unaffected. With no claim
every address renders exactly as before.
---
 testing/ci-local.sh                           | 78 +++++++++++++++----
 .../docker-compose.gateway-external-net.yml   | 15 ++++
 testing/static/docker-compose.yml             | 18 ++---
 testing/static/scripts/gateway-test.sh        | 40 ++++++----
 testing/static/scripts/generate-configs.sh    | 10 +++
 5 files changed, 122 insertions(+), 39 deletions(-)
 create mode 100644 testing/static/docker-compose.gateway-external-net.yml

diff --git a/testing/ci-local.sh b/testing/ci-local.sh
index 5bd9c99..21650b8 100755
--- a/testing/ci-local.sh
+++ b/testing/ci-local.sh
@@ -617,29 +617,77 @@ run_chaos() {
     return $rc
 }
 
+# Claim a free /64 for this run's gateway-lan network (Mechanism B).
+#
+# gateway-lan cannot float like fips-net: the LAN clients' resolv.conf pins the
+# gateway's LAN address as their nameserver, which must be a literal known
+# before they start. So instead of a fixed fd02::/64 that two concurrent runs
+# both request (the second failing on "Pool overlaps"), claim-and-advance a
+# candidate /64 and let docker's create be the atomic arbiter. The claimed
+# prefix is threaded — via the exported FIPS_GW_LAN6_PREFIX — into the compose
+# ipv6_address pins, the generated resolv.conf, and gateway-test.sh, so every
+# LAN address moves with the claim. i=0 renders today's fd02::/64.
+#
+# Mirrors sidecar/scripts/test-sidecar.sh:alloc_network. Deliberately does NOT
+# discard stderr: only an address-pool conflict is worth advancing on; any other
+# failure is real and burning through 256 candidates would bury the reason.
+claim_gateway_lan6() {
+    local net="$1" i err
+    for (( i = 0; i < 256; i++ )); do
+        if err=$(docker network create --ipv6 \
+                --subnet "fd02:0:0:${i}::/64" \
+                --label "$CI_LABEL" --label "$CI_LABEL_RUN" \
+                "$net" 2>&1); then
+            export FIPS_GW_LAN6_PREFIX="fd02:0:0:${i}"
+            info "[gateway] Claimed $net on fd02:0:0:${i}::/64"
+            return 0
+        fi
+        case "$err" in
+            *"Pool overlaps"*|*"pool overlaps"*) continue ;;
+            *) fail "[gateway] docker network create: $err"; return 1 ;;
+        esac
+    done
+    fail "[gateway] no free /64 in fd02:0:0:0-255::/64 after 256 attempts"
+    return 1
+}
+
 # Run gateway integration test
 run_gateway() {
     local compose="testing/static/docker-compose.yml"
+    local ext="testing/static/docker-compose.gateway-external-net.yml"
     local rc=0
     export COMPOSE_PROJECT_NAME="$(ci_project static)"
+    export FIPS_GW_LAN_NET="fips-gateway-lan${FIPS_CI_NAME_SUFFIX:-}"
 
-    info "[gateway] Generating configs"
-    bash testing/static/scripts/generate-configs.sh gateway gateway-test || { record "gateway" 1; return; }
-    bash testing/static/scripts/gateway-test.sh inject-config || { record "gateway" 1; return; }
-
-    info "[gateway] Starting containers"
-    docker compose -f "$compose" --profile gateway up -d || { record "gateway" 1; return; }
-
-    info "[gateway] Running gateway test"
-    if bash testing/static/scripts/gateway-test.sh; then
-        rc=0
-    else
-        rc=1
-        info "[gateway] Collecting failure logs"
-        docker compose -f "$compose" --profile gateway logs --no-color 2>&1 | tail -100
+    # Claim the LAN /64 first: generate-configs writes resolv.conf from the
+    # claimed prefix, and inject-config renders the port-forward targets from
+    # it, so both must see FIPS_GW_LAN6_PREFIX before they run.
+    info "[gateway] Claiming LAN network"
+    if ! claim_gateway_lan6 "$FIPS_GW_LAN_NET"; then
+        record "gateway" 1
+        return
     fi
 
-    docker compose -f "$compose" --profile gateway down --volumes --remove-orphans 2>/dev/null
+    info "[gateway] Generating configs"
+    if bash testing/static/scripts/generate-configs.sh gateway gateway-test \
+        && bash testing/static/scripts/gateway-test.sh inject-config \
+        && { info "[gateway] Starting containers"; \
+             docker compose -f "$compose" -f "$ext" --profile gateway up -d; }; then
+        info "[gateway] Running gateway test"
+        if bash testing/static/scripts/gateway-test.sh; then
+            rc=0
+        else
+            rc=1
+            info "[gateway] Collecting failure logs"
+            docker compose -f "$compose" -f "$ext" --profile gateway logs --no-color 2>&1 | tail -100
+        fi
+    else
+        rc=1
+    fi
+
+    # compose down does not remove an external network, so drop it explicitly.
+    docker compose -f "$compose" -f "$ext" --profile gateway down --volumes --remove-orphans 2>/dev/null
+    docker network rm "$FIPS_GW_LAN_NET" >/dev/null 2>&1 || true
     record "gateway" $rc
 }
 
diff --git a/testing/static/docker-compose.gateway-external-net.yml b/testing/static/docker-compose.gateway-external-net.yml
new file mode 100644
index 0000000..fa1d40d
--- /dev/null
+++ b/testing/static/docker-compose.gateway-external-net.yml
@@ -0,0 +1,15 @@
+# Override: attach gateway-lan to a pre-created external network instead of
+# letting compose create it from the base file's fd02::/64 pin.
+#
+# Applied only by ci-local.sh's run_gateway, which claims a free /64 per run
+# (claim_gateway_lan6) and exports FIPS_GW_LAN_NET / FIPS_GW_LAN6_PREFIX before
+# `up`. This is what makes two concurrent local gateway runs collision-safe:
+# each claims a distinct /64, so neither requests a fixed range the other holds.
+#
+# The GitHub matrix and any standalone `docker compose up` do NOT apply this
+# overlay; they use the base file's normal gateway-lan network unchanged. This
+# mirrors sidecar/docker-compose.external-net.yml.
+networks:
+  gateway-lan:
+    external: true
+    name: ${FIPS_GW_LAN_NET:-fips-gateway-lan}
diff --git a/testing/static/docker-compose.yml b/testing/static/docker-compose.yml
index 0e94542..602c037 100644
--- a/testing/static/docker-compose.yml
+++ b/testing/static/docker-compose.yml
@@ -8,6 +8,10 @@ networks:
     driver: bridge
     labels:
       - "com.corganlabs.fips-ci=1"
+  # IPv4 is dropped so docker auto-assigns it (nothing reads a LAN IPv4 — the
+  # whole gateway LAN path is IPv6). The fd02::/64 pin stays for the standalone /
+  # GitHub path; concurrent local runs replace this network with a per-run
+  # claimed /64 via docker-compose.gateway-external-net.yml (see run_gateway).
   gateway-lan:
     driver: bridge
     labels:
@@ -15,7 +19,6 @@ networks:
     enable_ipv6: true
     ipam:
       config:
-        - subnet: 172.20.1.0/24
         - subnet: fd02::/64
 
 x-fips-common: &fips-common
@@ -475,8 +478,7 @@ services:
     networks:
       fips-net:
       gateway-lan:
-        ipv4_address: 172.20.1.10
-        ipv6_address: fd02::10
+        ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::10
 
   gw-server:
     <<: *fips-common
@@ -513,11 +515,10 @@ services:
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
     volumes:
-      - ./configs/gateway-resolv.conf:/etc/resolv.conf:ro
+      - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/gateway/resolv.conf:/etc/resolv.conf:ro
     networks:
       gateway-lan:
-        ipv4_address: 172.20.1.20
-        ipv6_address: fd02::20
+        ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::20
     restart: "no"
     env_file:
       - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env
@@ -535,11 +536,10 @@ services:
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=0
     volumes:
-      - ./configs/gateway-resolv.conf:/etc/resolv.conf:ro
+      - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/gateway/resolv.conf:/etc/resolv.conf:ro
     networks:
       gateway-lan:
-        ipv4_address: 172.20.1.21
-        ipv6_address: fd02::21
+        ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::21
     restart: "no"
     env_file:
       - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env
diff --git a/testing/static/scripts/gateway-test.sh b/testing/static/scripts/gateway-test.sh
index 16fc9c2..9cb8a60 100755
--- a/testing/static/scripts/gateway-test.sh
+++ b/testing/static/scripts/gateway-test.sh
@@ -26,6 +26,16 @@ SERVER2="fips-gw-server-2${FIPS_CI_NAME_SUFFIX:-}"
 CLIENT="fips-gw-client${FIPS_CI_NAME_SUFFIX:-}"
 CLIENT2="fips-gw-client-2${FIPS_CI_NAME_SUFFIX:-}"
 
+# LAN-side IPv6 addressing. run_gateway claims a per-run /64 and exports
+# FIPS_GW_LAN6_PREFIX; unset (standalone / GitHub) these render the base
+# compose's fd02:: addresses, byte-identical to before. GW_DNS is the gateway's
+# LAN address (nameserver + route next-hop); GW_CLIENT_LAN is gw-client's LAN
+# address (inbound port-forward target). fd01::/112 (the virtual pool) is NOT
+# claimed and stays literal below.
+GW_LAN6_PREFIX="${FIPS_GW_LAN6_PREFIX:-fd02}"
+GW_DNS="${GW_LAN6_PREFIX}::10"
+GW_CLIENT_LAN="${GW_LAN6_PREFIX}::20"
+
 # ── inject-config subcommand ─────────────────────────────────────────────
 
 inject_gateway_config() {
@@ -58,21 +68,21 @@ cfg['gateway'] = {
         {
             'listen_port': 18080,
             'proto': 'tcp',
-            'target': '[fd02::20]:8080',
+            'target': '[${GW_CLIENT_LAN}]:8080',
         },
         # 6B: second TCP forward — exercises multiple simultaneous TCP
         # rules sharing the same LAN backend on a different listen port.
         {
             'listen_port': 18082,
             'proto': 'tcp',
-            'target': '[fd02::20]:8081',
+            'target': '[${GW_CLIENT_LAN}]:8081',
         },
         # 6A: UDP forward — exercises the runtime UDP DNAT path (rule
         # shape + conntrack handling) end-to-end.
         {
             'listen_port': 18081,
             'proto': 'udp',
-            'target': '[fd02::20]:8081',
+            'target': '[${GW_CLIENT_LAN}]:8081',
         },
     ],
 }
@@ -130,7 +140,7 @@ for i in $(seq 1 30); do
     # Try resolving the server's npub via the gateway DNS from the client.
     # Match fd01:: specifically (the pool prefix) to avoid false-positive
     # matches on error messages containing fd02::10.
-    local_result=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @fd02::10 2>/dev/null || true)
+    local_result=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @${GW_DNS} 2>/dev/null || true)
     if echo "$local_result" | grep -q "^fd01::"; then
         echo "  Gateway DNS responding after ${i}s"
         DNS_READY=true
@@ -146,23 +156,23 @@ fi
 # Phase 3: Client network setup — route virtual IP pool via gateway
 echo ""
 echo "Phase 3: Client network setup"
-docker exec "$CLIENT" ip -6 route add fd01::/112 via fd02::10 2>/dev/null || true
-echo "  Added route fd01::/112 via fd02::10 on $CLIENT"
-docker exec "$CLIENT2" ip -6 route add fd01::/112 via fd02::10 2>/dev/null || true
-echo "  Added route fd01::/112 via fd02::10 on $CLIENT2"
+docker exec "$CLIENT" ip -6 route add fd01::/112 via ${GW_DNS} 2>/dev/null || true
+echo "  Added route fd01::/112 via ${GW_DNS} on $CLIENT"
+docker exec "$CLIENT2" ip -6 route add fd01::/112 via ${GW_DNS} 2>/dev/null || true
+echo "  Added route fd01::/112 via ${GW_DNS} on $CLIENT2"
 
 # Phase 4: DNS resolution test — resolve server npub from both clients,
 # exercising concurrent multi-client mappings.
 echo ""
 echo "Phase 4: DNS resolution"
-VIRTUAL_IP=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @fd02::10 2>/dev/null | head -1)
+VIRTUAL_IP=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @${GW_DNS} 2>/dev/null | head -1)
 if [ -n "$VIRTUAL_IP" ] && echo "$VIRTUAL_IP" | grep -q "fd01"; then
     check "Resolve ${NPUB_B:0:20}...fips on $CLIENT → $VIRTUAL_IP" 0
 else
     check "Resolve ${NPUB_B:0:20}...fips on $CLIENT (got: '$VIRTUAL_IP')" 1
 fi
 
-VIRTUAL_IP_2=$(docker exec "$CLIENT2" dig +short AAAA "${NPUB_C}.fips" @fd02::10 2>/dev/null | head -1)
+VIRTUAL_IP_2=$(docker exec "$CLIENT2" dig +short AAAA "${NPUB_C}.fips" @${GW_DNS} 2>/dev/null | head -1)
 if [ -n "$VIRTUAL_IP_2" ] && echo "$VIRTUAL_IP_2" | grep -q "fd01"; then
     check "Resolve ${NPUB_C:0:20}...fips on $CLIENT2 → $VIRTUAL_IP_2" 0
 else
@@ -357,7 +367,7 @@ else
     # 8080 backend serves "inbound-forward-ok" (no -2 suffix) — distinct
     # from the 8081 backend so a misrouted response would be detectable.
     if echo "$FWD_RESPONSE" | grep -qE '^inbound-forward-ok$'; then
-        check "Inbound HTTP via TCP forward 18080 → [fd02::20]:8080" 0
+        check "Inbound HTTP via TCP forward 18080 → [${GW_CLIENT_LAN}]:8080" 0
     else
         check "Inbound HTTP via TCP forward 18080 (response: '${FWD_RESPONSE:0:80}')" 1
     fi
@@ -365,7 +375,7 @@ else
     FWD_RESPONSE_2=$(docker exec "$SERVER" curl -6 -s --max-time 10 \
         "http://[${GW_MESH_IP}]:18082/" 2>&1) || true
     if echo "$FWD_RESPONSE_2" | grep -q "inbound-forward-ok-2"; then
-        check "Inbound HTTP via TCP forward 18082 → [fd02::20]:8081 (6B)" 0
+        check "Inbound HTTP via TCP forward 18082 → [${GW_CLIENT_LAN}]:8081 (6B)" 0
     else
         check "Inbound HTTP via TCP forward 18082 (response: '${FWD_RESPONSE_2:0:80}')" 1
     fi
@@ -384,7 +394,7 @@ except Exception as e:
     sys.stdout.write('ERR: ' + str(e))
 " 2>&1) || true
     if echo "$UDP_RESPONSE" | grep -q "udp-forward-ok:ping-via-udp-fwd"; then
-        check "Inbound UDP via forward 18081 → [fd02::20]:8081 (6A)" 0
+        check "Inbound UDP via forward 18081 → [${GW_CLIENT_LAN}]:8081 (6A)" 0
     else
         check "Inbound UDP via forward 18081 (response: '${UDP_RESPONSE:0:80}')" 1
     fi
@@ -444,8 +454,8 @@ docker exec "$GATEWAY" pkill -f "^fips --config" 2>/dev/null || true
 sleep 2
 
 # Gateway upstream timeout is 5s, so dig must wait longer than that.
-SERVFAIL_RESULT=$(docker exec "$CLIENT" dig +short +tries=1 +time=8 AAAA "test-servfail.fips" @fd02::10 2>&1 || true)
-SERVFAIL_STATUS=$(docker exec "$CLIENT" dig +tries=1 +time=8 AAAA "test-servfail.fips" @fd02::10 2>&1 | grep -c "SERVFAIL" || true)
+SERVFAIL_RESULT=$(docker exec "$CLIENT" dig +short +tries=1 +time=8 AAAA "test-servfail.fips" @${GW_DNS} 2>&1 || true)
+SERVFAIL_STATUS=$(docker exec "$CLIENT" dig +tries=1 +time=8 AAAA "test-servfail.fips" @${GW_DNS} 2>&1 | grep -c "SERVFAIL" || true)
 if [ "$SERVFAIL_STATUS" -ge 1 ]; then
     check "SERVFAIL when daemon DNS is down" 0
 else
diff --git a/testing/static/scripts/generate-configs.sh b/testing/static/scripts/generate-configs.sh
index a1d1a04..529ec12 100755
--- a/testing/static/scripts/generate-configs.sh
+++ b/testing/static/scripts/generate-configs.sh
@@ -274,6 +274,16 @@ generate_topology() {
         echo "${var_name}=$(get_key RESOLVED_NPUB "$node_id")" >> "$env_file"
     done
     echo "  ✓ Generated $env_file"
+
+    # Phase 4 (gateway only): write the LAN-client resolv.conf. Its nameserver
+    # is the gateway's LAN address, which must be a literal known before the
+    # client starts. run_gateway claims a per-run /64 and exports
+    # FIPS_GW_LAN6_PREFIX before calling this; unset (standalone / GitHub) it
+    # renders the base compose's fd02::10.
+    if [ "$topology_name" = "gateway" ]; then
+        echo "nameserver ${FIPS_GW_LAN6_PREFIX:-fd02}::10" > "$output_dir/resolv.conf"
+        echo "  ✓ Generated $output_dir/resolv.conf"
+    fi
 }
 
 main() {