From fa49dc1210f039d068986046855b25c3ac137c30 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 13:54:14 +0000 Subject: [PATCH 01/11] Retire the mesh-public static test topology The mesh-public profile ran in neither runner: ci-local's static suite list carries only static-mesh and static-chain, and the GitHub matrix has only mesh and chain. It also added no coverage over static-mesh. ping-test.sh and iperf-test.sh branched mesh and mesh-public together and exercised the same 20 directed pairs among node-a through node-e, and no script referenced the external node at all. The convergence waits even used mesh's peer counts rather than mesh-public's, so the extra link to the public node was never counted, let alone asserted. Running it would therefore have added a dependency on a live internet host (test-us01.fips.network) in exchange for zero additional assertions. Remove the topology, its five compose services, the script branches that aliased it to mesh, and the documentation rows. An invocation using the old profile name now fails on ping-test.sh's unknown-profile guard instead of silently behaving as mesh. The config generator's external-node support (external_ip, is_external_node) stays. It has no consumer now, but it is woven into the config path every remaining topology uses, so removing it would put the gating suites at risk for no present gain. --- testing/README.md | 1 - testing/static/README.md | 5 +- .../configs/topologies/mesh-public.yaml | 51 ----------------- testing/static/docker-compose.yml | 56 ------------------- testing/static/scripts/bench-multirun.sh | 2 +- testing/static/scripts/generate-configs.sh | 2 +- testing/static/scripts/iperf-test.sh | 4 +- testing/static/scripts/ping-test.sh | 6 +- 8 files changed, 8 insertions(+), 119 deletions(-) delete mode 100644 testing/static/configs/topologies/mesh-public.yaml diff --git a/testing/README.md b/testing/README.md index 6933a43..bd1ba48 100644 --- a/testing/README.md +++ b/testing/README.md @@ -16,7 +16,6 @@ configurations. | ----------- | ----- | --------- | -------------------------------- | | mesh | 5 | UDP | Sparse mesh, 6 links, multi-hop | | chain | 5 | UDP | Linear chain, max 4-hop paths | -| mesh-public | 5+1 | UDP | Mesh with external public node | | tcp-chain | 3 | TCP | Linear chain over TCP (port 8443) | | rekey | 5 | UDP | Rekey integration test topology | diff --git a/testing/static/README.md b/testing/static/README.md index bde3f05..27980ed 100644 --- a/testing/static/README.md +++ b/testing/static/README.md @@ -119,15 +119,12 @@ testing/static/ │ └── topologies/ │ ├── mesh.yaml # Mesh topology definition │ ├── chain.yaml # Chain topology definition -│ ├── mesh-public.yaml # Mesh + external public node │ ├── tcp-chain.yaml # TCP chain (3 nodes, port 8443) │ └── rekey.yaml # Rekey integration test (5 nodes) ├── generated-configs/ # Auto-generated, run-scoped (gitignored) │ ├── npubs.env # NPUB_A=..., NPUB_B=..., etc. │ ├── mesh/ │ │ ├── node-a.yaml ... node-e.yaml -│ ├── mesh-public/ -│ │ ├── node-a.yaml ... node-e.yaml │ ├── chain/ │ │ ├── node-a.yaml ... node-e.yaml │ └── tcp-chain/ @@ -219,7 +216,7 @@ each mesh needs unique node identities to avoid key conflicts. The optional # Or generate configs directly ./testing/static/scripts/generate-configs.sh mesh my-mesh-1 -./testing/static/scripts/generate-configs.sh mesh-public my-mesh-1 +./testing/static/scripts/generate-configs.sh chain my-mesh-1 ``` ### How It Works diff --git a/testing/static/configs/topologies/mesh-public.yaml b/testing/static/configs/topologies/mesh-public.yaml deleted file mode 100644 index 1630241..0000000 --- a/testing/static/configs/topologies/mesh-public.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# Mesh Topology Definition -# -# Five nodes with 6 bidirectional UDP links forming a sparse, fully connected -# graph. Not all nodes are direct peers — non-adjacent pairs require -# discovery-driven multi-hop routing to establish end-to-end sessions. - -nodes: - a: - nsec: "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" - npub: "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m" - docker_ip: "172.20.0.10" - peers: [d, e, pub] - - b: - nsec: "b102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fb0" - npub: "npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le" - docker_ip: "172.20.0.11" - peers: [c, pub] - - c: - nsec: "c102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fc0" - npub: "npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6" - docker_ip: "172.20.0.12" - peers: [b, d, e, pub] - - d: - nsec: "d102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fd0" - npub: "npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl" - docker_ip: "172.20.0.13" - peers: [a, c, e] - - e: - nsec: "e102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fe0" - npub: "npub1wf8akf8lu2zdkjkmwhl75pqvven654mpv4sz2x2tprl5265mgrzq8nhak4" - docker_ip: "172.20.0.14" - peers: [a, c, d] - - # External/public node (not a Docker container) - pub: - npub: "npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98" - external_ip: "test-us01.fips.network" - peers: [a, b, c] - -# Spanning Tree Structure (rooted at node A): -# - A — D (tree edge, D's parent is A) -# - A — E (tree edge, E's parent is A) -# - C — D (tree edge, C's parent is D) -# - B — C (tree edge, B's parent is C) -# - D — E (non-tree link) -# - C — E (non-tree link) - diff --git a/testing/static/docker-compose.yml b/testing/static/docker-compose.yml index 602c037..74b7912 100644 --- a/testing/static/docker-compose.yml +++ b/testing/static/docker-compose.yml @@ -98,62 +98,6 @@ services: networks: fips-net: - # ── Mesh-public topology (mesh + external public node) ──────── - pub-a: - <<: *fips-common - profiles: ["mesh-public"] - container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-} - hostname: node-a - volumes: - - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/mesh-public/node-a.yaml:/etc/fips/fips.yaml:ro - networks: - fips-net: - - pub-b: - <<: *fips-common - profiles: ["mesh-public"] - container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-} - hostname: node-b - volumes: - - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/mesh-public/node-b.yaml:/etc/fips/fips.yaml:ro - networks: - fips-net: - - pub-c: - <<: *fips-common - profiles: ["mesh-public"] - container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-} - hostname: node-c - volumes: - - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/mesh-public/node-c.yaml:/etc/fips/fips.yaml:ro - networks: - fips-net: - - pub-d: - <<: *fips-common - profiles: ["mesh-public"] - container_name: fips-node-d${FIPS_CI_NAME_SUFFIX:-} - hostname: node-d - volumes: - - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/mesh-public/node-d.yaml:/etc/fips/fips.yaml:ro - networks: - fips-net: - - pub-e: - <<: *fips-common - profiles: ["mesh-public"] - container_name: fips-node-e${FIPS_CI_NAME_SUFFIX:-} - hostname: node-e - volumes: - - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/mesh-public/node-e.yaml:/etc/fips/fips.yaml:ro - networks: - fips-net: - # ── Chain topology (A-B-C-D-E) ──────────────────────────────── chain-a: <<: *fips-common diff --git a/testing/static/scripts/bench-multirun.sh b/testing/static/scripts/bench-multirun.sh index d2a138b..a96186a 100755 --- a/testing/static/scripts/bench-multirun.sh +++ b/testing/static/scripts/bench-multirun.sh @@ -137,7 +137,7 @@ ping_path() { # steady state, which is what real-world FIPS deployments see. To # force on-wire multihop, isolate nodes on distinct docker networks. case "$PROFILE" in - mesh|mesh-public) + mesh) # Static-peer paths only — without mDNS / a Nostr relay in the # test container set, non-adjacent pairs (A↔B, A↔C) can't # establish direct UDP and the bench would either fail diff --git a/testing/static/scripts/generate-configs.sh b/testing/static/scripts/generate-configs.sh index 529ec12..696a432 100755 --- a/testing/static/scripts/generate-configs.sh +++ b/testing/static/scripts/generate-configs.sh @@ -2,7 +2,7 @@ # Generate FIPS node configuration files from template and topology definition. # # Usage: ./generate-configs.sh [mesh-name] -# topology: mesh, mesh-public, chain, etc. +# topology: mesh, chain, etc. # mesh-name: optional; when given, docker node identities are derived # deterministically via sha256(mesh-name|node-id) diff --git a/testing/static/scripts/iperf-test.sh b/testing/static/scripts/iperf-test.sh index 124803e..9cf30ad 100755 --- a/testing/static/scripts/iperf-test.sh +++ b/testing/static/scripts/iperf-test.sh @@ -89,7 +89,7 @@ echo "" # nodes — labels DO NOT encode hop count, since peer discovery # converges every same-subnet pair onto a direct UDP path within a # few ticks regardless of the static `peers:` list. -if [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then +if [ "$PROFILE" = "mesh" ]; then echo "Topology (static \`peers:\` from mesh.yaml):" echo " A peers with: D, E" echo " B peers with: C" @@ -109,7 +109,7 @@ echo "" echo "Waiting ${SETTLE_SECONDS}s for mesh convergence..." sleep "$SETTLE_SECONDS" -if [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then +if [ "$PROFILE" = "mesh" ]; then echo "" echo "Testing mesh topology paths:" iperf_test node-d node-a "$NPUB_D" "A→D" diff --git a/testing/static/scripts/ping-test.sh b/testing/static/scripts/ping-test.sh index fb6069c..2c2e383 100755 --- a/testing/static/scripts/ping-test.sh +++ b/testing/static/scripts/ping-test.sh @@ -93,7 +93,7 @@ if [ "$PROFILE" = "chain" ]; then wait_for_peers fips-node-c${FIPS_CI_NAME_SUFFIX:-} 2 20 || true wait_for_peers fips-node-d${FIPS_CI_NAME_SUFFIX:-} 2 20 || true wait_for_peers fips-node-e${FIPS_CI_NAME_SUFFIX:-} 1 20 || true -elif [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then +elif [ "$PROFILE" = "mesh" ]; then # Mesh: check all nodes reach their configured peer counts wait_for_peers fips-node-a${FIPS_CI_NAME_SUFFIX:-} 2 20 || true wait_for_peers fips-node-b${FIPS_CI_NAME_SUFFIX:-} 1 20 || true @@ -105,7 +105,7 @@ else # section below is guarded by these same profile names, so the script # ran no assertion at all and exited 0. A typo in the caller's profile # argument produced a green run that tested nothing. - echo "ERROR: unknown profile '$PROFILE' (expected: chain, mesh, mesh-public)" >&2 + echo "ERROR: unknown profile '$PROFILE' (expected: chain, mesh)" >&2 exit 2 fi # Wait for full pairwise connectivity, progress-aware: the actual pings @@ -119,7 +119,7 @@ wait_until_connected ping_all_quiet 45 15 || true PASSED=0 FAILED=0 -if [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then +if [ "$PROFILE" = "mesh" ]; then # Sparse mesh topology: A-B, B-C, C-D, D-E, E-A, A-D # Test all 20 directed pairs (5 nodes × 4 targets each) echo "" From 44e04eb2e220f87743d5b4b51bed609ce988c593 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 14:02:12 +0000 Subject: [PATCH 02/11] Retire the tcp-chain static test topology The tcp-chain profile has never been runnable. ping-test.sh dispatches on chain and mesh only, and the string tcp-chain has never appeared in that script in its history, so invoking the profile has always fallen through to the unknown-profile branch: silently asserting nothing before that branch was made to fail, and exiting 2 since. Neither runner referenced it either. What it would have covered is already covered. The chaos tcp-mesh scenario runs in local CI and gives TCP a discriminating gate: n04's only edges are TCP, so if the transport were broken n04 could not parent and the scenario's baseline assertion would fail. It also carries a pure-TCP two-hop path, n01 to n04 to n05, under netem and link flaps. The only residual tcp-chain would have added is a mesh with no UDP present anywhere, which is not worth a fixture that has never run. Remove the topology, its three compose services, and the documentation rows. Every profile the static compose file still defines is now exercised by a suite. Also correct the chaos README, which has listed tcp-only and tcp-chain in its transport table since those two scenarios were deleted in 9e63b42. Neither has existed for months. --- testing/README.md | 1 - testing/chaos/README.md | 7 ---- testing/static/README.md | 7 ++-- .../static/configs/topologies/tcp-chain.yaml | 29 ---------------- testing/static/docker-compose.yml | 34 ------------------- 5 files changed, 2 insertions(+), 76 deletions(-) delete mode 100644 testing/static/configs/topologies/tcp-chain.yaml diff --git a/testing/README.md b/testing/README.md index bd1ba48..0cab275 100644 --- a/testing/README.md +++ b/testing/README.md @@ -16,7 +16,6 @@ configurations. | ----------- | ----- | --------- | -------------------------------- | | mesh | 5 | UDP | Sparse mesh, 6 links, multi-hop | | chain | 5 | UDP | Linear chain, max 4-hop paths | -| tcp-chain | 3 | TCP | Linear chain over TCP (port 8443) | | rekey | 5 | UDP | Rekey integration test topology | ### [tor/](tor/) -- Tor Transport Integration diff --git a/testing/chaos/README.md b/testing/chaos/README.md index 1733a9b..994ae38 100644 --- a/testing/chaos/README.md +++ b/testing/chaos/README.md @@ -67,19 +67,12 @@ Explicit topologies exercising non-UDP transports. | ------------- | ----- | -------------- | ----- | -------- | ----- | ---------- | ------------------------------------------ | | ethernet-only | 4 | Ethernet | Ring | 90s | yes | -- | AF_PACKET transport with beacon discovery | | ethernet-mesh | 6 | UDP + Ethernet | Mesh | 120s | yes | yes | Mixed UDP/Ethernet, netem mutation + flaps | -| tcp-only | 4 | TCP | Ring | 90s | yes | -- | TCP transport with static peer config | -| tcp-chain | 4 | TCP | Chain | 90s | yes | -- | TCP multi-hop routing through chain | | tcp-mesh | 6 | UDP + TCP | Mesh | 120s | yes | yes | Mixed UDP/TCP, netem mutation + flaps | - **ethernet-only**: 4-node ring on raw Ethernet (AF_PACKET). Peers discovered via beacons, not static config. Minimal netem (1-5ms delay). - **ethernet-mesh**: Mirrors `tcp-mesh` topology but with Ethernet instead of TCP. UDP edges use static config; Ethernet edges use beacon discovery. -- **tcp-only**: 4-node ring using TCP on port 8443. Tests connect-on-send, - FMP framing over TCP, and reconnection. Netem enabled (1-10ms delay, 0-1% - loss). -- **tcp-chain**: 4-node linear chain, all TCP. Tests multi-hop routing over - TCP-only mesh. - **tcp-mesh**: 6-node mesh with 4 UDP and 3 TCP edges. Both transports use static peer config. Netem mutation (30% fraction, every 20-40s) and link flaps (1 link max, 10-20s down). diff --git a/testing/static/README.md b/testing/static/README.md index 27980ed..1924ec9 100644 --- a/testing/static/README.md +++ b/testing/static/README.md @@ -119,16 +119,13 @@ testing/static/ │ └── topologies/ │ ├── mesh.yaml # Mesh topology definition │ ├── chain.yaml # Chain topology definition -│ ├── tcp-chain.yaml # TCP chain (3 nodes, port 8443) │ └── rekey.yaml # Rekey integration test (5 nodes) ├── generated-configs/ # Auto-generated, run-scoped (gitignored) │ ├── npubs.env # NPUB_A=..., NPUB_B=..., etc. │ ├── mesh/ │ │ ├── node-a.yaml ... node-e.yaml -│ ├── chain/ -│ │ ├── node-a.yaml ... node-e.yaml -│ └── tcp-chain/ -│ ├── node-a.yaml ... node-c.yaml +│ └── chain/ +│ ├── node-a.yaml ... node-e.yaml ├── scripts/ │ ├── build.sh # Build binary + generate configs │ ├── generate-configs.sh # Generate node configs from topology diff --git a/testing/static/configs/topologies/tcp-chain.yaml b/testing/static/configs/topologies/tcp-chain.yaml deleted file mode 100644 index a36fa6d..0000000 --- a/testing/static/configs/topologies/tcp-chain.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# TCP Chain Topology Definition -# -# Three nodes with TCP links forming a linear chain. Tests basic TCP -# transport connectivity, spanning tree convergence, and multi-hop -# routing over TCP. All peer connections use TCP on port 443. -# -# default_transport: tcp tells the config generator to use TCP -# transport and port 443 instead of the default UDP/2121. - -default_transport: tcp - -nodes: - a: - nsec: "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20" - npub: "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m" - docker_ip: "172.20.0.10" - peers: [b] - - b: - nsec: "b102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fb0" - npub: "npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le" - docker_ip: "172.20.0.11" - peers: [a, c] - - c: - nsec: "c102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1fc0" - npub: "npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6" - docker_ip: "172.20.0.12" - peers: [b] diff --git a/testing/static/docker-compose.yml b/testing/static/docker-compose.yml index 74b7912..7701000 100644 --- a/testing/static/docker-compose.yml +++ b/testing/static/docker-compose.yml @@ -365,40 +365,6 @@ services: networks: fips-net: - # ── TCP chain topology (A-B-C) ─────────────────────────────── - tcp-a: - <<: *fips-common - profiles: ["tcp-chain"] - container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-} - hostname: node-a - volumes: - - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/tcp-chain/node-a.yaml:/etc/fips/fips.yaml:ro - networks: - fips-net: - - tcp-b: - <<: *fips-common - profiles: ["tcp-chain"] - container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-} - hostname: node-b - volumes: - - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/tcp-chain/node-b.yaml:/etc/fips/fips.yaml:ro - networks: - fips-net: - - tcp-c: - <<: *fips-common - profiles: ["tcp-chain"] - container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-} - hostname: node-c - volumes: - - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/tcp-chain/node-c.yaml:/etc/fips/fips.yaml:ro - networks: - fips-net: - # ── Gateway integration test (gateway + server + non-FIPS client) ─ gw-gateway: <<: *fips-common From 4315328f2be47e94e5b473e786e54ff82da1e1bf Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 14:24:44 +0000 Subject: [PATCH 03/11] Stop the log analyzer double-counting outbound peer promotions The shared analyzer matched two strings for one event: "Peer promoted to active", emitted at info, and "Outbound handshake completed", emitted at debug. Both come from handle_msg2 on the same call path for the same outbound promotion, so any run at debug level counted every outbound promotion twice. Measured rather than argued. On the archived debug-level churn-mixed run of 2026-07-25, the node logs carry 20 of each string; the old matcher reported 40 promotions and the new one reports 20. The neighbouring counters are unmoved at 13 sessions and 33 parent switches. Info-level runs are unaffected, which was checked rather than assumed: on the tcp-mesh run of the same date the debug string occurs zero times, and the old and new matchers both report 7 against 7 raw occurrences. Three scenarios run at debug, one of them gating, so this corrects the promotion figure in their artifacts. No assertion reads the counter, so no suite's verdict changes. The earlier fix that introduced this added the live string alongside the dead one where it needed to replace it. --- testing/lib/log_analysis.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/testing/lib/log_analysis.py b/testing/lib/log_analysis.py index 7de162f..27039cf 100644 --- a/testing/lib/log_analysis.py +++ b/testing/lib/log_analysis.py @@ -148,8 +148,11 @@ def _analyze_lines(result: AnalysisResult, source: str, log_text: str): # Session establishment if "Session established" in line: result.sessions_established.append((source, line)) - # Peer promotion - if "Peer promoted to active" in line or "Outbound handshake completed" in line: + # Peer promotion. Match only the info-level string. "Outbound handshake + # completed" is emitted from handle_msg2 on the same call path for the + # same promotion, so matching it too double-counted every outbound + # promotion in any run at debug level. + if "Peer promoted to active" in line: result.peers_promoted.append((source, line)) # Peer removal if "Peer removed" in line: From cf129cde77eb2b25eb8cdfa73f16f94636fc14e3 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 14:41:47 +0000 Subject: [PATCH 04/11] Retire the three ignored Ethernet tests, which covered nothing unique All three required root or CAP_NET_RAW, neither runner passes --ignored, and so none had ever executed in CI. Before deleting them I checked what each covered and whether anything else covered it: - The two-node handshake is covered by the ethernet-only chaos scenario, whose baseline assertion cannot pass unless handshakes complete over AF_PACKET. - Tree convergence and root election by smallest NodeAddr are covered by ethernet-only's single-root assertion and by the in-process spanning-tree tests respectively. - The mixed-transport coexistence property is covered twice over by test_tcp_mixed_transport_coexistence and test_ble_mixed_transport, which call the same verify_tree_convergence_components helper with the same two-component shape and both run today. So the only thing running them would have added is an AF_PACKET variant of a property already proven on two other transports. The counts confirm nothing running was lost: 1392 tests pass before and after, and the ignored count falls from 7 to 4. The ethernet-only comment block cited these tests as the reason its assertion was the only Ethernet coverage in CI. It now records the sharper fact found while reading them: that coverage is control plane only. Both Ethernet scenarios disable traffic, so no datagram crosses an Ethernet link anywhere, which leaves the frame length field that trims NIC minimum frame padding ahead of AEAD verification unexercised. Deleting these tests does not widen that gap, because the test named for data exchange did not exchange any. --- src/node/tests/ethernet.rs | 214 --------------------- src/node/tests/mod.rs | 2 - testing/chaos/scenarios/ethernet-only.yaml | 9 +- 3 files changed, 5 insertions(+), 220 deletions(-) delete mode 100644 src/node/tests/ethernet.rs diff --git a/src/node/tests/ethernet.rs b/src/node/tests/ethernet.rs deleted file mode 100644 index 227ef3a..0000000 --- a/src/node/tests/ethernet.rs +++ /dev/null @@ -1,214 +0,0 @@ -//! Ethernet transport integration tests. -//! -//! Tests that the Ethernet transport works end-to-end using veth pairs. -//! All tests require root or CAP_NET_RAW and are marked `#[ignore]`. - -use super::*; -use crate::config::EthernetConfig; -use crate::transport::ethernet::EthernetTransport; -use crate::transport::{TransportAddr, TransportHandle, TransportId, packet_channel}; -use spanning_tree::{TestNode, cleanup_nodes, drain_all_packets, initiate_handshake}; - -use std::process::Command; -use std::sync::atomic::{AtomicU32, Ordering}; - -/// Atomic counter for unique veth names across tests. -static VETH_COUNTER: AtomicU32 = AtomicU32::new(0); - -/// RAII wrapper for a veth pair. -/// -/// Creates a pair of connected virtual Ethernet interfaces. Destroying -/// one end automatically destroys the other. -struct VethPair { - name_a: String, - name_b: String, -} - -impl VethPair { - /// Create a new veth pair with unique interface names. - /// - /// Names are kept under 15 chars (IFNAMSIZ limit). Format: `ftXXa`/`ftXXb` - /// where XX is an atomic counter combined with PID for cross-process uniqueness. - fn create() -> Self { - let id = VETH_COUNTER.fetch_add(1, Ordering::Relaxed); - let pid = std::process::id() % 10000; - let name_a = format!("ft{}{}a", pid, id); - let name_b = format!("ft{}{}b", pid, id); - - assert!(name_a.len() <= 15, "veth name too long: {}", name_a); - assert!(name_b.len() <= 15, "veth name too long: {}", name_b); - - // Create veth pair - let status = Command::new("ip") - .args([ - "link", "add", &name_a, "type", "veth", "peer", "name", &name_b, - ]) - .status() - .expect("failed to run 'ip link add'"); - assert!(status.success(), "failed to create veth pair"); - - // Bring both ends up - let status = Command::new("ip") - .args(["link", "set", &name_a, "up"]) - .status() - .expect("failed to run 'ip link set up'"); - assert!(status.success(), "failed to bring up {}", name_a); - - let status = Command::new("ip") - .args(["link", "set", &name_b, "up"]) - .status() - .expect("failed to run 'ip link set up'"); - assert!(status.success(), "failed to bring up {}", name_b); - - VethPair { name_a, name_b } - } -} - -impl Drop for VethPair { - fn drop(&mut self) { - // Deleting one end destroys both - let _ = Command::new("ip") - .args(["link", "delete", &self.name_a]) - .status(); - } -} - -/// Create a test node with a live Ethernet transport on the given interface. -/// -/// Parallel to `make_test_node()` in spanning_tree.rs but uses -/// EthernetTransport instead of UDP. -async fn make_test_node_ethernet(interface: &str) -> TestNode { - let mut node = make_node(); - let transport_id = TransportId::new(1); - - let config = EthernetConfig { - interface: interface.to_string(), - discovery: Some(false), - announce: Some(false), - accept_connections: Some(true), - ..Default::default() - }; - - let (packet_tx, packet_rx) = packet_channel(256); - let mut transport = EthernetTransport::new(transport_id, None, config, packet_tx); - transport.start_async().await.unwrap(); - - let mac = transport - .local_mac() - .expect("transport should have MAC after start"); - let addr = TransportAddr::from_bytes(&mac); - - node.transports - .insert(transport_id, TransportHandle::Ethernet(transport)); - - TestNode { - node, - transport_id, - packet_rx: spanning_tree::bridge_to_unbounded(packet_rx), - addr, - } -} - -/// Two nodes on a veth pair complete a Noise handshake and establish peering. -#[tokio::test] -#[ignore] // Requires root or CAP_NET_RAW -async fn test_ethernet_two_node_handshake() { - let veth = VethPair::create(); - - let mut nodes = vec![ - make_test_node_ethernet(&veth.name_a).await, - make_test_node_ethernet(&veth.name_b).await, - ]; - - // Initiate handshake from node 0 to node 1 - initiate_handshake(&mut nodes, 0, 1).await; - - // Drain all packets (handshake + tree announce) - let total = drain_all_packets(&mut nodes, false).await; - assert!(total > 0, "should have processed packets"); - - // Verify bidirectional peering - let addr_0 = *nodes[0].node.node_addr(); - let addr_1 = *nodes[1].node.node_addr(); - assert!( - nodes[0].node.get_peer(&addr_1).is_some(), - "node 0 should have node 1 as peer" - ); - assert!( - nodes[1].node.get_peer(&addr_0).is_some(), - "node 1 should have node 0 as peer" - ); - - cleanup_nodes(&mut nodes).await; -} - -/// Two Ethernet nodes converge to a correct spanning tree (2-node tree). -#[tokio::test] -#[ignore] // Requires root or CAP_NET_RAW -async fn test_ethernet_data_exchange() { - use spanning_tree::verify_tree_convergence; - - let veth = VethPair::create(); - - let mut nodes = vec![ - make_test_node_ethernet(&veth.name_a).await, - make_test_node_ethernet(&veth.name_b).await, - ]; - - initiate_handshake(&mut nodes, 0, 1).await; - let total = drain_all_packets(&mut nodes, false).await; - assert!(total > 0); - - // Verify spanning tree convergence - verify_tree_convergence(&nodes); - - // The root should be the node with the smallest NodeAddr - let expected_root = std::cmp::min(*nodes[0].node.node_addr(), *nodes[1].node.node_addr()); - assert_eq!(*nodes[0].node.tree_state().root(), expected_root); - assert_eq!(*nodes[1].node.tree_state().root(), expected_root); - - cleanup_nodes(&mut nodes).await; -} - -/// Mixed transport: 2 Ethernet nodes + 2 UDP nodes coexist. -/// -/// Each transport forms its own connected component. Validates that -/// `process_available_packets()` handles heterogeneous transport types. -#[tokio::test] -#[ignore] // Requires root or CAP_NET_RAW -async fn test_mixed_transport_coexistence() { - use spanning_tree::{make_test_node, verify_tree_convergence_components}; - - let veth = VethPair::create(); - - // Create 2 Ethernet nodes and 2 UDP nodes - let eth_0 = make_test_node_ethernet(&veth.name_a).await; - let eth_1 = make_test_node_ethernet(&veth.name_b).await; - let udp_0 = make_test_node().await; - let udp_1 = make_test_node().await; - - let mut nodes = vec![eth_0, eth_1, udp_0, udp_1]; - - // Handshake within each component - initiate_handshake(&mut nodes, 0, 1).await; // Ethernet pair - initiate_handshake(&mut nodes, 2, 3).await; // UDP pair - - // Drain all packets across both transports - let total = drain_all_packets(&mut nodes, false).await; - assert!(total > 0); - - // Verify each component converges independently - verify_tree_convergence_components(&nodes, &[vec![0, 1], vec![2, 3]]); - - // Ethernet component has its own root - let eth_root = std::cmp::min(*nodes[0].node.node_addr(), *nodes[1].node.node_addr()); - assert_eq!(*nodes[0].node.tree_state().root(), eth_root); - assert_eq!(*nodes[1].node.tree_state().root(), eth_root); - - // UDP component has its own root - let udp_root = std::cmp::min(*nodes[2].node.node_addr(), *nodes[3].node.node_addr()); - assert_eq!(*nodes[2].node.tree_state().root(), udp_root); - assert_eq!(*nodes[3].node.tree_state().root(), udp_root); - - cleanup_nodes(&mut nodes).await; -} diff --git a/src/node/tests/mod.rs b/src/node/tests/mod.rs index c6484ec..8c00841 100644 --- a/src/node/tests/mod.rs +++ b/src/node/tests/mod.rs @@ -13,8 +13,6 @@ mod bootstrap; mod decrypt_failure; mod disconnect; mod discovery; -#[cfg(target_os = "linux")] -mod ethernet; mod forwarding; mod handshake; mod heartbeat; diff --git a/testing/chaos/scenarios/ethernet-only.yaml b/testing/chaos/scenarios/ethernet-only.yaml index 39a823b..1b11f2f 100644 --- a/testing/chaos/scenarios/ethernet-only.yaml +++ b/testing/chaos/scenarios/ethernet-only.yaml @@ -50,10 +50,11 @@ traffic: # A 4-node mesh forms a spanning tree: one root and three nodes # with a parent. All six provably-completed archived runs show exactly # that, so these are the shape of a converged mesh rather than a -# tolerance fitted to observations. This is currently the only -# assertion covering Ethernet transport anywhere in CI: the three -# tests in src/node/tests/ethernet.rs are #[ignore]d for requiring -# CAP_NET_RAW and neither runner passes --ignored. +# tolerance fitted to observations. This is the only assertion covering +# Ethernet transport anywhere in CI, and it covers the control plane +# only: traffic is disabled above, so no datagram crosses an Ethernet +# link in any test. Framing, the length field that trims NIC minimum- +# frame padding, and AEAD over Ethernet are all unexercised as a result. assertions: baseline: min_nodes_reporting: 4 From 24bf4d46d48cb0ce0573b542ecd22e3e0db465a4 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 15:22:43 +0000 Subject: [PATCH 05/11] Widen the churn-mixed baseline bounds to sit outside the scenario's own spread The root-count ceiling of 4 was set from six runs whose observed maximum was 3. Fourteen runs that recorded a baseline verdict put the real distribution at 1 to 5 roots, so the ceiling sat at roughly the 93rd percentile of the scenario's own variance: one sample in fourteen exceeded it and four sat at or above it. It therefore reddened a share of runs whatever the daemon did, which is a calibration fault rather than a convergence one. Every sample ran the file's fixed seed, so the spread is container timing and not a differing chaos schedule. The ceiling is now 6, one step beyond the observed maximum, and the parented floor moves to its complement at 4. That still fails a mesh where seven or more of ten nodes are islanded, which is the collapse case the assertion exists to catch; driving evaluate_baseline directly confirms all fourteen samples pass and 7, 8 and 10 roots still fail. min_sessions is left at 10 against an observed minimum of 12. It has never fired, but the comment now records the margin as thin so a future failure there is read as calibration first. --- testing/chaos/scenarios/churn-mixed.yaml | 41 +++++++++++++++++++----- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/testing/chaos/scenarios/churn-mixed.yaml b/testing/chaos/scenarios/churn-mixed.yaml index 9c75dd3..23142f0 100644 --- a/testing/chaos/scenarios/churn-mixed.yaml +++ b/testing/chaos/scenarios/churn-mixed.yaml @@ -75,12 +75,37 @@ bandwidth: # # This one is calibrated rather than derived, because a scenario that # stops and starts nodes on purpose does not hold a single spanning -# tree. The six provably-completed archived runs end with two or three -# distinct roots, seven or eight of ten nodes parented, and fifteen to -# twenty sessions. The floors below sit one step outside those ranges -# so an unlucky run does not go red, which leaves them catching a mesh -# that collapsed rather than one that churned. Tighten them only -# against a larger sample than six runs. +# tree. +# +# Calibrated 2026-07-26 against fourteen runs that recorded a baseline +# verdict, read from their assertions.txt files. Every one used this +# file's fixed seed 42 and therefore an identical chaos schedule, so the +# spread below is container timing rather than differing scenarios: +# +# distinct roots 1 2 3 4 5 -> 2 3 5 3 1 runs +# nodes parented 9 8 7 6 5 -> the exact complement, in all 14 +# sessions 12 to 20, minimum 12 +# +# The previous ceiling of 4 roots sat at roughly the 93rd percentile of +# that distribution: one run in fourteen exceeded it and four sat at or +# above it, so it reddened a share of runs whatever the daemon did. It +# was set from six runs whose observed maximum was 3, which is how a +# threshold one step outside a small sample ends up inside the real one. +# +# The ceiling is now 6, one step beyond the observed maximum of 5, and +# the parented floor is its complement at 4. That still fails a mesh +# that collapsed — seven or more of ten nodes islanded — which is the +# only thing this assertion was ever meant to catch. Do not read a pass +# as convergence: four of the six gating scenarios assert a floor of +# this kind, and it means the mesh formed and nobody errored. +# +# min_sessions stays at 10 against an observed minimum of 12. It has +# never fired and there is no evidence it is mis-set, but the margin is +# thin and a future failure there should be read as calibration before +# it is read as a defect. +# +# Tighten any of these only against a larger sample, and against one +# gathered at the invocation CI actually runs. # # READ THIS BEFORE RETUNING: the numbers above describe the invocation CI # gates on, which is not this file's own defaults. ci-local runs it as @@ -98,8 +123,8 @@ bandwidth: assertions: baseline: min_nodes_reporting: 10 - max_roots: 4 - min_nodes_parented: 6 + max_roots: 6 + min_nodes_parented: 4 min_sessions: 10 logging: From cbbdf2c13c9c9e0250c4431d772f13d408ce9136 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 15:37:50 +0000 Subject: [PATCH 06/11] Stop the firewall and ACL suites rebuilding an image their caller supplied Both suites ran `docker compose up -d --build` unconditionally, and their compose files carry both a build context and an image name, so the flag rebuilt and retagged the shared test image from the shared build context on every run. Their --skip-build flag did not cover it: it guarded only the Rust build above. Under a harness that has already built the image and handed it over, that rebuild can only replace the binaries under test with whatever the shared context currently holds. The nat scripts had the mirror-image problem. Their image guard built a replacement whenever the image was missing, which is right by hand and wrong under a harness: when the caller names an image it built, a miss means something upstream is broken, and manufacturing a substitute hides that. They now fail loudly in that case and keep building only on the hand path. The mesh-lab guard reads the same variable and its build hint moves to the script that still produces the shared tag. --- testing/acl-allowlist/test.sh | 10 +++++++++- testing/firewall/test.sh | 10 +++++++++- testing/mesh-lab/run-loop.sh | 7 ++++--- testing/nat/scripts/nat-test.sh | 17 ++++++++++++++--- testing/nat/scripts/nostr-relay-test.sh | 17 ++++++++++++++--- testing/nat/scripts/stun-faults-test.sh | 17 ++++++++++++++--- 6 files changed, 64 insertions(+), 14 deletions(-) diff --git a/testing/acl-allowlist/test.sh b/testing/acl-allowlist/test.sh index 1bf9905..79f3fc3 100755 --- a/testing/acl-allowlist/test.sh +++ b/testing/acl-allowlist/test.sh @@ -185,7 +185,15 @@ log "Generating ACL allowlist fixtures" log "Starting ACL allowlist harness" docker compose -f "$COMPOSE_FILE" down >/dev/null 2>&1 || true -docker compose -f "$COMPOSE_FILE" up -d --build +# --build only on the hand path. Under a harness, --skip-build means the caller +# has already built the image this compose file names, and rebuilding it here +# would overwrite that image from whatever the shared build context happens to +# hold — which is how a suite ends up certifying binaries it was never given. +if [ "$SKIP_BUILD" = false ]; then + docker compose -f "$COMPOSE_FILE" up -d --build +else + docker compose -f "$COMPOSE_FILE" up -d +fi log "Waiting for expected peer convergence" wait_for_peers_exact fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-} 3 40 diff --git a/testing/firewall/test.sh b/testing/firewall/test.sh index 833e9c8..418af19 100755 --- a/testing/firewall/test.sh +++ b/testing/firewall/test.sh @@ -191,7 +191,15 @@ log "Generating firewall fixtures" log "Starting firewall harness" docker compose -f "$COMPOSE_FILE" down >/dev/null 2>&1 || true -docker compose -f "$COMPOSE_FILE" up -d --build +# --build only on the hand path. Under a harness, --skip-build means the caller +# has already built the image this compose file names, and rebuilding it here +# would overwrite that image from whatever the shared build context happens to +# hold — which is how a suite ends up certifying binaries it was never given. +if [ "$SKIP_BUILD" = false ]; then + docker compose -f "$COMPOSE_FILE" up -d --build +else + docker compose -f "$COMPOSE_FILE" up -d +fi log "Waiting for fips0 on both nodes" wait_for_fips0 "$CONTAINER_A" 40 diff --git a/testing/mesh-lab/run-loop.sh b/testing/mesh-lab/run-loop.sh index 9fa9487..a640baa 100755 --- a/testing/mesh-lab/run-loop.sh +++ b/testing/mesh-lab/run-loop.sh @@ -135,9 +135,10 @@ require_docker() { } require_test_image() { - if ! docker image inspect fips-test:latest >/dev/null 2>&1; then - echo "ERROR: fips-test:latest not present" >&2 - echo "Build it once with: bash testing/ci-local.sh --build-only" >&2 + local img="${FIPS_TEST_IMAGE:-fips-test:latest}" + if ! docker image inspect "$img" >/dev/null 2>&1; then + echo "ERROR: $img not present" >&2 + echo "Build it once with: bash testing/scripts/build.sh" >&2 exit 2 fi } diff --git a/testing/nat/scripts/nat-test.sh b/testing/nat/scripts/nat-test.sh index 3628e87..c1eea64 100755 --- a/testing/nat/scripts/nat-test.sh +++ b/testing/nat/scripts/nat-test.sh @@ -262,10 +262,21 @@ dump_lan_diagnostics() { trap 'echo ""; echo "NAT test interrupted"; cleanup; exit 130' INT TERM require_test_image() { - if ! docker image inspect fips-test:latest >/dev/null 2>&1; then - echo "fips-test:latest not found; building test image" - "$BUILD_SCRIPT" + local img="${FIPS_TEST_IMAGE:-fips-test:latest}" + if docker image inspect "$img" >/dev/null 2>&1; then + return 0 fi + # Building here is right for a hand run and wrong under a harness. When + # FIPS_TEST_IMAGE is set the caller has already built the image it named, so + # a miss means something upstream is broken; building a substitute would + # hide that and run binaries nobody asked for. + if [ -n "${FIPS_TEST_IMAGE:-}" ]; then + echo "ERROR: $img not present, and FIPS_TEST_IMAGE names the caller's own image" >&2 + echo "The harness that set it is expected to have built it." >&2 + exit 1 + fi + echo "$img not found; building test image" + "$BUILD_SCRIPT" } require_docker_daemon() { diff --git a/testing/nat/scripts/nostr-relay-test.sh b/testing/nat/scripts/nostr-relay-test.sh index 812d879..718120d 100755 --- a/testing/nat/scripts/nostr-relay-test.sh +++ b/testing/nat/scripts/nostr-relay-test.sh @@ -49,10 +49,21 @@ require_docker_daemon() { } require_test_image() { - if ! docker image inspect fips-test:latest >/dev/null 2>&1; then - echo "fips-test:latest not found; building test image" - "$BUILD_SCRIPT" + local img="${FIPS_TEST_IMAGE:-fips-test:latest}" + if docker image inspect "$img" >/dev/null 2>&1; then + return 0 fi + # Building here is right for a hand run and wrong under a harness. When + # FIPS_TEST_IMAGE is set the caller has already built the image it named, so + # a miss means something upstream is broken; building a substitute would + # hide that and run binaries nobody asked for. + if [ -n "${FIPS_TEST_IMAGE:-}" ]; then + echo "ERROR: $img not present, and FIPS_TEST_IMAGE names the caller's own image" >&2 + echo "The harness that set it is expected to have built it." >&2 + exit 1 + fi + echo "$img not found; building test image" + "$BUILD_SCRIPT" } dump_diagnostics() { diff --git a/testing/nat/scripts/stun-faults-test.sh b/testing/nat/scripts/stun-faults-test.sh index c9dbbb5..bc508fe 100755 --- a/testing/nat/scripts/stun-faults-test.sh +++ b/testing/nat/scripts/stun-faults-test.sh @@ -60,10 +60,21 @@ require_docker_daemon() { } require_test_image() { - if ! docker image inspect fips-test:latest >/dev/null 2>&1; then - echo "fips-test:latest not found; building test image" - "$BUILD_SCRIPT" + local img="${FIPS_TEST_IMAGE:-fips-test:latest}" + if docker image inspect "$img" >/dev/null 2>&1; then + return 0 fi + # Building here is right for a hand run and wrong under a harness. When + # FIPS_TEST_IMAGE is set the caller has already built the image it named, so + # a miss means something upstream is broken; building a substitute would + # hide that and run binaries nobody asked for. + if [ -n "${FIPS_TEST_IMAGE:-}" ]; then + echo "ERROR: $img not present, and FIPS_TEST_IMAGE names the caller's own image" >&2 + echo "The harness that set it is expected to have built it." >&2 + exit 1 + fi + echo "$img not found; building test image" + "$BUILD_SCRIPT" } dump_diagnostics() { From a718cef8ced6cf05f31dde808d899fb17ec6ee2f Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 15:38:28 +0000 Subject: [PATCH 07/11] Have every suite compose file name the caller's test image Only the static family read FIPS_TEST_IMAGE; the firewall, ACL, nat, sidecar and both Tor compose files named the shared mutable tag directly, which is why the local CI runner has to retag its per-run image to that name. They now take the same defaulted form the static file already uses, so a harness run resolves the image it built and a bare hand run still resolves the shared tag exactly as before. Checked by rendering each file through `docker compose config` with the variables set: every service resolves to the supplied name and none is left naming the shared tag. --- testing/acl-allowlist/docker-compose.yml | 2 +- testing/firewall/docker-compose.yml | 2 +- testing/nat/docker-compose.yml | 4 ++-- testing/sidecar/docker-compose.yml | 4 ++-- testing/tor/directory-mode/docker-compose.yml | 4 ++-- testing/tor/socks5-outbound/docker-compose.yml | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/testing/acl-allowlist/docker-compose.yml b/testing/acl-allowlist/docker-compose.yml index 7d731d9..9bcb02e 100644 --- a/testing/acl-allowlist/docker-compose.yml +++ b/testing/acl-allowlist/docker-compose.yml @@ -20,7 +20,7 @@ networks: x-fips-common: &fips-common build: context: ../docker - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} entrypoint: ["/usr/local/bin/entrypoint.sh"] cap_add: - NET_ADMIN diff --git a/testing/firewall/docker-compose.yml b/testing/firewall/docker-compose.yml index 88110dd..9cedf41 100644 --- a/testing/firewall/docker-compose.yml +++ b/testing/firewall/docker-compose.yml @@ -20,7 +20,7 @@ networks: x-fips-common: &fips-common build: context: ../docker - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} entrypoint: ["/usr/local/bin/entrypoint.sh"] cap_add: - NET_ADMIN diff --git a/testing/nat/docker-compose.yml b/testing/nat/docker-compose.yml index a9a2c58..d428ecc 100644 --- a/testing/nat/docker-compose.yml +++ b/testing/nat/docker-compose.yml @@ -18,7 +18,7 @@ volumes: relay-data: x-fips-common: &fips-common - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} cap_add: - NET_ADMIN devices: @@ -325,7 +325,7 @@ services: ipv4_address: 172.31.10.51 stun-fault-shim: - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} profiles: ["stun-faults"] container_name: fips-nat-stun-fault-shim${FIPS_CI_NAME_SUFFIX:-} depends_on: diff --git a/testing/sidecar/docker-compose.yml b/testing/sidecar/docker-compose.yml index 801da5b..e7dded2 100644 --- a/testing/sidecar/docker-compose.yml +++ b/testing/sidecar/docker-compose.yml @@ -14,7 +14,7 @@ networks: services: fips: - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} hostname: fips-sidecar labels: - "com.corganlabs.fips-ci=1" @@ -40,7 +40,7 @@ services: ipv4_address: ${FIPS_IPV4:-172.20.1.20} app: - image: fips-test-app:latest + image: ${FIPS_TEST_APP_IMAGE:-fips-test-app:latest} labels: - "com.corganlabs.fips-ci=1" network_mode: "service:fips" diff --git a/testing/tor/directory-mode/docker-compose.yml b/testing/tor/directory-mode/docker-compose.yml index 9b09c64..5e9f874 100644 --- a/testing/tor/directory-mode/docker-compose.yml +++ b/testing/tor/directory-mode/docker-compose.yml @@ -16,7 +16,7 @@ networks: services: fips-a: - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} cap_add: - NET_ADMIN devices: @@ -36,7 +36,7 @@ services: dir-test: fips-b: - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} cap_add: - NET_ADMIN devices: diff --git a/testing/tor/socks5-outbound/docker-compose.yml b/testing/tor/socks5-outbound/docker-compose.yml index 58baa5e..4bb64ae 100644 --- a/testing/tor/socks5-outbound/docker-compose.yml +++ b/testing/tor/socks5-outbound/docker-compose.yml @@ -24,7 +24,7 @@ services: tor-test: fips-a: - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} cap_add: - NET_ADMIN devices: @@ -45,7 +45,7 @@ services: tor-test: fips-b: - image: fips-test:latest + image: ${FIPS_TEST_IMAGE:-fips-test:latest} cap_add: - NET_ADMIN devices: From af847c68b55b2d48a04721284a9be9b6f9e6bbe8 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 15:40:03 +0000 Subject: [PATCH 08/11] Stop the chaos simulation rebuilding the shared test image per scenario The runner rebuilt the shared tag from the shared build context at the start of every scenario, unconditionally. Local CI runs chaos scenarios in parallel, so that was several concurrent builds into one name inside a single run, before any second run is considered, and it silently replaced whatever image the harness had already built and handed over. It now uses the caller's image when one is named, and asserts the image is present rather than building a substitute, because under a harness a miss means something upstream is broken and building would hide it. A bare run still builds the shared tag exactly as before, reading the run's own build context when one is set. --- testing/chaos/sim/compose.py | 9 +++++++-- testing/chaos/sim/runner.py | 34 +++++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/testing/chaos/sim/compose.py b/testing/chaos/sim/compose.py index 3594ead..461212e 100644 --- a/testing/chaos/sim/compose.py +++ b/testing/chaos/sim/compose.py @@ -10,8 +10,13 @@ from .scenario import Scenario from .topology import SimTopology # Image name for the pre-built FIPS test image. -# The runner builds this once before starting containers. -FIPS_SIM_IMAGE = "fips-test:latest" +# +# A harness that has already built an image passes it in FIPS_TEST_IMAGE, and +# it is then the caller's image: the runner uses it and must not rebuild it. +# Unset means a bare run, where the shared tag is the right name and the runner +# still builds it. Read at import, which is safe because the simulation always +# starts as a child process with the environment already set. +FIPS_SIM_IMAGE = os.environ.get("FIPS_TEST_IMAGE", "fips-test:latest") # Jinja2 template for the compose file. # Uses a pre-built image instead of per-service build to support large topologies. diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index ee9c8c6..8f661ca 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -268,14 +268,34 @@ class SimRunner: ) log.info("Wrote %s", self.compose_file) - # 4. Build the test image once (avoids per-service build at scale) - log.info("Building Docker image...") + # 4. Obtain the test image (once, rather than per-service at scale). + # + # Building it is right for a bare run and wrong under a harness. When + # FIPS_TEST_IMAGE is set the image belongs to the caller, and every + # scenario of a parallel run would otherwise rebuild it into one shared + # name from one shared context — so assert it exists and fail loudly if + # it does not, rather than manufacture a substitute nobody asked for. from .compose import FIPS_SIM_IMAGE - docker_dir = os.path.join(os.path.dirname(__file__), "..", "..", "docker") - subprocess.run( - ["docker", "build", "-t", FIPS_SIM_IMAGE, docker_dir], - check=True, - ) + if os.environ.get("FIPS_TEST_IMAGE"): + log.info("Using caller-supplied image %s", FIPS_SIM_IMAGE) + probe = subprocess.run( + ["docker", "image", "inspect", FIPS_SIM_IMAGE], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + if probe.returncode != 0: + raise RuntimeError( + f"FIPS_TEST_IMAGE names {FIPS_SIM_IMAGE}, which is not present. " + "The harness that set it is expected to have built it." + ) + else: + log.info("Building Docker image...") + docker_dir = os.environ.get("FIPS_BUILD_CONTEXT") or os.path.join( + os.path.dirname(__file__), "..", "..", "docker" + ) + subprocess.run( + ["docker", "build", "-t", FIPS_SIM_IMAGE, docker_dir], + check=True, + ) # 5. Start containers log.info("Starting %d containers...", len(self.topology.nodes)) From 49163befd57d0cf52153082ae392233906d4c9a8 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 15:42:48 +0000 Subject: [PATCH 09/11] Give each local CI run its own docker build context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scoping the image tag was never sufficient on its own. Every build read one unscoped directory in the working tree, into which each run copies the binaries it just built, so two concurrent runs raced on the contents of the context as well as on the name of the result — and a run could produce a correctly-per-run-tagged image built from the other run's binaries. The run now copies the context's tracked files into its own directory, installs its binaries there, and builds from it, exporting the path so every other consumer follows. Deliberately not carried over: a previous run's binaries, since inheriting them is the failure this prevents. The path is absolute because compose resolves a relative build context against the compose file's own directory rather than the working directory, which was measured rather than assumed. The chaos entry script gated the whole simulation on a binary in the shared directory by literal path, so it moves in this same commit: left behind, it would have failed on a clean checkout and, worse on a host with leftovers, passed while reading a binary that was not the one under test. Teardown removes the directory on red runs as well as green, since it holds only reproducible content and is never the evidence of a failure. The worker's SIGKILL runs no trap, so the cleanup script also sweeps contexts left by a preempted run, and the ignore rule keeps a concurrent run from showing up as untracked working-tree noise. --- .gitignore | 5 +++ testing/acl-allowlist/docker-compose.yml | 6 ++- testing/chaos/scripts/chaos.sh | 6 ++- testing/ci-cleanup.sh | 22 +++++++++++ testing/ci-local.sh | 48 ++++++++++++++++++++---- testing/firewall/docker-compose.yml | 6 ++- testing/sidecar/scripts/test-sidecar.sh | 2 +- 7 files changed, 84 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index ce9b98e..d3bbb5e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,11 @@ __pycache__/ *.egg-info/ *.egg +# Per-run build contexts created by testing/ci-local.sh. Its teardown normally +# removes them, but the CI worker's SIGKILL runs no trap, so one can survive a +# preempted run; ci-cleanup.sh sweeps the survivors. +/testing/docker-*/ + # Runtime artifacts from running fips in-tree during local testing. # Root-anchored so legitimately-tracked fips.yaml under packaging/ and # examples/ stays included. diff --git a/testing/acl-allowlist/docker-compose.yml b/testing/acl-allowlist/docker-compose.yml index 9bcb02e..a20428a 100644 --- a/testing/acl-allowlist/docker-compose.yml +++ b/testing/acl-allowlist/docker-compose.yml @@ -19,7 +19,11 @@ networks: x-fips-common: &fips-common build: - context: ../docker + # The harness scopes its build context per run and passes it here; the + # shared directory is the hand-run default. Compose resolves a relative + # value against THIS file's directory, so the harness must export an + # absolute path. + context: ${FIPS_BUILD_CONTEXT:-../docker} image: ${FIPS_TEST_IMAGE:-fips-test:latest} entrypoint: ["/usr/local/bin/entrypoint.sh"] cap_add: diff --git a/testing/chaos/scripts/chaos.sh b/testing/chaos/scripts/chaos.sh index 0ffdbb9..2a22d2b 100755 --- a/testing/chaos/scripts/chaos.sh +++ b/testing/chaos/scripts/chaos.sh @@ -125,7 +125,11 @@ if ! docker info &> /dev/null; then exit 1 fi -DOCKER_DIR="$CHAOS_DIR/../docker" +# A harness that scopes its build context per run passes it in +# FIPS_BUILD_CONTEXT and stops writing to the shared directory, so checking the +# shared one would either fail on a clean checkout or, worse, pass while +# reading a stale binary that is not the one under test. +DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$CHAOS_DIR/../docker}" if [ ! -f "$DOCKER_DIR/fips" ]; then echo "Error: FIPS binary not found at $DOCKER_DIR/fips" >&2 echo "Run testing/scripts/build.sh first" >&2 diff --git a/testing/ci-cleanup.sh b/testing/ci-cleanup.sh index fbcb5ec..0f25243 100755 --- a/testing/ci-cleanup.sh +++ b/testing/ci-cleanup.sh @@ -247,6 +247,27 @@ reap_images() { timeout "$TMO" docker rmi -f "${imgs[@]}" >/dev/null 2>&1 || true } +# Per-run build contexts left in the working tree. ci-local.sh removes its own +# from the EXIT trap, but the CI worker sends SIGKILL after SIGTERM and a +# SIGKILL runs no trap, so a preempted run can leave an 18 MB directory behind +# with nothing else that would ever notice it. +# +# Scoped mode takes only the named run's. Broad mode cannot tell a live run's +# context from an abandoned one by name, so it goes by age instead: a run lasts +# well under an hour, and a day is far outside that. +reap_build_contexts() { + local dir + if [[ -n "$RUN_ID" ]]; then + dir="$SCRIPT_DIR/docker-$RUN_ID" + [[ -d "$dir" ]] && rm -rf "$dir" + return 0 + fi + while IFS= read -r dir; do + [[ -n "$dir" ]] && rm -rf "$dir" + done < <(find "$SCRIPT_DIR" -maxdepth 1 -type d -name 'docker-*' -mtime +0 2>/dev/null) + return 0 +} + # Order matters: containers reference networks/volumes, so drop them first, and # the veth sweep needs an image to run ip(8) in, so it precedes the image reap. reap_containers @@ -254,5 +275,6 @@ reap_networks reap_volumes reap_veths reap_images +reap_build_contexts exit 0 diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 3fe7796..670795e 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -318,6 +318,14 @@ CI_LABEL_RUN="com.corganlabs.fips-ci.run=${CI_RUN_ID}" export FIPS_CI_RUN_ID="$CI_RUN_ID" export FIPS_TEST_IMAGE="$CI_IMAGE_TEST" export FIPS_TEST_APP_IMAGE="$CI_IMAGE_APP" +# The build context is this run's too. testing/docker/ is a single directory in +# the working tree, so two runs racing on its CONTENTS produce a per-run-tagged +# image built from the other run's binaries — scoping the tag alone does not +# close that. Absolute, and it has to be: compose interpolates this into +# build.context but resolves a relative result against the compose FILE's +# directory, not the working directory. +CI_BUILD_CONTEXT="$SCRIPT_DIR/docker-${CI_RUN_ID}" +export FIPS_BUILD_CONTEXT="$CI_BUILD_CONTEXT" # Docker container names are GLOBAL — a compose project name does not scope # them — so the suite compose files append this suffix to every explicit # container_name, and the suite scripts append it wherever they address a @@ -405,6 +413,15 @@ ci_teardown() { rm -rf "$SCRIPT_DIR/static/generated-configs${CI_RUN_NAME_SUFFIX}" rm -rf "$SCRIPT_DIR/firewall/generated-configs${CI_RUN_NAME_SUFFIX}" fi + + # 4. This run's build context. Unlike the generated configs it is removed + # on a red run too: it holds the binaries and the Dockerfiles, both + # reproducible from the commit, so it is never the evidence of a + # failure. Guarded on the path having been derived at all, so an early + # exit cannot turn this into `rm -rf $SCRIPT_DIR/docker-`. + if [[ -n "${CI_BUILD_CONTEXT:-}" && "$CI_BUILD_CONTEXT" != "$SCRIPT_DIR/docker-" ]]; then + rm -rf "$CI_BUILD_CONTEXT" + fi } on_signal() { @@ -871,22 +888,39 @@ run_tor_directory() { run_integration() { stage "Stage 3: Integration Tests" - # Install binaries to shared docker context + # Populate THIS run's build context, then install the binaries into it. + # Everything but the binaries is copied from the tracked context directory; + # the binaries are installed fresh, and a previous run's are deliberately + # not carried over, since inheriting them is the failure this scoping + # exists to prevent. + info "Preparing build context $CI_BUILD_CONTEXT" + rm -rf "$CI_BUILD_CONTEXT" + mkdir -p "$CI_BUILD_CONTEXT" || { record "docker-build" 1; return; } + local _f + for _f in "$SCRIPT_DIR"/docker/*; do + case "$(basename "$_f")" in + fips|fipsctl|fipstop|fips-gateway) continue ;; + esac + cp -a "$_f" "$CI_BUILD_CONTEXT/" || { record "docker-build" 1; return; } + done + info "Installing release binaries" - install_binaries testing/docker + install_binaries "$CI_BUILD_CONTEXT" # Build unified test image once (used by all harnesses). Tag per-run # (fips-test:${run}) so a build killed mid-flight never wedges the next # run's rebuild, and concurrent runs never clobber each other's image. - # Then retag :latest for the compose files / harness scripts that still - # reference fips-test:latest directly; the retag happens only after BOTH - # builds succeed, so :latest never points at a half-built image. info "Building $CI_IMAGE_TEST Docker image" - docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" testing/docker --quiet \ + docker build -t "$CI_IMAGE_TEST" --label "$CI_LABEL" --label "$CI_LABEL_RUN" "$CI_BUILD_CONTEXT" --quiet \ || { record "docker-build" 1; return; } docker build -t "$CI_IMAGE_APP" --label "$CI_LABEL" --label "$CI_LABEL_RUN" \ - -f testing/docker/Dockerfile.app testing/docker --quiet \ + -f "$CI_BUILD_CONTEXT/Dockerfile.app" "$CI_BUILD_CONTEXT" --quiet \ || { record "docker-build-app" 1; return; } + # The remaining bridge back to the shared mutable tag, for any consumer not + # yet reading FIPS_TEST_IMAGE. Removed in the following commit, which is the + # step that makes a missed consumer fail loudly instead of silently + # resolving another run's binaries. Both builds have succeeded by here, so + # :latest never points at a half-built image. docker tag "$CI_IMAGE_TEST" fips-test:latest docker tag "$CI_IMAGE_APP" fips-test-app:latest diff --git a/testing/firewall/docker-compose.yml b/testing/firewall/docker-compose.yml index 9cedf41..fe65773 100644 --- a/testing/firewall/docker-compose.yml +++ b/testing/firewall/docker-compose.yml @@ -19,7 +19,11 @@ networks: x-fips-common: &fips-common build: - context: ../docker + # The harness scopes its build context per run and passes it here; the + # shared directory is the hand-run default. Compose resolves a relative + # value against THIS file's directory, so the harness must export an + # absolute path. + context: ${FIPS_BUILD_CONTEXT:-../docker} image: ${FIPS_TEST_IMAGE:-fips-test:latest} entrypoint: ["/usr/local/bin/entrypoint.sh"] cap_add: diff --git a/testing/sidecar/scripts/test-sidecar.sh b/testing/sidecar/scripts/test-sidecar.sh index 2cbd114..975973f 100755 --- a/testing/sidecar/scripts/test-sidecar.sh +++ b/testing/sidecar/scripts/test-sidecar.sh @@ -115,7 +115,7 @@ trap cleanup EXIT if [[ "${1:-}" != "--skip-build" ]]; then log "Building test images..." - DOCKER_DIR="$(cd "$SIDECAR_DIR/../docker" && pwd)" + DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$(cd "$SIDECAR_DIR/../docker" && pwd)}" docker build -t fips-test:latest "$DOCKER_DIR" docker build -t fips-test-app:latest -f "$DOCKER_DIR/Dockerfile.app" "$DOCKER_DIR" fi From cdda660f1098d624c0fc7933f44cfa84164c562d Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 15:47:22 +0000 Subject: [PATCH 10/11] Stop local CI writing the shared mutable test image tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run built per-run images and then retagged them to fips-test:latest as a compatibility bridge for the consumers that had never been migrated. The bridge was the defect: while it existed, two concurrent runs shared one binding, so a suite could start containers from the other run's binaries and the verdict was recorded against a commit whose code never ran. It fails silently by construction — the run is green either way, and nothing compares a running container's binary against the commit under test. Every consumer now reads the run's image, so the retag is deleted rather than kept. That is the point of deleting it: a consumer that was missed fails loudly instead of quietly resolving whichever run wrote the tag last. Two consequences handled here. The cleanup script ran ip(8) inside fips-test:latest to reap simulation interfaces, and nothing writes that tag any more, so it now takes the caller's image, then the run's, then any surviving test image — the last of which is what keeps an unscoped --reap working, since that path execs before the run identity is exported. And a guard checks statically that nothing names the shared tag, because on a host with a hand-built copy lying around a reintroduction would run green; it is break-checked against a reintroduced compose consumer and a reintroduced default. Both runners gate on it, as they do the other guards. --- .github/workflows/ci.yml | 2 + testing/README.md | 17 +++++- testing/check-image-scoping.sh | 102 +++++++++++++++++++++++++++++++++ testing/ci-cleanup.sh | 31 ++++++++-- testing/ci-local.sh | 29 +++++++--- 5 files changed, 165 insertions(+), 16 deletions(-) create mode 100755 testing/check-image-scoping.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b74943b..391ba90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,8 @@ jobs: run: python3 testing/check-log-strings.py - name: Check no tested function's exit status is a log call's run: python3 testing/check-trailing-log.py + - name: Check nothing resolves the shared mutable test image + run: bash testing/check-image-scoping.sh # Hermetic: synthetic ping functions, no containers, ~45s. Lives beside # the other two so both runners gate on it identically — putting it in # only one would create exactly the drift check-ci-parity.sh exists to diff --git a/testing/README.md b/testing/README.md index 0cab275..f7d35e3 100644 --- a/testing/README.md +++ b/testing/README.md @@ -93,8 +93,21 @@ flight) never collide: - **Compose projects** are named `fipsci__`, so container, network, and volume names are all prefixed per run. - **Build images** are tagged `fips-test:` and - `fips-test-app:` (exported as `FIPS_TEST_IMAGE` / - `FIPS_TEST_APP_IMAGE` for the compose consumers). + `fips-test-app:`, exported as `FIPS_TEST_IMAGE` / + `FIPS_TEST_APP_IMAGE`, and **every** compose file and suite script reads + those. The run does not write `fips-test:latest` at all: a bridge back to + that shared mutable name would let a consumer that had been missed keep + working while resolving whichever concurrent run wrote the tag last. + `:latest` stays the hand-build name, produced by + `testing/scripts/build.sh`, and remains the default every consumer falls + back to when the variables are unset. +- **The build context** is a per-run copy at `testing/docker-/`, + exported as `FIPS_BUILD_CONTEXT`. It is absolute because compose resolves + a relative build context against the compose file's own directory rather + than the working directory. `testing/docker/` is the hand-run context and + a CI run does not write to it. Without this, two runs race on the contents + of one directory and either can build a correctly-per-run-tagged image + from the other's binaries. - Each parallel chaos child gets a unique, non-overlapping `/24` in `10.30.x` (via the sim `--subnet` override). `10.30.x` sits outside Docker's default address pool and the fixed-subnet suites' `172.x` diff --git a/testing/check-image-scoping.sh b/testing/check-image-scoping.sh new file mode 100755 index 0000000..1190184 --- /dev/null +++ b/testing/check-image-scoping.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# ── Test-image scoping guard ──────────────────────────────────────────────── +# A local CI run builds fips-test: and hands it to every suite through +# FIPS_TEST_IMAGE / FIPS_TEST_APP_IMAGE. It does NOT write fips-test:latest, +# deliberately: while a bridge back to that shared mutable name existed, a +# consumer that named it directly kept working while resolving whichever +# concurrent run wrote the tag last, and the verdict was then recorded against +# a commit whose binaries had not run. Nothing in the harness compares a +# running container's binary against the commit under test, so that failure is +# silent and leaves no artifact. +# +# The bridge is gone, so a consumer that names the shared tag now fails loudly +# at run time. This guard is the static half: it stops one being reintroduced, +# because the reintroduction is invisible on any host where a hand build has +# left an fips-test:latest lying around. +# +# What counts as a violation: a reference to the shared tag that is neither a +# comment, nor a documented default of the ${FIPS_TEST_IMAGE:-...} form, nor in +# a file on the allowlist below. +# +# Exit 0 = clean. Exit 1 = an unexpected reference. Exit 2 = the guard could +# not run; never treated as a pass. +# ───────────────────────────────────────────────────────────────────────────── +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Files permitted to name the shared tag outright, each for a stated reason. +# Every one of these is a hand-run path: none is reachable from ci-local.sh +# with FIPS_TEST_IMAGE set. +# +# scripts/build.sh the developer build; :latest IS its product +# sidecar/scripts/test-sidecar.sh hand build, behind its --skip-build guard, +# which ci-local always passes +# static/scripts/iperf-compare-refs.sh hand-run A/B against two refs +# ci-cleanup.sh last-resort name for the image ip(8) runs in, +# after the caller's and the run's have failed +# check-image-scoping.sh this guard, which has to name what it looks +# for; it resolves no image +ALLOWED=( + "scripts/build.sh" + "sidecar/scripts/test-sidecar.sh" + "static/scripts/iperf-compare-refs.sh" + "ci-cleanup.sh" + "check-image-scoping.sh" +) + +if ! command -v git >/dev/null 2>&1; then + echo "check-image-scoping: git not available, cannot sweep" >&2 + exit 2 +fi +if [[ ! -d "$SCRIPT_DIR" ]]; then + echo "check-image-scoping: $SCRIPT_DIR missing" >&2 + exit 2 +fi + +# Tracked files only, and no documentation: prose naming the tag is describing +# it, not resolving it. +mapfile -t files < <(git -C "$SCRIPT_DIR/.." ls-files -- testing/ | grep -vE '\.md$') +if [[ ${#files[@]} -eq 0 ]]; then + echo "check-image-scoping: no tracked files under testing/, refusing to pass" >&2 + exit 2 +fi + +violations=0 +for f in "${files[@]}"; do + rel="${f#testing/}" + skip=0 + for a in "${ALLOWED[@]}"; do + [[ "$rel" == "$a" ]] && skip=1 && break + done + [[ $skip -eq 1 ]] && continue + [[ -f "$SCRIPT_DIR/../$f" ]] || continue + + while IFS= read -r hit; do + n="${hit%%:*}" + text="${hit#*:}" + # A comment line is describing the tag, not resolving it. Shell, python + # and yaml all use #; nothing under testing/ uses // for comments. + [[ "$text" =~ ^[[:space:]]*# ]] && continue + # The documented indirection: the shared tag as a FALLBACK, which is + # what a bare hand run is supposed to get. + [[ "$text" == *'${FIPS_TEST_IMAGE:-fips-test:latest}'* ]] && continue + [[ "$text" == *'${FIPS_TEST_APP_IMAGE:-fips-test-app:latest}'* ]] && continue + [[ "$text" == *'os.environ.get("FIPS_TEST_IMAGE", "fips-test:latest")'* ]] && continue + echo "FAIL $f:$n names the shared test image directly:" + echo " $text" + violations=$((violations + 1)) + done < <(grep -n 'fips-test:latest\|fips-test-app:latest' "$SCRIPT_DIR/../$f" 2>/dev/null) +done + +if [[ $violations -gt 0 ]]; then + echo "" + echo "check-image-scoping: $violations reference(s) to the shared mutable test image." + echo "Read FIPS_TEST_IMAGE (default \${FIPS_TEST_IMAGE:-fips-test:latest}) instead." + echo "A run that resolves the shared tag can execute a concurrent run's binaries" + echo "and record the verdict against this commit." + exit 1 +fi + +echo "check-image-scoping: no unscoped references to the shared test image" +exit 0 diff --git a/testing/ci-cleanup.sh b/testing/ci-cleanup.sh index 0f25243..fa32a86 100755 --- a/testing/ci-cleanup.sh +++ b/testing/ci-cleanup.sh @@ -66,9 +66,15 @@ RUN_ID="" # broad default: every CI run IMAGES="" VETH_SUFFIXES="" # empty AND no --run-id: every simulation veth name # ip(8) runs inside this image, the same way the simulation creates the -# interfaces, so the reap works wherever the simulation does. The chaos -# simulation builds it, and it carries iproute2. -VETH_IMAGE="fips-test:latest" +# interfaces, so the reap works wherever the simulation does. Any fips test +# image will do; it is wanted only for its iproute2. +# +# Empty here and resolved after the argument loop, because the resolution has +# to consider --veth-image. The old default of fips-test:latest is no longer +# safe on its own: ci-local.sh does not write that tag, so on a host that has +# only ever run the harness it need not exist at all, and the reap this script +# advertises as the remedy for orphaned interfaces would be a permanent no-op. +VETH_IMAGE="" while [[ $# -gt 0 ]]; do case "$1" in @@ -77,11 +83,26 @@ while [[ $# -gt 0 ]]; do --run-id) RUN_ID="$2"; shift 2 ;; --images) IMAGES="$2"; shift 2 ;; --veth-suffixes) VETH_SUFFIXES="$2"; shift 2 ;; + --veth-image) VETH_IMAGE="$2"; shift 2 ;; -h|--help) sed -n '2,/^set /{ /^set /d; s/^# \?//; p }' "$0"; exit 0 ;; *) echo "Unknown option: $1" >&2; exit 2 ;; esac done +# Resolve the image to run ip(8) in: the caller's choice, then the run's own +# image, then any surviving fips test image. That last fallback is what keeps +# an unscoped `ci-local.sh --reap` working — it execs this script from inside +# its own argument loop, before the run identity is exported, so it can pass +# neither. The empty case is handled at the point of use, which already warns +# and skips rather than failing the sweep. +if [[ -z "$VETH_IMAGE" ]]; then + VETH_IMAGE="${FIPS_TEST_IMAGE:-}" +fi +if [[ -z "$VETH_IMAGE" ]] && command -v docker >/dev/null 2>&1; then + VETH_IMAGE="$(timeout 10 docker image ls --format '{{.Repository}}:{{.Tag}}' fips-test 2>/dev/null | head -n1)" +fi +[[ -z "$VETH_IMAGE" ]] && VETH_IMAGE="fips-test:latest" + if ! command -v docker >/dev/null 2>&1; then # No docker, nothing to reap. exit 0 @@ -217,8 +238,8 @@ reap_veths() { [[ -z "$pattern" ]] && return 0 # Without the image there is no way to run ip(8). Orphans can outlive it — # `docker image prune -a`, a build host that prunes between runs, or a run - # aborted before ci-local.sh retags :latest all remove it while interfaces - # are still up — so this is a real skip, not "nothing was ever run here". + # whose per-run image was already reaped all remove it while interfaces are + # still up — so this is a real skip, not "nothing was ever run here". if ! docker image inspect "$VETH_IMAGE" >/dev/null 2>&1; then veth_warn "image $VETH_IMAGE not present, cannot run ip(8)" return 0 diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 670795e..3c01dd8 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -280,8 +280,9 @@ record() { # # This script may be preempted (a CI worker sends SIGTERM, waits ~30s, then # SIGKILL) so it can restart on a newer tip. To make that safe: -# * every docker resource is namespaced to THIS run (compose project prefix -# + per-run image tags) so a restart never collides with a dying run; +# * every docker resource is namespaced to THIS run (compose project prefix, +# per-run image tags, per-run build context) so a restart never collides +# with a dying run, and neither does a concurrent one; # * a trap tears down everything this run created on signal/exit, bounded by # `timeout` so a stuck `down` cannot wedge the trap (SIGKILL is the backstop). @@ -916,13 +917,11 @@ run_integration() { docker build -t "$CI_IMAGE_APP" --label "$CI_LABEL" --label "$CI_LABEL_RUN" \ -f "$CI_BUILD_CONTEXT/Dockerfile.app" "$CI_BUILD_CONTEXT" --quiet \ || { record "docker-build-app" 1; return; } - # The remaining bridge back to the shared mutable tag, for any consumer not - # yet reading FIPS_TEST_IMAGE. Removed in the following commit, which is the - # step that makes a missed consumer fail loudly instead of silently - # resolving another run's binaries. Both builds have succeeded by here, so - # :latest never points at a half-built image. - docker tag "$CI_IMAGE_TEST" fips-test:latest - docker tag "$CI_IMAGE_APP" fips-test-app:latest + # Deliberately NOT retagged to fips-test:latest. Every consumer reads + # FIPS_TEST_IMAGE, and a bridge back to the shared mutable name would let a + # consumer that does not keep working silently — resolving whichever + # concurrent run wrote the tag last, and recording that run's binaries under + # this run's commit. Without the bridge a missed consumer fails loudly. # Single suite mode if [[ -n "$ONLY_SUITE" ]]; then @@ -1141,6 +1140,17 @@ run_ci_parity() { record "ci-parity" $rc } +# Nothing may resolve the shared mutable test image. This run does not write +# fips-test:latest, so a consumer naming it fails loudly here and now — but only +# on a host with no hand-built copy lying around, which is not a property to +# rely on. This is the static half of that. +run_image_scoping() { + local rc=0 + info "[image-scoping] Checking that nothing names the shared test image" + "$SCRIPT_DIR/check-image-scoping.sh" || rc=$? + record "image-scoping" $rc +} + # Every daemon log string a test matches on must still be emitted by src/. # A stale one does not fail — it stops observing, and an expect-zero assertion # built on it then passes for the wrong reason. @@ -1194,6 +1204,7 @@ main() { run_ci_parity run_log_strings run_trailing_log + run_image_scoping run_wait_converge if [[ "$TEST_ONLY" == true ]]; then From 93800a503e2b52ac86c34912ba1841a5af076295 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 26 Jul 2026 16:51:23 +0000 Subject: [PATCH 11/11] Retire the bloom-storm chaos scenario from both CI runners The scenario guarded a real regression: a mid-chain tree update that changed neither root nor depth leaking downstream as a sustained bloom announce storm. But it was never once run against the regressed binary, and its per-node ceiling was inferred from a post-mortem harness that no longer exists in the tree. On the only surviving regressed measurement the tail node's rate scales to roughly 7 sends per 30s, well under the scenario's ceiling of 40, so it was never established that the assertion could fire on its own bug class. The ceiling is also uniform per node, calibrated against the flap target that is legitimately busy rather than against the tail where the storm actually shows. This removes coverage rather than relocating it, unlike the two earlier retirements above it, and the comment in ci-local.sh records that gap explicitly. The scenario, its README, the link_swap sim primitive and the mesh-lab dispatch all stay on disk, so it remains runnable by hand. Parity holds at 24 legs a side. The parity guard was break-checked by re-adding the GitHub leg alone, which correctly reported the asymmetry. --- .github/workflows/ci.yml | 3 --- testing/ci-local.sh | 20 ++++++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 391ba90..9083806 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -408,9 +408,6 @@ jobs: - suite: congestion-stress type: chaos scenario: congestion-stress - - suite: bloom-storm - type: chaos - scenario: bloom-storm # ── Sidecar deployment ────────────────────────────────────────── - suite: sidecar type: sidecar diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 3c01dd8..d0330df 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -32,7 +32,6 @@ # nat-lan, nostr-publish-consume, stun-faults, # chaos-churn-mixed-10, chaos-ethernet-mesh, # chaos-ethernet-only, chaos-tcp-mesh, chaos-congestion-stress, -# chaos-bloom-storm, # sidecar, dns-resolver, deb-install # # Opt-in (require --with-tor; depend on live Tor network): @@ -141,7 +140,6 @@ CHAOS_SUITES=( "ethernet-only ethernet-only" "tcp-mesh tcp-mesh" "congestion-stress congestion-stress" - "bloom-storm bloom-storm" ) # Scenarios retired 2026-07-23 because their subject was a pure decision the # Docker harness could not test reliably, now covered by sans-IO unit tests: @@ -164,6 +162,24 @@ CHAOS_SUITES=( # plus end-to-end datagram delivery (src/node/tests/forwarding.rs). Real- # UDP convergence smoke still runs via static-mesh and the other chaos # scenarios' baseline assertions, so no Docker coverage is lost. +# +# Retired 2026-07-26 for a third reason — the guard's power was never +# established, and unlike the two blocks above this one DOES lose coverage: +# - bloom-storm: guarded the regression in 0caef2a (fixed by 4cdf382), where +# a mid-chain tree update changing neither root nor depth leaked downstream +# as a sustained bloom announce storm. It was never once run against that +# regressed binary; its ceiling was inferred from a separate post-mortem +# harness that no longer exists in the tree. On the only regressed +# measurement that survives, the tail node's rate scales to ~7 sends per +# 30s, well under the scenario's ceiling of 40 — so it is not established +# that the assertion could ever have fired on its own bug class. The +# ceiling is also uniform per node, calibrated against the flap target +# (legitimately ~24) while the storm's actual signature is at the tail, +# which sits at 0 on fixed code. COVERAGE GAP: nothing now exercises +# downstream containment of a mid-chain ancestor swap. The scenario, its +# README, the link_swap sim primitive and the mesh-lab dispatch all stay +# on disk and it remains runnable by hand via +# testing/chaos/scripts/chaos.sh bloom-storm. GATEWAY_SUITES=(gateway) SIDECAR_SUITES=(sidecar) FIREWALL_SUITES=(firewall)