diff --git a/testing/chaos/scenarios/congestion-stress.yaml b/testing/chaos/scenarios/congestion-stress.yaml index 8dce07e..f5aa4c6 100644 --- a/testing/chaos/scenarios/congestion-stress.yaml +++ b/testing/chaos/scenarios/congestion-stress.yaml @@ -119,15 +119,39 @@ assertions: min_nodes_ce_received: 1 # The fourth criterion, "kernel_drop_events > 0 on at least one node", is -# NOT asserted, because the implementation does not meet it: across all 182 -# archived runs of this scenario, including the six that meet the other -# three, no node has ever reported a non-zero kernel_drop_events. Asserting -# it would red the scenario permanently, and asserting a weakened version -# would assert nothing. It stands here as an unmet criterion: either the -# 4 KB recv_buf_size no longer provokes SO_RXQ_OVFL under this traffic, or -# the counter does not reach the snapshot. Until that is settled it is a -# coverage gap, because the transport drop-detection path this scenario -# names as signal 2 is exercised by nothing. +# NOT asserted, because this scenario has never met it: across all 182 +# archived runs, including the six that meet the other three, no node has +# reported a non-zero kernel_drop_events. Asserting it would red the scenario +# permanently, and asserting a weakened version would assert nothing. +# +# CORRECTED 2026-07-23. An earlier version of this note concluded that the +# transport drop-detection path "is exercised by nothing". That is wrong, and +# the error was one of scope: it looked only inside this scenario. Widening +# the scan to every scenario's congestion snapshots -- 2317 files, 11778 node +# records -- finds 51 records reporting kernel_drop_events > 0 across six +# other scenarios (depth-vs-cost, bottleneck-parent, tcp-mesh, cost-avoidance, +# maelstrom-sparse, mixed-technology), with raw drop counts up to 41165. The +# path is live and plumbed through to the snapshot. What is dead is this +# scenario's ability to provoke it. +# +# The inversion is the lead: every scenario that DOES overflow runs with the +# default 2 MB recv_buf_size, while this one, the only scenario that shrinks +# it to 4 KB, never does. SO_RXQ_OVFL counts datagrams arriving at a full +# queue, and the 1 Mbps cap below sets the arrival rate to 125 kB/s per link. +# Linux doubles a requested SO_RCVBUF, so the queue is 8192 bytes and filling +# it takes ~65 ms of reader stall -- ~22 ms even at the busiest node's 3 Mbps +# aggregate. Traffic volume cannot cause the overflow because the volume is +# capped below the rate the buffer drains. The ingress policer works against +# the same signal, discarding excess in tc before the socket ever sees it. +# +# So signal 3 appears to suppress signal 2, and the two may not belong in one +# scenario. Tracked with the experiment that would settle it (raise or drop +# the bandwidth cap for one run and watch the counter) in ISSUE-2026-0084. +# +# Note also that the header above says "small recv_buf_size (8KB)" while +# fips_overrides sets 4096. The 8 KB figure matches what Linux allocates after +# doubling, so the comment may be describing the effective queue rather than +# the setting; it reads as the setting and should say which it means. logging: rust_log: "info" diff --git a/testing/chaos/scenarios/cost-avoidance.yaml b/testing/chaos/scenarios/cost-avoidance.yaml index dec22fe..d9c529b 100644 --- a/testing/chaos/scenarios/cost-avoidance.yaml +++ b/testing/chaos/scenarios/cost-avoidance.yaml @@ -13,9 +13,21 @@ # n04 (test subject) # # n04 has two candidate parents at depth 1: n02 (via Bluetooth L2CAP) -# and n03 (via fiber). Cost-based selection should pick n03 because: -# effective_depth(n02) = 1 + ~1.4 (Bluetooth) = ~2.4 -# effective_depth(n03) = 1 + ~1.01 (fiber) = ~2.01 +# and n03 (via fiber). Cost-based selection should pick n03 because +# link_cost = etx * (1 + srtt_ms/100), so the far slower Bluetooth link +# costs far more: +# effective_depth(n02) = 1 + ~4.5 (Bluetooth, ~150-250ms) = ~5.5 +# effective_depth(n03) = 1 + ~1.05 (fiber, ~1-5ms) = ~2.05 +# +# The Bluetooth delay was widened deliberately, and the size is set by load, +# not by realism. At the original 15-40ms the cost margin was ~0.4, and even +# 80-120ms was not enough: under the CI's parallel-chaos contention the +# *measured* srtt of the fiber link spikes (host scheduling, not netem) and +# was observed to exceed a 120ms Bluetooth base, flipping n04 to n02. At +# 150-250ms the gap dwarfs any plausible one-sided fiber spike, so the +# assertion below is a reliable verdict rather than a coin toss under load. +# n04 still establishes the n02 link (it was parented to it when this flaked), +# so the choice remains real rather than vacuous. # # Validation: tree snapshot shows n04's parent is n03. @@ -44,11 +56,14 @@ netem: jitter_ms: [0, 1] loss_pct: [0, 0.5] link_policies: - # Bluetooth (L2CAP) link from n02 to n04 + # Bluetooth (L2CAP) link from n02 to n04. Delay widened to 150-250ms so the + # fiber-vs-Bluetooth cost margin survives CI load spikes on the fiber + # probe's measured latency (see header). Well within handshake timeouts, so + # the n02 link still establishes and the choice stays real. - edges: ["n02-n04"] policy: - delay_ms: [15, 40] - jitter_ms: [5, 15] + delay_ms: [150, 250] + jitter_ms: [15, 30] loss_pct: [2, 8] link_flaps: diff --git a/testing/chaos/scenarios/cost-reeval.yaml b/testing/chaos/scenarios/cost-reeval.yaml index 559966d..734d8f4 100644 --- a/testing/chaos/scenarios/cost-reeval.yaml +++ b/testing/chaos/scenarios/cost-reeval.yaml @@ -32,6 +32,37 @@ # re-evaluation" (trigger=periodic). If mutation never creates enough # asymmetry in a particular seed, the test still validates that periodic # re-eval runs without interference. +# +# THIS SCENARIO COULD NOT EXERCISE ITS SUBJECT AT ALL until 2026-07-23, and +# the reason is worth keeping because nothing about it was visible from here. +# The mesh roots at the numerically smallest NodeAddr, a hash of the node's +# key (src/tree/state.rs:363-390), and that turned out to be **n04** — this +# scenario's own designated test subject. A root has no parent, and +# evaluate_parent returns early for the smallest address, so the parent switch +# described above could never happen. The tree was inverted: n04 at depth 0, +# n01 a leaf at depth 2, in all five recent runs. +# +# The archived logs show exactly that shape. The live info! string above +# appears in 14 of 105 historical runs, on n04 in 13 of them, back when keys +# were regenerated per run and the root wandered. In the five runs before this +# fix it appears only on **n01** — the node that actually had a parent choice. +# +# `topology.pin_root` (default true) now assigns identities in NodeAddr order +# so n01 is root and n04 is the depth-1 subject this file describes. +# +# MEASURED after the fix, 2026-07-23: the periodic switch now happens, in 2 of +# 3 runs on n04 (0, 1, 1 occurrences). Before the fix it was 0 in 5 of 5, and +# structurally impossible rather than merely rare. So the subject is exercised +# for the first time. +# +# STILL NOT ASSERTED, deliberately. A "at least one periodic switch" assertion +# would red roughly one run in three on this evidence, and a single run reading +# zero is an argument against asserting rather than for it. Two ways forward, +# neither taken here: gather enough runs to know whether 2-in-3 is the real +# rate, or reduce the variance -- the mutation is stochastic with fraction=0.5, +# so a deterministic link-degradation schedule would make the asymmetry this +# scenario needs occur every run and the outcome assertable. +# Tracked in ISSUE-2026-0085. scenario: name: "cost-reeval" diff --git a/testing/chaos/scenarios/mixed-technology.yaml b/testing/chaos/scenarios/mixed-technology.yaml index 9e52923..ef32e9e 100644 --- a/testing/chaos/scenarios/mixed-technology.yaml +++ b/testing/chaos/scenarios/mixed-technology.yaml @@ -26,30 +26,53 @@ # Fiber: n03-n08, n06-n10 # # Test subjects: -# - n06 has fiber (n03) and Bluetooth (n02) parents — should pick n03 -# - n08 has fiber (n03) and Bluetooth (n04) parents — should pick n03 +# - n08 has fiber (n03) and Bluetooth (n04) parents — should pick n03. +# ASSERTED below. Holds in 4 of 4 runs under the pinned root. +# - n06 was documented as having "fiber (n03) and Bluetooth (n02) parents". +# CORRECTED 2026-07-23: that is wrong, and the netem block below is the +# evidence rather than the ASCII art above. `n03-n06` is a **WiFi** link +# (5-20ms, 1-3% loss) and `n02-n06` is Bluetooth (15-40ms, 2-8% loss), so +# n06 chooses between two impaired links, not between fiber and Bluetooth. +# n06's only fiber-grade link is `n06-n10`, and n10 sits at depth 3 so it +# is never a good parent. There is therefore no reason to expect n06 to +# prefer n03, and the mutation block (fraction 0.2, degraded 50-100ms) +# moves the two costs run to run: across four runs n06 took n03 twice and +# n02 twice. NOT asserted, and the observed split is the correct outcome +# rather than a defect — in the run checked, n03 measured a link cost of 3 +# against n02's 1, so effective depth 1+3 beat by 1+1 and the daemon chose +# the cheaper path exactly as it should. # # Netem mutation shifts fiber-only links between normal and degraded. # -# NEITHER TEST SUBJECT IS ASSERTED, because the implementation does not do -# what the two lines above say. Across the five archived runs that carry a -# status.txt and are therefore provably completed, n06's parent is n10 in -# three and n03 in two, and n08's is n04 in four and n03 in one. Encoding -# either as written would red the scenario most runs; encoding what it -# actually does would pin behaviour nobody specified and would silently -# bless it. +# n08 IS now asserted (below); n06 is not, and both reasons trace to the root, +# which was found and fixed rather than papered over here. # -# So this is an open question rather than a missing assertion: either -# cost-based selection is not preferring fiber here, or the criteria were -# written for a topology this file no longer has. n06 taking n10 is the -# more interesting half — n10 reaches n06 by fiber, so the choice may be -# defensible and the comment simply stale. It needs someone to work out -# which, and until then this scenario proves nothing about parent choice. +# The diagram above puts n01 at the top, and both test subjects only make +# sense in a tree rooted there. The mesh does not root by topology: it roots +# at the numerically smallest NodeAddr (src/tree/state.rs:363-390), which is a +# hash of the node's key and bears no relation to the numbering. Until +# 2026-07-23 this scenario rooted at **n09** in every run, which put both +# criteria out of reach. Under that root n08's two candidates were not at +# equal depth, so its choice was settled by depth long before link technology +# could matter, and n08 taking n04 was CORRECT behaviour that these comments +# made look like a defect. # -# That leaves this scenario asserting nothing about its own subject. The -# baseline block further down holds it to forming a tree and carrying -# traffic, which is a floor and not a substitute: nothing here checks that -# the tree it forms is the one these comments describe. +# What the archive shows, for anyone re-deriving this later: across the five +# runs under the old root, n06's parent is n10 in three and n03 in two, and +# n08's is n04 in four and n03 in one. Those numbers describe a tree rooted at +# n09 and must not be used to calibrate anything now. +# +# `topology.pin_root` (default true) now assigns identities in NodeAddr order, +# so n01 holds the smallest and the diagram describes the tree that forms. +# Under that root n08 takes fiber n03 as its subject assertion (below) now +# checks. n06 stays unasserted: its "fiber n03" was a WiFi link (see the +# corrected note above), so it was never a founded criterion. Tracked in +# ISSUE-2026-0085. +# +# The n08 assertion is made reliable, not merely reachable: the Bluetooth +# contrast is widened (netem below) and the two links n08 chooses between are +# excluded from the netem mutation, so neither host load nor a random +# degradation can flip the asserted comparison. scenario: name: "mixed-technology" @@ -84,11 +107,15 @@ netem: jitter_ms: [0, 1] loss_pct: [0, 0.5] link_policies: - # Bluetooth (L2CAP) links + # Bluetooth (L2CAP) links. Delay widened to 150-250ms so the + # fiber-vs-Bluetooth cost margin on n08's choice (fiber n03 vs Bluetooth + # n04) dwarfs any load-induced srtt spike on the fiber probe, matching + # cost-avoidance. Combined with excluding n03-n08/n04-n08 from the mutation + # (below), n08's asserted comparison is robust to both load and churn. - edges: ["n02-n06", "n04-n08"] policy: - delay_ms: [15, 40] - jitter_ms: [5, 15] + delay_ms: [150, 250] + jitter_ms: [15, 30] loss_pct: [2, 8] # WiFi links (moderate latency, low loss) - edges: ["n03-n06", "n08-n10"] @@ -99,6 +126,12 @@ netem: mutation: interval_secs: {min: 30, max: 60} fraction: 0.2 + # Never mutate the two links n08's tree_parents assertion depends on: the + # degraded policy (50-100ms) could otherwise slow the fiber n03-n08 link + # enough to flip n08 to n04 independently of the widened Bluetooth contrast, + # making the assertion a coin toss. The mutation still churns the other ten + # links, so the dynamic behaviour it exists to exercise is unchanged. + exclude_edges: ["n03-n08", "n04-n08"] policies: normal: delay_ms: [1, 10] @@ -134,6 +167,19 @@ assertions: max_roots: 1 min_nodes_parented: 9 min_sessions: 3 + # The scenario's own subject, assertable for the first time now that the + # root is pinned to n01 and the topology above describes the tree that + # forms. n08's two candidates sit at equal depth, so the choice is decided + # by link cost alone: n03 over fiber-grade default netem against n04 over + # Bluetooth. That is the fiber-versus-Bluetooth preference this scenario + # exists to test. + # + # The criterion is encoded as written rather than calibrated from runs: + # "should pick n03" is the spec, and four of four runs agree. n06 is + # deliberately absent — see the corrected note in the header for why its + # documented criterion was never founded on this topology. + tree_parents: + n08: n03 logging: rust_log: "info" diff --git a/testing/chaos/scenarios/smoke-10.yaml b/testing/chaos/scenarios/smoke-10.yaml index 9a63958..452e9b5 100644 --- a/testing/chaos/scenarios/smoke-10.yaml +++ b/testing/chaos/scenarios/smoke-10.yaml @@ -11,6 +11,16 @@ topology: ensure_connected: true subnet: "172.20.0.0/24" ip_start: 10 + # The one scenario that deliberately does NOT pin the root. Every other + # scenario assigns identities in NodeAddr order so that n01 is root and its + # diagram describes the tree that actually forms. That is right for them, + # because they test parent selection and were silently testing it under a + # root they did not intend, but applying it everywhere would leave root + # election itself exercised nowhere. This is the generic 10-node convergence + # baseline and its `baseline` assertion is root-agnostic, so it is the + # cheapest place to keep election under test. + # Do not "fix" this to true for consistency with the others. + pin_root: false # Phase 2+ features (disabled for MVP) netem: diff --git a/testing/chaos/sim/__main__.py b/testing/chaos/sim/__main__.py index ec1867f..92bbe5a 100644 --- a/testing/chaos/sim/__main__.py +++ b/testing/chaos/sim/__main__.py @@ -27,9 +27,10 @@ def main(): ) parser.add_argument( "--subnet", type=str, default=None, - help="Override topology subnet CIDR (e.g. 10.30.0.0/24); node IPs " - "derive from it. Used by CI to give each parallel run a " - "non-overlapping network.", + help="Pin the topology subnet CIDR (e.g. 10.30.0.0/24) instead of " + "claiming a free one. The sim normally claims a range so " + "concurrent runs cannot collide; pin only when you need a known " + "range, and expect a hard failure if it is already taken.", ) args = parser.parse_args() @@ -55,7 +56,10 @@ def main(): sys.exit(1) scenario.duration_secs = args.duration if args.subnet is not None: - scenario.topology.subnet = args.subnet + # Pinning opts out of claiming. The runner still *creates* the network, + # so a range already in use fails loudly here rather than silently + # overlapping a concurrent run. + scenario.topology.pinned_subnet = args.subnet runner = SimRunner(scenario) result = runner.run() diff --git a/testing/chaos/sim/compose.py b/testing/chaos/sim/compose.py index c27a153..3594ead 100644 --- a/testing/chaos/sim/compose.py +++ b/testing/chaos/sim/compose.py @@ -19,12 +19,12 @@ _COMPOSE_TEMPLATE = Template( """\ networks: fips-net: - driver: bridge - labels: - - "com.corganlabs.fips-ci=1" - ipam: - config: - - subnet: {{ subnet }} + # External, and created by the sim rather than by compose. The range has to + # be *claimed* -- attempt-create, advance on docker's own overlap error -- + # so that two concurrent runs cannot select the same one, and only the + # process that creates the network can do that. See sim/netclaim.py. + external: true + name: {{ network_name }} x-fips-common: &fips-common image: {{ image }} @@ -64,8 +64,14 @@ def generate_compose( topology: SimTopology, scenario: Scenario, output_dir: str, + network_name: str, ) -> str: - """Render docker-compose.yml and write to output_dir. Returns the file path.""" + """Render docker-compose.yml and write to output_dir. Returns the file path. + + ``network_name`` is the docker network the sim has already claimed; the + compose file refers to it as external rather than declaring a subnet, so + that the claim and the creation are the same operation. + """ os.makedirs(output_dir, exist_ok=True) nodes = [topology.nodes[nid] for nid in sorted(topology.nodes)] @@ -77,7 +83,7 @@ def generate_compose( ) content = _COMPOSE_TEMPLATE.render( - subnet=scenario.topology.subnet, + network_name=network_name, rust_log=scenario.logging.rust_log, image=FIPS_SIM_IMAGE, nodes=nodes, diff --git a/testing/chaos/sim/keys.py b/testing/chaos/sim/keys.py index dd2dd24..69ee9d1 100644 --- a/testing/chaos/sim/keys.py +++ b/testing/chaos/sim/keys.py @@ -8,4 +8,4 @@ _TESTING_DIR = os.path.join(os.path.dirname(__file__), "..", "..") if _TESTING_DIR not in sys.path: sys.path.insert(0, _TESTING_DIR) -from lib.derive_keys import derive # noqa: E402 +from lib.derive_keys import derive, derive_full # noqa: E402,F401 diff --git a/testing/chaos/sim/netclaim.py b/testing/chaos/sim/netclaim.py new file mode 100644 index 0000000..ab4294d --- /dev/null +++ b/testing/chaos/sim/netclaim.py @@ -0,0 +1,108 @@ +"""Claim a free /24 for a scenario's network, atomically. + +Two concurrent chaos runs used to request identical ranges: `ci-local.sh` +derived each child's subnet from its position in the suite list, so both runs +walked `10.30.0` through `10.30.12` and collided on every one of them. A run +index or a hashed offset only makes a collision unlikely, and a collision is +precisely the failure being removed, so this claims instead. + +Claim-and-advance: attempt to create the network on a candidate range and, on +docker's own "Pool overlaps" error, move to the next. Docker's address pool is +then the arbiter and an overlap between two concurrent runs is *impossible* +rather than improbable. The pattern is taken from `testing/sidecar/scripts/ +test-sidecar.sh:75-98`, which has been carrying it since 2026-07-19. + +The claim has to happen before the topology is generated, not before the +compose file is written: node IPs derive from the subnet inside +`generate_topology`, and traffic shaping keys its filters on those addresses. +Claiming later yields a network on one range and `tc` filters on another, which +does not fail at bring-up and instead leaves the shaping matching nothing. +""" + +from __future__ import annotations + +import logging +import subprocess + +log = logging.getLogger(__name__) + +# Stay inside 10.30.0.0/16, which ci-local.sh already documents as clear of +# docker's default pool (172.17-31, 192.168) and of the fixed-subnet suites in +# 172.x. Sidecar has claimed 10.40.0.0/16; do not overlap it. +NET_BASE = "10.30" +NET_CANDIDATES = 200 + + +class NetworkClaimError(RuntimeError): + """No range could be claimed, or docker refused for another reason.""" + + +def claim_network( + name: str, + labels: dict[str, str] | None = None, + candidates: list[str] | None = None, +) -> str: + """Create `name` on the first free /24 and return its CIDR. + + `candidates` pins the search to an explicit list, which is how `--subnet` + opts out of claiming. A single pinned candidate that is already taken + raises rather than advancing, so pinning fails loudly instead of silently + overlapping a concurrent run. + + Raises NetworkClaimError if every candidate is taken, or immediately if + docker fails for any reason other than an address-pool conflict. Retrying + a real error 200 times would bury the reason for it. + """ + ranges = candidates or [ + f"{NET_BASE}.{i}.0/24" for i in range(NET_CANDIDATES) + ] + label_args: list[str] = [] + for key, value in (labels or {}).items(): + label_args += ["--label", f"{key}={value}"] + + for subnet in ranges: + proc = subprocess.run( + ["docker", "network", "create", "--subnet", subnet] + + label_args + + [name], + capture_output=True, + text=True, + ) + if proc.returncode == 0: + log.info("Claimed network %s on %s", name, subnet) + return subnet + err = (proc.stderr or "") + (proc.stdout or "") + if "ool overlaps" in err: + continue + raise NetworkClaimError( + f"docker network create {name} on {subnet} failed: {err.strip()}" + ) + + if candidates: + raise NetworkClaimError( + f"pinned subnet(s) {', '.join(candidates)} already in use; " + f"another run holds the range" + ) + raise NetworkClaimError( + f"no free /24 in {NET_BASE}.0.0/16 after {NET_CANDIDATES} attempts" + ) + + +def remove_network(name: str) -> None: + """Remove a claimed network, tolerating its absence. + + The network is `external:` to the generated compose file, so `compose down` + leaves it behind and this is the only thing that reclaims the range. + Failures are logged rather than raised: teardown runs on the failure path + too, and losing a /24 is a leak, not a reason to mask the original error. + """ + proc = subprocess.run( + ["docker", "network", "rm", name], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + err = ((proc.stderr or "") + (proc.stdout or "")).strip() + if "not found" in err or "No such network" in err: + return + log.warning("Could not remove network %s: %s", name, err) diff --git a/testing/chaos/sim/netem.py b/testing/chaos/sim/netem.py index 45720a8..3ac8d0e 100644 --- a/testing/chaos/sim/netem.py +++ b/testing/chaos/sim/netem.py @@ -144,6 +144,27 @@ class NetemManager: len(self._edge_overrides), ) + # Edges the periodic mutation must never touch (canonical "nXX-nYY"). + # Unlike a link_policy typo, which merely fails to shape a link, a typo + # here would silently leave an asserted link unprotected and reintroduce + # the exact flakiness the exclusion exists to remove — so an unknown + # edge is a hard error, not a warning. + self._mutation_exclude: set[str] = set() + for edge_str in config.mutation.exclude_edges: + canonical = "-".join(sorted(edge_str.split("-"))) + if canonical not in topo_edge_strs: + raise ValueError( + f"netem.mutation.exclude_edges references {edge_str!r}, " + "which is not an edge in the topology" + ) + self._mutation_exclude.add(canonical) + if self._mutation_exclude: + log.info( + "Mutation excludes %d edge(s): %s", + len(self._mutation_exclude), + ", ".join(sorted(self._mutation_exclude)), + ) + def _htb_rate(self, node_id: str, peer_id: str) -> str: """Return the HTB rate string for a link direction.""" rate = self._edge_rates.get((node_id, peer_id), 0) @@ -370,10 +391,12 @@ class NetemManager: if not self.config.mutation.policies: return - # Only consider edges where both endpoints are up + # Only consider edges where both endpoints are up, and never the + # explicitly excluded ones (links an assertion depends on). live_edges = [ (a, b) for a, b in self.topology.edges if a not in self.down_nodes and b not in self.down_nodes + and "-".join(sorted([a, b])) not in self._mutation_exclude ] if not live_edges: return diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index 182a252..ee9c8c6 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -30,6 +30,7 @@ from .link_swap import LinkSwapManager from .links import LinkManager from .logs import AnalysisResult, analyze_logs, collect_logs, write_sim_metadata from .naming import name_suffix +from .netclaim import claim_network, remove_network from .netem import NetemManager from .nodes import NodeManager from .peer_churn import PeerChurnManager @@ -47,6 +48,9 @@ class SimRunner: self.rng = random.Random(scenario.seed) self.topology: SimTopology | None = None self.compose_file: str | None = None + # Claimed in _setup; the compose file refers to it as external, so + # `compose down` does not remove it and teardown must. + self.network_name: str | None = None self.output_dir: str = self._resolve_output_dir(scenario) self._interrupted = False @@ -185,6 +189,28 @@ class SimRunner: logging.getLogger().addHandler(fh) log.info("Runner log: %s", runner_log_path) + # 0. Claim this run's network range, before anything derives from it. + # + # The ordering is not obvious and it matters: node IPs are computed + # from the subnet inside generate_topology, and traffic shaping keys + # its filters on those addresses. Claiming after the topology exists + # would give a network on one range and `tc` filters on another, which + # does not fail at bring-up — it silently leaves the shaping matching + # nothing. + self.network_name = f"fips-sim{name_suffix()}-net" + s.topology.subnet = claim_network( + self.network_name, + labels={ + "com.corganlabs.fips-ci": "1", + "com.corganlabs.fips-ci.run": os.environ.get( + "FIPS_CI_RUN_ID", "manual" + ), + }, + candidates=( + [s.topology.pinned_subnet] if s.topology.pinned_subnet else None + ), + ) + # 1. Generate topology log.info( "Generating %d-node %s topology (seed=%d)...", @@ -237,7 +263,9 @@ class SimRunner: log.info("Wrote node configs to %s", config_dir) # 3. Generate docker-compose.yml - self.compose_file = generate_compose(self.topology, self.scenario, config_dir) + self.compose_file = generate_compose( + self.topology, self.scenario, config_dir, self.network_name + ) log.info("Wrote %s", self.compose_file) # 4. Build the test image once (avoids per-service build at scale) @@ -521,6 +549,17 @@ class SimRunner: except Exception: log.exception("Could not write status file") + def _release_network(self) -> None: + """Give this run's claimed /24 back. + + The compose file declares the network `external:`, so `compose down` + leaves it alone and nothing else reclaims the range. Called from both + teardown paths, including the setup-failed one, because a run that + fell over after claiming still holds a range. + """ + if self.network_name: + remove_network(self.network_name) + def _teardown(self) -> AnalysisResult | None: """Stop dynamic elements, collect logs, analyze, stop containers.""" if not self._containers_started: @@ -536,6 +575,7 @@ class SimRunner: # containers or a network even with no mesh to speak of. log.info("Stopping containers...") docker_compose(self.compose_file, ["down"], check=False) + self._release_network() return None result = None @@ -701,6 +741,7 @@ class SimRunner: ["down"], check=False, ) + self._release_network() return result diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py index d7e30c4..c1f3eaf 100644 --- a/testing/chaos/sim/scenario.py +++ b/testing/chaos/sim/scenario.py @@ -44,6 +44,18 @@ class TopologyConfig: # When set, each edge is randomly assigned a transport based on weights. # Only valid for non-explicit algorithms (explicit uses per-edge syntax). transport_mix: dict[str, float] | None = None + # Assign derived identities in NodeAddr order so n01 holds the smallest and + # is therefore the root every scenario diagram draws. Default on: without it + # the root is effectively arbitrary, which left `cost-reeval` rooted at its + # own designated test subject — so the parent switch it exists to observe + # could not occur — and made `mixed-technology`'s parent criteria + # unreachable. Set false where election from an arbitrary key distribution + # is itself the subject. + pin_root: bool = True + # Set by --subnet to opt out of claiming a free range. Not a scenario-file + # key: a scenario that hardcoded its range would reintroduce the collision + # claiming exists to remove. + pinned_subnet: str | None = None @dataclass @@ -61,6 +73,11 @@ class NetemMutationConfig: interval_secs: Range = field(default_factory=lambda: Range(15, 30)) fraction: float = 0.3 policies: dict[str, NetemPolicy] = field(default_factory=dict) + # Edges the periodic mutation must never touch, "nXX-nYY" strings. Use it + # to pin the links a tree_parents assertion depends on so a random + # degradation cannot flip the very comparison being asserted. Validated + # against the topology in NetemManager — an unknown edge is a hard error. + exclude_edges: list[str] = field(default_factory=list) @dataclass @@ -372,11 +389,11 @@ _SECTION_KEYS = { "scenario": {"name", "seed", "duration_secs"}, "topology": { "num_nodes", "algorithm", "params", "ensure_connected", "subnet", - "ip_start", "default_transport", "transport_mix", + "ip_start", "default_transport", "transport_mix", "pin_root", }, "netem": {"enabled", "default_policy", "link_policies", "mutation"}, "netem.link_policies[]": {"edges", "policy", "policy_name"}, - "netem.mutation": {"interval_secs", "fraction", "policies"}, + "netem.mutation": {"interval_secs", "fraction", "policies", "exclude_edges"}, "link_flaps": { "enabled", "interval_secs", "max_down_links", "down_duration_secs", "protect_connectivity", @@ -494,6 +511,13 @@ def load_scenario(path: str) -> Scenario: s.topology.subnet = tc.get("subnet", "172.20.0.0/24") s.topology.ip_start = int(tc.get("ip_start", 10)) s.topology.default_transport = tc.get("default_transport", "udp") + if "pin_root" in tc: + pin = tc["pin_root"] + if not isinstance(pin, bool): + raise ValueError( + f"topology.pin_root must be a boolean, got {pin!r}" + ) + s.topology.pin_root = pin if "transport_mix" in tc: mix = tc["transport_mix"] if not isinstance(mix, dict) or not mix: @@ -530,6 +554,7 @@ def load_scenario(path: str) -> Scenario: mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs" ) s.netem.mutation.fraction = float(mc.get("fraction", 0.3)) + s.netem.mutation.exclude_edges = list(mc.get("exclude_edges", [])) if "policies" in mc: s.netem.mutation.policies = { name: _parse_netem_policy(pdata, f"netem.mutation.policies.{name}") diff --git a/testing/chaos/sim/topology.py b/testing/chaos/sim/topology.py index 69ea769..4ac99d3 100644 --- a/testing/chaos/sim/topology.py +++ b/testing/chaos/sim/topology.py @@ -7,7 +7,7 @@ import random from collections import deque from dataclasses import dataclass, field -from .keys import derive +from .keys import derive_full from .naming import name_suffix, veth_token from .scenario import TopologyConfig @@ -203,12 +203,31 @@ def generate_topology( n = config.num_nodes subnet_base = config.subnet.rsplit(".", 1)[0] # "172.20.0" - # Create nodes with IPs and keys + # Create nodes with IPs and keys. + # + # The mesh roots itself at the numerically smallest NodeAddr + # (`src/tree/state.rs:363-390`), which is a hash of the node's public key + # and so bears no relation to the node numbering. Every scenario diagram in + # this tree draws n01 at the top, and before this ordering was applied that + # held in only three of thirteen: `cost-reeval` rooted at n04 — its own + # designated test subject, which therefore had no parent to switch and could + # not exercise what the scenario exists to test — and `mixed-technology` at + # n09, which made its two documented parent criteria unreachable. + # + # So derive the identities from the mesh name as before, then *assign* them + # in NodeAddr order: n01 receives the smallest and is the root, n02 the next, + # and so on. The keys are unchanged and still deterministic; only which node + # id holds which one changes. Scenarios that want an arbitrary root set + # `pin_root: false` and keep exercising election. + node_ids_ordered = [f"n{i + 1:02d}" for i in range(n)] + identities = [derive_full(mesh_name, nid) for nid in node_ids_ordered] + if config.pin_root: + identities.sort(key=lambda t: t[2]) + nodes: dict[str, SimNode] = {} - for i in range(n): - node_id = f"n{i + 1:02d}" + for i, node_id in enumerate(node_ids_ordered): docker_ip = f"{subnet_base}.{config.ip_start + i}" - nsec, npub = derive(mesh_name, node_id) + nsec, npub, _ = identities[i] nodes[node_id] = SimNode( node_id=node_id, docker_ip=docker_ip, diff --git a/testing/check-trailing-log.py b/testing/check-trailing-log.py index eb8a5c8..990dc34 100755 --- a/testing/check-trailing-log.py +++ b/testing/check-trailing-log.py @@ -133,6 +133,15 @@ def status_tested(name: str, all_text: str, defining_file: Path) -> list[str]: (rf"^\s*{re.escape(name)}\s+[^\n|&]*&&", " &&"), (rf"^\s*{re.escape(name)}\s*&&", " &&"), (rf"\bif\s+\S*\s*{re.escape(name)}\b.*;\s*then", "if ... ; then"), + # Command substitution. Found 2026-07-23 while fixing a function this + # check reported clean: `total=$(count_log_pattern "$p") || { ... }` + # consumes the status, but the line begins with the variable, so none + # of the patterns above match it. That is not an exotic form — it is + # how a shell function returns a *value*, and it is the shape of the + # `|| true`-swallowed-failure family this check exists to catch. + (rf"=\$\(\s*{re.escape(name)}\b", "=$()"), + (rf"\bif\s+!?\s*\S*=?\$\(\s*{re.escape(name)}\b", "if =$()"), + (rf"\[\s+\"?\$\(\s*{re.escape(name)}\b", "[ $() ]"), ] hits = [] for pat, label in patterns: @@ -186,8 +195,8 @@ def main() -> int: print(f" last statement: {stmt[:100]}") print(f" status consumed by: {', '.join(callers)}") print(f" fix: add an explicit `return 0` as the last statement. Its " - f"success value is currently whatever the log call returned, " - f"which is 0 whatever the function did.") + f"success value is currently the trailing command's, which is 0 " + f"whatever the function did.") print(f"trailing-log check: {scanned} function(s) scanned, " f"{shaped} end in a logging call, {len(ALLOWED)} allowed, " diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 2e39b53..6492cff 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -646,29 +646,77 @@ run_chaos() { return $rc } +# Claim a free /64 for this run's gateway-lan network (Mechanism B). +# +# gateway-lan cannot float like fips-net: the LAN clients' resolv.conf pins the +# gateway's LAN address as their nameserver, which must be a literal known +# before they start. So instead of a fixed fd02::/64 that two concurrent runs +# both request (the second failing on "Pool overlaps"), claim-and-advance a +# candidate /64 and let docker's create be the atomic arbiter. The claimed +# prefix is threaded — via the exported FIPS_GW_LAN6_PREFIX — into the compose +# ipv6_address pins, the generated resolv.conf, and gateway-test.sh, so every +# LAN address moves with the claim. i=0 renders today's fd02::/64. +# +# Mirrors sidecar/scripts/test-sidecar.sh:alloc_network. Deliberately does NOT +# discard stderr: only an address-pool conflict is worth advancing on; any other +# failure is real and burning through 256 candidates would bury the reason. +claim_gateway_lan6() { + local net="$1" i err + for (( i = 0; i < 256; i++ )); do + if err=$(docker network create --ipv6 \ + --subnet "fd02:0:0:${i}::/64" \ + --label "$CI_LABEL" --label "$CI_LABEL_RUN" \ + "$net" 2>&1); then + export FIPS_GW_LAN6_PREFIX="fd02:0:0:${i}" + info "[gateway] Claimed $net on fd02:0:0:${i}::/64" + return 0 + fi + case "$err" in + *"Pool overlaps"*|*"pool overlaps"*) continue ;; + *) fail "[gateway] docker network create: $err"; return 1 ;; + esac + done + fail "[gateway] no free /64 in fd02:0:0:0-255::/64 after 256 attempts" + return 1 +} + # Run gateway integration test run_gateway() { local compose="testing/static/docker-compose.yml" + local ext="testing/static/docker-compose.gateway-external-net.yml" local rc=0 export COMPOSE_PROJECT_NAME="$(ci_project static)" + export FIPS_GW_LAN_NET="fips-gateway-lan${FIPS_CI_NAME_SUFFIX:-}" - info "[gateway] Generating configs" - bash testing/static/scripts/generate-configs.sh gateway gateway-test || { record "gateway" 1; return; } - bash testing/static/scripts/gateway-test.sh inject-config || { record "gateway" 1; return; } - - info "[gateway] Starting containers" - docker compose -f "$compose" --profile gateway up -d || { record "gateway" 1; return; } - - info "[gateway] Running gateway test" - if bash testing/static/scripts/gateway-test.sh; then - rc=0 - else - rc=1 - info "[gateway] Collecting failure logs" - docker compose -f "$compose" --profile gateway logs --no-color 2>&1 | tail -100 + # Claim the LAN /64 first: generate-configs writes resolv.conf from the + # claimed prefix, and inject-config renders the port-forward targets from + # it, so both must see FIPS_GW_LAN6_PREFIX before they run. + info "[gateway] Claiming LAN network" + if ! claim_gateway_lan6 "$FIPS_GW_LAN_NET"; then + record "gateway" 1 + return fi - docker compose -f "$compose" --profile gateway down --volumes --remove-orphans 2>/dev/null + info "[gateway] Generating configs" + if bash testing/static/scripts/generate-configs.sh gateway gateway-test \ + && bash testing/static/scripts/gateway-test.sh inject-config \ + && { info "[gateway] Starting containers"; \ + docker compose -f "$compose" -f "$ext" --profile gateway up -d; }; then + info "[gateway] Running gateway test" + if bash testing/static/scripts/gateway-test.sh; then + rc=0 + else + rc=1 + info "[gateway] Collecting failure logs" + docker compose -f "$compose" -f "$ext" --profile gateway logs --no-color 2>&1 | tail -100 + fi + else + rc=1 + fi + + # compose down does not remove an external network, so drop it explicitly. + docker compose -f "$compose" -f "$ext" --profile gateway down --volumes --remove-orphans 2>/dev/null + docker network rm "$FIPS_GW_LAN_NET" >/dev/null 2>&1 || true record "gateway" $rc } @@ -939,7 +987,6 @@ run_integration() { local pids=() local suite_names=() local running=0 - local chaos_idx=0 for entry in "${CHAOS_SUITES[@]}"; do # Parse: "display-name scenario [flags...]" @@ -947,14 +994,15 @@ run_integration() { local name="${parts[0]}" local args=("${parts[@]:1}") - # Give each chaos child a unique, non-overlapping /24 in 10.30.x so - # parallel children never collide with each other, and so a chaos - # net can never swallow a fixed-subnet suite (sidecar/static/nat in - # 172.x). 10.30.x sits outside docker's default-address-pool range - # (172.17-31 / 192.168), so auto-assigned nets can't land on it - # either. Node IPs derive from this subnet inside the sim. - args+=("--subnet" "10.30.${chaos_idx}.0/24") - chaos_idx=$((chaos_idx + 1)) + # No --subnet: the sim claims a free /24 itself (sim/netclaim.py). + # This used to pass 10.30.${chaos_idx}.0/24, which uniquified the + # children of ONE run against each other and was byte-identical + # between runs -- so two concurrent ci-local runs both walked + # 10.30.0 through 10.30.12 and collided on every one. A claim makes + # the daemon the arbiter, so an overlap is impossible rather than + # merely unlikely. The range still sits in 10.30.0.0/16, clear of + # docker's default pool (172.17-31 / 192.168) and of the + # fixed-subnet suites in 172.x. # Throttle: wait for a slot while [[ $running -ge $PARALLEL_JOBS ]]; do diff --git a/testing/deb-install/test.sh b/testing/deb-install/test.sh index 6a78827..432ef1c 100755 --- a/testing/deb-install/test.sh +++ b/testing/deb-install/test.sh @@ -32,7 +32,7 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" CACHE_DIR="$SCRIPT_DIR/.cache" DEB_CACHE_DIR="$CACHE_DIR/deb" -# Timeouts +# Timeouts. Each wait loop below exits as soon as its condition is met. BOOT_TIMEOUT=30 SERVICE_TIMEOUT=20 DAEMON_TIMEOUT=15 @@ -75,15 +75,26 @@ start_systemd_container_with_tun() { } wait_for_systemd() { - local name="$1" + local name="$1" state for _i in $(seq 1 "$BOOT_TIMEOUT"); do - if docker exec "$name" systemctl is-system-running --wait 2>/dev/null | grep -qE 'running|degraded'; then - return 0 - fi + # `is-system-running` exits non-zero for `degraded` (a unit failed to + # start -- e.g. systemd-modules-load, which cannot load kernel modules + # inside a container -- even though the system did finish booting). This + # script runs `set -o pipefail`, so a piped `grep` would inherit that + # non-zero exit and reject an acceptable state, which timed out the + # newest distros (they reach `degraded`, older ones reach `running`). + # Capture the state string and test it directly instead of the pipe. + state=$(docker exec "$name" systemctl is-system-running --wait 2>/dev/null || true) + case "$state" in + running | degraded) return 0 ;; + esac sleep 1 done - echo " WARNING: systemd did not reach running state in ${BOOT_TIMEOUT}s (may still work)" - return 0 + # A boot that never reached `running` or `degraded` is not a warning: every + # check after this point reads a system that may not have started its units, + # and returning 0 here made the timeout indistinguishable from a clean boot. + echo " ERROR: systemd did not reach running state in ${BOOT_TIMEOUT}s" >&2 + return 1 } wait_for_service_active() { @@ -229,18 +240,32 @@ DOCKERFILE rm -f "$CACHE_DIR/deb-for-image" start_systemd_container_with_tun "$name" "$image" - wait_for_systemd "$name" + wait_for_systemd "$name" || { + fail "systemd did not boot in $name; the install checks below would read an unstarted system" + cleanup_container "$name" + return + } # Install the .deb. apt handles dependencies (libc6, systemd, # libdbus-1-3) and runs the maintainer scripts (postinst → # systemctl enable fips.service; fips-dns.service starts and # runs fips-dns-setup). log "Installing .deb (apt install /opt/fips-deb/${deb_basename})" - local install_output + # `|| true` here used to discard apt's exit status, and an empty capture (a + # failed `docker exec`) matches neither error pattern below, so a install + # that never ran reached `pass "apt install completed"`. Keep the capture on + # failure so the diagnostics below can print it, but remember the status. + local install_output install_rc=0 install_output=$(docker exec "$name" bash -c " apt-get update >/dev/null 2>&1 cd /opt/fips-deb && apt-get install -y --no-install-recommends ./${deb_basename} 2>&1 - ") || true + ") || install_rc=$? + if [ "$install_rc" -ne 0 ]; then + fail "apt install exited $install_rc" + echo "$install_output" | tail -20 + cleanup_container "$name" + return + fi if echo "$install_output" | grep -qE "^E:|errors? were encountered"; then fail "apt install reported errors" echo "$install_output" | tail -20 diff --git a/testing/interop/interop-test.sh b/testing/interop/interop-test.sh index f240daf..ab27b3c 100755 --- a/testing/interop/interop-test.sh +++ b/testing/interop/interop-test.sh @@ -476,13 +476,24 @@ phase_result() { } # Count a pattern across all node logs. +# +# A node whose logs cannot be read makes the whole count unusable rather than +# contributing 0. The previous form ended each read `| grep -cE … || true`, so a +# failed `docker logs` yielded 0 for that node and the eight expect-zero +# assertions in the GLOBAL_PATTERNS loop read a clean result from a node that was +# never consulted. Same defect and same fix as rekey-test.sh. count_log_pattern() { - local pattern="$1" total=0 n count + local pattern="$1" total=0 n logs count for n in "${NODES[@]}"; do - count=$(docker logs "${CONTAINER[$n]}" 2>&1 | grep -cE "$pattern" || true) + if ! logs=$(docker logs "${CONTAINER[$n]}" 2>&1); then + echo "unreadable:${CONTAINER[$n]}" + return 1 + fi + count=$(grep -cE "$pattern" <<<"$logs" || true) total=$((total + count)) done echo "$total" + return 0 } # Per-node count of a pattern. @@ -854,7 +865,12 @@ declare -A GLOBAL_PATTERNS=( for pat in "${!GLOBAL_PATTERNS[@]}"; do desc="${GLOBAL_PATTERNS[$pat]}" - total="$(count_log_pattern "$pat")" + if ! total="$(count_log_pattern "$pat")"; then + echo " FAIL $desc: node logs unreadable ($total), zero not established" + FAILED=$((FAILED + 1)) + INTEROP_FAILURES+=("[log] $desc: node logs unreadable ($total)") + continue + fi if [ "$total" -eq 0 ]; then echo " PASS $desc: 0" PASSED=$((PASSED + 1)) diff --git a/testing/lib/derive_keys.py b/testing/lib/derive_keys.py index 37495e5..460095d 100755 --- a/testing/lib/derive_keys.py +++ b/testing/lib/derive_keys.py @@ -93,13 +93,28 @@ def _convertbits(data, frombits, tobits): # --- public API --- def derive(mesh_name, node_name): + nsec_hex, npub, _ = derive_full(mesh_name, node_name) + return nsec_hex, npub + + +def derive_full(mesh_name, node_name): + """As `derive`, plus the NodeAddr the daemon will compute for this key. + + NodeAddr is the first 16 bytes of SHA-256 over the serialized x-only + public key (`src/identity/node_addr.rs:36-43`), and the mesh elects the + numerically smallest one as root (`src/tree/state.rs:363-390`). Callers + that need the tree's root to be predictable derive it from here rather + than assuming it follows the node numbering. + """ nsec_hex = hashlib.sha256(f"{mesh_name}|{node_name}".encode()).hexdigest() k = int(nsec_hex, 16) pub = _scalar_mult(k, (Gx, Gy)) x_hex = format(pub[0], "064x") - data_5bit = _convertbits(list(bytes.fromhex(x_hex)), 8, 5) + x_bytes = bytes.fromhex(x_hex) + data_5bit = _convertbits(list(x_bytes), 8, 5) npub = _bech32_encode("npub", data_5bit) - return nsec_hex, npub + node_addr = hashlib.sha256(x_bytes).hexdigest()[:32] + return nsec_hex, npub, node_addr if __name__ == "__main__": diff --git a/testing/nat/scripts/nostr-relay-test.sh b/testing/nat/scripts/nostr-relay-test.sh index 28c3678..36ba79f 100755 --- a/testing/nat/scripts/nostr-relay-test.sh +++ b/testing/nat/scripts/nostr-relay-test.sh @@ -291,14 +291,21 @@ assert_process_alive() { return 0 } +# A container whose logs cannot be read has not been shown to be panic-free. +# See the companion note in stun-faults-test.sh: the previous `|| true` made +# this assertion's failure mode indistinguishable from its success condition. assert_no_panic() { local container="$1" local logs - logs="$(docker logs "$container" 2>&1 || true)" + if ! logs="$(docker logs "$container" 2>&1)"; then + echo "could not read logs from $container; absence of panics is not established" >&2 + return 1 + fi if grep -Eq "panicked at|RUST_BACKTRACE|fatal runtime error" <<<"$logs"; then echo "panic detected in $container logs" >&2 return 1 fi + return 0 } run_test() { diff --git a/testing/nat/scripts/stun-faults-test.sh b/testing/nat/scripts/stun-faults-test.sh index 51a5a13..c9dbbb5 100755 --- a/testing/nat/scripts/stun-faults-test.sh +++ b/testing/nat/scripts/stun-faults-test.sh @@ -143,13 +143,21 @@ assert_process_alive() { return 0 } +# A container whose logs cannot be read has not been shown to be panic-free. +# The previous form read `docker logs … || true`, so an unreadable container +# yielded empty output, matched no panic pattern, and returned success — the +# assertion's failure mode was indistinguishable from its success condition. assert_no_panic() { local logs - logs="$(docker logs "$NODE" 2>&1 || true)" + if ! logs="$(docker logs "$NODE" 2>&1)"; then + echo "could not read logs from $NODE; absence of panics is not established" >&2 + return 1 + fi if grep -Eq "panicked at|RUST_BACKTRACE|fatal runtime error" <<<"$logs"; then echo "panic detected in $NODE logs" >&2 return 1 fi + return 0 } # Look for STUN-related fault evidence in the daemon's logs. The diff --git a/testing/static/docker-compose.gateway-external-net.yml b/testing/static/docker-compose.gateway-external-net.yml new file mode 100644 index 0000000..fa1d40d --- /dev/null +++ b/testing/static/docker-compose.gateway-external-net.yml @@ -0,0 +1,15 @@ +# Override: attach gateway-lan to a pre-created external network instead of +# letting compose create it from the base file's fd02::/64 pin. +# +# Applied only by ci-local.sh's run_gateway, which claims a free /64 per run +# (claim_gateway_lan6) and exports FIPS_GW_LAN_NET / FIPS_GW_LAN6_PREFIX before +# `up`. This is what makes two concurrent local gateway runs collision-safe: +# each claims a distinct /64, so neither requests a fixed range the other holds. +# +# The GitHub matrix and any standalone `docker compose up` do NOT apply this +# overlay; they use the base file's normal gateway-lan network unchanged. This +# mirrors sidecar/docker-compose.external-net.yml. +networks: + gateway-lan: + external: true + name: ${FIPS_GW_LAN_NET:-fips-gateway-lan} diff --git a/testing/static/docker-compose.yml b/testing/static/docker-compose.yml index d2c49fe..a682da8 100644 --- a/testing/static/docker-compose.yml +++ b/testing/static/docker-compose.yml @@ -8,6 +8,10 @@ networks: driver: bridge labels: - "com.corganlabs.fips-ci=1" + # IPv4 is dropped so docker auto-assigns it (nothing reads a LAN IPv4 — the + # whole gateway LAN path is IPv6). The fd02::/64 pin stays for the standalone / + # GitHub path; concurrent local runs replace this network with a per-run + # claimed /64 via docker-compose.gateway-external-net.yml (see run_gateway). gateway-lan: driver: bridge labels: @@ -15,7 +19,6 @@ networks: enable_ipv6: true ipam: config: - - subnet: 172.20.1.0/24 - subnet: fd02::/64 x-fips-common: &fips-common @@ -523,8 +526,7 @@ services: networks: fips-net: gateway-lan: - ipv4_address: 172.20.1.10 - ipv6_address: fd02::10 + ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::10 gw-server: <<: *fips-common @@ -561,11 +563,10 @@ services: sysctls: - net.ipv6.conf.all.disable_ipv6=0 volumes: - - ./configs/gateway-resolv.conf:/etc/resolv.conf:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/gateway/resolv.conf:/etc/resolv.conf:ro networks: gateway-lan: - ipv4_address: 172.20.1.20 - ipv6_address: fd02::20 + ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::20 restart: "no" env_file: - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env @@ -583,11 +584,10 @@ services: sysctls: - net.ipv6.conf.all.disable_ipv6=0 volumes: - - ./configs/gateway-resolv.conf:/etc/resolv.conf:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/gateway/resolv.conf:/etc/resolv.conf:ro networks: gateway-lan: - ipv4_address: 172.20.1.21 - ipv6_address: fd02::21 + ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::21 restart: "no" env_file: - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env diff --git a/testing/static/scripts/bench-multirun.sh b/testing/static/scripts/bench-multirun.sh index d9a8547..d2a138b 100755 --- a/testing/static/scripts/bench-multirun.sh +++ b/testing/static/scripts/bench-multirun.sh @@ -120,6 +120,7 @@ ping_path() { return fi echo "${rtt//\// } ${loss:-N/A}" + return 0 } # ── Path definitions ─────────────────────────────────────────────────────── diff --git a/testing/static/scripts/gateway-test.sh b/testing/static/scripts/gateway-test.sh index 16fc9c2..9cb8a60 100755 --- a/testing/static/scripts/gateway-test.sh +++ b/testing/static/scripts/gateway-test.sh @@ -26,6 +26,16 @@ SERVER2="fips-gw-server-2${FIPS_CI_NAME_SUFFIX:-}" CLIENT="fips-gw-client${FIPS_CI_NAME_SUFFIX:-}" CLIENT2="fips-gw-client-2${FIPS_CI_NAME_SUFFIX:-}" +# LAN-side IPv6 addressing. run_gateway claims a per-run /64 and exports +# FIPS_GW_LAN6_PREFIX; unset (standalone / GitHub) these render the base +# compose's fd02:: addresses, byte-identical to before. GW_DNS is the gateway's +# LAN address (nameserver + route next-hop); GW_CLIENT_LAN is gw-client's LAN +# address (inbound port-forward target). fd01::/112 (the virtual pool) is NOT +# claimed and stays literal below. +GW_LAN6_PREFIX="${FIPS_GW_LAN6_PREFIX:-fd02}" +GW_DNS="${GW_LAN6_PREFIX}::10" +GW_CLIENT_LAN="${GW_LAN6_PREFIX}::20" + # ── inject-config subcommand ───────────────────────────────────────────── inject_gateway_config() { @@ -58,21 +68,21 @@ cfg['gateway'] = { { 'listen_port': 18080, 'proto': 'tcp', - 'target': '[fd02::20]:8080', + 'target': '[${GW_CLIENT_LAN}]:8080', }, # 6B: second TCP forward — exercises multiple simultaneous TCP # rules sharing the same LAN backend on a different listen port. { 'listen_port': 18082, 'proto': 'tcp', - 'target': '[fd02::20]:8081', + 'target': '[${GW_CLIENT_LAN}]:8081', }, # 6A: UDP forward — exercises the runtime UDP DNAT path (rule # shape + conntrack handling) end-to-end. { 'listen_port': 18081, 'proto': 'udp', - 'target': '[fd02::20]:8081', + 'target': '[${GW_CLIENT_LAN}]:8081', }, ], } @@ -130,7 +140,7 @@ for i in $(seq 1 30); do # Try resolving the server's npub via the gateway DNS from the client. # Match fd01:: specifically (the pool prefix) to avoid false-positive # matches on error messages containing fd02::10. - local_result=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @fd02::10 2>/dev/null || true) + local_result=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @${GW_DNS} 2>/dev/null || true) if echo "$local_result" | grep -q "^fd01::"; then echo " Gateway DNS responding after ${i}s" DNS_READY=true @@ -146,23 +156,23 @@ fi # Phase 3: Client network setup — route virtual IP pool via gateway echo "" echo "Phase 3: Client network setup" -docker exec "$CLIENT" ip -6 route add fd01::/112 via fd02::10 2>/dev/null || true -echo " Added route fd01::/112 via fd02::10 on $CLIENT" -docker exec "$CLIENT2" ip -6 route add fd01::/112 via fd02::10 2>/dev/null || true -echo " Added route fd01::/112 via fd02::10 on $CLIENT2" +docker exec "$CLIENT" ip -6 route add fd01::/112 via ${GW_DNS} 2>/dev/null || true +echo " Added route fd01::/112 via ${GW_DNS} on $CLIENT" +docker exec "$CLIENT2" ip -6 route add fd01::/112 via ${GW_DNS} 2>/dev/null || true +echo " Added route fd01::/112 via ${GW_DNS} on $CLIENT2" # Phase 4: DNS resolution test — resolve server npub from both clients, # exercising concurrent multi-client mappings. echo "" echo "Phase 4: DNS resolution" -VIRTUAL_IP=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @fd02::10 2>/dev/null | head -1) +VIRTUAL_IP=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @${GW_DNS} 2>/dev/null | head -1) if [ -n "$VIRTUAL_IP" ] && echo "$VIRTUAL_IP" | grep -q "fd01"; then check "Resolve ${NPUB_B:0:20}...fips on $CLIENT → $VIRTUAL_IP" 0 else check "Resolve ${NPUB_B:0:20}...fips on $CLIENT (got: '$VIRTUAL_IP')" 1 fi -VIRTUAL_IP_2=$(docker exec "$CLIENT2" dig +short AAAA "${NPUB_C}.fips" @fd02::10 2>/dev/null | head -1) +VIRTUAL_IP_2=$(docker exec "$CLIENT2" dig +short AAAA "${NPUB_C}.fips" @${GW_DNS} 2>/dev/null | head -1) if [ -n "$VIRTUAL_IP_2" ] && echo "$VIRTUAL_IP_2" | grep -q "fd01"; then check "Resolve ${NPUB_C:0:20}...fips on $CLIENT2 → $VIRTUAL_IP_2" 0 else @@ -357,7 +367,7 @@ else # 8080 backend serves "inbound-forward-ok" (no -2 suffix) — distinct # from the 8081 backend so a misrouted response would be detectable. if echo "$FWD_RESPONSE" | grep -qE '^inbound-forward-ok$'; then - check "Inbound HTTP via TCP forward 18080 → [fd02::20]:8080" 0 + check "Inbound HTTP via TCP forward 18080 → [${GW_CLIENT_LAN}]:8080" 0 else check "Inbound HTTP via TCP forward 18080 (response: '${FWD_RESPONSE:0:80}')" 1 fi @@ -365,7 +375,7 @@ else FWD_RESPONSE_2=$(docker exec "$SERVER" curl -6 -s --max-time 10 \ "http://[${GW_MESH_IP}]:18082/" 2>&1) || true if echo "$FWD_RESPONSE_2" | grep -q "inbound-forward-ok-2"; then - check "Inbound HTTP via TCP forward 18082 → [fd02::20]:8081 (6B)" 0 + check "Inbound HTTP via TCP forward 18082 → [${GW_CLIENT_LAN}]:8081 (6B)" 0 else check "Inbound HTTP via TCP forward 18082 (response: '${FWD_RESPONSE_2:0:80}')" 1 fi @@ -384,7 +394,7 @@ except Exception as e: sys.stdout.write('ERR: ' + str(e)) " 2>&1) || true if echo "$UDP_RESPONSE" | grep -q "udp-forward-ok:ping-via-udp-fwd"; then - check "Inbound UDP via forward 18081 → [fd02::20]:8081 (6A)" 0 + check "Inbound UDP via forward 18081 → [${GW_CLIENT_LAN}]:8081 (6A)" 0 else check "Inbound UDP via forward 18081 (response: '${UDP_RESPONSE:0:80}')" 1 fi @@ -444,8 +454,8 @@ docker exec "$GATEWAY" pkill -f "^fips --config" 2>/dev/null || true sleep 2 # Gateway upstream timeout is 5s, so dig must wait longer than that. -SERVFAIL_RESULT=$(docker exec "$CLIENT" dig +short +tries=1 +time=8 AAAA "test-servfail.fips" @fd02::10 2>&1 || true) -SERVFAIL_STATUS=$(docker exec "$CLIENT" dig +tries=1 +time=8 AAAA "test-servfail.fips" @fd02::10 2>&1 | grep -c "SERVFAIL" || true) +SERVFAIL_RESULT=$(docker exec "$CLIENT" dig +short +tries=1 +time=8 AAAA "test-servfail.fips" @${GW_DNS} 2>&1 || true) +SERVFAIL_STATUS=$(docker exec "$CLIENT" dig +tries=1 +time=8 AAAA "test-servfail.fips" @${GW_DNS} 2>&1 | grep -c "SERVFAIL" || true) if [ "$SERVFAIL_STATUS" -ge 1 ]; then check "SERVFAIL when daemon DNS is down" 0 else diff --git a/testing/static/scripts/generate-configs.sh b/testing/static/scripts/generate-configs.sh index 9be398b..529ec12 100755 --- a/testing/static/scripts/generate-configs.sh +++ b/testing/static/scripts/generate-configs.sh @@ -74,6 +74,7 @@ docker_host_name() { local host host=$(get_node_attr "$topology_file" "$node_id" "docker_host") echo "${host:-node-$node_id}" + return 0 } # Get peers list from topology @@ -118,6 +119,7 @@ get_default_transport() { local topology_file="$1" local transport=$(grep "^default_transport:" "$topology_file" | head -1 | sed 's/.*: *\([a-z]*\).*/\1/') echo "${transport:-udp}" + return 0 } # Get the port for a given transport type @@ -272,6 +274,16 @@ generate_topology() { echo "${var_name}=$(get_key RESOLVED_NPUB "$node_id")" >> "$env_file" done echo " ✓ Generated $env_file" + + # Phase 4 (gateway only): write the LAN-client resolv.conf. Its nameserver + # is the gateway's LAN address, which must be a literal known before the + # client starts. run_gateway claims a per-run /64 and exports + # FIPS_GW_LAN6_PREFIX before calling this; unset (standalone / GitHub) it + # renders the base compose's fd02::10. + if [ "$topology_name" = "gateway" ]; then + echo "nameserver ${FIPS_GW_LAN6_PREFIX:-fd02}::10" > "$output_dir/resolv.conf" + echo " ✓ Generated $output_dir/resolv.conf" + fi } main() { diff --git a/testing/static/scripts/netem.sh b/testing/static/scripts/netem.sh index 866a20f..30d1787 100755 --- a/testing/static/scripts/netem.sh +++ b/testing/static/scripts/netem.sh @@ -159,6 +159,7 @@ build_netem_params() { fi echo "$params" + return 0 } # Check if a container is running diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index eab2b2a..bc78113 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -285,15 +285,31 @@ phase_result() { fi } -# Count occurrences of a pattern across all node logs +# Count occurrences of a pattern across all node logs. +# +# A node whose logs cannot be read makes the whole count unusable rather than +# contributing 0. The previous form ended each read `| grep -c "$pat" || true`, +# so a failed `docker logs` yielded 0 for that node and the six assert_zero_count +# callers below read a clean result from a node that was never consulted — one +# unreadable node silently weakened the assertion instead of voiding it. +# +# Returns non-zero and prints a sentinel naming the container. Every caller must +# split the declaration from the assignment (`local c` then `c=$(...)`), because +# `local c=$(...)` takes `local`'s exit status and discards this one. count_log_pattern() { local pattern="$1" local total=0 + local node logs count for node in $NODES; do - local count=$(docker logs "fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" 2>&1 | grep -c "$pattern" || true) + if ! logs=$(docker logs "fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" 2>&1); then + echo "unreadable:fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" + return 1 + fi + count=$(grep -c "$pattern" <<<"$logs" || true) total=$((total + count)) done echo "$total" + return 0 } wait_for_log_pattern_count() { @@ -325,7 +341,12 @@ assert_min_count() { local pattern="$1" local min_count="$2" local description="$3" - local count=$(count_log_pattern "$pattern") + local count + count=$(count_log_pattern "$pattern") || { + echo " ✗ $description: node logs unreadable ($count), count not established" + FAILED=$((FAILED + 1)) + return + } if [ "$count" -ge "$min_count" ]; then echo " ✓ $description: $count (>= $min_count)" PASSED=$((PASSED + 1)) @@ -339,7 +360,12 @@ assert_min_count() { assert_zero_count() { local pattern="$1" local description="$2" - local count=$(count_log_pattern "$pattern") + local count + count=$(count_log_pattern "$pattern") || { + echo " ✗ $description: node logs unreadable ($count), zero not established" + FAILED=$((FAILED + 1)) + return + } if [ "$count" -eq 0 ]; then echo " ✓ $description: 0" PASSED=$((PASSED + 1))