Merge branch 'maint'

.github/workflows/ci.yml

@@ -38,12 +38,12 @@ env:
 #                    unreliable on GitHub-hosted runners.
 #   tor-directory  — same; live Tor dependency.
 #
-# Granularity-only differences (same coverage, different matrix shape —
+# The two runners express the same work in different matrix shapes, and the
-# NOT a divergence):
+# parity guard compares through that shape rather than around it: chaos legs
-#   deb-install    — split here into per-distro legs (debian12/debian13/
+# are compared per scenario (and per flag) via their `scenario:` field,
-#                    ubuntu22/ubuntu24/ubuntu26) for parallelism; local runs the
+# deb-install legs per distro. The one leg still compared at leg granularity
-#                    same distro set in one suite.
+# is dns-resolver — a single leg here, running all of its scenarios
-#   dns-resolver   — single leg here; runs all scenarios (same as local).
+# internally, exactly as the local suite does.
 # ─────────────────────────────────────────────────────────────────────────────
 
 # ─────────────────────────────────────────────────────────────────────────────
@@ -52,6 +52,16 @@ env:
 # Builds on Linux x86_64, Linux aarch64, and macOS.
 # ─────────────────────────────────────────────────────────────────────────────
 jobs:
+  ci-parity:
+    name: CI parity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install Python deps
+        run: pip3 install --quiet pyyaml
+      - name: Check local and GitHub runners cover the same work
+        run: bash testing/check-ci-parity.sh
+
   fmt:
     name: Format check
     runs-on: ubuntu-latest

testing/README.md

@@ -79,8 +79,11 @@ a per-commit gate; not part of `ci-local.sh`.
 clippy, unit tests, and the integration suites (including the chaos
 scenarios) — mirroring the GitHub `ci.yml` integration matrix. Run
 `./ci-local.sh --help` for the full option list and `--list` for the
-available suites. `--check-parity` verifies the local suite set matches
+available suites. Every run starts with a parity check that verifies the
-the GitHub matrix (see [check-ci-parity.sh](check-ci-parity.sh)).
+local suite set covers the same work as the GitHub matrix, per scenario for
+chaos and per distro for deb-install; a divergence fails the run. GitHub
+runs the same check as its own `ci-parity` job. `--check-parity` runs it
+alone (see [check-ci-parity.sh](check-ci-parity.sh)).
 
 ### Per-run isolation and the `FIPS_CI_RUN_ID` override
 

testing/chaos/sim/scenario.py

@@ -239,15 +239,91 @@ class Scenario:
     fips_overrides: dict = field(default_factory=dict)
 
 
+# Keys each fixed-schema section understands. A key absent from these sets is a
+# typo, and silently ignoring it leaves the block default-constructed while the
+# YAML still looks correct — a mistyped "assertion:" disarms a scenario's only
+# assertions, a mistyped "link_flap:" turns off the chaos it was meant to inject.
+#
+# Three mappings are deliberately NOT listed, because their keys are names the
+# scenario author chooses rather than a schema: netem.mutation.policies,
+# link_swap.policies and topology.transport_mix. Two sub-trees are passed through
+# whole and are likewise not checked: fips_overrides and topology.params.
+#
+# NOTE: adding a new assertion type means registering it in TWO places below —
+# _SECTION_KEYS["assertions"] (so the block accepts its name) and _ASSERTION_KEYS
+# (so its own members are checked) — or scenarios using it are rejected at load.
+_TOP_KEYS = {
+    "scenario", "topology", "netem", "link_flaps", "traffic", "node_churn",
+    "peer_churn", "bandwidth", "ingress", "link_swap", "assertions", "logging",
+    "fips_overrides",
+}
+_SECTION_KEYS = {
+    "scenario": {"name", "seed", "duration_secs"},
+    "topology": {
+        "num_nodes", "algorithm", "params", "ensure_connected", "subnet",
+        "ip_start", "default_transport", "transport_mix",
+    },
+    "netem": {"enabled", "default_policy", "link_policies", "mutation"},
+    "netem.link_policies[]": {"edges", "policy", "policy_name"},
+    "netem.mutation": {"interval_secs", "fraction", "policies"},
+    "link_flaps": {
+        "enabled", "interval_secs", "max_down_links", "down_duration_secs",
+        "protect_connectivity",
+    },
+    "traffic": {
+        "enabled", "max_concurrent", "interval_secs", "duration_secs",
+        "parallel_streams",
+    },
+    "node_churn": {
+        "enabled", "interval_secs", "max_down_nodes", "down_duration_secs",
+        "protect_connectivity",
+    },
+    "peer_churn": {"enabled", "interval_secs", "ephemeral_fraction"},
+    "bandwidth": {"enabled", "tiers_mbps"},
+    "ingress": {"enabled", "tiers_kbps", "burst_bytes"},
+    "link_swap": {"enabled", "interval_secs", "policies", "edges"},
+    "link_swap.edges[]": {"edge", "policy"},
+    "assertions": {"bloom_send_rate", "min_parent_switches"},
+    "logging": {"rust_log", "output_dir"},
+}
+_ASSERTION_KEYS = {
+    "bloom_send_rate": {"window_secs", "max_per_node"},
+    "min_parent_switches": {"min_total"},
+}
+_NETEM_POLICY_KEYS = {
+    "delay_ms", "jitter_ms", "loss_pct", "duplicate_pct", "reorder_pct",
+    "corrupt_pct",
+}
+_RANGE_KEYS = {"min", "max"}
+
+
+def _reject_unknown(data, known, context: str):
+    """Raise unless every key in `data` is one the loader understands."""
+    if data is None:
+        raise ValueError(
+            f"{context}: section is present but empty; give it a body or remove it"
+        )
+    if not isinstance(data, dict):
+        raise ValueError(f"{context}: expected a mapping, got {type(data).__name__}")
+    unknown = sorted(str(k) for k in set(data) - set(known))
+    if unknown:
+        raise ValueError(
+            f"{context}: unknown key(s): {', '.join(unknown)} "
+            f"(known: {', '.join(sorted(known))})"
+        )
+
+
 def _parse_range(data, name: str) -> Range:
     """Parse a {min, max} dict into a Range."""
     if isinstance(data, dict):
+        _reject_unknown(data, _RANGE_KEYS, name)
         return Range(min=float(data["min"]), max=float(data["max"]))
     raise ValueError(f"{name}: expected {{min, max}} dict, got {type(data).__name__}")
 
 
-def _parse_netem_policy(data: dict) -> NetemPolicy:
+def _parse_netem_policy(data: dict, name: str = "netem policy") -> NetemPolicy:
     """Parse a netem policy from a dict with [min, max] lists or {min, max} dicts."""
+    _reject_unknown(data, _NETEM_POLICY_KEYS, name)
     policy = NetemPolicy()
     for attr in (
         "delay_ms",
@@ -273,16 +349,22 @@ def load_scenario(path: str) -> Scenario:
     with open(path) as f:
         raw = yaml.safe_load(f)
 
+    if raw is None:
+        raise ValueError(f"{path}: file is empty")
+    _reject_unknown(raw, _TOP_KEYS, "(top level)")
+
     s = Scenario()
 
     # Scenario section
     sc = raw.get("scenario", {})
+    _reject_unknown(sc, _SECTION_KEYS["scenario"], "scenario")
     s.name = sc.get("name", os.path.splitext(os.path.basename(path))[0])
     s.seed = int(sc.get("seed", 42))
     s.duration_secs = int(sc.get("duration_secs", 120))
 
     # Topology section
     tc = raw.get("topology", {})
+    _reject_unknown(tc, _SECTION_KEYS["topology"], "topology")
     s.topology.num_nodes = int(tc.get("num_nodes", 10))
     s.topology.algorithm = tc.get("algorithm", "random_geometric")
     s.topology.params = tc.get("params", {})
@@ -298,33 +380,43 @@ def load_scenario(path: str) -> Scenario:
 
     # Netem section
     nc = raw.get("netem", {})
+    _reject_unknown(nc, _SECTION_KEYS["netem"], "netem")
     s.netem.enabled = nc.get("enabled", False)
     if "default_policy" in nc:
-        s.netem.default_policy = _parse_netem_policy(nc["default_policy"])
+        s.netem.default_policy = _parse_netem_policy(
+            nc["default_policy"], "netem.default_policy"
+        )
     if "link_policies" in nc:
         for lp_data in nc["link_policies"]:
+            _reject_unknown(
+                lp_data, _SECTION_KEYS["netem.link_policies[]"], "netem.link_policies[]"
+            )
             override = LinkPolicyOverride(
                 edges=lp_data.get("edges", []),
             )
             if "policy" in lp_data:
-                override.policy = _parse_netem_policy(lp_data["policy"])
+                override.policy = _parse_netem_policy(
+                    lp_data["policy"], "netem.link_policies[].policy"
+                )
             if "policy_name" in lp_data:
                 override.policy_name = lp_data["policy_name"]
             s.netem.link_policies.append(override)
     if "mutation" in nc:
         mc = nc["mutation"]
+        _reject_unknown(mc, _SECTION_KEYS["netem.mutation"], "netem.mutation")
         s.netem.mutation.interval_secs = _parse_range(
             mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs"
         )
         s.netem.mutation.fraction = float(mc.get("fraction", 0.3))
         if "policies" in mc:
             s.netem.mutation.policies = {
-                name: _parse_netem_policy(pdata)
+                name: _parse_netem_policy(pdata, f"netem.mutation.policies.{name}")
                 for name, pdata in mc["policies"].items()
             }
 
     # Link flaps section
     lf = raw.get("link_flaps", {})
+    _reject_unknown(lf, _SECTION_KEYS["link_flaps"], "link_flaps")
     s.link_flaps.enabled = lf.get("enabled", False)
     if "interval_secs" in lf:
         s.link_flaps.interval_secs = _parse_range(lf["interval_secs"], "link_flaps.interval_secs")
@@ -337,6 +429,7 @@ def load_scenario(path: str) -> Scenario:
 
     # Traffic section
     tf = raw.get("traffic", {})
+    _reject_unknown(tf, _SECTION_KEYS["traffic"], "traffic")
     s.traffic.enabled = tf.get("enabled", False)
     s.traffic.max_concurrent = int(tf.get("max_concurrent", 3))
     if "interval_secs" in tf:
@@ -347,6 +440,7 @@ def load_scenario(path: str) -> Scenario:
 
     # Node churn section
     nc2 = raw.get("node_churn", {})
+    _reject_unknown(nc2, _SECTION_KEYS["node_churn"], "node_churn")
     s.node_churn.enabled = nc2.get("enabled", False)
     if "interval_secs" in nc2:
         s.node_churn.interval_secs = _parse_range(nc2["interval_secs"], "node_churn.interval_secs")
@@ -359,6 +453,7 @@ def load_scenario(path: str) -> Scenario:
 
     # Peer churn section
     pc = raw.get("peer_churn", {})
+    _reject_unknown(pc, _SECTION_KEYS["peer_churn"], "peer_churn")
     s.peer_churn.enabled = pc.get("enabled", False)
     if "interval_secs" in pc:
         s.peer_churn.interval_secs = _parse_range(pc["interval_secs"], "peer_churn.interval_secs")
@@ -366,6 +461,7 @@ def load_scenario(path: str) -> Scenario:
 
     # Bandwidth section
     bw = raw.get("bandwidth", {})
+    _reject_unknown(bw, _SECTION_KEYS["bandwidth"], "bandwidth")
     s.bandwidth.enabled = bw.get("enabled", False)
     if "tiers_mbps" in bw:
         tiers = bw["tiers_mbps"]
@@ -375,6 +471,7 @@ def load_scenario(path: str) -> Scenario:
 
     # Ingress section
     ig = raw.get("ingress", {})
+    _reject_unknown(ig, _SECTION_KEYS["ingress"], "ingress")
     s.ingress.enabled = ig.get("enabled", False)
     if "tiers_kbps" in ig:
         tiers = ig["tiers_kbps"]
@@ -385,18 +482,22 @@ def load_scenario(path: str) -> Scenario:
 
     # Link swap section (deterministic asymmetric link-cost flapping).
     ls = raw.get("link_swap", {})
+    _reject_unknown(ls, _SECTION_KEYS["link_swap"], "link_swap")
     s.link_swap.enabled = ls.get("enabled", False)
     if "interval_secs" in ls:
         s.link_swap.interval_secs = float(ls["interval_secs"])
     if "policies" in ls:
         s.link_swap.policies = {
-            name: _parse_netem_policy(pdata)
+            name: _parse_netem_policy(pdata, f"link_swap.policies.{name}")
             for name, pdata in ls["policies"].items()
         }
     if "edges" in ls:
         for edata in ls["edges"]:
             if not isinstance(edata, dict):
                 raise ValueError("link_swap.edges entries must be dicts")
+            _reject_unknown(
+                edata, _SECTION_KEYS["link_swap.edges[]"], "link_swap.edges[]"
+            )
             edge = str(edata.get("edge", ""))
             policy = str(edata.get("policy", ""))
             if not edge or not policy:
@@ -405,20 +506,29 @@ def load_scenario(path: str) -> Scenario:
 
     # Assertions section (post-run control-socket-based checks).
     asrt = raw.get("assertions", {})
+    _reject_unknown(asrt, _SECTION_KEYS["assertions"], "assertions")
     if "bloom_send_rate" in asrt:
         bsr = asrt["bloom_send_rate"]
+        _reject_unknown(
+            bsr, _ASSERTION_KEYS["bloom_send_rate"], "assertions.bloom_send_rate"
+        )
         s.assertions.bloom_send_rate = BloomSendRateAssertion(
             window_secs=int(bsr.get("window_secs", 30)),
             max_per_node=int(bsr.get("max_per_node", 30)),
         )
     if "min_parent_switches" in asrt:
         mps = asrt["min_parent_switches"]
+        _reject_unknown(
+            mps, _ASSERTION_KEYS["min_parent_switches"],
+            "assertions.min_parent_switches",
+        )
         s.assertions.min_parent_switches = MinParentSwitchesAssertion(
             min_total=int(mps.get("min_total", 1)),
         )
 
     # Logging section
     lg = raw.get("logging", {})
+    _reject_unknown(lg, _SECTION_KEYS["logging"], "logging")
     s.logging.rust_log = lg.get("rust_log", "info")
     s.logging.output_dir = lg.get("output_dir", "./sim-results")
 

testing/check-ci-parity.sh

@@ -11,19 +11,30 @@
 #                    unreliable on GitHub-hosted runners.
 #   tor-directory  — same; live Tor dependency.
 #
-# Granularity-only differences folded before comparison (same coverage,
+# What is compared, and at what granularity:
-# different matrix shape — NOT a divergence):
+#   chaos          — per scenario, plus its flags. GitHub fans each scenario
-#   deb-install    — GitHub splits into per-distro legs (deb-install-debian12/
+#                    into its own matrix leg carrying `scenario:` (and
-#                    debian13/ubuntu22/ubuntu24/ubuntu26); local runs the same
+#                    optionally `chaos_flags:`); local lists the same scenarios
-#                    distro set in one suite. Folded to "deb-install".
+#                    in CHAOS_SUITES as "display scenario flags". The `suite:`
-#   chaos-*        — GitHub fans each chaos scenario into its own matrix leg
+#                    names differ cosmetically between runners and are ignored
-#                    (type: chaos); local runs them all via the one CHAOS_SUITES
+#                    — `scenario:` is the identity.
-#                    path. The individual scenario names also differ cosmetically
+#   deb-install    — per distro. GitHub splits into per-distro legs carrying
-#                    between runners (e.g. chaos-smoke-10 vs churn-mixed-10).
+#                    `scenario:`; local runs the same distro set in one suite,
-#                    Folded to a single "chaos" token on both sides.
+#                    enumerated by ALL_SCENARIOS in deb-install/test.sh.
-#   dns-resolver   — single leg / single suite both sides; runs all scenarios.
+#   everything else — per suite name.
 #
-# Exit 0 = parity clean. Exit 1 = unexpected divergence (suite names printed).
+#   dns-resolver is the one leg still compared at leg granularity rather than
+#   per scenario: it is a single leg and a single suite on both sides, and it
+#   runs all of its scenarios internally. Its scenario list is NOT cross-checked.
+#
+# The local suite set is discovered by sweeping ci-local.sh for *_SUITES arrays
+# rather than from a hardcoded list of variable names, and every run_suite
+# dispatch arm is then checked to have a backing array — a suite dispatched
+# without one is invisible to a name-list sweep, which is how a real divergence
+# went unnoticed.
+#
+# Exit 0 = parity clean. Exit 1 = unexpected divergence. Exit 2 = the guard
+# could not run (missing file or missing dependency); never treated as a pass.
 # ─────────────────────────────────────────────────────────────────────────────
 set -uo pipefail
 
@@ -32,113 +43,235 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 
 CI_LOCAL="$SCRIPT_DIR/ci-local.sh"
 CI_YML="$PROJECT_ROOT/.github/workflows/ci.yml"
+DEB_TEST="$SCRIPT_DIR/deb-install/test.sh"
 
 # Deliberate local-only allowlist (suites intentionally absent from GitHub).
 ALLOWLIST="tor-socks5 tor-directory"
 
-for f in "$CI_LOCAL" "$CI_YML"; do
+for f in "$CI_LOCAL" "$CI_YML" "$DEB_TEST"; do
     if [[ ! -f "$f" ]]; then
         echo "check-ci-parity: missing file: $f" >&2
         exit 2
     fi
 done
 
-# Extract and normalize both suite sets in Python (robust YAML parse of the
+if ! command -v python3 >/dev/null 2>&1; then
-# matrix; regex extraction of the bash suite arrays). Folding rules above are
+    echo "check-ci-parity: python3 not found; cannot verify CI parity" >&2
-# applied identically to both sides so only genuine divergence surfaces.
+    exit 2
-python3 - "$CI_LOCAL" "$CI_YML" "$ALLOWLIST" <<'PY'
+fi
+if ! python3 -c "import yaml" >/dev/null 2>&1; then
+    echo "check-ci-parity: python3 module 'yaml' not found; cannot verify CI parity" >&2
+    echo "check-ci-parity: install it with 'pip3 install pyyaml'" >&2
+    exit 2
+fi
+
+python3 - "$CI_LOCAL" "$CI_YML" "$DEB_TEST" "$ALLOWLIST" <<'PY'
 import re
 import sys
 
-ci_local_path, ci_yml_path, allowlist_raw = sys.argv[1], sys.argv[2], sys.argv[3]
+import yaml
+
+ci_local_path, ci_yml_path, deb_test_path, allowlist_raw = sys.argv[1:5]
 allowlist = set(allowlist_raw.split())
 
-
-def fold(name):
-    """Collapse granularity-only matrix shape into canonical suite identity."""
-    if name.startswith("chaos-") or name == "chaos":
-        return "chaos"
-    if name.startswith("deb-install"):
-        return "deb-install"
-    return name
-
-
-# ── Local: parse the suite arrays from ci-local.sh ───────────────────────────
 with open(ci_local_path, encoding="utf-8") as fh:
     local_src = fh.read()
+with open(deb_test_path, encoding="utf-8") as fh:
+    deb_src = fh.read()
 
 
-def bash_array(var):
+def bash_array_entries(var):
+    """Full entries of a bash array, in order, quotes stripped."""
     m = re.search(rf"^{var}=\((.*?)\)", local_src, re.MULTILINE | re.DOTALL)
     if not m:
         return []
     body = m.group(1)
-    # Quoted entries (chaos uses "display scenario flags"): first token is name.
     quoted = re.findall(r'"([^"]*)"', body)
     if quoted:
-        return [entry.split()[0] for entry in quoted if entry.strip()]
+        return [e.strip() for e in quoted if e.strip()]
     return [tok for tok in body.split() if tok.strip()]
 
 
+def discovered_arrays():
+    """Every *_SUITES array in ci-local.sh, by name.
+
+    Swept rather than hardcoded: a suite whose array is not on a fixed list
+    would otherwise be invisible to this guard.
+    """
+    return {
+        name + "_SUITES": bash_array_entries(name + "_SUITES")
+        for name in re.findall(r"^([A-Z_]+)_SUITES=\(", local_src, re.MULTILINE)
+    }
+
+
+arrays = discovered_arrays()
+
+# ── Local side ───────────────────────────────────────────────────────────────
+# Chaos: "display scenario flags" — compare the scenario and its flags.
+local_chaos = {}
+for entry in arrays.get("CHAOS_SUITES", []):
+    parts = entry.split()
+    if len(parts) < 2:
+        continue
+    local_chaos[parts[1]] = " ".join(parts[2:])
+
+# deb-install: one local suite that runs the distro set enumerated in its script.
+m = re.search(r'^ALL_SCENARIOS="([^"]*)"', deb_src, re.MULTILINE)
+local_deb = set(m.group(1).split()) if m else set()
+
+# Everything else: suite names, with NAT stored bare and prefixed at use.
 local = set()
-# Static, rekey, gateway, sidecar, acl, firewall, nostr, stun, dns, deb.
+for name, entries in arrays.items():
-for var in ("STATIC_SUITES", "REKEY_SUITES", "ADMISSION_SUITES",
+    if name in ("CHAOS_SUITES", "DEB_INSTALL_SUITES"):
-            "GATEWAY_SUITES", "SIDECAR_SUITES", "ACL_SUITES",
+        continue
-            "FIREWALL_SUITES", "NOSTR_RELAY_SUITES", "STUN_FAULTS_SUITES",
+    names = [e.split()[0] for e in entries]
-            "DNS_RESOLVER_SUITES", "DEB_INSTALL_SUITES"):
+    if name == "NAT_SUITES":
-    local.update(bash_array(var))
+        local |= {f"nat-{n}" for n in names}
-# Chaos display names → fold to "chaos".
+    else:
-for _ in bash_array("CHAOS_SUITES"):
+        local |= set(names)
-    local.add("chaos")
-# NAT scenarios are stored bare (cone/symmetric/lan) and prefixed nat- at use.
-for scen in bash_array("NAT_SUITES"):
-    local.add(f"nat-{scen}")
-# TOR_SUITES is the deliberate local-only set — excluded from the default path.
-
-local = {fold(n) for n in local}
-
-# ── GitHub: parse the integration matrix suite: values from ci.yml ───────────
-import yaml  # noqa: E402
 
+# ── GitHub side ──────────────────────────────────────────────────────────────
 with open(ci_yml_path, encoding="utf-8") as fh:
     doc = yaml.safe_load(fh)
 
 include = doc["jobs"]["integration"]["strategy"]["matrix"]["include"]
-github = set()
+github_chaos, github_deb, github = {}, set(), set()
+malformed = []
 for leg in include:
-    if "suite" not in leg:
+    if "suite" not in leg and "scenario" not in leg:
         continue
-    # Chaos legs carry inconsistent suite: names (chaos-smoke-10 vs
+    kind = str(leg.get("type", ""))
-    # churn-mixed-10) but a uniform type: chaos — fold via type, not name.
+    if kind in ("chaos", "deb-install"):
-    if str(leg.get("type", "")) == "chaos":
+        # scenario: is the identity for these; suite: is cosmetic.
-        github.add("chaos")
+        if "scenario" not in leg:
+            malformed.append(f"{leg.get('suite', '(unnamed leg)')} has type "
+                             f"{kind} but no scenario:")
+            continue
+        if kind == "chaos":
+            github_chaos[str(leg["scenario"])] = str(leg.get("chaos_flags", ""))
+        else:
+            github_deb.add(str(leg["scenario"]))
+    elif "suite" in leg:
+        github.add(str(leg["suite"]))
     else:
-        github.add(fold(str(leg["suite"])))
+        malformed.append(f"leg with scenario {leg['scenario']} has no suite: "
+                         f"and no chaos/deb-install type")
 
-# ── Diff (subtract allowlist from local before comparison) ───────────────────
+# ── Dispatch cross-check: every run_suite arm needs a backing array ──────────
+# A suite dispatched without an array is invisible to the sweep above, so the
+# guard would report it as GitHub-only forever without ever naming the cause.
+body = re.search(r"^run_suite\(\).*?^\}", local_src, re.MULTILINE | re.DOTALL)
+dispatch_uncovered = []
+if body is None:
+    print("check-ci-parity: could not locate run_suite() in ci-local.sh", file=sys.stderr)
+    sys.exit(2)
+
+# Arms sit at one fixed indentation inside the case block. Pin to it, taken from
+# the first arm rather than assumed, so a body line that happens to end in ')'
+# cannot be read as an arm.
+arm_re = re.compile(r"^([ \t]+)['\"]?([a-z0-9|*_.-]+)['\"]?\)", re.MULTILINE)
+first = arm_re.search(body.group(0))
+if first is None:
+    print("check-ci-parity: no dispatch arms found in run_suite()", file=sys.stderr)
+    sys.exit(2)
+
+indent = first.group(1)
+# An arm-shaped line at a different indent is not skipped silently: it would
+# make this check quietly stop covering a suite, which is the failure mode the
+# check exists to prevent.
+odd_arms = [
+    m.group(2) for m in arm_re.finditer(body.group(0)) if m.group(1) != indent
+]
+if odd_arms:
+    print("check-ci-parity: run_suite has arm-shaped lines at an unexpected "
+          f"indent, so the dispatch check cannot be trusted: {', '.join(odd_arms)}",
+          file=sys.stderr)
+    sys.exit(2)
+known = (set(local) | set(local_chaos) | local_deb
+         | {e.split()[0] for e in arrays.get("DEB_INSTALL_SUITES", [])})
+for m in arm_re.finditer(body.group(0)):
+    if m.group(1) != indent:
+        continue
+    for arm in m.group(2).split("|"):
+        if arm == "*":
+            continue              # the unknown-suite error arm, not a suite
+        if arm == "chaos-*":
+            # Dispatches any chaos-<name> through a fallback, so its vocabulary
+            # is unbounded; the chaos scenario comparison covers it instead.
+            continue
+        if arm not in known:
+            dispatch_uncovered.append(arm)
+
+# ── Diff ─────────────────────────────────────────────────────────────────────
 local_cmp = {n for n in local if n not in allowlist}
 
 local_only = sorted(local_cmp - github)
 github_only = sorted(github - local_cmp)
+chaos_local_only = sorted(set(local_chaos) - set(github_chaos))
+chaos_github_only = sorted(set(github_chaos) - set(local_chaos))
+chaos_flag_drift = sorted(
+    (s, local_chaos[s], github_chaos[s])
+    for s in set(local_chaos) & set(github_chaos)
+    if local_chaos[s] != github_chaos[s]
+)
+deb_local_only = sorted(local_deb - github_deb)
+deb_github_only = sorted(github_deb - local_deb)
 
-if local_only or github_only:
+problems = (local_only or github_only or chaos_local_only or chaos_github_only
-    print("CI parity FAILED: integration suite sets diverge.\n")
+            or chaos_flag_drift or deb_local_only or deb_github_only
+            or dispatch_uncovered or malformed)
+
+if problems:
+    print("CI parity FAILED: the two runners do not cover the same work.\n")
     if local_only:
-        print("  Local-only (in ci-local.sh, missing from ci.yml, "
+        print("  Suites local-only (in ci-local.sh, missing from ci.yml, "
-              "not in deliberate allowlist):")
+              "not in the deliberate allowlist):")
         for n in local_only:
             print(f"    - {n}")
     if github_only:
-        print("  GitHub-only (in ci.yml, missing from local default path):")
+        print("  Suites GitHub-only (in ci.yml, missing from the local default path):")
         for n in github_only:
             print(f"    - {n}")
-    print("\n  Resolve by adding the suite to the other runner, or by adding "
+    if chaos_local_only:
-          "it\n  to the deliberate local-only allowlist in "
+        print("  Chaos scenarios local-only:")
-          "check-ci-parity.sh with a\n  stated reason.")
+        for n in chaos_local_only:
+            print(f"    - {n}")
+    if chaos_github_only:
+        print("  Chaos scenarios GitHub-only:")
+        for n in chaos_github_only:
+            print(f"    - {n}")
+    if chaos_flag_drift:
+        print("  Chaos scenarios whose flags differ between runners:")
+        for name, lflags, gflags in chaos_flag_drift:
+            print(f"    - {name}: local '{lflags}' vs GitHub '{gflags}'")
+    if deb_local_only:
+        print("  deb-install distros local-only:")
+        for n in deb_local_only:
+            print(f"    - {n}")
+    if deb_github_only:
+        print("  deb-install distros GitHub-only:")
+        for n in deb_github_only:
+            print(f"    - {n}")
+    if malformed:
+        print("  Matrix legs this guard cannot identify:")
+        for n in malformed:
+            print(f"    - {n}")
+    if dispatch_uncovered:
+        print("  run_suite dispatches these with no backing *_SUITES array, so "
+              "this guard\n  cannot see them in the local set:")
+        for n in dispatch_uncovered:
+            print(f"    - {n}")
+    print("\n  Resolve by adding the suite to the other runner, by giving a "
+          "dispatchable\n  suite a *_SUITES array, or by adding it to the "
+          "deliberate local-only\n  allowlist in check-ci-parity.sh with a "
+          "stated reason.")
     sys.exit(1)
 
-print("CI parity OK: integration suite sets match "
+total = len(github) + len(github_chaos) + len(github_deb)
+print("CI parity OK: both runners cover the same work "
       "(allowlist: " + ", ".join(sorted(allowlist)) + ").")
-print(f"  {len(github)} canonical suites compared on each side.")
+print(f"  {len(github)} suites, {len(github_chaos)} chaos scenarios "
+      f"(flags compared), {len(github_deb)} deb-install distros "
+      f"— {total} legs on each side.")
 sys.exit(0)
 PY

testing/ci-local.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
-# Run the CI pipeline locally: build, unit tests, integration tests.
+# Run the CI pipeline locally: CI parity check, build, unit tests,
+# integration tests.
 #
 # Usage: ./ci-local.sh [options]
 #
@@ -53,19 +54,19 @@
 # (.github/workflows/ci.yml) MUST run the same integration suites, EXCEPT for
 # the deliberate local-only entries below. Adding a suite to one runner
 # without the other means "local green" and "GitHub green" stop being
-# equivalent. testing/check-ci-parity.sh enforces this and fails on drift.
+# equivalent. testing/check-ci-parity.sh enforces this and fails on drift; it
+# runs as the first stage of every local run, before the build.
 #
 # Deliberate local-only (NOT on the GitHub gate), with reason:
 #   tor-socks5     — requires live Tor network; opt-in via --with-tor,
 #                    unreliable on GitHub-hosted runners.
 #   tor-directory  — same; live Tor dependency.
 #
-# Granularity-only differences (same coverage, different matrix shape —
+# The two runners express the same work in different matrix shapes, and the
-# NOT a divergence):
+# guard compares through that shape rather than around it: chaos legs are
-#   deb-install    — local runs all distros sequentially in one suite; GitHub
+# compared per scenario (and per flag), deb-install legs per distro. The one
-#                    splits into per-distro legs (debian12/debian13/ubuntu22/
+# leg still compared at leg granularity is dns-resolver — a single suite on
-#                    ubuntu24/ubuntu26 — the same distro set).
+# both sides that runs all of its scenarios internally.
-#   dns-resolver   — single suite both sides; runs all scenarios.
 # ─────────────────────────────────────────────────────────────────────────────
 set -uo pipefail
 
@@ -1034,6 +1035,16 @@ print_summary() {
     echo ""
 }
 
+# Verify the local default suite set and the GitHub matrix still cover the
+# same work. Runs first: it takes about a second, and a divergence should be
+# reported before a half-hour suite rather than after it.
+run_ci_parity() {
+    local rc=0
+    info "[ci-parity] Comparing the local suite set against the GitHub matrix"
+    "$SCRIPT_DIR/check-ci-parity.sh" || rc=$?
+    record "ci-parity" $rc
+}
+
 # ── Main ───────────────────────────────────────────────────────────────────
 
 main() {
@@ -1042,6 +1053,11 @@ main() {
     stage "FIPS Local CI"
     info "Project root: $PROJECT_ROOT"
 
+    # Above the mode branches deliberately, so --only, --test-only and
+    # --build-only are gated too: a divergence invalidates any claim that a
+    # local run means what a GitHub run means, whichever subset was asked for.
+    run_ci_parity
+
     if [[ "$TEST_ONLY" == true ]]; then
         run_tests
     elif [[ "$BUILD_ONLY" == true ]]; then

testing/dns-resolver/test.sh

@@ -270,13 +270,49 @@ DOCKERFILE
     fi
 
     log "Extracting fips + fips-gateway binaries from builder image"
-    local cid
+    # Drop the previous run's binaries before extracting. Without this, a failed
-    cid=$(docker create "$builder_tag")
+    # extraction below leaves them in place, they satisfy the caller's -x check,
-    docker cp "$cid:/src/target/release/fips" "$FIPS_BIN_CACHE" >/dev/null 2>&1
+    # and the e2e scenarios silently exercise the previous commit's code.
-    docker cp "$cid:/src/target/release/fips-gateway" "$FIPS_GATEWAY_BIN_CACHE" >/dev/null 2>&1
+    rm -f "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"
+
+    # stderr goes to its own file rather than into $cid: docker prints
+    # warnings (a platform mismatch, say) on success too, and folding them
+    # into the id would leave every later reference pointing at nothing.
+    local cid err errfile
+    errfile=$(mktemp)
+    if ! cid=$(docker create "$builder_tag" 2>"$errfile"); then
+        echo "  ERROR: docker create failed: $(cat "$errfile")"
+        rm -f "$errfile"
+        return 1
+    fi
+    rm -f "$errfile"
+
+    local rc=0 spec bin dest
+    for spec in "fips:$FIPS_BIN_CACHE" "fips-gateway:$FIPS_GATEWAY_BIN_CACHE"; do
+        bin="${spec%%:*}"
+        dest="${spec#*:}"
+        if ! err=$(docker cp "$cid:/src/target/release/$bin" "$dest" 2>&1); then
+            echo "  ERROR: extracting $bin from the builder image failed: $err"
+            rc=1
+        fi
+    done
     docker rm "$cid" >/dev/null
-    chmod +x "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"
+    [ "$rc" -eq 0 ] || return 1
+
+    if ! chmod +x "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"; then
+        echo "  ERROR: chmod +x failed on the extracted binaries"
+        return 1
+    fi
+
+    for dest in "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"; do
+        if [ ! -s "$dest" ] || [ ! -x "$dest" ]; then
+            echo "  ERROR: extracted binary missing, empty or not executable: $dest"
+            return 1
+        fi
+    done
+
     log "Cached fips ($(stat -c %s "$FIPS_BIN_CACHE") bytes) + fips-gateway ($(stat -c %s "$FIPS_GATEWAY_BIN_CACHE") bytes)"
+    return 0
 }
 
 # ─────────────────────────────────────────────────────────────────────