diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fce01a1..7db211f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,12 +38,12 @@ env: # unreliable on GitHub-hosted runners. # tor-directory — same; live Tor dependency. # -# Granularity-only differences (same coverage, different matrix shape — -# NOT a divergence): -# deb-install — split here into per-distro legs (debian12/debian13/ -# ubuntu22/ubuntu24/ubuntu26) for parallelism; local runs the -# same distro set in one suite. -# dns-resolver — single leg here; runs all scenarios (same as local). +# The two runners express the same work in different matrix shapes, and the +# parity guard compares through that shape rather than around it: chaos legs +# are compared per scenario (and per flag) via their `scenario:` field, +# deb-install legs per distro. The one leg still compared at leg granularity +# is dns-resolver — a single leg here, running all of its scenarios +# internally, exactly as the local suite does. # ───────────────────────────────────────────────────────────────────────────── # ───────────────────────────────────────────────────────────────────────────── @@ -52,6 +52,16 @@ env: # Builds on Linux x86_64, Linux aarch64, and macOS. # ───────────────────────────────────────────────────────────────────────────── jobs: + ci-parity: + name: CI parity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Install Python deps + run: pip3 install --quiet pyyaml + - name: Check local and GitHub runners cover the same work + run: bash testing/check-ci-parity.sh + fmt: name: Format check runs-on: ubuntu-latest diff --git a/testing/README.md b/testing/README.md index 2dd2760..6933a43 100644 --- a/testing/README.md +++ b/testing/README.md @@ -79,8 +79,11 @@ a per-commit gate; not part of `ci-local.sh`. clippy, unit tests, and the integration suites (including the chaos scenarios) — mirroring the GitHub `ci.yml` integration matrix. Run `./ci-local.sh --help` for the full option list and `--list` for the -available suites. `--check-parity` verifies the local suite set matches -the GitHub matrix (see [check-ci-parity.sh](check-ci-parity.sh)). +available suites. Every run starts with a parity check that verifies the +local suite set covers the same work as the GitHub matrix, per scenario for +chaos and per distro for deb-install; a divergence fails the run. GitHub +runs the same check as its own `ci-parity` job. `--check-parity` runs it +alone (see [check-ci-parity.sh](check-ci-parity.sh)). ### Per-run isolation and the `FIPS_CI_RUN_ID` override diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py index 79a7228..3d925d3 100644 --- a/testing/chaos/sim/scenario.py +++ b/testing/chaos/sim/scenario.py @@ -239,15 +239,91 @@ class Scenario: fips_overrides: dict = field(default_factory=dict) +# Keys each fixed-schema section understands. A key absent from these sets is a +# typo, and silently ignoring it leaves the block default-constructed while the +# YAML still looks correct — a mistyped "assertion:" disarms a scenario's only +# assertions, a mistyped "link_flap:" turns off the chaos it was meant to inject. +# +# Three mappings are deliberately NOT listed, because their keys are names the +# scenario author chooses rather than a schema: netem.mutation.policies, +# link_swap.policies and topology.transport_mix. Two sub-trees are passed through +# whole and are likewise not checked: fips_overrides and topology.params. +# +# NOTE: adding a new assertion type means registering it in TWO places below — +# _SECTION_KEYS["assertions"] (so the block accepts its name) and _ASSERTION_KEYS +# (so its own members are checked) — or scenarios using it are rejected at load. +_TOP_KEYS = { + "scenario", "topology", "netem", "link_flaps", "traffic", "node_churn", + "peer_churn", "bandwidth", "ingress", "link_swap", "assertions", "logging", + "fips_overrides", +} +_SECTION_KEYS = { + "scenario": {"name", "seed", "duration_secs"}, + "topology": { + "num_nodes", "algorithm", "params", "ensure_connected", "subnet", + "ip_start", "default_transport", "transport_mix", + }, + "netem": {"enabled", "default_policy", "link_policies", "mutation"}, + "netem.link_policies[]": {"edges", "policy", "policy_name"}, + "netem.mutation": {"interval_secs", "fraction", "policies"}, + "link_flaps": { + "enabled", "interval_secs", "max_down_links", "down_duration_secs", + "protect_connectivity", + }, + "traffic": { + "enabled", "max_concurrent", "interval_secs", "duration_secs", + "parallel_streams", + }, + "node_churn": { + "enabled", "interval_secs", "max_down_nodes", "down_duration_secs", + "protect_connectivity", + }, + "peer_churn": {"enabled", "interval_secs", "ephemeral_fraction"}, + "bandwidth": {"enabled", "tiers_mbps"}, + "ingress": {"enabled", "tiers_kbps", "burst_bytes"}, + "link_swap": {"enabled", "interval_secs", "policies", "edges"}, + "link_swap.edges[]": {"edge", "policy"}, + "assertions": {"bloom_send_rate", "min_parent_switches"}, + "logging": {"rust_log", "output_dir"}, +} +_ASSERTION_KEYS = { + "bloom_send_rate": {"window_secs", "max_per_node"}, + "min_parent_switches": {"min_total"}, +} +_NETEM_POLICY_KEYS = { + "delay_ms", "jitter_ms", "loss_pct", "duplicate_pct", "reorder_pct", + "corrupt_pct", +} +_RANGE_KEYS = {"min", "max"} + + +def _reject_unknown(data, known, context: str): + """Raise unless every key in `data` is one the loader understands.""" + if data is None: + raise ValueError( + f"{context}: section is present but empty; give it a body or remove it" + ) + if not isinstance(data, dict): + raise ValueError(f"{context}: expected a mapping, got {type(data).__name__}") + unknown = sorted(str(k) for k in set(data) - set(known)) + if unknown: + raise ValueError( + f"{context}: unknown key(s): {', '.join(unknown)} " + f"(known: {', '.join(sorted(known))})" + ) + + def _parse_range(data, name: str) -> Range: """Parse a {min, max} dict into a Range.""" if isinstance(data, dict): + _reject_unknown(data, _RANGE_KEYS, name) return Range(min=float(data["min"]), max=float(data["max"])) raise ValueError(f"{name}: expected {{min, max}} dict, got {type(data).__name__}") -def _parse_netem_policy(data: dict) -> NetemPolicy: +def _parse_netem_policy(data: dict, name: str = "netem policy") -> NetemPolicy: """Parse a netem policy from a dict with [min, max] lists or {min, max} dicts.""" + _reject_unknown(data, _NETEM_POLICY_KEYS, name) policy = NetemPolicy() for attr in ( "delay_ms", @@ -273,16 +349,22 @@ def load_scenario(path: str) -> Scenario: with open(path) as f: raw = yaml.safe_load(f) + if raw is None: + raise ValueError(f"{path}: file is empty") + _reject_unknown(raw, _TOP_KEYS, "(top level)") + s = Scenario() # Scenario section sc = raw.get("scenario", {}) + _reject_unknown(sc, _SECTION_KEYS["scenario"], "scenario") s.name = sc.get("name", os.path.splitext(os.path.basename(path))[0]) s.seed = int(sc.get("seed", 42)) s.duration_secs = int(sc.get("duration_secs", 120)) # Topology section tc = raw.get("topology", {}) + _reject_unknown(tc, _SECTION_KEYS["topology"], "topology") s.topology.num_nodes = int(tc.get("num_nodes", 10)) s.topology.algorithm = tc.get("algorithm", "random_geometric") s.topology.params = tc.get("params", {}) @@ -298,33 +380,43 @@ def load_scenario(path: str) -> Scenario: # Netem section nc = raw.get("netem", {}) + _reject_unknown(nc, _SECTION_KEYS["netem"], "netem") s.netem.enabled = nc.get("enabled", False) if "default_policy" in nc: - s.netem.default_policy = _parse_netem_policy(nc["default_policy"]) + s.netem.default_policy = _parse_netem_policy( + nc["default_policy"], "netem.default_policy" + ) if "link_policies" in nc: for lp_data in nc["link_policies"]: + _reject_unknown( + lp_data, _SECTION_KEYS["netem.link_policies[]"], "netem.link_policies[]" + ) override = LinkPolicyOverride( edges=lp_data.get("edges", []), ) if "policy" in lp_data: - override.policy = _parse_netem_policy(lp_data["policy"]) + override.policy = _parse_netem_policy( + lp_data["policy"], "netem.link_policies[].policy" + ) if "policy_name" in lp_data: override.policy_name = lp_data["policy_name"] s.netem.link_policies.append(override) if "mutation" in nc: mc = nc["mutation"] + _reject_unknown(mc, _SECTION_KEYS["netem.mutation"], "netem.mutation") s.netem.mutation.interval_secs = _parse_range( mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs" ) s.netem.mutation.fraction = float(mc.get("fraction", 0.3)) if "policies" in mc: s.netem.mutation.policies = { - name: _parse_netem_policy(pdata) + name: _parse_netem_policy(pdata, f"netem.mutation.policies.{name}") for name, pdata in mc["policies"].items() } # Link flaps section lf = raw.get("link_flaps", {}) + _reject_unknown(lf, _SECTION_KEYS["link_flaps"], "link_flaps") s.link_flaps.enabled = lf.get("enabled", False) if "interval_secs" in lf: s.link_flaps.interval_secs = _parse_range(lf["interval_secs"], "link_flaps.interval_secs") @@ -337,6 +429,7 @@ def load_scenario(path: str) -> Scenario: # Traffic section tf = raw.get("traffic", {}) + _reject_unknown(tf, _SECTION_KEYS["traffic"], "traffic") s.traffic.enabled = tf.get("enabled", False) s.traffic.max_concurrent = int(tf.get("max_concurrent", 3)) if "interval_secs" in tf: @@ -347,6 +440,7 @@ def load_scenario(path: str) -> Scenario: # Node churn section nc2 = raw.get("node_churn", {}) + _reject_unknown(nc2, _SECTION_KEYS["node_churn"], "node_churn") s.node_churn.enabled = nc2.get("enabled", False) if "interval_secs" in nc2: s.node_churn.interval_secs = _parse_range(nc2["interval_secs"], "node_churn.interval_secs") @@ -359,6 +453,7 @@ def load_scenario(path: str) -> Scenario: # Peer churn section pc = raw.get("peer_churn", {}) + _reject_unknown(pc, _SECTION_KEYS["peer_churn"], "peer_churn") s.peer_churn.enabled = pc.get("enabled", False) if "interval_secs" in pc: s.peer_churn.interval_secs = _parse_range(pc["interval_secs"], "peer_churn.interval_secs") @@ -366,6 +461,7 @@ def load_scenario(path: str) -> Scenario: # Bandwidth section bw = raw.get("bandwidth", {}) + _reject_unknown(bw, _SECTION_KEYS["bandwidth"], "bandwidth") s.bandwidth.enabled = bw.get("enabled", False) if "tiers_mbps" in bw: tiers = bw["tiers_mbps"] @@ -375,6 +471,7 @@ def load_scenario(path: str) -> Scenario: # Ingress section ig = raw.get("ingress", {}) + _reject_unknown(ig, _SECTION_KEYS["ingress"], "ingress") s.ingress.enabled = ig.get("enabled", False) if "tiers_kbps" in ig: tiers = ig["tiers_kbps"] @@ -385,18 +482,22 @@ def load_scenario(path: str) -> Scenario: # Link swap section (deterministic asymmetric link-cost flapping). ls = raw.get("link_swap", {}) + _reject_unknown(ls, _SECTION_KEYS["link_swap"], "link_swap") s.link_swap.enabled = ls.get("enabled", False) if "interval_secs" in ls: s.link_swap.interval_secs = float(ls["interval_secs"]) if "policies" in ls: s.link_swap.policies = { - name: _parse_netem_policy(pdata) + name: _parse_netem_policy(pdata, f"link_swap.policies.{name}") for name, pdata in ls["policies"].items() } if "edges" in ls: for edata in ls["edges"]: if not isinstance(edata, dict): raise ValueError("link_swap.edges entries must be dicts") + _reject_unknown( + edata, _SECTION_KEYS["link_swap.edges[]"], "link_swap.edges[]" + ) edge = str(edata.get("edge", "")) policy = str(edata.get("policy", "")) if not edge or not policy: @@ -405,20 +506,29 @@ def load_scenario(path: str) -> Scenario: # Assertions section (post-run control-socket-based checks). asrt = raw.get("assertions", {}) + _reject_unknown(asrt, _SECTION_KEYS["assertions"], "assertions") if "bloom_send_rate" in asrt: bsr = asrt["bloom_send_rate"] + _reject_unknown( + bsr, _ASSERTION_KEYS["bloom_send_rate"], "assertions.bloom_send_rate" + ) s.assertions.bloom_send_rate = BloomSendRateAssertion( window_secs=int(bsr.get("window_secs", 30)), max_per_node=int(bsr.get("max_per_node", 30)), ) if "min_parent_switches" in asrt: mps = asrt["min_parent_switches"] + _reject_unknown( + mps, _ASSERTION_KEYS["min_parent_switches"], + "assertions.min_parent_switches", + ) s.assertions.min_parent_switches = MinParentSwitchesAssertion( min_total=int(mps.get("min_total", 1)), ) # Logging section lg = raw.get("logging", {}) + _reject_unknown(lg, _SECTION_KEYS["logging"], "logging") s.logging.rust_log = lg.get("rust_log", "info") s.logging.output_dir = lg.get("output_dir", "./sim-results") diff --git a/testing/check-ci-parity.sh b/testing/check-ci-parity.sh index deadc6f..a369063 100755 --- a/testing/check-ci-parity.sh +++ b/testing/check-ci-parity.sh @@ -11,19 +11,30 @@ # unreliable on GitHub-hosted runners. # tor-directory — same; live Tor dependency. # -# Granularity-only differences folded before comparison (same coverage, -# different matrix shape — NOT a divergence): -# deb-install — GitHub splits into per-distro legs (deb-install-debian12/ -# debian13/ubuntu22/ubuntu24/ubuntu26); local runs the same -# distro set in one suite. Folded to "deb-install". -# chaos-* — GitHub fans each chaos scenario into its own matrix leg -# (type: chaos); local runs them all via the one CHAOS_SUITES -# path. The individual scenario names also differ cosmetically -# between runners (e.g. chaos-smoke-10 vs churn-mixed-10). -# Folded to a single "chaos" token on both sides. -# dns-resolver — single leg / single suite both sides; runs all scenarios. +# What is compared, and at what granularity: +# chaos — per scenario, plus its flags. GitHub fans each scenario +# into its own matrix leg carrying `scenario:` (and +# optionally `chaos_flags:`); local lists the same scenarios +# in CHAOS_SUITES as "display scenario flags". The `suite:` +# names differ cosmetically between runners and are ignored +# — `scenario:` is the identity. +# deb-install — per distro. GitHub splits into per-distro legs carrying +# `scenario:`; local runs the same distro set in one suite, +# enumerated by ALL_SCENARIOS in deb-install/test.sh. +# everything else — per suite name. # -# Exit 0 = parity clean. Exit 1 = unexpected divergence (suite names printed). +# dns-resolver is the one leg still compared at leg granularity rather than +# per scenario: it is a single leg and a single suite on both sides, and it +# runs all of its scenarios internally. Its scenario list is NOT cross-checked. +# +# The local suite set is discovered by sweeping ci-local.sh for *_SUITES arrays +# rather than from a hardcoded list of variable names, and every run_suite +# dispatch arm is then checked to have a backing array — a suite dispatched +# without one is invisible to a name-list sweep, which is how a real divergence +# went unnoticed. +# +# Exit 0 = parity clean. Exit 1 = unexpected divergence. Exit 2 = the guard +# could not run (missing file or missing dependency); never treated as a pass. # ───────────────────────────────────────────────────────────────────────────── set -uo pipefail @@ -32,113 +43,235 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" CI_LOCAL="$SCRIPT_DIR/ci-local.sh" CI_YML="$PROJECT_ROOT/.github/workflows/ci.yml" +DEB_TEST="$SCRIPT_DIR/deb-install/test.sh" # Deliberate local-only allowlist (suites intentionally absent from GitHub). ALLOWLIST="tor-socks5 tor-directory" -for f in "$CI_LOCAL" "$CI_YML"; do +for f in "$CI_LOCAL" "$CI_YML" "$DEB_TEST"; do if [[ ! -f "$f" ]]; then echo "check-ci-parity: missing file: $f" >&2 exit 2 fi done -# Extract and normalize both suite sets in Python (robust YAML parse of the -# matrix; regex extraction of the bash suite arrays). Folding rules above are -# applied identically to both sides so only genuine divergence surfaces. -python3 - "$CI_LOCAL" "$CI_YML" "$ALLOWLIST" <<'PY' +if ! command -v python3 >/dev/null 2>&1; then + echo "check-ci-parity: python3 not found; cannot verify CI parity" >&2 + exit 2 +fi +if ! python3 -c "import yaml" >/dev/null 2>&1; then + echo "check-ci-parity: python3 module 'yaml' not found; cannot verify CI parity" >&2 + echo "check-ci-parity: install it with 'pip3 install pyyaml'" >&2 + exit 2 +fi + +python3 - "$CI_LOCAL" "$CI_YML" "$DEB_TEST" "$ALLOWLIST" <<'PY' import re import sys -ci_local_path, ci_yml_path, allowlist_raw = sys.argv[1], sys.argv[2], sys.argv[3] +import yaml + +ci_local_path, ci_yml_path, deb_test_path, allowlist_raw = sys.argv[1:5] allowlist = set(allowlist_raw.split()) - -def fold(name): - """Collapse granularity-only matrix shape into canonical suite identity.""" - if name.startswith("chaos-") or name == "chaos": - return "chaos" - if name.startswith("deb-install"): - return "deb-install" - return name - - -# ── Local: parse the suite arrays from ci-local.sh ─────────────────────────── with open(ci_local_path, encoding="utf-8") as fh: local_src = fh.read() +with open(deb_test_path, encoding="utf-8") as fh: + deb_src = fh.read() -def bash_array(var): +def bash_array_entries(var): + """Full entries of a bash array, in order, quotes stripped.""" m = re.search(rf"^{var}=\((.*?)\)", local_src, re.MULTILINE | re.DOTALL) if not m: return [] body = m.group(1) - # Quoted entries (chaos uses "display scenario flags"): first token is name. quoted = re.findall(r'"([^"]*)"', body) if quoted: - return [entry.split()[0] for entry in quoted if entry.strip()] + return [e.strip() for e in quoted if e.strip()] return [tok for tok in body.split() if tok.strip()] +def discovered_arrays(): + """Every *_SUITES array in ci-local.sh, by name. + + Swept rather than hardcoded: a suite whose array is not on a fixed list + would otherwise be invisible to this guard. + """ + return { + name + "_SUITES": bash_array_entries(name + "_SUITES") + for name in re.findall(r"^([A-Z_]+)_SUITES=\(", local_src, re.MULTILINE) + } + + +arrays = discovered_arrays() + +# ── Local side ─────────────────────────────────────────────────────────────── +# Chaos: "display scenario flags" — compare the scenario and its flags. +local_chaos = {} +for entry in arrays.get("CHAOS_SUITES", []): + parts = entry.split() + if len(parts) < 2: + continue + local_chaos[parts[1]] = " ".join(parts[2:]) + +# deb-install: one local suite that runs the distro set enumerated in its script. +m = re.search(r'^ALL_SCENARIOS="([^"]*)"', deb_src, re.MULTILINE) +local_deb = set(m.group(1).split()) if m else set() + +# Everything else: suite names, with NAT stored bare and prefixed at use. local = set() -# Static, rekey, gateway, sidecar, acl, firewall, nostr, stun, dns, deb. -for var in ("STATIC_SUITES", "REKEY_SUITES", "ADMISSION_SUITES", - "GATEWAY_SUITES", "SIDECAR_SUITES", "ACL_SUITES", - "FIREWALL_SUITES", "NOSTR_RELAY_SUITES", "STUN_FAULTS_SUITES", - "DNS_RESOLVER_SUITES", "DEB_INSTALL_SUITES"): - local.update(bash_array(var)) -# Chaos display names → fold to "chaos". -for _ in bash_array("CHAOS_SUITES"): - local.add("chaos") -# NAT scenarios are stored bare (cone/symmetric/lan) and prefixed nat- at use. -for scen in bash_array("NAT_SUITES"): - local.add(f"nat-{scen}") -# TOR_SUITES is the deliberate local-only set — excluded from the default path. - -local = {fold(n) for n in local} - -# ── GitHub: parse the integration matrix suite: values from ci.yml ─────────── -import yaml # noqa: E402 +for name, entries in arrays.items(): + if name in ("CHAOS_SUITES", "DEB_INSTALL_SUITES"): + continue + names = [e.split()[0] for e in entries] + if name == "NAT_SUITES": + local |= {f"nat-{n}" for n in names} + else: + local |= set(names) +# ── GitHub side ────────────────────────────────────────────────────────────── with open(ci_yml_path, encoding="utf-8") as fh: doc = yaml.safe_load(fh) include = doc["jobs"]["integration"]["strategy"]["matrix"]["include"] -github = set() +github_chaos, github_deb, github = {}, set(), set() +malformed = [] for leg in include: - if "suite" not in leg: + if "suite" not in leg and "scenario" not in leg: continue - # Chaos legs carry inconsistent suite: names (chaos-smoke-10 vs - # churn-mixed-10) but a uniform type: chaos — fold via type, not name. - if str(leg.get("type", "")) == "chaos": - github.add("chaos") + kind = str(leg.get("type", "")) + if kind in ("chaos", "deb-install"): + # scenario: is the identity for these; suite: is cosmetic. + if "scenario" not in leg: + malformed.append(f"{leg.get('suite', '(unnamed leg)')} has type " + f"{kind} but no scenario:") + continue + if kind == "chaos": + github_chaos[str(leg["scenario"])] = str(leg.get("chaos_flags", "")) + else: + github_deb.add(str(leg["scenario"])) + elif "suite" in leg: + github.add(str(leg["suite"])) else: - github.add(fold(str(leg["suite"]))) + malformed.append(f"leg with scenario {leg['scenario']} has no suite: " + f"and no chaos/deb-install type") -# ── Diff (subtract allowlist from local before comparison) ─────────────────── +# ── Dispatch cross-check: every run_suite arm needs a backing array ────────── +# A suite dispatched without an array is invisible to the sweep above, so the +# guard would report it as GitHub-only forever without ever naming the cause. +body = re.search(r"^run_suite\(\).*?^\}", local_src, re.MULTILINE | re.DOTALL) +dispatch_uncovered = [] +if body is None: + print("check-ci-parity: could not locate run_suite() in ci-local.sh", file=sys.stderr) + sys.exit(2) + +# Arms sit at one fixed indentation inside the case block. Pin to it, taken from +# the first arm rather than assumed, so a body line that happens to end in ')' +# cannot be read as an arm. +arm_re = re.compile(r"^([ \t]+)['\"]?([a-z0-9|*_.-]+)['\"]?\)", re.MULTILINE) +first = arm_re.search(body.group(0)) +if first is None: + print("check-ci-parity: no dispatch arms found in run_suite()", file=sys.stderr) + sys.exit(2) + +indent = first.group(1) +# An arm-shaped line at a different indent is not skipped silently: it would +# make this check quietly stop covering a suite, which is the failure mode the +# check exists to prevent. +odd_arms = [ + m.group(2) for m in arm_re.finditer(body.group(0)) if m.group(1) != indent +] +if odd_arms: + print("check-ci-parity: run_suite has arm-shaped lines at an unexpected " + f"indent, so the dispatch check cannot be trusted: {', '.join(odd_arms)}", + file=sys.stderr) + sys.exit(2) +known = (set(local) | set(local_chaos) | local_deb + | {e.split()[0] for e in arrays.get("DEB_INSTALL_SUITES", [])}) +for m in arm_re.finditer(body.group(0)): + if m.group(1) != indent: + continue + for arm in m.group(2).split("|"): + if arm == "*": + continue # the unknown-suite error arm, not a suite + if arm == "chaos-*": + # Dispatches any chaos- through a fallback, so its vocabulary + # is unbounded; the chaos scenario comparison covers it instead. + continue + if arm not in known: + dispatch_uncovered.append(arm) + +# ── Diff ───────────────────────────────────────────────────────────────────── local_cmp = {n for n in local if n not in allowlist} local_only = sorted(local_cmp - github) github_only = sorted(github - local_cmp) +chaos_local_only = sorted(set(local_chaos) - set(github_chaos)) +chaos_github_only = sorted(set(github_chaos) - set(local_chaos)) +chaos_flag_drift = sorted( + (s, local_chaos[s], github_chaos[s]) + for s in set(local_chaos) & set(github_chaos) + if local_chaos[s] != github_chaos[s] +) +deb_local_only = sorted(local_deb - github_deb) +deb_github_only = sorted(github_deb - local_deb) -if local_only or github_only: - print("CI parity FAILED: integration suite sets diverge.\n") +problems = (local_only or github_only or chaos_local_only or chaos_github_only + or chaos_flag_drift or deb_local_only or deb_github_only + or dispatch_uncovered or malformed) + +if problems: + print("CI parity FAILED: the two runners do not cover the same work.\n") if local_only: - print(" Local-only (in ci-local.sh, missing from ci.yml, " - "not in deliberate allowlist):") + print(" Suites local-only (in ci-local.sh, missing from ci.yml, " + "not in the deliberate allowlist):") for n in local_only: print(f" - {n}") if github_only: - print(" GitHub-only (in ci.yml, missing from local default path):") + print(" Suites GitHub-only (in ci.yml, missing from the local default path):") for n in github_only: print(f" - {n}") - print("\n Resolve by adding the suite to the other runner, or by adding " - "it\n to the deliberate local-only allowlist in " - "check-ci-parity.sh with a\n stated reason.") + if chaos_local_only: + print(" Chaos scenarios local-only:") + for n in chaos_local_only: + print(f" - {n}") + if chaos_github_only: + print(" Chaos scenarios GitHub-only:") + for n in chaos_github_only: + print(f" - {n}") + if chaos_flag_drift: + print(" Chaos scenarios whose flags differ between runners:") + for name, lflags, gflags in chaos_flag_drift: + print(f" - {name}: local '{lflags}' vs GitHub '{gflags}'") + if deb_local_only: + print(" deb-install distros local-only:") + for n in deb_local_only: + print(f" - {n}") + if deb_github_only: + print(" deb-install distros GitHub-only:") + for n in deb_github_only: + print(f" - {n}") + if malformed: + print(" Matrix legs this guard cannot identify:") + for n in malformed: + print(f" - {n}") + if dispatch_uncovered: + print(" run_suite dispatches these with no backing *_SUITES array, so " + "this guard\n cannot see them in the local set:") + for n in dispatch_uncovered: + print(f" - {n}") + print("\n Resolve by adding the suite to the other runner, by giving a " + "dispatchable\n suite a *_SUITES array, or by adding it to the " + "deliberate local-only\n allowlist in check-ci-parity.sh with a " + "stated reason.") sys.exit(1) -print("CI parity OK: integration suite sets match " +total = len(github) + len(github_chaos) + len(github_deb) +print("CI parity OK: both runners cover the same work " "(allowlist: " + ", ".join(sorted(allowlist)) + ").") -print(f" {len(github)} canonical suites compared on each side.") +print(f" {len(github)} suites, {len(github_chaos)} chaos scenarios " + f"(flags compared), {len(github_deb)} deb-install distros " + f"— {total} legs on each side.") sys.exit(0) PY diff --git a/testing/ci-local.sh b/testing/ci-local.sh index ffdb60d..c8a4efc 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -1,5 +1,6 @@ #!/bin/bash -# Run the CI pipeline locally: build, unit tests, integration tests. +# Run the CI pipeline locally: CI parity check, build, unit tests, +# integration tests. # # Usage: ./ci-local.sh [options] # @@ -53,19 +54,19 @@ # (.github/workflows/ci.yml) MUST run the same integration suites, EXCEPT for # the deliberate local-only entries below. Adding a suite to one runner # without the other means "local green" and "GitHub green" stop being -# equivalent. testing/check-ci-parity.sh enforces this and fails on drift. +# equivalent. testing/check-ci-parity.sh enforces this and fails on drift; it +# runs as the first stage of every local run, before the build. # # Deliberate local-only (NOT on the GitHub gate), with reason: # tor-socks5 — requires live Tor network; opt-in via --with-tor, # unreliable on GitHub-hosted runners. # tor-directory — same; live Tor dependency. # -# Granularity-only differences (same coverage, different matrix shape — -# NOT a divergence): -# deb-install — local runs all distros sequentially in one suite; GitHub -# splits into per-distro legs (debian12/debian13/ubuntu22/ -# ubuntu24/ubuntu26 — the same distro set). -# dns-resolver — single suite both sides; runs all scenarios. +# The two runners express the same work in different matrix shapes, and the +# guard compares through that shape rather than around it: chaos legs are +# compared per scenario (and per flag), deb-install legs per distro. The one +# leg still compared at leg granularity is dns-resolver — a single suite on +# both sides that runs all of its scenarios internally. # ───────────────────────────────────────────────────────────────────────────── set -uo pipefail @@ -1034,6 +1035,16 @@ print_summary() { echo "" } +# Verify the local default suite set and the GitHub matrix still cover the +# same work. Runs first: it takes about a second, and a divergence should be +# reported before a half-hour suite rather than after it. +run_ci_parity() { + local rc=0 + info "[ci-parity] Comparing the local suite set against the GitHub matrix" + "$SCRIPT_DIR/check-ci-parity.sh" || rc=$? + record "ci-parity" $rc +} + # ── Main ─────────────────────────────────────────────────────────────────── main() { @@ -1042,6 +1053,11 @@ main() { stage "FIPS Local CI" info "Project root: $PROJECT_ROOT" + # Above the mode branches deliberately, so --only, --test-only and + # --build-only are gated too: a divergence invalidates any claim that a + # local run means what a GitHub run means, whichever subset was asked for. + run_ci_parity + if [[ "$TEST_ONLY" == true ]]; then run_tests elif [[ "$BUILD_ONLY" == true ]]; then diff --git a/testing/dns-resolver/test.sh b/testing/dns-resolver/test.sh index c81e30d..1c799ff 100755 --- a/testing/dns-resolver/test.sh +++ b/testing/dns-resolver/test.sh @@ -270,13 +270,49 @@ DOCKERFILE fi log "Extracting fips + fips-gateway binaries from builder image" - local cid - cid=$(docker create "$builder_tag") - docker cp "$cid:/src/target/release/fips" "$FIPS_BIN_CACHE" >/dev/null 2>&1 - docker cp "$cid:/src/target/release/fips-gateway" "$FIPS_GATEWAY_BIN_CACHE" >/dev/null 2>&1 + # Drop the previous run's binaries before extracting. Without this, a failed + # extraction below leaves them in place, they satisfy the caller's -x check, + # and the e2e scenarios silently exercise the previous commit's code. + rm -f "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE" + + # stderr goes to its own file rather than into $cid: docker prints + # warnings (a platform mismatch, say) on success too, and folding them + # into the id would leave every later reference pointing at nothing. + local cid err errfile + errfile=$(mktemp) + if ! cid=$(docker create "$builder_tag" 2>"$errfile"); then + echo " ERROR: docker create failed: $(cat "$errfile")" + rm -f "$errfile" + return 1 + fi + rm -f "$errfile" + + local rc=0 spec bin dest + for spec in "fips:$FIPS_BIN_CACHE" "fips-gateway:$FIPS_GATEWAY_BIN_CACHE"; do + bin="${spec%%:*}" + dest="${spec#*:}" + if ! err=$(docker cp "$cid:/src/target/release/$bin" "$dest" 2>&1); then + echo " ERROR: extracting $bin from the builder image failed: $err" + rc=1 + fi + done docker rm "$cid" >/dev/null - chmod +x "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE" + [ "$rc" -eq 0 ] || return 1 + + if ! chmod +x "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"; then + echo " ERROR: chmod +x failed on the extracted binaries" + return 1 + fi + + for dest in "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"; do + if [ ! -s "$dest" ] || [ ! -x "$dest" ]; then + echo " ERROR: extracted binary missing, empty or not executable: $dest" + return 1 + fi + done + log "Cached fips ($(stat -c %s "$FIPS_BIN_CACHE") bytes) + fips-gateway ($(stat -c %s "$FIPS_GATEWAY_BIN_CACHE") bytes)" + return 0 } # ─────────────────────────────────────────────────────────────────────