diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py index 139c2e0..3d925d3 100644 --- a/testing/chaos/sim/scenario.py +++ b/testing/chaos/sim/scenario.py @@ -249,8 +249,9 @@ class Scenario: # link_swap.policies and topology.transport_mix. Two sub-trees are passed through # whole and are likewise not checked: fips_overrides and topology.params. # -# NOTE: adding a new assertion type means adding it to _ASSERTION_KEYS below and -# giving it an entry here, or scenarios using it will be rejected at load. +# NOTE: adding a new assertion type means registering it in TWO places below — +# _SECTION_KEYS["assertions"] (so the block accepts its name) and _ASSERTION_KEYS +# (so its own members are checked) — or scenarios using it are rejected at load. _TOP_KEYS = { "scenario", "topology", "netem", "link_flaps", "traffic", "node_churn", "peer_churn", "bandwidth", "ingress", "link_swap", "assertions", "logging", diff --git a/testing/check-ci-parity.sh b/testing/check-ci-parity.sh index f66e377..a369063 100755 --- a/testing/check-ci-parity.sh +++ b/testing/check-ci-parity.sh @@ -136,16 +136,26 @@ with open(ci_yml_path, encoding="utf-8") as fh: include = doc["jobs"]["integration"]["strategy"]["matrix"]["include"] github_chaos, github_deb, github = {}, set(), set() +malformed = [] for leg in include: - if "suite" not in leg: + if "suite" not in leg and "scenario" not in leg: continue kind = str(leg.get("type", "")) - if kind == "chaos": - github_chaos[str(leg["scenario"])] = str(leg.get("chaos_flags", "")) - elif kind == "deb-install": - github_deb.add(str(leg["scenario"])) - else: + if kind in ("chaos", "deb-install"): + # scenario: is the identity for these; suite: is cosmetic. + if "scenario" not in leg: + malformed.append(f"{leg.get('suite', '(unnamed leg)')} has type " + f"{kind} but no scenario:") + continue + if kind == "chaos": + github_chaos[str(leg["scenario"])] = str(leg.get("chaos_flags", "")) + else: + github_deb.add(str(leg["scenario"])) + elif "suite" in leg: github.add(str(leg["suite"])) + else: + malformed.append(f"leg with scenario {leg['scenario']} has no suite: " + f"and no chaos/deb-install type") # ── Dispatch cross-check: every run_suite arm needs a backing array ────────── # A suite dispatched without an array is invisible to the sweep above, so the @@ -159,14 +169,26 @@ if body is None: # Arms sit at one fixed indentation inside the case block. Pin to it, taken from # the first arm rather than assumed, so a body line that happens to end in ')' # cannot be read as an arm. -arm_re = re.compile(r"^([ \t]+)([a-z0-9|*_-]+)\)", re.MULTILINE) +arm_re = re.compile(r"^([ \t]+)['\"]?([a-z0-9|*_.-]+)['\"]?\)", re.MULTILINE) first = arm_re.search(body.group(0)) if first is None: print("check-ci-parity: no dispatch arms found in run_suite()", file=sys.stderr) sys.exit(2) indent = first.group(1) -known = set(local) | set(local_chaos) | local_deb | {"deb-install"} +# An arm-shaped line at a different indent is not skipped silently: it would +# make this check quietly stop covering a suite, which is the failure mode the +# check exists to prevent. +odd_arms = [ + m.group(2) for m in arm_re.finditer(body.group(0)) if m.group(1) != indent +] +if odd_arms: + print("check-ci-parity: run_suite has arm-shaped lines at an unexpected " + f"indent, so the dispatch check cannot be trusted: {', '.join(odd_arms)}", + file=sys.stderr) + sys.exit(2) +known = (set(local) | set(local_chaos) | local_deb + | {e.split()[0] for e in arrays.get("DEB_INSTALL_SUITES", [])}) for m in arm_re.finditer(body.group(0)): if m.group(1) != indent: continue @@ -197,7 +219,7 @@ deb_github_only = sorted(github_deb - local_deb) problems = (local_only or github_only or chaos_local_only or chaos_github_only or chaos_flag_drift or deb_local_only or deb_github_only - or dispatch_uncovered) + or dispatch_uncovered or malformed) if problems: print("CI parity FAILED: the two runners do not cover the same work.\n") @@ -230,6 +252,10 @@ if problems: print(" deb-install distros GitHub-only:") for n in deb_github_only: print(f" - {n}") + if malformed: + print(" Matrix legs this guard cannot identify:") + for n in malformed: + print(f" - {n}") if dispatch_uncovered: print(" run_suite dispatches these with no backing *_SUITES array, so " "this guard\n cannot see them in the local set:") diff --git a/testing/dns-resolver/test.sh b/testing/dns-resolver/test.sh index 905954e..faf071a 100755 --- a/testing/dns-resolver/test.sh +++ b/testing/dns-resolver/test.sh @@ -275,11 +275,17 @@ DOCKERFILE # and the e2e scenarios silently exercise the previous commit's code. rm -f "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE" - local cid err - if ! cid=$(docker create "$builder_tag" 2>&1); then - echo " ERROR: docker create failed: $cid" + # stderr goes to its own file rather than into $cid: docker prints + # warnings (a platform mismatch, say) on success too, and folding them + # into the id would leave every later reference pointing at nothing. + local cid err errfile + errfile=$(mktemp) + if ! cid=$(docker create "$builder_tag" 2>"$errfile"); then + echo " ERROR: docker create failed: $(cat "$errfile")" + rm -f "$errfile" return 1 fi + rm -f "$errfile" local rc=0 spec bin dest for spec in "fips:$FIPS_BIN_CACHE" "fips-gateway:$FIPS_GATEWAY_BIN_CACHE"; do