From 5d3a3d7cad0bf182a62bce2a7c35618683ad9694 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Thu, 23 Jul 2026 07:09:36 +0000 Subject: [PATCH] Give the assertion-free chaos scenarios a convergence floor Five scenarios named by the audit carried no assertions at all, so a run in which the mesh never formed exited 0 and reported green. Add a baseline assertion covering how many nodes answered, how many distinct roots they agreed on, how many took a parent, and optionally how many sessions were established, and apply it to those five plus the two cost scenarios left unasserted by the previous commit. For ethernet-only, ethernet-mesh, tcp-mesh, smoke-10, mixed-technology and depth-vs-cost the values are not calibrated: one root and N-1 parented nodes is what a spanning tree is, and all provably-completed archived runs of each show exactly that. churn-mixed is different and says so at the site. A scenario that stops and starts nodes on purpose does not hold a single tree, and its six completed runs end with two or three roots and seven or eight of ten nodes parented, so its floors sit one step outside the observed range. That leaves them catching a mesh that collapsed rather than one that churned, which is the most a sample of six supports. This gives Ethernet transport its only assertion anywhere in CI. The three tests in src/node/tests/ethernet.rs are ignored for requiring CAP_NET_RAW and neither runner passes --ignored, so until now nothing exercised that transport with a verdict attached. The floor is deliberately weak and deliberately not a substitute: it says the mesh formed, not that it formed the tree the scenario describes. Where those differ the scenario now says so in its own comments. --- testing/chaos/scenarios/churn-mixed.yaml | 21 ++++++ testing/chaos/scenarios/depth-vs-cost.yaml | 15 +++++ testing/chaos/scenarios/ethernet-mesh.yaml | 15 +++++ testing/chaos/scenarios/ethernet-only.yaml | 19 ++++++ testing/chaos/scenarios/mixed-technology.yaml | 22 ++++++- testing/chaos/scenarios/smoke-10.yaml | 14 ++++ testing/chaos/scenarios/tcp-mesh.yaml | 13 ++++ testing/chaos/sim/assertions.py | 58 +++++++++++++++++ testing/chaos/sim/runner.py | 14 +++- testing/chaos/sim/scenario.py | 64 ++++++++++++++++++- 10 files changed, 251 insertions(+), 4 deletions(-) diff --git a/testing/chaos/scenarios/churn-mixed.yaml b/testing/chaos/scenarios/churn-mixed.yaml index 961ccf2..54a0864 100644 --- a/testing/chaos/scenarios/churn-mixed.yaml +++ b/testing/chaos/scenarios/churn-mixed.yaml @@ -67,6 +67,27 @@ bandwidth: enabled: true tiers_mbps: [1, 10, 100, 1000] +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about which nodes survive the churn; it exists so that +# a run in which the mesh never formed cannot report success, which +# until now it could, because this scenario carried no assertions at +# all. +# +# This one is calibrated rather than derived, because a scenario that +# stops and starts nodes on purpose does not hold a single spanning +# tree. The six provably-completed archived runs end with two or three +# distinct roots, seven or eight of ten nodes parented, and fifteen to +# twenty sessions. The floors below sit one step outside those ranges +# so an unlucky run does not go red, which leaves them catching a mesh +# that collapsed rather than one that churned. Tighten them only +# against a larger sample than six runs. +assertions: + baseline: + min_nodes_reporting: 10 + max_roots: 4 + min_nodes_parented: 6 + min_sessions: 10 + logging: rust_log: "debug" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/depth-vs-cost.yaml b/testing/chaos/scenarios/depth-vs-cost.yaml index 4f779c2..47d5e4a 100644 --- a/testing/chaos/scenarios/depth-vs-cost.yaml +++ b/testing/chaos/scenarios/depth-vs-cost.yaml @@ -83,6 +83,21 @@ traffic: duration_secs: {min: 5, max: 10} parallel_streams: 2 +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about which parent n04 chooses, which is unspecified; +# it exists so that a run in which the mesh never formed cannot report +# success, which until now it could, +# because this scenario carried no assertions at all. +# +# Four nodes, one root, three parented and four sessions in all five +# provably-completed archived runs. +assertions: + baseline: + min_nodes_reporting: 4 + max_roots: 1 + min_nodes_parented: 3 + min_sessions: 2 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/ethernet-mesh.yaml b/testing/chaos/scenarios/ethernet-mesh.yaml index 00cf009..d1ceb44 100644 --- a/testing/chaos/scenarios/ethernet-mesh.yaml +++ b/testing/chaos/scenarios/ethernet-mesh.yaml @@ -62,6 +62,21 @@ link_flaps: traffic: enabled: false +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about Ethernet link behaviour under flaps; it exists +# so that a run in which the mesh never formed cannot report success, +# which until now it could, because this scenario carried no assertions +# at all. +# +# Six nodes, one root, five parented in all six provably-completed +# archived runs. The other assertion covering Ethernet transport in +# CI; see ethernet-only for why that matters. +assertions: + baseline: + min_nodes_reporting: 6 + max_roots: 1 + min_nodes_parented: 5 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/ethernet-only.yaml b/testing/chaos/scenarios/ethernet-only.yaml index ee34089..39a823b 100644 --- a/testing/chaos/scenarios/ethernet-only.yaml +++ b/testing/chaos/scenarios/ethernet-only.yaml @@ -41,6 +41,25 @@ link_flaps: traffic: enabled: false +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about what Ethernet transport does; it exists so that +# a run in which the mesh never formed cannot report success, which +# until now it could, because this scenario carried no assertions at +# all. +# +# A 4-node mesh forms a spanning tree: one root and three nodes +# with a parent. All six provably-completed archived runs show exactly +# that, so these are the shape of a converged mesh rather than a +# tolerance fitted to observations. This is currently the only +# assertion covering Ethernet transport anywhere in CI: the three +# tests in src/node/tests/ethernet.rs are #[ignore]d for requiring +# CAP_NET_RAW and neither runner passes --ignored. +assertions: + baseline: + min_nodes_reporting: 4 + max_roots: 1 + min_nodes_parented: 3 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/mixed-technology.yaml b/testing/chaos/scenarios/mixed-technology.yaml index bf52c35..9e52923 100644 --- a/testing/chaos/scenarios/mixed-technology.yaml +++ b/testing/chaos/scenarios/mixed-technology.yaml @@ -47,8 +47,9 @@ # which, and until then this scenario proves nothing about parent choice. # # That leaves this scenario asserting nothing about its own subject. The -# baseline convergence assertion is the right floor for it and is tracked -# separately; this note is here so the gap is visible until that lands. +# baseline block further down holds it to forming a tree and carrying +# traffic, which is a floor and not a substitute: nothing here checks that +# the tree it forms is the one these comments describe. scenario: name: "mixed-technology" @@ -117,6 +118,23 @@ traffic: duration_secs: {min: 5, max: 15} parallel_streams: 4 +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about parent choice, which this scenario cannot +# currently assert; it exists so that a run in which the mesh never formed +# cannot report success, which until now it could, +# because this scenario carried no assertions at all. +# +# Ten nodes, one root, nine parented and six to eight sessions across +# the five provably-completed archived runs. The session floor is set +# below the observed minimum deliberately: it is here to catch a mesh +# that carries no traffic at all, not to pin a throughput. +assertions: + baseline: + min_nodes_reporting: 10 + max_roots: 1 + min_nodes_parented: 9 + min_sessions: 3 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/smoke-10.yaml b/testing/chaos/scenarios/smoke-10.yaml index cc36915..9a63958 100644 --- a/testing/chaos/scenarios/smoke-10.yaml +++ b/testing/chaos/scenarios/smoke-10.yaml @@ -22,6 +22,20 @@ link_flaps: traffic: enabled: false +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about any particular subsystem; it exists so that a +# run in which the mesh never formed cannot report success, which until +# now it could, because this scenario carried no assertions at all. +# +# Ten nodes, one root, nine parented in all six provably-completed +# archived runs. As the smoke test this is the scenario whose silent +# success was least defensible. +assertions: + baseline: + min_nodes_reporting: 10 + max_roots: 1 + min_nodes_parented: 9 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/tcp-mesh.yaml b/testing/chaos/scenarios/tcp-mesh.yaml index 0010a29..09dda34 100644 --- a/testing/chaos/scenarios/tcp-mesh.yaml +++ b/testing/chaos/scenarios/tcp-mesh.yaml @@ -61,6 +61,19 @@ link_flaps: traffic: enabled: false +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about TCP transport specifically; it exists so that a +# run in which the mesh never formed cannot report success, which until +# now it could, because this scenario carried no assertions at all. +# +# Six nodes, one root, five parented in all six provably-completed +# archived runs. +assertions: + baseline: + min_nodes_reporting: 6 + max_roots: 1 + min_nodes_parented: 5 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/sim/assertions.py b/testing/chaos/sim/assertions.py index 467da25..1fc636a 100644 --- a/testing/chaos/sim/assertions.py +++ b/testing/chaos/sim/assertions.py @@ -19,6 +19,7 @@ from dataclasses import dataclass from .control import snapshot_all_bloom from .scenario import ( + BaselineAssertion, BloomSendRateAssertion, CongestionSignalsAssertion, MaxErrorsAssertion, @@ -173,6 +174,63 @@ def evaluate_max_parent_switches( ) +def evaluate_baseline( + cfg: BaselineAssertion, + snapshot: dict | None, + sessions: int, +) -> AssertionOutcome: + """Floor on the mesh having formed: nodes answered, agreed a root, took parents.""" + if not snapshot: + return AssertionOutcome( + name="baseline", + passed=False, + detail=( + "FAIL baseline: no final tree snapshot was taken, so nothing " + "about the mesh was observed. This is a harness failure." + ), + ) + + reporting = len(snapshot) + roots = {v.get("root") for v in snapshot.values() if v.get("root")} + parented = sum( + 1 for v in snapshot.values() + if v.get("parent") and v.get("parent") != v.get("my_node_addr") + ) + + parts, failures = [], [] + + def note(ok, text): + parts.append(text) + if not ok: + failures.append(text) + + if cfg.min_nodes_reporting is not None: + note(reporting >= cfg.min_nodes_reporting, + f"{reporting} node(s) answered (need {cfg.min_nodes_reporting})") + if cfg.max_roots is not None: + note(len(roots) <= cfg.max_roots and len(roots) >= 1, + f"{len(roots)} distinct root(s) (allowed {cfg.max_roots})") + if cfg.min_nodes_parented is not None: + note(parented >= cfg.min_nodes_parented, + f"{parented} node(s) have a parent (need {cfg.min_nodes_parented})") + if cfg.min_sessions is not None: + note(sessions >= cfg.min_sessions, + f"{sessions} session(s) established (need {cfg.min_sessions})") + + summary = "; ".join(parts) + if failures: + return AssertionOutcome( + name="baseline", + passed=False, + detail=( + f"FAIL baseline: {'; '.join(failures)}. Full: {summary}" + ), + ) + return AssertionOutcome( + name="baseline", passed=True, detail=f"PASS baseline: {summary}" + ) + + def evaluate_tree_parents( cfg: TreeParentsAssertion, snapshot: dict | None, diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index 0327508..182a252 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -15,11 +15,12 @@ from datetime import datetime from .assertions import ( AssertionOutcome, BloomSendRateMonitor, + evaluate_baseline, evaluate_congestion_signals, evaluate_max_errors, - evaluate_tree_parents, evaluate_max_parent_switches, evaluate_min_parent_switches, + evaluate_tree_parents, ) from .compose import generate_compose from .config_gen import write_configs @@ -624,6 +625,17 @@ class SimRunner: else: log.error("%s", outcome.detail) + bl_cfg = self.scenario.assertions.baseline + if bl_cfg is not None: + outcome = evaluate_baseline( + bl_cfg, self.final_tree, len(result.sessions_established) + ) + self.assertion_outcomes.append(outcome) + if outcome.passed: + log.info("%s", outcome.detail) + else: + log.error("%s", outcome.detail) + tp_cfg = self.scenario.assertions.tree_parents if tp_cfg is not None: outcome = evaluate_tree_parents(tp_cfg, self.final_tree) diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py index 80d80f7..d7e30c4 100644 --- a/testing/chaos/sim/scenario.py +++ b/testing/chaos/sim/scenario.py @@ -224,6 +224,28 @@ class MaxParentSwitchesAssertion: node: str | None = None +@dataclass +class BaselineAssertion: + """Floor on the mesh having formed at all. + + Deliberately weak and deliberately universal. It does not describe any + scenario's subject; it says the nodes came up, agreed on a root, and + took parents. Its value is that a scenario with no other assertion + still cannot pass while the mesh is dead, which is the state twelve of + thirteen scenarios were previously unable to distinguish from success. + + ``max_roots`` is how many distinct root values the snapshot may carry. + One means the whole mesh agreed; a churn scenario that partitions on + purpose needs a higher number, and setting it above one is a statement + that partition is expected rather than an oversight. + """ + + min_nodes_reporting: int | None = None + max_roots: int | None = None + min_nodes_parented: int | None = None + min_sessions: int | None = None + + @dataclass class TreeParentsAssertion: """Expected parent per node in the final tree snapshot. @@ -294,6 +316,7 @@ class AssertionsConfig: max_errors: MaxErrorsAssertion | None = None congestion_signals: CongestionSignalsAssertion | None = None tree_parents: TreeParentsAssertion | None = None + baseline: BaselineAssertion | None = None @dataclass @@ -373,7 +396,7 @@ _SECTION_KEYS = { "link_swap.edges[]": {"edge", "policy"}, "assertions": { "bloom_send_rate", "min_parent_switches", "max_parent_switches", - "max_errors", "congestion_signals", "tree_parents", + "max_errors", "congestion_signals", "tree_parents", "baseline", }, "logging": {"rust_log", "output_dir"}, } @@ -385,6 +408,9 @@ _ASSERTION_KEYS = { "congestion_signals": { "min_nodes_detected", "min_nodes_ce_forwarded", "min_nodes_ce_received", }, + "baseline": { + "min_nodes_reporting", "max_roots", "min_nodes_parented", "min_sessions", + }, } _NETEM_POLICY_KEYS = { "delay_ms", "jitter_ms", "loss_pct", "duplicate_pct", "reorder_pct", @@ -758,6 +784,42 @@ def load_scenario(path: str) -> Scenario: ) expected[child] = parent s.assertions.tree_parents = TreeParentsAssertion(expected=expected) + if "baseline" in asrt: + bl = asrt["baseline"] + _reject_unknown(bl, _ASSERTION_KEYS["baseline"], "assertions.baseline") + vals = {} + for key in _ASSERTION_KEYS["baseline"]: + if key not in bl: + continue + val = bl[key] + if isinstance(val, bool) or not isinstance(val, int): + raise ValueError( + f"assertions.baseline.{key}: must be an integer, " + f"got {val!r}" + ) + floor = 1 if key == "max_roots" else 0 + if val < floor: + raise ValueError( + f"assertions.baseline.{key}: must be at least {floor}, " + f"got {val}" + ) + vals[key] = val + if not vals: + raise ValueError( + "assertions.baseline: set at least one of " + + ", ".join(sorted(_ASSERTION_KEYS["baseline"])) + + "; a block with none asserts nothing" + ) + if vals.get("min_nodes_parented", 0) > 0 or vals.get("max_roots"): + n = s.topology.num_nodes + if vals.get("min_nodes_parented", 0) > n - 1: + raise ValueError( + f"assertions.baseline.min_nodes_parented: " + f"{vals['min_nodes_parented']} exceeds {n - 1}, the most a " + f"{n}-node mesh can reach — the root is its own parent, so " + f"this could never pass" + ) + s.assertions.baseline = BaselineAssertion(**vals) # Logging section lg = raw.get("logging", {})