diff --git a/testing/chaos/scenarios/churn-mixed.yaml b/testing/chaos/scenarios/churn-mixed.yaml index 961ccf2..54a0864 100644 --- a/testing/chaos/scenarios/churn-mixed.yaml +++ b/testing/chaos/scenarios/churn-mixed.yaml @@ -67,6 +67,27 @@ bandwidth: enabled: true tiers_mbps: [1, 10, 100, 1000] +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about which nodes survive the churn; it exists so that +# a run in which the mesh never formed cannot report success, which +# until now it could, because this scenario carried no assertions at +# all. +# +# This one is calibrated rather than derived, because a scenario that +# stops and starts nodes on purpose does not hold a single spanning +# tree. The six provably-completed archived runs end with two or three +# distinct roots, seven or eight of ten nodes parented, and fifteen to +# twenty sessions. The floors below sit one step outside those ranges +# so an unlucky run does not go red, which leaves them catching a mesh +# that collapsed rather than one that churned. Tighten them only +# against a larger sample than six runs. +assertions: + baseline: + min_nodes_reporting: 10 + max_roots: 4 + min_nodes_parented: 6 + min_sessions: 10 + logging: rust_log: "debug" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/depth-vs-cost.yaml b/testing/chaos/scenarios/depth-vs-cost.yaml index 4f779c2..47d5e4a 100644 --- a/testing/chaos/scenarios/depth-vs-cost.yaml +++ b/testing/chaos/scenarios/depth-vs-cost.yaml @@ -83,6 +83,21 @@ traffic: duration_secs: {min: 5, max: 10} parallel_streams: 2 +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about which parent n04 chooses, which is unspecified; +# it exists so that a run in which the mesh never formed cannot report +# success, which until now it could, +# because this scenario carried no assertions at all. +# +# Four nodes, one root, three parented and four sessions in all five +# provably-completed archived runs. +assertions: + baseline: + min_nodes_reporting: 4 + max_roots: 1 + min_nodes_parented: 3 + min_sessions: 2 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/ethernet-mesh.yaml b/testing/chaos/scenarios/ethernet-mesh.yaml index 00cf009..d1ceb44 100644 --- a/testing/chaos/scenarios/ethernet-mesh.yaml +++ b/testing/chaos/scenarios/ethernet-mesh.yaml @@ -62,6 +62,21 @@ link_flaps: traffic: enabled: false +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about Ethernet link behaviour under flaps; it exists +# so that a run in which the mesh never formed cannot report success, +# which until now it could, because this scenario carried no assertions +# at all. +# +# Six nodes, one root, five parented in all six provably-completed +# archived runs. The other assertion covering Ethernet transport in +# CI; see ethernet-only for why that matters. +assertions: + baseline: + min_nodes_reporting: 6 + max_roots: 1 + min_nodes_parented: 5 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/ethernet-only.yaml b/testing/chaos/scenarios/ethernet-only.yaml index ee34089..39a823b 100644 --- a/testing/chaos/scenarios/ethernet-only.yaml +++ b/testing/chaos/scenarios/ethernet-only.yaml @@ -41,6 +41,25 @@ link_flaps: traffic: enabled: false +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about what Ethernet transport does; it exists so that +# a run in which the mesh never formed cannot report success, which +# until now it could, because this scenario carried no assertions at +# all. +# +# A 4-node mesh forms a spanning tree: one root and three nodes +# with a parent. All six provably-completed archived runs show exactly +# that, so these are the shape of a converged mesh rather than a +# tolerance fitted to observations. This is currently the only +# assertion covering Ethernet transport anywhere in CI: the three +# tests in src/node/tests/ethernet.rs are #[ignore]d for requiring +# CAP_NET_RAW and neither runner passes --ignored. +assertions: + baseline: + min_nodes_reporting: 4 + max_roots: 1 + min_nodes_parented: 3 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/mixed-technology.yaml b/testing/chaos/scenarios/mixed-technology.yaml index bf52c35..9e52923 100644 --- a/testing/chaos/scenarios/mixed-technology.yaml +++ b/testing/chaos/scenarios/mixed-technology.yaml @@ -47,8 +47,9 @@ # which, and until then this scenario proves nothing about parent choice. # # That leaves this scenario asserting nothing about its own subject. The -# baseline convergence assertion is the right floor for it and is tracked -# separately; this note is here so the gap is visible until that lands. +# baseline block further down holds it to forming a tree and carrying +# traffic, which is a floor and not a substitute: nothing here checks that +# the tree it forms is the one these comments describe. scenario: name: "mixed-technology" @@ -117,6 +118,23 @@ traffic: duration_secs: {min: 5, max: 15} parallel_streams: 4 +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about parent choice, which this scenario cannot +# currently assert; it exists so that a run in which the mesh never formed +# cannot report success, which until now it could, +# because this scenario carried no assertions at all. +# +# Ten nodes, one root, nine parented and six to eight sessions across +# the five provably-completed archived runs. The session floor is set +# below the observed minimum deliberately: it is here to catch a mesh +# that carries no traffic at all, not to pin a throughput. +assertions: + baseline: + min_nodes_reporting: 10 + max_roots: 1 + min_nodes_parented: 9 + min_sessions: 3 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/smoke-10.yaml b/testing/chaos/scenarios/smoke-10.yaml index cc36915..9a63958 100644 --- a/testing/chaos/scenarios/smoke-10.yaml +++ b/testing/chaos/scenarios/smoke-10.yaml @@ -22,6 +22,20 @@ link_flaps: traffic: enabled: false +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about any particular subsystem; it exists so that a +# run in which the mesh never formed cannot report success, which until +# now it could, because this scenario carried no assertions at all. +# +# Ten nodes, one root, nine parented in all six provably-completed +# archived runs. As the smoke test this is the scenario whose silent +# success was least defensible. +assertions: + baseline: + min_nodes_reporting: 10 + max_roots: 1 + min_nodes_parented: 9 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/tcp-mesh.yaml b/testing/chaos/scenarios/tcp-mesh.yaml index 0010a29..09dda34 100644 --- a/testing/chaos/scenarios/tcp-mesh.yaml +++ b/testing/chaos/scenarios/tcp-mesh.yaml @@ -61,6 +61,19 @@ link_flaps: traffic: enabled: false +# Baseline: the mesh came up, agreed on a root, and took parents. This +# asserts nothing about TCP transport specifically; it exists so that a +# run in which the mesh never formed cannot report success, which until +# now it could, because this scenario carried no assertions at all. +# +# Six nodes, one root, five parented in all six provably-completed +# archived runs. +assertions: + baseline: + min_nodes_reporting: 6 + max_roots: 1 + min_nodes_parented: 5 + logging: rust_log: "info" output_dir: "./sim-results" diff --git a/testing/chaos/sim/assertions.py b/testing/chaos/sim/assertions.py index 467da25..1fc636a 100644 --- a/testing/chaos/sim/assertions.py +++ b/testing/chaos/sim/assertions.py @@ -19,6 +19,7 @@ from dataclasses import dataclass from .control import snapshot_all_bloom from .scenario import ( + BaselineAssertion, BloomSendRateAssertion, CongestionSignalsAssertion, MaxErrorsAssertion, @@ -173,6 +174,63 @@ def evaluate_max_parent_switches( ) +def evaluate_baseline( + cfg: BaselineAssertion, + snapshot: dict | None, + sessions: int, +) -> AssertionOutcome: + """Floor on the mesh having formed: nodes answered, agreed a root, took parents.""" + if not snapshot: + return AssertionOutcome( + name="baseline", + passed=False, + detail=( + "FAIL baseline: no final tree snapshot was taken, so nothing " + "about the mesh was observed. This is a harness failure." + ), + ) + + reporting = len(snapshot) + roots = {v.get("root") for v in snapshot.values() if v.get("root")} + parented = sum( + 1 for v in snapshot.values() + if v.get("parent") and v.get("parent") != v.get("my_node_addr") + ) + + parts, failures = [], [] + + def note(ok, text): + parts.append(text) + if not ok: + failures.append(text) + + if cfg.min_nodes_reporting is not None: + note(reporting >= cfg.min_nodes_reporting, + f"{reporting} node(s) answered (need {cfg.min_nodes_reporting})") + if cfg.max_roots is not None: + note(len(roots) <= cfg.max_roots and len(roots) >= 1, + f"{len(roots)} distinct root(s) (allowed {cfg.max_roots})") + if cfg.min_nodes_parented is not None: + note(parented >= cfg.min_nodes_parented, + f"{parented} node(s) have a parent (need {cfg.min_nodes_parented})") + if cfg.min_sessions is not None: + note(sessions >= cfg.min_sessions, + f"{sessions} session(s) established (need {cfg.min_sessions})") + + summary = "; ".join(parts) + if failures: + return AssertionOutcome( + name="baseline", + passed=False, + detail=( + f"FAIL baseline: {'; '.join(failures)}. Full: {summary}" + ), + ) + return AssertionOutcome( + name="baseline", passed=True, detail=f"PASS baseline: {summary}" + ) + + def evaluate_tree_parents( cfg: TreeParentsAssertion, snapshot: dict | None, diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index 0327508..182a252 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -15,11 +15,12 @@ from datetime import datetime from .assertions import ( AssertionOutcome, BloomSendRateMonitor, + evaluate_baseline, evaluate_congestion_signals, evaluate_max_errors, - evaluate_tree_parents, evaluate_max_parent_switches, evaluate_min_parent_switches, + evaluate_tree_parents, ) from .compose import generate_compose from .config_gen import write_configs @@ -624,6 +625,17 @@ class SimRunner: else: log.error("%s", outcome.detail) + bl_cfg = self.scenario.assertions.baseline + if bl_cfg is not None: + outcome = evaluate_baseline( + bl_cfg, self.final_tree, len(result.sessions_established) + ) + self.assertion_outcomes.append(outcome) + if outcome.passed: + log.info("%s", outcome.detail) + else: + log.error("%s", outcome.detail) + tp_cfg = self.scenario.assertions.tree_parents if tp_cfg is not None: outcome = evaluate_tree_parents(tp_cfg, self.final_tree) diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py index 80d80f7..d7e30c4 100644 --- a/testing/chaos/sim/scenario.py +++ b/testing/chaos/sim/scenario.py @@ -224,6 +224,28 @@ class MaxParentSwitchesAssertion: node: str | None = None +@dataclass +class BaselineAssertion: + """Floor on the mesh having formed at all. + + Deliberately weak and deliberately universal. It does not describe any + scenario's subject; it says the nodes came up, agreed on a root, and + took parents. Its value is that a scenario with no other assertion + still cannot pass while the mesh is dead, which is the state twelve of + thirteen scenarios were previously unable to distinguish from success. + + ``max_roots`` is how many distinct root values the snapshot may carry. + One means the whole mesh agreed; a churn scenario that partitions on + purpose needs a higher number, and setting it above one is a statement + that partition is expected rather than an oversight. + """ + + min_nodes_reporting: int | None = None + max_roots: int | None = None + min_nodes_parented: int | None = None + min_sessions: int | None = None + + @dataclass class TreeParentsAssertion: """Expected parent per node in the final tree snapshot. @@ -294,6 +316,7 @@ class AssertionsConfig: max_errors: MaxErrorsAssertion | None = None congestion_signals: CongestionSignalsAssertion | None = None tree_parents: TreeParentsAssertion | None = None + baseline: BaselineAssertion | None = None @dataclass @@ -373,7 +396,7 @@ _SECTION_KEYS = { "link_swap.edges[]": {"edge", "policy"}, "assertions": { "bloom_send_rate", "min_parent_switches", "max_parent_switches", - "max_errors", "congestion_signals", "tree_parents", + "max_errors", "congestion_signals", "tree_parents", "baseline", }, "logging": {"rust_log", "output_dir"}, } @@ -385,6 +408,9 @@ _ASSERTION_KEYS = { "congestion_signals": { "min_nodes_detected", "min_nodes_ce_forwarded", "min_nodes_ce_received", }, + "baseline": { + "min_nodes_reporting", "max_roots", "min_nodes_parented", "min_sessions", + }, } _NETEM_POLICY_KEYS = { "delay_ms", "jitter_ms", "loss_pct", "duplicate_pct", "reorder_pct", @@ -758,6 +784,42 @@ def load_scenario(path: str) -> Scenario: ) expected[child] = parent s.assertions.tree_parents = TreeParentsAssertion(expected=expected) + if "baseline" in asrt: + bl = asrt["baseline"] + _reject_unknown(bl, _ASSERTION_KEYS["baseline"], "assertions.baseline") + vals = {} + for key in _ASSERTION_KEYS["baseline"]: + if key not in bl: + continue + val = bl[key] + if isinstance(val, bool) or not isinstance(val, int): + raise ValueError( + f"assertions.baseline.{key}: must be an integer, " + f"got {val!r}" + ) + floor = 1 if key == "max_roots" else 0 + if val < floor: + raise ValueError( + f"assertions.baseline.{key}: must be at least {floor}, " + f"got {val}" + ) + vals[key] = val + if not vals: + raise ValueError( + "assertions.baseline: set at least one of " + + ", ".join(sorted(_ASSERTION_KEYS["baseline"])) + + "; a block with none asserts nothing" + ) + if vals.get("min_nodes_parented", 0) > 0 or vals.get("max_roots"): + n = s.topology.num_nodes + if vals.get("min_nodes_parented", 0) > n - 1: + raise ValueError( + f"assertions.baseline.min_nodes_parented: " + f"{vals['min_nodes_parented']} exceeds {n - 1}, the most a " + f"{n}-node mesh can reach — the root is its own parent, so " + f"this could never pass" + ) + s.assertions.baseline = BaselineAssertion(**vals) # Logging section lg = raw.get("logging", {})