From 2bc0345174fddd6f6d64b4b2fbd31cfc6a53a14f Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Thu, 23 Jul 2026 18:58:49 +0000 Subject: [PATCH] Make the cost-based chaos assertions reliable under parallel CI load cost-avoidance and mixed-technology decide a parent by measured link cost (etx * (1 + srtt_ms/100)). Under the local CI's 4-way-parallel chaos the fast fiber link's measured srtt spikes from host scheduling and can exceed the Bluetooth link's, flipping the choice and reddening a run that proves nothing about the daemon. The old margin was ~0.4 cost units, which a ~24ms one-sided scheduling blip closes. Widen the Bluetooth delay to 150-250ms so the margin dwarfs any plausible one-sided spike. The link still establishes well within the handshake budget (a ~500ms RTT against a 30s stale-handshake window), so the losing candidate is still a real, established peer rather than an absent one. Add a netem mutation exclude_edges option and use it in mixed-technology to pin the two links n08 chooses between, so a random degradation can't flip the asserted comparison either. An unknown excluded edge is a hard error, since a silent no-op would reintroduce the flakiness it exists to remove. Validated by running both scenarios repeatedly under heavier-than-CI concurrent load: the parent choice holds every time. --- testing/chaos/scenarios/cost-avoidance.yaml | 27 +++++++++++---- testing/chaos/scenarios/mixed-technology.yaml | 34 +++++++++++++------ testing/chaos/sim/netem.py | 25 +++++++++++++- testing/chaos/sim/scenario.py | 8 ++++- 4 files changed, 75 insertions(+), 19 deletions(-) diff --git a/testing/chaos/scenarios/cost-avoidance.yaml b/testing/chaos/scenarios/cost-avoidance.yaml index dec22fe..d9c529b 100644 --- a/testing/chaos/scenarios/cost-avoidance.yaml +++ b/testing/chaos/scenarios/cost-avoidance.yaml @@ -13,9 +13,21 @@ # n04 (test subject) # # n04 has two candidate parents at depth 1: n02 (via Bluetooth L2CAP) -# and n03 (via fiber). Cost-based selection should pick n03 because: -# effective_depth(n02) = 1 + ~1.4 (Bluetooth) = ~2.4 -# effective_depth(n03) = 1 + ~1.01 (fiber) = ~2.01 +# and n03 (via fiber). Cost-based selection should pick n03 because +# link_cost = etx * (1 + srtt_ms/100), so the far slower Bluetooth link +# costs far more: +# effective_depth(n02) = 1 + ~4.5 (Bluetooth, ~150-250ms) = ~5.5 +# effective_depth(n03) = 1 + ~1.05 (fiber, ~1-5ms) = ~2.05 +# +# The Bluetooth delay was widened deliberately, and the size is set by load, +# not by realism. At the original 15-40ms the cost margin was ~0.4, and even +# 80-120ms was not enough: under the CI's parallel-chaos contention the +# *measured* srtt of the fiber link spikes (host scheduling, not netem) and +# was observed to exceed a 120ms Bluetooth base, flipping n04 to n02. At +# 150-250ms the gap dwarfs any plausible one-sided fiber spike, so the +# assertion below is a reliable verdict rather than a coin toss under load. +# n04 still establishes the n02 link (it was parented to it when this flaked), +# so the choice remains real rather than vacuous. # # Validation: tree snapshot shows n04's parent is n03. @@ -44,11 +56,14 @@ netem: jitter_ms: [0, 1] loss_pct: [0, 0.5] link_policies: - # Bluetooth (L2CAP) link from n02 to n04 + # Bluetooth (L2CAP) link from n02 to n04. Delay widened to 150-250ms so the + # fiber-vs-Bluetooth cost margin survives CI load spikes on the fiber + # probe's measured latency (see header). Well within handshake timeouts, so + # the n02 link still establishes and the choice stays real. - edges: ["n02-n04"] policy: - delay_ms: [15, 40] - jitter_ms: [5, 15] + delay_ms: [150, 250] + jitter_ms: [15, 30] loss_pct: [2, 8] link_flaps: diff --git a/testing/chaos/scenarios/mixed-technology.yaml b/testing/chaos/scenarios/mixed-technology.yaml index fb9afd5..ef32e9e 100644 --- a/testing/chaos/scenarios/mixed-technology.yaml +++ b/testing/chaos/scenarios/mixed-technology.yaml @@ -44,8 +44,8 @@ # # Netem mutation shifts fiber-only links between normal and degraded. # -# NEITHER TEST SUBJECT IS ASSERTED YET, and the reason has been found and -# fixed at the root rather than papered over here. +# n08 IS now asserted (below); n06 is not, and both reasons trace to the root, +# which was found and fixed rather than papered over here. # # The diagram above puts n01 at the top, and both test subjects only make # sense in a tree rooted there. The mesh does not root by topology: it roots @@ -64,13 +64,15 @@ # # `topology.pin_root` (default true) now assigns identities in NodeAddr order, # so n01 holds the smallest and the diagram describes the tree that forms. -# Both criteria should therefore become reachable and assertable. +# Under that root n08 takes fiber n03 as its subject assertion (below) now +# checks. n06 stays unasserted: its "fiber n03" was a WiFi link (see the +# corrected note above), so it was never a founded criterion. Tracked in +# ISSUE-2026-0085. # -# STILL TO DO: run this under the pinned root, confirm n06 and n08 both take -# n03, and encode `tree_parents` for them. Until then the baseline block below -# is a floor and not a substitute — it holds the mesh to forming a tree and -# carrying traffic, and nothing here yet checks that the tree is the one these -# comments describe. Tracked in ISSUE-2026-0085. +# The n08 assertion is made reliable, not merely reachable: the Bluetooth +# contrast is widened (netem below) and the two links n08 chooses between are +# excluded from the netem mutation, so neither host load nor a random +# degradation can flip the asserted comparison. scenario: name: "mixed-technology" @@ -105,11 +107,15 @@ netem: jitter_ms: [0, 1] loss_pct: [0, 0.5] link_policies: - # Bluetooth (L2CAP) links + # Bluetooth (L2CAP) links. Delay widened to 150-250ms so the + # fiber-vs-Bluetooth cost margin on n08's choice (fiber n03 vs Bluetooth + # n04) dwarfs any load-induced srtt spike on the fiber probe, matching + # cost-avoidance. Combined with excluding n03-n08/n04-n08 from the mutation + # (below), n08's asserted comparison is robust to both load and churn. - edges: ["n02-n06", "n04-n08"] policy: - delay_ms: [15, 40] - jitter_ms: [5, 15] + delay_ms: [150, 250] + jitter_ms: [15, 30] loss_pct: [2, 8] # WiFi links (moderate latency, low loss) - edges: ["n03-n06", "n08-n10"] @@ -120,6 +126,12 @@ netem: mutation: interval_secs: {min: 30, max: 60} fraction: 0.2 + # Never mutate the two links n08's tree_parents assertion depends on: the + # degraded policy (50-100ms) could otherwise slow the fiber n03-n08 link + # enough to flip n08 to n04 independently of the widened Bluetooth contrast, + # making the assertion a coin toss. The mutation still churns the other ten + # links, so the dynamic behaviour it exists to exercise is unchanged. + exclude_edges: ["n03-n08", "n04-n08"] policies: normal: delay_ms: [1, 10] diff --git a/testing/chaos/sim/netem.py b/testing/chaos/sim/netem.py index 45720a8..3ac8d0e 100644 --- a/testing/chaos/sim/netem.py +++ b/testing/chaos/sim/netem.py @@ -144,6 +144,27 @@ class NetemManager: len(self._edge_overrides), ) + # Edges the periodic mutation must never touch (canonical "nXX-nYY"). + # Unlike a link_policy typo, which merely fails to shape a link, a typo + # here would silently leave an asserted link unprotected and reintroduce + # the exact flakiness the exclusion exists to remove — so an unknown + # edge is a hard error, not a warning. + self._mutation_exclude: set[str] = set() + for edge_str in config.mutation.exclude_edges: + canonical = "-".join(sorted(edge_str.split("-"))) + if canonical not in topo_edge_strs: + raise ValueError( + f"netem.mutation.exclude_edges references {edge_str!r}, " + "which is not an edge in the topology" + ) + self._mutation_exclude.add(canonical) + if self._mutation_exclude: + log.info( + "Mutation excludes %d edge(s): %s", + len(self._mutation_exclude), + ", ".join(sorted(self._mutation_exclude)), + ) + def _htb_rate(self, node_id: str, peer_id: str) -> str: """Return the HTB rate string for a link direction.""" rate = self._edge_rates.get((node_id, peer_id), 0) @@ -370,10 +391,12 @@ class NetemManager: if not self.config.mutation.policies: return - # Only consider edges where both endpoints are up + # Only consider edges where both endpoints are up, and never the + # explicitly excluded ones (links an assertion depends on). live_edges = [ (a, b) for a, b in self.topology.edges if a not in self.down_nodes and b not in self.down_nodes + and "-".join(sorted([a, b])) not in self._mutation_exclude ] if not live_edges: return diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py index eb2e66e..c1f3eaf 100644 --- a/testing/chaos/sim/scenario.py +++ b/testing/chaos/sim/scenario.py @@ -73,6 +73,11 @@ class NetemMutationConfig: interval_secs: Range = field(default_factory=lambda: Range(15, 30)) fraction: float = 0.3 policies: dict[str, NetemPolicy] = field(default_factory=dict) + # Edges the periodic mutation must never touch, "nXX-nYY" strings. Use it + # to pin the links a tree_parents assertion depends on so a random + # degradation cannot flip the very comparison being asserted. Validated + # against the topology in NetemManager — an unknown edge is a hard error. + exclude_edges: list[str] = field(default_factory=list) @dataclass @@ -388,7 +393,7 @@ _SECTION_KEYS = { }, "netem": {"enabled", "default_policy", "link_policies", "mutation"}, "netem.link_policies[]": {"edges", "policy", "policy_name"}, - "netem.mutation": {"interval_secs", "fraction", "policies"}, + "netem.mutation": {"interval_secs", "fraction", "policies", "exclude_edges"}, "link_flaps": { "enabled", "interval_secs", "max_down_links", "down_duration_secs", "protect_connectivity", @@ -549,6 +554,7 @@ def load_scenario(path: str) -> Scenario: mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs" ) s.netem.mutation.fraction = float(mc.get("fraction", 0.3)) + s.netem.mutation.exclude_edges = list(mc.get("exclude_edges", [])) if "policies" in mc: s.netem.mutation.policies = { name: _parse_netem_policy(pdata, f"netem.mutation.policies.{name}")