Files
fips/testing/chaos/sim/scenario.py
T
Johnathan Corgan 73e6917341 Give cost-stability the parent-switch ceiling its criterion described
The scenario's success criterion existed only as a comment: count
"Parent switched" in n04's log, expect at most 5. Nothing read it, and
the scenario declared no assertions at all, so its exit code could only
report that the mesh came up and nothing crashed.

Add a max_parent_switches assertion and wire that criterion to it.

The assertion takes an optional node scope, which is the part that
matters. The existing sibling counts mesh-wide, and a criterion written
about one node's log is a different quantity from the sum over every
node: across archived runs of this scenario n04 is 1 or 2 while the
mesh-wide total ranges 3 to 6. Asserting the sum against a per-node
threshold would check something other than what was specified, and at
this threshold would also have failed several runs that were fine.

Counting nothing must not read as stability. A node id absent from the
topology fails explicitly rather than matching zero log lines and
sailing under the ceiling, and a scenario that declares a parent-switch
assertion while setting a log level that suppresses the events it counts
is rejected at load rather than passing vacuously after a full run. The
parse rejects a mistyped key, a missing, negative, boolean or
non-integer ceiling, and a node written empty or as a non-string -- that
last one because YAML renders a bare `node:` as null, which would
silently revert to the mesh-wide count.

The scenario comment now records what the threshold is worth against
the 11 completed archived runs rather than a single sample: 5 is well
above anything observed, the daemon's own parent hold-down caps
switches near 6 per run, and nearly every switch lands during initial
tree formation before any mutation fires. So this catches only
near-pathological reparenting. Tightening to 3 would make it a real
detector, but that changes the stated criterion rather than fixing it,
so it is left as a decision.

Verified against a live run: the assertion evaluates n04 at 2 against
mesh-wide 4 and passes, and with the ceiling temporarily set to 0 the
scenario exits 3, the assertion-failure code, which it previously could
not reach.
2026-07-23 05:19:36 +00:00

770 lines
30 KiB
Python

"""Scenario YAML loading and validation."""
from __future__ import annotations
import os
from dataclasses import dataclass, field
import yaml
@dataclass
class Range:
"""A min/max range for stochastic parameters."""
min: float
max: float
def validate(self, name: str):
if self.min > self.max:
raise ValueError(f"{name}: min ({self.min}) > max ({self.max})")
if self.min < 0:
raise ValueError(f"{name}: min ({self.min}) must be >= 0")
VALID_TRANSPORTS = ("udp", "ethernet", "tcp")
@dataclass
class TopologyConfig:
num_nodes: int = 10
algorithm: str = "random_geometric"
params: dict = field(default_factory=dict)
ensure_connected: bool = True
subnet: str = "172.20.0.0/24"
ip_start: int = 10
default_transport: str = "udp"
# Optional transport mix for random topologies: {transport: weight}.
# When set, each edge is randomly assigned a transport based on weights.
# Only valid for non-explicit algorithms (explicit uses per-edge syntax).
transport_mix: dict[str, float] | None = None
@dataclass
class NetemPolicy:
delay_ms: tuple[float, float] = (0, 0)
jitter_ms: tuple[float, float] = (0, 0)
loss_pct: tuple[float, float] = (0, 0)
duplicate_pct: tuple[float, float] = (0, 0)
reorder_pct: tuple[float, float] = (0, 0)
corrupt_pct: tuple[float, float] = (0, 0)
@dataclass
class NetemMutationConfig:
interval_secs: Range = field(default_factory=lambda: Range(15, 30))
fraction: float = 0.3
policies: dict[str, NetemPolicy] = field(default_factory=dict)
@dataclass
class LinkPolicyOverride:
"""Per-edge netem policy override.
Edges are specified as "nXX-nYY" strings in canonical form (sorted
alphabetically). Either ``policy`` (inline) or ``policy_name``
(reference to a named mutation policy) must be set, not both.
"""
edges: list[str] = field(default_factory=list)
policy: NetemPolicy | None = None
policy_name: str | None = None
@dataclass
class NetemConfig:
enabled: bool = False
default_policy: NetemPolicy = field(default_factory=NetemPolicy)
link_policies: list[LinkPolicyOverride] = field(default_factory=list)
mutation: NetemMutationConfig = field(default_factory=NetemMutationConfig)
@dataclass
class LinkFlapsConfig:
enabled: bool = False
interval_secs: Range = field(default_factory=lambda: Range(20, 60))
max_down_links: int = 2
down_duration_secs: Range = field(default_factory=lambda: Range(10, 30))
protect_connectivity: bool = True
@dataclass
class TrafficConfig:
enabled: bool = False
max_concurrent: int = 3
interval_secs: Range = field(default_factory=lambda: Range(10, 30))
duration_secs: Range = field(default_factory=lambda: Range(5, 15))
parallel_streams: int = 4
@dataclass
class NodeChurnConfig:
enabled: bool = False
interval_secs: Range = field(default_factory=lambda: Range(60, 180))
max_down_nodes: int = 1
down_duration_secs: Range = field(default_factory=lambda: Range(30, 90))
protect_connectivity: bool = True
@dataclass
class PeerChurnConfig:
"""Peer-level topology churn via connect/disconnect commands.
When enabled, periodically disconnects a random active link and
connects a random unconnected node pair, causing the mesh topology
to evolve over time.
"""
enabled: bool = False
interval_secs: Range = field(default_factory=lambda: Range(8, 12))
ephemeral_fraction: float = 0.0
@dataclass
class BandwidthConfig:
"""Per-link bandwidth pacing via HTB rate limiting.
When enabled, each link is randomly assigned a rate from the tiers list.
# TODO: Add structured bandwidth assignment (e.g., per-node roles,
# asymmetric uplink/downlink, tiered by topology distance, or
# time-varying bandwidth mutations similar to netem mutation).
"""
enabled: bool = False
tiers_mbps: list[int] = field(default_factory=lambda: [1, 10, 100, 1000])
@dataclass
class IngressConfig:
"""Per-link ingress policing via tc ingress qdisc + policer.
When enabled, each link gets an ingress policer that drops inbound
packets exceeding the configured rate. Unlike egress HTB (which
paces packets smoothly), the policer drops excess packets at the
kernel level, creating bursty arrival patterns that can overflow
small socket buffers and trigger SO_RXQ_OVFL kernel drops.
"""
enabled: bool = False
tiers_kbps: list[int] = field(default_factory=lambda: [1000])
burst_bytes: int = 32000
@dataclass
class LinkSwapEdge:
"""One edge in a link-swap rotation.
Edge is a canonical "nXX-nYY" string. ``policy`` names a policy
in ``LinkSwapConfig.policies``.
"""
edge: str = ""
policy: str = ""
@dataclass
class LinkSwapConfig:
"""Deterministic asymmetric link-cost flapping.
On each ``interval_secs``, the policies on every pair of edges in
``edges`` are swapped (cyclically rotated by one position). This
differs from ``link_flaps`` (random link-down events) and
``netem.mutation`` (random per-edge policy mutation) by being a
deterministic, periodic flip between two named netem policies on
a fixed set of edges.
Used to drive a downstream node to repeatedly switch parents on
a fixed cadence — exercises the spanning-tree rebalance path
without the noise of a random mutation walk.
"""
enabled: bool = False
interval_secs: float = 4.0
policies: dict[str, NetemPolicy] = field(default_factory=dict)
edges: list[LinkSwapEdge] = field(default_factory=list)
@dataclass
class BloomSendRateAssertion:
"""Trailing-window ceiling on per-node ``stats.bloom.sent`` delta."""
window_secs: int = 30
max_per_node: int = 30
@dataclass
class MinParentSwitchesAssertion:
"""Sanity guard: total parent switches across the run must be at
least ``min_total``. Used to detect a misconfigured harness where
the flap inducer is firing but the topology never produces a real
parent-switch event (e.g., wrong root election).
"""
min_total: int = 1
@dataclass
class MaxParentSwitchesAssertion:
"""Stability ceiling: fail if parent switches exceed ``max_total``.
``node`` scopes the count to a single node id (e.g. ``n04``) instead
of the mesh-wide total. The distinction is not cosmetic: a criterion
written about one node's log is a different number from the sum over
every node, and checking the sum against a per-node threshold
silently tests something other than what was specified.
"""
max_total: int = 0
node: str | None = None
@dataclass
class AssertionsConfig:
"""Optional post-run assertions evaluated against control-socket data."""
bloom_send_rate: BloomSendRateAssertion | None = None
min_parent_switches: MinParentSwitchesAssertion | None = None
max_parent_switches: MaxParentSwitchesAssertion | None = None
@dataclass
class LoggingConfig:
rust_log: str = "info"
output_dir: str = "./sim-results"
@dataclass
class Scenario:
name: str = "unnamed"
seed: int = 42
duration_secs: int = 120
topology: TopologyConfig = field(default_factory=TopologyConfig)
netem: NetemConfig = field(default_factory=NetemConfig)
link_flaps: LinkFlapsConfig = field(default_factory=LinkFlapsConfig)
traffic: TrafficConfig = field(default_factory=TrafficConfig)
node_churn: NodeChurnConfig = field(default_factory=NodeChurnConfig)
peer_churn: PeerChurnConfig = field(default_factory=PeerChurnConfig)
bandwidth: BandwidthConfig = field(default_factory=BandwidthConfig)
ingress: IngressConfig = field(default_factory=IngressConfig)
link_swap: LinkSwapConfig = field(default_factory=LinkSwapConfig)
assertions: AssertionsConfig = field(default_factory=AssertionsConfig)
logging: LoggingConfig = field(default_factory=LoggingConfig)
# Raw YAML dict appended to each generated FIPS node config.
# Allows scenarios to override any FIPS config parameter
# (e.g., node.tree.reeval_interval_secs).
fips_overrides: dict = field(default_factory=dict)
# Keys each fixed-schema section understands. A key absent from these sets is a
# typo, and silently ignoring it leaves the block default-constructed while the
# YAML still looks correct — a mistyped "assertion:" disarms a scenario's only
# assertions, a mistyped "link_flap:" turns off the chaos it was meant to inject.
#
# Three mappings are deliberately NOT listed, because their keys are names the
# scenario author chooses rather than a schema: netem.mutation.policies,
# link_swap.policies and topology.transport_mix. Two sub-trees are passed through
# whole and are likewise not checked: fips_overrides and topology.params.
#
# NOTE: adding a new assertion type means registering it in TWO places below —
# _SECTION_KEYS["assertions"] (so the block accepts its name) and _ASSERTION_KEYS
# (so its own members are checked) — or scenarios using it are rejected at load.
_TOP_KEYS = {
"scenario", "topology", "netem", "link_flaps", "traffic", "node_churn",
"peer_churn", "bandwidth", "ingress", "link_swap", "assertions", "logging",
"fips_overrides",
}
_SECTION_KEYS = {
"scenario": {"name", "seed", "duration_secs"},
"topology": {
"num_nodes", "algorithm", "params", "ensure_connected", "subnet",
"ip_start", "default_transport", "transport_mix",
},
"netem": {"enabled", "default_policy", "link_policies", "mutation"},
"netem.link_policies[]": {"edges", "policy", "policy_name"},
"netem.mutation": {"interval_secs", "fraction", "policies"},
"link_flaps": {
"enabled", "interval_secs", "max_down_links", "down_duration_secs",
"protect_connectivity",
},
"traffic": {
"enabled", "max_concurrent", "interval_secs", "duration_secs",
"parallel_streams",
},
"node_churn": {
"enabled", "interval_secs", "max_down_nodes", "down_duration_secs",
"protect_connectivity",
},
"peer_churn": {"enabled", "interval_secs", "ephemeral_fraction"},
"bandwidth": {"enabled", "tiers_mbps"},
"ingress": {"enabled", "tiers_kbps", "burst_bytes"},
"link_swap": {"enabled", "interval_secs", "policies", "edges"},
"link_swap.edges[]": {"edge", "policy"},
"assertions": {
"bloom_send_rate", "min_parent_switches", "max_parent_switches",
},
"logging": {"rust_log", "output_dir"},
}
_ASSERTION_KEYS = {
"bloom_send_rate": {"window_secs", "max_per_node"},
"min_parent_switches": {"min_total"},
"max_parent_switches": {"max_total", "node"},
}
_NETEM_POLICY_KEYS = {
"delay_ms", "jitter_ms", "loss_pct", "duplicate_pct", "reorder_pct",
"corrupt_pct",
}
_RANGE_KEYS = {"min", "max"}
def _reject_unknown(data, known, context: str):
"""Raise unless every key in `data` is one the loader understands."""
if data is None:
raise ValueError(
f"{context}: section is present but empty; give it a body or remove it"
)
if not isinstance(data, dict):
raise ValueError(f"{context}: expected a mapping, got {type(data).__name__}")
unknown = sorted(str(k) for k in set(data) - set(known))
if unknown:
raise ValueError(
f"{context}: unknown key(s): {', '.join(unknown)} "
f"(known: {', '.join(sorted(known))})"
)
def _parse_range(data, name: str) -> Range:
"""Parse a {min, max} dict into a Range."""
if isinstance(data, dict):
_reject_unknown(data, _RANGE_KEYS, name)
return Range(min=float(data["min"]), max=float(data["max"]))
raise ValueError(f"{name}: expected {{min, max}} dict, got {type(data).__name__}")
def _parse_netem_policy(data: dict, name: str = "netem policy") -> NetemPolicy:
"""Parse a netem policy from a dict with [min, max] lists or {min, max} dicts."""
_reject_unknown(data, _NETEM_POLICY_KEYS, name)
policy = NetemPolicy()
for attr in (
"delay_ms",
"jitter_ms",
"loss_pct",
"duplicate_pct",
"reorder_pct",
"corrupt_pct",
):
if attr in data:
val = data[attr]
if isinstance(val, list) and len(val) == 2:
setattr(policy, attr, (float(val[0]), float(val[1])))
elif isinstance(val, dict):
setattr(policy, attr, (float(val["min"]), float(val["max"])))
else:
raise ValueError(f"netem policy {attr}: expected [min, max] or {{min, max}}")
return policy
def load_scenario(path: str) -> Scenario:
"""Load and validate a scenario from a YAML file."""
with open(path) as f:
raw = yaml.safe_load(f)
if raw is None:
raise ValueError(f"{path}: file is empty")
_reject_unknown(raw, _TOP_KEYS, "(top level)")
s = Scenario()
# Scenario section
sc = raw.get("scenario", {})
_reject_unknown(sc, _SECTION_KEYS["scenario"], "scenario")
s.name = sc.get("name", os.path.splitext(os.path.basename(path))[0])
s.seed = int(sc.get("seed", 42))
s.duration_secs = int(sc.get("duration_secs", 120))
# Topology section
tc = raw.get("topology", {})
_reject_unknown(tc, _SECTION_KEYS["topology"], "topology")
s.topology.num_nodes = int(tc.get("num_nodes", 10))
s.topology.algorithm = tc.get("algorithm", "random_geometric")
s.topology.params = tc.get("params", {})
s.topology.ensure_connected = tc.get("ensure_connected", True)
s.topology.subnet = tc.get("subnet", "172.20.0.0/24")
s.topology.ip_start = int(tc.get("ip_start", 10))
s.topology.default_transport = tc.get("default_transport", "udp")
if "transport_mix" in tc:
mix = tc["transport_mix"]
if not isinstance(mix, dict) or not mix:
raise ValueError("topology.transport_mix must be a non-empty dict")
s.topology.transport_mix = {str(k): float(v) for k, v in mix.items()}
# Netem section
nc = raw.get("netem", {})
_reject_unknown(nc, _SECTION_KEYS["netem"], "netem")
s.netem.enabled = nc.get("enabled", False)
if "default_policy" in nc:
s.netem.default_policy = _parse_netem_policy(
nc["default_policy"], "netem.default_policy"
)
if "link_policies" in nc:
for lp_data in nc["link_policies"]:
_reject_unknown(
lp_data, _SECTION_KEYS["netem.link_policies[]"], "netem.link_policies[]"
)
override = LinkPolicyOverride(
edges=lp_data.get("edges", []),
)
if "policy" in lp_data:
override.policy = _parse_netem_policy(
lp_data["policy"], "netem.link_policies[].policy"
)
if "policy_name" in lp_data:
override.policy_name = lp_data["policy_name"]
s.netem.link_policies.append(override)
if "mutation" in nc:
mc = nc["mutation"]
_reject_unknown(mc, _SECTION_KEYS["netem.mutation"], "netem.mutation")
s.netem.mutation.interval_secs = _parse_range(
mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs"
)
s.netem.mutation.fraction = float(mc.get("fraction", 0.3))
if "policies" in mc:
s.netem.mutation.policies = {
name: _parse_netem_policy(pdata, f"netem.mutation.policies.{name}")
for name, pdata in mc["policies"].items()
}
# Link flaps section
lf = raw.get("link_flaps", {})
_reject_unknown(lf, _SECTION_KEYS["link_flaps"], "link_flaps")
s.link_flaps.enabled = lf.get("enabled", False)
if "interval_secs" in lf:
s.link_flaps.interval_secs = _parse_range(lf["interval_secs"], "link_flaps.interval_secs")
s.link_flaps.max_down_links = int(lf.get("max_down_links", 2))
if "down_duration_secs" in lf:
s.link_flaps.down_duration_secs = _parse_range(
lf["down_duration_secs"], "link_flaps.down_duration_secs"
)
s.link_flaps.protect_connectivity = lf.get("protect_connectivity", True)
# Traffic section
tf = raw.get("traffic", {})
_reject_unknown(tf, _SECTION_KEYS["traffic"], "traffic")
s.traffic.enabled = tf.get("enabled", False)
s.traffic.max_concurrent = int(tf.get("max_concurrent", 3))
if "interval_secs" in tf:
s.traffic.interval_secs = _parse_range(tf["interval_secs"], "traffic.interval_secs")
if "duration_secs" in tf:
s.traffic.duration_secs = _parse_range(tf["duration_secs"], "traffic.duration_secs")
s.traffic.parallel_streams = int(tf.get("parallel_streams", 4))
# Node churn section
nc2 = raw.get("node_churn", {})
_reject_unknown(nc2, _SECTION_KEYS["node_churn"], "node_churn")
s.node_churn.enabled = nc2.get("enabled", False)
if "interval_secs" in nc2:
s.node_churn.interval_secs = _parse_range(nc2["interval_secs"], "node_churn.interval_secs")
s.node_churn.max_down_nodes = int(nc2.get("max_down_nodes", 1))
if "down_duration_secs" in nc2:
s.node_churn.down_duration_secs = _parse_range(
nc2["down_duration_secs"], "node_churn.down_duration_secs"
)
s.node_churn.protect_connectivity = nc2.get("protect_connectivity", True)
# Peer churn section
pc = raw.get("peer_churn", {})
_reject_unknown(pc, _SECTION_KEYS["peer_churn"], "peer_churn")
s.peer_churn.enabled = pc.get("enabled", False)
if "interval_secs" in pc:
s.peer_churn.interval_secs = _parse_range(pc["interval_secs"], "peer_churn.interval_secs")
s.peer_churn.ephemeral_fraction = float(pc.get("ephemeral_fraction", 0.0))
# Bandwidth section
bw = raw.get("bandwidth", {})
_reject_unknown(bw, _SECTION_KEYS["bandwidth"], "bandwidth")
s.bandwidth.enabled = bw.get("enabled", False)
if "tiers_mbps" in bw:
tiers = bw["tiers_mbps"]
if not isinstance(tiers, list) or not tiers:
raise ValueError("bandwidth.tiers_mbps must be a non-empty list")
s.bandwidth.tiers_mbps = [int(t) for t in tiers]
# Ingress section
ig = raw.get("ingress", {})
_reject_unknown(ig, _SECTION_KEYS["ingress"], "ingress")
s.ingress.enabled = ig.get("enabled", False)
if "tiers_kbps" in ig:
tiers = ig["tiers_kbps"]
if not isinstance(tiers, list) or not tiers:
raise ValueError("ingress.tiers_kbps must be a non-empty list")
s.ingress.tiers_kbps = [int(t) for t in tiers]
s.ingress.burst_bytes = int(ig.get("burst_bytes", 32000))
# Link swap section (deterministic asymmetric link-cost flapping).
ls = raw.get("link_swap", {})
_reject_unknown(ls, _SECTION_KEYS["link_swap"], "link_swap")
s.link_swap.enabled = ls.get("enabled", False)
if "interval_secs" in ls:
s.link_swap.interval_secs = float(ls["interval_secs"])
if "policies" in ls:
s.link_swap.policies = {
name: _parse_netem_policy(pdata, f"link_swap.policies.{name}")
for name, pdata in ls["policies"].items()
}
if "edges" in ls:
for edata in ls["edges"]:
if not isinstance(edata, dict):
raise ValueError("link_swap.edges entries must be dicts")
_reject_unknown(
edata, _SECTION_KEYS["link_swap.edges[]"], "link_swap.edges[]"
)
edge = str(edata.get("edge", ""))
policy = str(edata.get("policy", ""))
if not edge or not policy:
raise ValueError("link_swap.edges entries require 'edge' and 'policy'")
s.link_swap.edges.append(LinkSwapEdge(edge=edge, policy=policy))
# Assertions section (post-run control-socket-based checks).
asrt = raw.get("assertions", {})
_reject_unknown(asrt, _SECTION_KEYS["assertions"], "assertions")
if "bloom_send_rate" in asrt:
bsr = asrt["bloom_send_rate"]
_reject_unknown(
bsr, _ASSERTION_KEYS["bloom_send_rate"], "assertions.bloom_send_rate"
)
s.assertions.bloom_send_rate = BloomSendRateAssertion(
window_secs=int(bsr.get("window_secs", 30)),
max_per_node=int(bsr.get("max_per_node", 30)),
)
if "min_parent_switches" in asrt:
mps = asrt["min_parent_switches"]
_reject_unknown(
mps, _ASSERTION_KEYS["min_parent_switches"],
"assertions.min_parent_switches",
)
s.assertions.min_parent_switches = MinParentSwitchesAssertion(
min_total=int(mps.get("min_total", 1)),
)
if "max_parent_switches" in asrt:
xps = asrt["max_parent_switches"]
_reject_unknown(
xps, _ASSERTION_KEYS["max_parent_switches"],
"assertions.max_parent_switches",
)
if "max_total" not in xps:
raise ValueError(
"assertions.max_parent_switches: max_total is required "
"(a defaulted ceiling would assert an arbitrary number)"
)
max_total = xps["max_total"]
# bool is a subclass of int, and `max_total: yes` would coerce to 1
# -- a ceiling low enough to change the verdict, arrived at by typo.
if isinstance(max_total, bool) or not isinstance(max_total, int):
raise ValueError(
f"assertions.max_parent_switches: max_total must be a "
f"non-negative integer, got {max_total!r}"
)
if max_total < 0:
raise ValueError(
f"assertions.max_parent_switches: max_total must be "
f"non-negative, got {max_total}"
)
# Distinguish "key absent" from "key present but empty". Only the
# first means mesh-wide. YAML renders a bare `node:` as None, and
# treating that as absent would silently swap a per-node ceiling
# for a mesh-wide one -- the exact conflation this assertion was
# added to stop, and a live flake rather than a theoretical one:
# archived runs of this scenario reach 6 mesh-wide against an n04
# maximum of 2.
node = None
if "node" in xps:
node = xps["node"]
if not isinstance(node, str) or not node.strip():
raise ValueError(
f"assertions.max_parent_switches: node must be a node id "
f"string such as 'n04', got {node!r}. Omit the key "
f"entirely for the mesh-wide total."
)
node = node.strip()
s.assertions.max_parent_switches = MaxParentSwitchesAssertion(
max_total=max_total,
node=node,
)
# Logging section
lg = raw.get("logging", {})
_reject_unknown(lg, _SECTION_KEYS["logging"], "logging")
s.logging.rust_log = lg.get("rust_log", "info")
s.logging.output_dir = lg.get("output_dir", "./sim-results")
# FIPS config overrides (raw YAML dict appended to node configs)
s.fips_overrides = raw.get("fips_overrides", {})
# Validation
_validate(s)
return s
_SUPPRESSING_LOG_LEVELS = ("off", "error", "warn")
def _validate_parent_switch_observability(s: Scenario):
"""Refuse a parent-switch assertion the log level cannot observe.
The events these assertions count are emitted at ``info``. A scenario
that declares one while setting ``logging.rust_log`` to ``warn`` or
below counts zero switches: the minimum assertion then fails for the
wrong reason, and the maximum assertion passes without observing
anything, which is the failure mode the whole assertion effort exists
to remove. Catch it at load rather than after the run.
Deliberately conservative. It rejects only a default level that is
demonstrably too coarse, and says nothing about per-target directives
such as ``info,fips::node=debug``, which raise verbosity rather than
lower it.
"""
if (
s.assertions.min_parent_switches is None
and s.assertions.max_parent_switches is None
):
return
default_level = s.logging.rust_log.split(",")[0].strip().lower()
if default_level in _SUPPRESSING_LOG_LEVELS:
raise ValueError(
f"logging.rust_log is '{s.logging.rust_log}', whose default level "
f"'{default_level}' suppresses the info-level parent-switch events "
f"that a parent-switch assertion counts. The assertion would see "
f"zero switches regardless of what the tree did. Use 'info' or "
f"more verbose, or drop the assertion."
)
def _validate(s: Scenario):
"""Validate scenario constraints."""
_validate_parent_switch_observability(s)
if s.topology.num_nodes < 2:
raise ValueError("topology.num_nodes must be >= 2")
if s.topology.num_nodes > 250:
raise ValueError("topology.num_nodes must be <= 250 (subnet limit)")
if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain", "explicit"):
raise ValueError(f"Unknown topology algorithm: {s.topology.algorithm}")
if s.topology.default_transport not in VALID_TRANSPORTS:
raise ValueError(
f"topology.default_transport: '{s.topology.default_transport}' "
f"not in {VALID_TRANSPORTS}"
)
if s.topology.transport_mix is not None:
if s.topology.algorithm == "explicit":
raise ValueError(
"topology.transport_mix cannot be used with explicit algorithm "
"(use per-edge transport syntax instead)"
)
for transport, weight in s.topology.transport_mix.items():
if transport not in VALID_TRANSPORTS:
raise ValueError(
f"topology.transport_mix: '{transport}' not in {VALID_TRANSPORTS}"
)
if weight <= 0:
raise ValueError(
f"topology.transport_mix: weight for '{transport}' must be > 0"
)
if s.topology.algorithm == "explicit":
adj = s.topology.params.get("adjacency")
if not adj or not isinstance(adj, list):
raise ValueError("explicit topology requires params.adjacency list")
node_ids = set()
for i, entry in enumerate(adj):
if not isinstance(entry, (list, tuple)) or len(entry) not in (2, 3):
raise ValueError(
f"explicit adjacency[{i}]: expected [nodeA, nodeB] or "
f"[nodeA, nodeB, transport], got {entry}"
)
node_ids.update(str(p) for p in entry[:2])
if len(entry) == 3:
transport = str(entry[2])
if transport not in VALID_TRANSPORTS:
raise ValueError(
f"explicit adjacency[{i}]: transport '{transport}' "
f"not in {VALID_TRANSPORTS}"
)
if len(node_ids) != s.topology.num_nodes:
raise ValueError(
f"explicit adjacency references {len(node_ids)} nodes "
f"but num_nodes is {s.topology.num_nodes}"
)
if s.duration_secs < 1:
raise ValueError("duration_secs must be >= 1")
# Validate link_policies
for i, lp in enumerate(s.netem.link_policies):
if not lp.edges:
raise ValueError(f"netem.link_policies[{i}]: edges list is empty")
if lp.policy is not None and lp.policy_name is not None:
raise ValueError(
f"netem.link_policies[{i}]: specify policy or policy_name, not both"
)
if lp.policy is None and lp.policy_name is None:
raise ValueError(
f"netem.link_policies[{i}]: must specify policy or policy_name"
)
if lp.policy_name and lp.policy_name not in s.netem.mutation.policies:
raise ValueError(
f"netem.link_policies[{i}]: policy_name '{lp.policy_name}' "
f"not found in mutation.policies"
)
# Validate ranges
if s.netem.enabled and s.netem.mutation.policies:
s.netem.mutation.interval_secs.validate("netem.mutation.interval_secs")
if s.link_flaps.enabled:
s.link_flaps.interval_secs.validate("link_flaps.interval_secs")
s.link_flaps.down_duration_secs.validate("link_flaps.down_duration_secs")
if s.traffic.enabled:
s.traffic.interval_secs.validate("traffic.interval_secs")
s.traffic.duration_secs.validate("traffic.duration_secs")
if s.node_churn.enabled:
s.node_churn.interval_secs.validate("node_churn.interval_secs")
s.node_churn.down_duration_secs.validate("node_churn.down_duration_secs")
if s.node_churn.max_down_nodes >= s.topology.num_nodes:
raise ValueError("node_churn.max_down_nodes must be < topology.num_nodes")
if s.peer_churn.enabled:
s.peer_churn.interval_secs.validate("peer_churn.interval_secs")
if not 0.0 <= s.peer_churn.ephemeral_fraction <= 1.0:
raise ValueError("peer_churn.ephemeral_fraction must be between 0.0 and 1.0")
if s.bandwidth.enabled:
for tier in s.bandwidth.tiers_mbps:
if tier <= 0:
raise ValueError(f"bandwidth.tiers_mbps: all values must be > 0, got {tier}")
if s.ingress.enabled:
for tier in s.ingress.tiers_kbps:
if tier <= 0:
raise ValueError(f"ingress.tiers_kbps: all values must be > 0, got {tier}")
if s.ingress.burst_bytes <= 0:
raise ValueError(f"ingress.burst_bytes must be > 0, got {s.ingress.burst_bytes}")
# Validate link_swap
if s.link_swap.enabled:
if s.link_swap.interval_secs <= 0:
raise ValueError("link_swap.interval_secs must be > 0")
if len(s.link_swap.edges) < 2:
raise ValueError("link_swap.edges must list at least 2 edges to swap")
if not s.link_swap.policies:
raise ValueError("link_swap.policies must not be empty when link_swap.enabled")
for entry in s.link_swap.edges:
if entry.policy not in s.link_swap.policies:
raise ValueError(
f"link_swap.edges: policy '{entry.policy}' not in link_swap.policies"
)
# Validate assertions
if s.assertions.bloom_send_rate is not None:
bsr = s.assertions.bloom_send_rate
if bsr.window_secs < 1:
raise ValueError("assertions.bloom_send_rate.window_secs must be >= 1")
if bsr.max_per_node < 0:
raise ValueError("assertions.bloom_send_rate.max_per_node must be >= 0")
if bsr.window_secs > s.duration_secs:
raise ValueError(
"assertions.bloom_send_rate.window_secs must not exceed scenario duration"
)