mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
The scenario's success criterion existed only as a comment: count "Parent switched" in n04's log, expect at most 5. Nothing read it, and the scenario declared no assertions at all, so its exit code could only report that the mesh came up and nothing crashed. Add a max_parent_switches assertion and wire that criterion to it. The assertion takes an optional node scope, which is the part that matters. The existing sibling counts mesh-wide, and a criterion written about one node's log is a different quantity from the sum over every node: across archived runs of this scenario n04 is 1 or 2 while the mesh-wide total ranges 3 to 6. Asserting the sum against a per-node threshold would check something other than what was specified, and at this threshold would also have failed several runs that were fine. Counting nothing must not read as stability. A node id absent from the topology fails explicitly rather than matching zero log lines and sailing under the ceiling, and a scenario that declares a parent-switch assertion while setting a log level that suppresses the events it counts is rejected at load rather than passing vacuously after a full run. The parse rejects a mistyped key, a missing, negative, boolean or non-integer ceiling, and a node written empty or as a non-string -- that last one because YAML renders a bare `node:` as null, which would silently revert to the mesh-wide count. The scenario comment now records what the threshold is worth against the 11 completed archived runs rather than a single sample: 5 is well above anything observed, the daemon's own parent hold-down caps switches near 6 per run, and nearly every switch lands during initial tree formation before any mutation fires. So this catches only near-pathological reparenting. Tightening to 3 would make it a real detector, but that changes the stated criterion rather than fixing it, so it is left as a decision. Verified against a live run: the assertion evaluates n04 at 2 against mesh-wide 4 and passes, and with the ceiling temporarily set to 0 the scenario exits 3, the assertion-failure code, which it previously could not reach.
770 lines
30 KiB
Python
770 lines
30 KiB
Python
"""Scenario YAML loading and validation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
|
|
import yaml
|
|
|
|
|
|
@dataclass
|
|
class Range:
|
|
"""A min/max range for stochastic parameters."""
|
|
|
|
min: float
|
|
max: float
|
|
|
|
def validate(self, name: str):
|
|
if self.min > self.max:
|
|
raise ValueError(f"{name}: min ({self.min}) > max ({self.max})")
|
|
if self.min < 0:
|
|
raise ValueError(f"{name}: min ({self.min}) must be >= 0")
|
|
|
|
|
|
VALID_TRANSPORTS = ("udp", "ethernet", "tcp")
|
|
|
|
|
|
@dataclass
|
|
class TopologyConfig:
|
|
num_nodes: int = 10
|
|
algorithm: str = "random_geometric"
|
|
params: dict = field(default_factory=dict)
|
|
ensure_connected: bool = True
|
|
subnet: str = "172.20.0.0/24"
|
|
ip_start: int = 10
|
|
default_transport: str = "udp"
|
|
# Optional transport mix for random topologies: {transport: weight}.
|
|
# When set, each edge is randomly assigned a transport based on weights.
|
|
# Only valid for non-explicit algorithms (explicit uses per-edge syntax).
|
|
transport_mix: dict[str, float] | None = None
|
|
|
|
|
|
@dataclass
|
|
class NetemPolicy:
|
|
delay_ms: tuple[float, float] = (0, 0)
|
|
jitter_ms: tuple[float, float] = (0, 0)
|
|
loss_pct: tuple[float, float] = (0, 0)
|
|
duplicate_pct: tuple[float, float] = (0, 0)
|
|
reorder_pct: tuple[float, float] = (0, 0)
|
|
corrupt_pct: tuple[float, float] = (0, 0)
|
|
|
|
|
|
@dataclass
|
|
class NetemMutationConfig:
|
|
interval_secs: Range = field(default_factory=lambda: Range(15, 30))
|
|
fraction: float = 0.3
|
|
policies: dict[str, NetemPolicy] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass
|
|
class LinkPolicyOverride:
|
|
"""Per-edge netem policy override.
|
|
|
|
Edges are specified as "nXX-nYY" strings in canonical form (sorted
|
|
alphabetically). Either ``policy`` (inline) or ``policy_name``
|
|
(reference to a named mutation policy) must be set, not both.
|
|
"""
|
|
|
|
edges: list[str] = field(default_factory=list)
|
|
policy: NetemPolicy | None = None
|
|
policy_name: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class NetemConfig:
|
|
enabled: bool = False
|
|
default_policy: NetemPolicy = field(default_factory=NetemPolicy)
|
|
link_policies: list[LinkPolicyOverride] = field(default_factory=list)
|
|
mutation: NetemMutationConfig = field(default_factory=NetemMutationConfig)
|
|
|
|
|
|
@dataclass
|
|
class LinkFlapsConfig:
|
|
enabled: bool = False
|
|
interval_secs: Range = field(default_factory=lambda: Range(20, 60))
|
|
max_down_links: int = 2
|
|
down_duration_secs: Range = field(default_factory=lambda: Range(10, 30))
|
|
protect_connectivity: bool = True
|
|
|
|
|
|
@dataclass
|
|
class TrafficConfig:
|
|
enabled: bool = False
|
|
max_concurrent: int = 3
|
|
interval_secs: Range = field(default_factory=lambda: Range(10, 30))
|
|
duration_secs: Range = field(default_factory=lambda: Range(5, 15))
|
|
parallel_streams: int = 4
|
|
|
|
|
|
@dataclass
|
|
class NodeChurnConfig:
|
|
enabled: bool = False
|
|
interval_secs: Range = field(default_factory=lambda: Range(60, 180))
|
|
max_down_nodes: int = 1
|
|
down_duration_secs: Range = field(default_factory=lambda: Range(30, 90))
|
|
protect_connectivity: bool = True
|
|
|
|
|
|
@dataclass
|
|
class PeerChurnConfig:
|
|
"""Peer-level topology churn via connect/disconnect commands.
|
|
|
|
When enabled, periodically disconnects a random active link and
|
|
connects a random unconnected node pair, causing the mesh topology
|
|
to evolve over time.
|
|
"""
|
|
|
|
enabled: bool = False
|
|
interval_secs: Range = field(default_factory=lambda: Range(8, 12))
|
|
ephemeral_fraction: float = 0.0
|
|
|
|
|
|
@dataclass
|
|
class BandwidthConfig:
|
|
"""Per-link bandwidth pacing via HTB rate limiting.
|
|
|
|
When enabled, each link is randomly assigned a rate from the tiers list.
|
|
# TODO: Add structured bandwidth assignment (e.g., per-node roles,
|
|
# asymmetric uplink/downlink, tiered by topology distance, or
|
|
# time-varying bandwidth mutations similar to netem mutation).
|
|
"""
|
|
|
|
enabled: bool = False
|
|
tiers_mbps: list[int] = field(default_factory=lambda: [1, 10, 100, 1000])
|
|
|
|
|
|
@dataclass
|
|
class IngressConfig:
|
|
"""Per-link ingress policing via tc ingress qdisc + policer.
|
|
|
|
When enabled, each link gets an ingress policer that drops inbound
|
|
packets exceeding the configured rate. Unlike egress HTB (which
|
|
paces packets smoothly), the policer drops excess packets at the
|
|
kernel level, creating bursty arrival patterns that can overflow
|
|
small socket buffers and trigger SO_RXQ_OVFL kernel drops.
|
|
"""
|
|
|
|
enabled: bool = False
|
|
tiers_kbps: list[int] = field(default_factory=lambda: [1000])
|
|
burst_bytes: int = 32000
|
|
|
|
|
|
@dataclass
|
|
class LinkSwapEdge:
|
|
"""One edge in a link-swap rotation.
|
|
|
|
Edge is a canonical "nXX-nYY" string. ``policy`` names a policy
|
|
in ``LinkSwapConfig.policies``.
|
|
"""
|
|
|
|
edge: str = ""
|
|
policy: str = ""
|
|
|
|
|
|
@dataclass
|
|
class LinkSwapConfig:
|
|
"""Deterministic asymmetric link-cost flapping.
|
|
|
|
On each ``interval_secs``, the policies on every pair of edges in
|
|
``edges`` are swapped (cyclically rotated by one position). This
|
|
differs from ``link_flaps`` (random link-down events) and
|
|
``netem.mutation`` (random per-edge policy mutation) by being a
|
|
deterministic, periodic flip between two named netem policies on
|
|
a fixed set of edges.
|
|
|
|
Used to drive a downstream node to repeatedly switch parents on
|
|
a fixed cadence — exercises the spanning-tree rebalance path
|
|
without the noise of a random mutation walk.
|
|
"""
|
|
|
|
enabled: bool = False
|
|
interval_secs: float = 4.0
|
|
policies: dict[str, NetemPolicy] = field(default_factory=dict)
|
|
edges: list[LinkSwapEdge] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class BloomSendRateAssertion:
|
|
"""Trailing-window ceiling on per-node ``stats.bloom.sent`` delta."""
|
|
|
|
window_secs: int = 30
|
|
max_per_node: int = 30
|
|
|
|
|
|
@dataclass
|
|
class MinParentSwitchesAssertion:
|
|
"""Sanity guard: total parent switches across the run must be at
|
|
least ``min_total``. Used to detect a misconfigured harness where
|
|
the flap inducer is firing but the topology never produces a real
|
|
parent-switch event (e.g., wrong root election).
|
|
"""
|
|
|
|
min_total: int = 1
|
|
|
|
|
|
@dataclass
|
|
class MaxParentSwitchesAssertion:
|
|
"""Stability ceiling: fail if parent switches exceed ``max_total``.
|
|
|
|
``node`` scopes the count to a single node id (e.g. ``n04``) instead
|
|
of the mesh-wide total. The distinction is not cosmetic: a criterion
|
|
written about one node's log is a different number from the sum over
|
|
every node, and checking the sum against a per-node threshold
|
|
silently tests something other than what was specified.
|
|
"""
|
|
|
|
max_total: int = 0
|
|
node: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class AssertionsConfig:
|
|
"""Optional post-run assertions evaluated against control-socket data."""
|
|
|
|
bloom_send_rate: BloomSendRateAssertion | None = None
|
|
min_parent_switches: MinParentSwitchesAssertion | None = None
|
|
max_parent_switches: MaxParentSwitchesAssertion | None = None
|
|
|
|
|
|
@dataclass
|
|
class LoggingConfig:
|
|
rust_log: str = "info"
|
|
output_dir: str = "./sim-results"
|
|
|
|
|
|
@dataclass
|
|
class Scenario:
|
|
name: str = "unnamed"
|
|
seed: int = 42
|
|
duration_secs: int = 120
|
|
topology: TopologyConfig = field(default_factory=TopologyConfig)
|
|
netem: NetemConfig = field(default_factory=NetemConfig)
|
|
link_flaps: LinkFlapsConfig = field(default_factory=LinkFlapsConfig)
|
|
traffic: TrafficConfig = field(default_factory=TrafficConfig)
|
|
node_churn: NodeChurnConfig = field(default_factory=NodeChurnConfig)
|
|
peer_churn: PeerChurnConfig = field(default_factory=PeerChurnConfig)
|
|
bandwidth: BandwidthConfig = field(default_factory=BandwidthConfig)
|
|
ingress: IngressConfig = field(default_factory=IngressConfig)
|
|
link_swap: LinkSwapConfig = field(default_factory=LinkSwapConfig)
|
|
assertions: AssertionsConfig = field(default_factory=AssertionsConfig)
|
|
logging: LoggingConfig = field(default_factory=LoggingConfig)
|
|
# Raw YAML dict appended to each generated FIPS node config.
|
|
# Allows scenarios to override any FIPS config parameter
|
|
# (e.g., node.tree.reeval_interval_secs).
|
|
fips_overrides: dict = field(default_factory=dict)
|
|
|
|
|
|
# Keys each fixed-schema section understands. A key absent from these sets is a
|
|
# typo, and silently ignoring it leaves the block default-constructed while the
|
|
# YAML still looks correct — a mistyped "assertion:" disarms a scenario's only
|
|
# assertions, a mistyped "link_flap:" turns off the chaos it was meant to inject.
|
|
#
|
|
# Three mappings are deliberately NOT listed, because their keys are names the
|
|
# scenario author chooses rather than a schema: netem.mutation.policies,
|
|
# link_swap.policies and topology.transport_mix. Two sub-trees are passed through
|
|
# whole and are likewise not checked: fips_overrides and topology.params.
|
|
#
|
|
# NOTE: adding a new assertion type means registering it in TWO places below —
|
|
# _SECTION_KEYS["assertions"] (so the block accepts its name) and _ASSERTION_KEYS
|
|
# (so its own members are checked) — or scenarios using it are rejected at load.
|
|
_TOP_KEYS = {
|
|
"scenario", "topology", "netem", "link_flaps", "traffic", "node_churn",
|
|
"peer_churn", "bandwidth", "ingress", "link_swap", "assertions", "logging",
|
|
"fips_overrides",
|
|
}
|
|
_SECTION_KEYS = {
|
|
"scenario": {"name", "seed", "duration_secs"},
|
|
"topology": {
|
|
"num_nodes", "algorithm", "params", "ensure_connected", "subnet",
|
|
"ip_start", "default_transport", "transport_mix",
|
|
},
|
|
"netem": {"enabled", "default_policy", "link_policies", "mutation"},
|
|
"netem.link_policies[]": {"edges", "policy", "policy_name"},
|
|
"netem.mutation": {"interval_secs", "fraction", "policies"},
|
|
"link_flaps": {
|
|
"enabled", "interval_secs", "max_down_links", "down_duration_secs",
|
|
"protect_connectivity",
|
|
},
|
|
"traffic": {
|
|
"enabled", "max_concurrent", "interval_secs", "duration_secs",
|
|
"parallel_streams",
|
|
},
|
|
"node_churn": {
|
|
"enabled", "interval_secs", "max_down_nodes", "down_duration_secs",
|
|
"protect_connectivity",
|
|
},
|
|
"peer_churn": {"enabled", "interval_secs", "ephemeral_fraction"},
|
|
"bandwidth": {"enabled", "tiers_mbps"},
|
|
"ingress": {"enabled", "tiers_kbps", "burst_bytes"},
|
|
"link_swap": {"enabled", "interval_secs", "policies", "edges"},
|
|
"link_swap.edges[]": {"edge", "policy"},
|
|
"assertions": {
|
|
"bloom_send_rate", "min_parent_switches", "max_parent_switches",
|
|
},
|
|
"logging": {"rust_log", "output_dir"},
|
|
}
|
|
_ASSERTION_KEYS = {
|
|
"bloom_send_rate": {"window_secs", "max_per_node"},
|
|
"min_parent_switches": {"min_total"},
|
|
"max_parent_switches": {"max_total", "node"},
|
|
}
|
|
_NETEM_POLICY_KEYS = {
|
|
"delay_ms", "jitter_ms", "loss_pct", "duplicate_pct", "reorder_pct",
|
|
"corrupt_pct",
|
|
}
|
|
_RANGE_KEYS = {"min", "max"}
|
|
|
|
|
|
def _reject_unknown(data, known, context: str):
|
|
"""Raise unless every key in `data` is one the loader understands."""
|
|
if data is None:
|
|
raise ValueError(
|
|
f"{context}: section is present but empty; give it a body or remove it"
|
|
)
|
|
if not isinstance(data, dict):
|
|
raise ValueError(f"{context}: expected a mapping, got {type(data).__name__}")
|
|
unknown = sorted(str(k) for k in set(data) - set(known))
|
|
if unknown:
|
|
raise ValueError(
|
|
f"{context}: unknown key(s): {', '.join(unknown)} "
|
|
f"(known: {', '.join(sorted(known))})"
|
|
)
|
|
|
|
|
|
def _parse_range(data, name: str) -> Range:
|
|
"""Parse a {min, max} dict into a Range."""
|
|
if isinstance(data, dict):
|
|
_reject_unknown(data, _RANGE_KEYS, name)
|
|
return Range(min=float(data["min"]), max=float(data["max"]))
|
|
raise ValueError(f"{name}: expected {{min, max}} dict, got {type(data).__name__}")
|
|
|
|
|
|
def _parse_netem_policy(data: dict, name: str = "netem policy") -> NetemPolicy:
|
|
"""Parse a netem policy from a dict with [min, max] lists or {min, max} dicts."""
|
|
_reject_unknown(data, _NETEM_POLICY_KEYS, name)
|
|
policy = NetemPolicy()
|
|
for attr in (
|
|
"delay_ms",
|
|
"jitter_ms",
|
|
"loss_pct",
|
|
"duplicate_pct",
|
|
"reorder_pct",
|
|
"corrupt_pct",
|
|
):
|
|
if attr in data:
|
|
val = data[attr]
|
|
if isinstance(val, list) and len(val) == 2:
|
|
setattr(policy, attr, (float(val[0]), float(val[1])))
|
|
elif isinstance(val, dict):
|
|
setattr(policy, attr, (float(val["min"]), float(val["max"])))
|
|
else:
|
|
raise ValueError(f"netem policy {attr}: expected [min, max] or {{min, max}}")
|
|
return policy
|
|
|
|
|
|
def load_scenario(path: str) -> Scenario:
|
|
"""Load and validate a scenario from a YAML file."""
|
|
with open(path) as f:
|
|
raw = yaml.safe_load(f)
|
|
|
|
if raw is None:
|
|
raise ValueError(f"{path}: file is empty")
|
|
_reject_unknown(raw, _TOP_KEYS, "(top level)")
|
|
|
|
s = Scenario()
|
|
|
|
# Scenario section
|
|
sc = raw.get("scenario", {})
|
|
_reject_unknown(sc, _SECTION_KEYS["scenario"], "scenario")
|
|
s.name = sc.get("name", os.path.splitext(os.path.basename(path))[0])
|
|
s.seed = int(sc.get("seed", 42))
|
|
s.duration_secs = int(sc.get("duration_secs", 120))
|
|
|
|
# Topology section
|
|
tc = raw.get("topology", {})
|
|
_reject_unknown(tc, _SECTION_KEYS["topology"], "topology")
|
|
s.topology.num_nodes = int(tc.get("num_nodes", 10))
|
|
s.topology.algorithm = tc.get("algorithm", "random_geometric")
|
|
s.topology.params = tc.get("params", {})
|
|
s.topology.ensure_connected = tc.get("ensure_connected", True)
|
|
s.topology.subnet = tc.get("subnet", "172.20.0.0/24")
|
|
s.topology.ip_start = int(tc.get("ip_start", 10))
|
|
s.topology.default_transport = tc.get("default_transport", "udp")
|
|
if "transport_mix" in tc:
|
|
mix = tc["transport_mix"]
|
|
if not isinstance(mix, dict) or not mix:
|
|
raise ValueError("topology.transport_mix must be a non-empty dict")
|
|
s.topology.transport_mix = {str(k): float(v) for k, v in mix.items()}
|
|
|
|
# Netem section
|
|
nc = raw.get("netem", {})
|
|
_reject_unknown(nc, _SECTION_KEYS["netem"], "netem")
|
|
s.netem.enabled = nc.get("enabled", False)
|
|
if "default_policy" in nc:
|
|
s.netem.default_policy = _parse_netem_policy(
|
|
nc["default_policy"], "netem.default_policy"
|
|
)
|
|
if "link_policies" in nc:
|
|
for lp_data in nc["link_policies"]:
|
|
_reject_unknown(
|
|
lp_data, _SECTION_KEYS["netem.link_policies[]"], "netem.link_policies[]"
|
|
)
|
|
override = LinkPolicyOverride(
|
|
edges=lp_data.get("edges", []),
|
|
)
|
|
if "policy" in lp_data:
|
|
override.policy = _parse_netem_policy(
|
|
lp_data["policy"], "netem.link_policies[].policy"
|
|
)
|
|
if "policy_name" in lp_data:
|
|
override.policy_name = lp_data["policy_name"]
|
|
s.netem.link_policies.append(override)
|
|
if "mutation" in nc:
|
|
mc = nc["mutation"]
|
|
_reject_unknown(mc, _SECTION_KEYS["netem.mutation"], "netem.mutation")
|
|
s.netem.mutation.interval_secs = _parse_range(
|
|
mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs"
|
|
)
|
|
s.netem.mutation.fraction = float(mc.get("fraction", 0.3))
|
|
if "policies" in mc:
|
|
s.netem.mutation.policies = {
|
|
name: _parse_netem_policy(pdata, f"netem.mutation.policies.{name}")
|
|
for name, pdata in mc["policies"].items()
|
|
}
|
|
|
|
# Link flaps section
|
|
lf = raw.get("link_flaps", {})
|
|
_reject_unknown(lf, _SECTION_KEYS["link_flaps"], "link_flaps")
|
|
s.link_flaps.enabled = lf.get("enabled", False)
|
|
if "interval_secs" in lf:
|
|
s.link_flaps.interval_secs = _parse_range(lf["interval_secs"], "link_flaps.interval_secs")
|
|
s.link_flaps.max_down_links = int(lf.get("max_down_links", 2))
|
|
if "down_duration_secs" in lf:
|
|
s.link_flaps.down_duration_secs = _parse_range(
|
|
lf["down_duration_secs"], "link_flaps.down_duration_secs"
|
|
)
|
|
s.link_flaps.protect_connectivity = lf.get("protect_connectivity", True)
|
|
|
|
# Traffic section
|
|
tf = raw.get("traffic", {})
|
|
_reject_unknown(tf, _SECTION_KEYS["traffic"], "traffic")
|
|
s.traffic.enabled = tf.get("enabled", False)
|
|
s.traffic.max_concurrent = int(tf.get("max_concurrent", 3))
|
|
if "interval_secs" in tf:
|
|
s.traffic.interval_secs = _parse_range(tf["interval_secs"], "traffic.interval_secs")
|
|
if "duration_secs" in tf:
|
|
s.traffic.duration_secs = _parse_range(tf["duration_secs"], "traffic.duration_secs")
|
|
s.traffic.parallel_streams = int(tf.get("parallel_streams", 4))
|
|
|
|
# Node churn section
|
|
nc2 = raw.get("node_churn", {})
|
|
_reject_unknown(nc2, _SECTION_KEYS["node_churn"], "node_churn")
|
|
s.node_churn.enabled = nc2.get("enabled", False)
|
|
if "interval_secs" in nc2:
|
|
s.node_churn.interval_secs = _parse_range(nc2["interval_secs"], "node_churn.interval_secs")
|
|
s.node_churn.max_down_nodes = int(nc2.get("max_down_nodes", 1))
|
|
if "down_duration_secs" in nc2:
|
|
s.node_churn.down_duration_secs = _parse_range(
|
|
nc2["down_duration_secs"], "node_churn.down_duration_secs"
|
|
)
|
|
s.node_churn.protect_connectivity = nc2.get("protect_connectivity", True)
|
|
|
|
# Peer churn section
|
|
pc = raw.get("peer_churn", {})
|
|
_reject_unknown(pc, _SECTION_KEYS["peer_churn"], "peer_churn")
|
|
s.peer_churn.enabled = pc.get("enabled", False)
|
|
if "interval_secs" in pc:
|
|
s.peer_churn.interval_secs = _parse_range(pc["interval_secs"], "peer_churn.interval_secs")
|
|
s.peer_churn.ephemeral_fraction = float(pc.get("ephemeral_fraction", 0.0))
|
|
|
|
# Bandwidth section
|
|
bw = raw.get("bandwidth", {})
|
|
_reject_unknown(bw, _SECTION_KEYS["bandwidth"], "bandwidth")
|
|
s.bandwidth.enabled = bw.get("enabled", False)
|
|
if "tiers_mbps" in bw:
|
|
tiers = bw["tiers_mbps"]
|
|
if not isinstance(tiers, list) or not tiers:
|
|
raise ValueError("bandwidth.tiers_mbps must be a non-empty list")
|
|
s.bandwidth.tiers_mbps = [int(t) for t in tiers]
|
|
|
|
# Ingress section
|
|
ig = raw.get("ingress", {})
|
|
_reject_unknown(ig, _SECTION_KEYS["ingress"], "ingress")
|
|
s.ingress.enabled = ig.get("enabled", False)
|
|
if "tiers_kbps" in ig:
|
|
tiers = ig["tiers_kbps"]
|
|
if not isinstance(tiers, list) or not tiers:
|
|
raise ValueError("ingress.tiers_kbps must be a non-empty list")
|
|
s.ingress.tiers_kbps = [int(t) for t in tiers]
|
|
s.ingress.burst_bytes = int(ig.get("burst_bytes", 32000))
|
|
|
|
# Link swap section (deterministic asymmetric link-cost flapping).
|
|
ls = raw.get("link_swap", {})
|
|
_reject_unknown(ls, _SECTION_KEYS["link_swap"], "link_swap")
|
|
s.link_swap.enabled = ls.get("enabled", False)
|
|
if "interval_secs" in ls:
|
|
s.link_swap.interval_secs = float(ls["interval_secs"])
|
|
if "policies" in ls:
|
|
s.link_swap.policies = {
|
|
name: _parse_netem_policy(pdata, f"link_swap.policies.{name}")
|
|
for name, pdata in ls["policies"].items()
|
|
}
|
|
if "edges" in ls:
|
|
for edata in ls["edges"]:
|
|
if not isinstance(edata, dict):
|
|
raise ValueError("link_swap.edges entries must be dicts")
|
|
_reject_unknown(
|
|
edata, _SECTION_KEYS["link_swap.edges[]"], "link_swap.edges[]"
|
|
)
|
|
edge = str(edata.get("edge", ""))
|
|
policy = str(edata.get("policy", ""))
|
|
if not edge or not policy:
|
|
raise ValueError("link_swap.edges entries require 'edge' and 'policy'")
|
|
s.link_swap.edges.append(LinkSwapEdge(edge=edge, policy=policy))
|
|
|
|
# Assertions section (post-run control-socket-based checks).
|
|
asrt = raw.get("assertions", {})
|
|
_reject_unknown(asrt, _SECTION_KEYS["assertions"], "assertions")
|
|
if "bloom_send_rate" in asrt:
|
|
bsr = asrt["bloom_send_rate"]
|
|
_reject_unknown(
|
|
bsr, _ASSERTION_KEYS["bloom_send_rate"], "assertions.bloom_send_rate"
|
|
)
|
|
s.assertions.bloom_send_rate = BloomSendRateAssertion(
|
|
window_secs=int(bsr.get("window_secs", 30)),
|
|
max_per_node=int(bsr.get("max_per_node", 30)),
|
|
)
|
|
if "min_parent_switches" in asrt:
|
|
mps = asrt["min_parent_switches"]
|
|
_reject_unknown(
|
|
mps, _ASSERTION_KEYS["min_parent_switches"],
|
|
"assertions.min_parent_switches",
|
|
)
|
|
s.assertions.min_parent_switches = MinParentSwitchesAssertion(
|
|
min_total=int(mps.get("min_total", 1)),
|
|
)
|
|
if "max_parent_switches" in asrt:
|
|
xps = asrt["max_parent_switches"]
|
|
_reject_unknown(
|
|
xps, _ASSERTION_KEYS["max_parent_switches"],
|
|
"assertions.max_parent_switches",
|
|
)
|
|
if "max_total" not in xps:
|
|
raise ValueError(
|
|
"assertions.max_parent_switches: max_total is required "
|
|
"(a defaulted ceiling would assert an arbitrary number)"
|
|
)
|
|
max_total = xps["max_total"]
|
|
# bool is a subclass of int, and `max_total: yes` would coerce to 1
|
|
# -- a ceiling low enough to change the verdict, arrived at by typo.
|
|
if isinstance(max_total, bool) or not isinstance(max_total, int):
|
|
raise ValueError(
|
|
f"assertions.max_parent_switches: max_total must be a "
|
|
f"non-negative integer, got {max_total!r}"
|
|
)
|
|
if max_total < 0:
|
|
raise ValueError(
|
|
f"assertions.max_parent_switches: max_total must be "
|
|
f"non-negative, got {max_total}"
|
|
)
|
|
# Distinguish "key absent" from "key present but empty". Only the
|
|
# first means mesh-wide. YAML renders a bare `node:` as None, and
|
|
# treating that as absent would silently swap a per-node ceiling
|
|
# for a mesh-wide one -- the exact conflation this assertion was
|
|
# added to stop, and a live flake rather than a theoretical one:
|
|
# archived runs of this scenario reach 6 mesh-wide against an n04
|
|
# maximum of 2.
|
|
node = None
|
|
if "node" in xps:
|
|
node = xps["node"]
|
|
if not isinstance(node, str) or not node.strip():
|
|
raise ValueError(
|
|
f"assertions.max_parent_switches: node must be a node id "
|
|
f"string such as 'n04', got {node!r}. Omit the key "
|
|
f"entirely for the mesh-wide total."
|
|
)
|
|
node = node.strip()
|
|
s.assertions.max_parent_switches = MaxParentSwitchesAssertion(
|
|
max_total=max_total,
|
|
node=node,
|
|
)
|
|
|
|
# Logging section
|
|
lg = raw.get("logging", {})
|
|
_reject_unknown(lg, _SECTION_KEYS["logging"], "logging")
|
|
s.logging.rust_log = lg.get("rust_log", "info")
|
|
s.logging.output_dir = lg.get("output_dir", "./sim-results")
|
|
|
|
# FIPS config overrides (raw YAML dict appended to node configs)
|
|
s.fips_overrides = raw.get("fips_overrides", {})
|
|
|
|
# Validation
|
|
_validate(s)
|
|
|
|
return s
|
|
|
|
|
|
_SUPPRESSING_LOG_LEVELS = ("off", "error", "warn")
|
|
|
|
|
|
def _validate_parent_switch_observability(s: Scenario):
|
|
"""Refuse a parent-switch assertion the log level cannot observe.
|
|
|
|
The events these assertions count are emitted at ``info``. A scenario
|
|
that declares one while setting ``logging.rust_log`` to ``warn`` or
|
|
below counts zero switches: the minimum assertion then fails for the
|
|
wrong reason, and the maximum assertion passes without observing
|
|
anything, which is the failure mode the whole assertion effort exists
|
|
to remove. Catch it at load rather than after the run.
|
|
|
|
Deliberately conservative. It rejects only a default level that is
|
|
demonstrably too coarse, and says nothing about per-target directives
|
|
such as ``info,fips::node=debug``, which raise verbosity rather than
|
|
lower it.
|
|
"""
|
|
if (
|
|
s.assertions.min_parent_switches is None
|
|
and s.assertions.max_parent_switches is None
|
|
):
|
|
return
|
|
default_level = s.logging.rust_log.split(",")[0].strip().lower()
|
|
if default_level in _SUPPRESSING_LOG_LEVELS:
|
|
raise ValueError(
|
|
f"logging.rust_log is '{s.logging.rust_log}', whose default level "
|
|
f"'{default_level}' suppresses the info-level parent-switch events "
|
|
f"that a parent-switch assertion counts. The assertion would see "
|
|
f"zero switches regardless of what the tree did. Use 'info' or "
|
|
f"more verbose, or drop the assertion."
|
|
)
|
|
|
|
|
|
def _validate(s: Scenario):
|
|
"""Validate scenario constraints."""
|
|
_validate_parent_switch_observability(s)
|
|
if s.topology.num_nodes < 2:
|
|
raise ValueError("topology.num_nodes must be >= 2")
|
|
if s.topology.num_nodes > 250:
|
|
raise ValueError("topology.num_nodes must be <= 250 (subnet limit)")
|
|
if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain", "explicit"):
|
|
raise ValueError(f"Unknown topology algorithm: {s.topology.algorithm}")
|
|
if s.topology.default_transport not in VALID_TRANSPORTS:
|
|
raise ValueError(
|
|
f"topology.default_transport: '{s.topology.default_transport}' "
|
|
f"not in {VALID_TRANSPORTS}"
|
|
)
|
|
if s.topology.transport_mix is not None:
|
|
if s.topology.algorithm == "explicit":
|
|
raise ValueError(
|
|
"topology.transport_mix cannot be used with explicit algorithm "
|
|
"(use per-edge transport syntax instead)"
|
|
)
|
|
for transport, weight in s.topology.transport_mix.items():
|
|
if transport not in VALID_TRANSPORTS:
|
|
raise ValueError(
|
|
f"topology.transport_mix: '{transport}' not in {VALID_TRANSPORTS}"
|
|
)
|
|
if weight <= 0:
|
|
raise ValueError(
|
|
f"topology.transport_mix: weight for '{transport}' must be > 0"
|
|
)
|
|
if s.topology.algorithm == "explicit":
|
|
adj = s.topology.params.get("adjacency")
|
|
if not adj or not isinstance(adj, list):
|
|
raise ValueError("explicit topology requires params.adjacency list")
|
|
node_ids = set()
|
|
for i, entry in enumerate(adj):
|
|
if not isinstance(entry, (list, tuple)) or len(entry) not in (2, 3):
|
|
raise ValueError(
|
|
f"explicit adjacency[{i}]: expected [nodeA, nodeB] or "
|
|
f"[nodeA, nodeB, transport], got {entry}"
|
|
)
|
|
node_ids.update(str(p) for p in entry[:2])
|
|
if len(entry) == 3:
|
|
transport = str(entry[2])
|
|
if transport not in VALID_TRANSPORTS:
|
|
raise ValueError(
|
|
f"explicit adjacency[{i}]: transport '{transport}' "
|
|
f"not in {VALID_TRANSPORTS}"
|
|
)
|
|
if len(node_ids) != s.topology.num_nodes:
|
|
raise ValueError(
|
|
f"explicit adjacency references {len(node_ids)} nodes "
|
|
f"but num_nodes is {s.topology.num_nodes}"
|
|
)
|
|
if s.duration_secs < 1:
|
|
raise ValueError("duration_secs must be >= 1")
|
|
|
|
# Validate link_policies
|
|
for i, lp in enumerate(s.netem.link_policies):
|
|
if not lp.edges:
|
|
raise ValueError(f"netem.link_policies[{i}]: edges list is empty")
|
|
if lp.policy is not None and lp.policy_name is not None:
|
|
raise ValueError(
|
|
f"netem.link_policies[{i}]: specify policy or policy_name, not both"
|
|
)
|
|
if lp.policy is None and lp.policy_name is None:
|
|
raise ValueError(
|
|
f"netem.link_policies[{i}]: must specify policy or policy_name"
|
|
)
|
|
if lp.policy_name and lp.policy_name not in s.netem.mutation.policies:
|
|
raise ValueError(
|
|
f"netem.link_policies[{i}]: policy_name '{lp.policy_name}' "
|
|
f"not found in mutation.policies"
|
|
)
|
|
|
|
# Validate ranges
|
|
if s.netem.enabled and s.netem.mutation.policies:
|
|
s.netem.mutation.interval_secs.validate("netem.mutation.interval_secs")
|
|
if s.link_flaps.enabled:
|
|
s.link_flaps.interval_secs.validate("link_flaps.interval_secs")
|
|
s.link_flaps.down_duration_secs.validate("link_flaps.down_duration_secs")
|
|
if s.traffic.enabled:
|
|
s.traffic.interval_secs.validate("traffic.interval_secs")
|
|
s.traffic.duration_secs.validate("traffic.duration_secs")
|
|
if s.node_churn.enabled:
|
|
s.node_churn.interval_secs.validate("node_churn.interval_secs")
|
|
s.node_churn.down_duration_secs.validate("node_churn.down_duration_secs")
|
|
if s.node_churn.max_down_nodes >= s.topology.num_nodes:
|
|
raise ValueError("node_churn.max_down_nodes must be < topology.num_nodes")
|
|
if s.peer_churn.enabled:
|
|
s.peer_churn.interval_secs.validate("peer_churn.interval_secs")
|
|
if not 0.0 <= s.peer_churn.ephemeral_fraction <= 1.0:
|
|
raise ValueError("peer_churn.ephemeral_fraction must be between 0.0 and 1.0")
|
|
if s.bandwidth.enabled:
|
|
for tier in s.bandwidth.tiers_mbps:
|
|
if tier <= 0:
|
|
raise ValueError(f"bandwidth.tiers_mbps: all values must be > 0, got {tier}")
|
|
if s.ingress.enabled:
|
|
for tier in s.ingress.tiers_kbps:
|
|
if tier <= 0:
|
|
raise ValueError(f"ingress.tiers_kbps: all values must be > 0, got {tier}")
|
|
if s.ingress.burst_bytes <= 0:
|
|
raise ValueError(f"ingress.burst_bytes must be > 0, got {s.ingress.burst_bytes}")
|
|
|
|
# Validate link_swap
|
|
if s.link_swap.enabled:
|
|
if s.link_swap.interval_secs <= 0:
|
|
raise ValueError("link_swap.interval_secs must be > 0")
|
|
if len(s.link_swap.edges) < 2:
|
|
raise ValueError("link_swap.edges must list at least 2 edges to swap")
|
|
if not s.link_swap.policies:
|
|
raise ValueError("link_swap.policies must not be empty when link_swap.enabled")
|
|
for entry in s.link_swap.edges:
|
|
if entry.policy not in s.link_swap.policies:
|
|
raise ValueError(
|
|
f"link_swap.edges: policy '{entry.policy}' not in link_swap.policies"
|
|
)
|
|
|
|
# Validate assertions
|
|
if s.assertions.bloom_send_rate is not None:
|
|
bsr = s.assertions.bloom_send_rate
|
|
if bsr.window_secs < 1:
|
|
raise ValueError("assertions.bloom_send_rate.window_secs must be >= 1")
|
|
if bsr.max_per_node < 0:
|
|
raise ValueError("assertions.bloom_send_rate.max_per_node must be >= 0")
|
|
if bsr.window_secs > s.duration_secs:
|
|
raise ValueError(
|
|
"assertions.bloom_send_rate.window_secs must not exceed scenario duration"
|
|
)
|