mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Implement cost-based parent selection with periodic re-evaluation
Cost-based parent selection: - Replace depth-only parent selection with effective_depth = depth + link_cost - link_cost computed from locally measured MMP metrics: etx * (1.0 + srtt_ms / 100.0) - Prevents bottleneck subtrees in heterogeneous networks where a LoRa link at depth 1 would otherwise always beat fiber at depth 2 - Configurable hysteresis (default 0.2) prevents marginal parent switches - Configurable hold-down timer (default 30s) suppresses re-evaluation after parent switch - Mandatory switches (parent lost, root change) bypass both safeguards - Link costs passed as HashMap parameter to keep TreeState pure Periodic re-evaluation: - evaluate_parent() was only called on TreeAnnounce receipt or parent loss; after tree stabilization, link degradation went undetected - Added timer-based re-evaluation (reeval_interval_secs, default 60s) that calls evaluate_parent() from the tick handler with current MMP link costs - Respects existing hold-down and hysteresis safeguards - Short-circuits when disabled or <2 peers Design documentation: - Update 7 design docs to reflect cost-based parent selection - Replace depth-only algorithm descriptions with effective_depth model - Replace rejected cumulative path cost spec with local-only design rationale - Rewrite Example 2 (heterogeneous links) for local-only cost model - Update config docs: parent_switch_threshold replaced by parent_hysteresis, hold_down_secs, reeval_interval_secs Chaos simulation enhancements: - fips_overrides with deep merge for per-scenario FIPS config customization - Explicit topology algorithm for deterministic test graphs - Control socket querying via fipsctl for tree/MMP snapshot collection - Edge existence validation in netem manager - Per-link netem policy overrides - 9 new chaos scenarios covering cost avoidance, depth-vs-cost tradeoffs, stability, mixed topologies, periodic re-evaluation, and bottleneck parent 12 new unit tests, 667 total passing, clippy clean.
This commit is contained in:
Johnathan Corgan
@@ -0,0 +1,91 @@
|
||||
# Bottleneck Parent: 10-node focused LoRa bottleneck test
|
||||
#
|
||||
# Explicit topology with two LoRa links that create bottleneck parent
|
||||
# candidates. Tests that nodes avoid choosing LoRa parents when fiber
|
||||
# alternatives exist at the same or slightly greater depth.
|
||||
#
|
||||
# Topology:
|
||||
#
|
||||
# n01 (root)
|
||||
# / | \ \
|
||||
# f f f f
|
||||
# / | \ \
|
||||
# n02 n03 n04 n05
|
||||
# | / | \ |
|
||||
# LoRa f f LoRa
|
||||
# | / | \ |
|
||||
# n06 n07 n08 n09
|
||||
# |
|
||||
# f
|
||||
# |
|
||||
# n10
|
||||
#
|
||||
# Edges and link types:
|
||||
# Fiber: n01-n02, n01-n03, n01-n04, n01-n05,
|
||||
# n03-n06, n03-n07, n04-n08, n08-n10
|
||||
# LoRa: n02-n06, n05-n09
|
||||
#
|
||||
# Cross-links: n03-n08 (fiber) — gives n08 a fiber alternative to n04
|
||||
#
|
||||
# Test subjects:
|
||||
# - n06 has LoRa (n02, depth 1) and fiber (n03, depth 1) — should pick n03
|
||||
# - n09 has only LoRa (n05) — no alternative, stuck with LoRa parent
|
||||
|
||||
scenario:
|
||||
name: "bottleneck-parent"
|
||||
seed: 42
|
||||
duration_secs: 120
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 10
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n01, n04]
|
||||
- [n01, n05]
|
||||
- [n02, n06]
|
||||
- [n03, n06]
|
||||
- [n03, n07]
|
||||
- [n03, n08]
|
||||
- [n04, n08]
|
||||
- [n05, n09]
|
||||
- [n08, n10]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# LoRa links (high delay, moderate loss)
|
||||
- edges: ["n02-n06", "n05-n09"]
|
||||
policy:
|
||||
delay_ms: [400, 600]
|
||||
jitter_ms: [50, 100]
|
||||
loss_pct: [5, 15]
|
||||
mutation:
|
||||
interval_secs: {min: 30, max: 60}
|
||||
fraction: 0.2
|
||||
policies:
|
||||
normal:
|
||||
delay_ms: [1, 5]
|
||||
loss_pct: [0, 0.5]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 2
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 10}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,66 @@
|
||||
# Cost-Based Parent Selection: Bottleneck Avoidance Test
|
||||
#
|
||||
# Topology (explicit 4-node diamond):
|
||||
#
|
||||
# n01 (root — smallest addr)
|
||||
# / \
|
||||
# fiber fiber
|
||||
# / \
|
||||
# n02 n03
|
||||
# \ /
|
||||
# LoRa fiber
|
||||
# \ /
|
||||
# n04 (test subject)
|
||||
#
|
||||
# n04 has two candidate parents at depth 1: n02 (via LoRa) and n03 (via
|
||||
# fiber). Cost-based selection should pick n03 because:
|
||||
# effective_depth(n02) = 1 + ~6.0 (LoRa) = ~7.0
|
||||
# effective_depth(n03) = 1 + ~1.01 (fiber) = ~2.01
|
||||
#
|
||||
# Validation: tree snapshot shows n04's parent is n03.
|
||||
|
||||
scenario:
|
||||
name: "cost-avoidance"
|
||||
seed: 42
|
||||
duration_secs: 120
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n02, n04]
|
||||
- [n03, n04]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# LoRa link from n02 to n04
|
||||
- edges: ["n02-n04"]
|
||||
policy:
|
||||
delay_ms: [400, 600]
|
||||
jitter_ms: [50, 100]
|
||||
loss_pct: [5, 15]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 2
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 10}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,78 @@
|
||||
# Cost-Based Parent Selection: Mixed Technology 7-Node Test
|
||||
#
|
||||
# Topology (explicit, multiple link types):
|
||||
#
|
||||
# n01 (root)
|
||||
# / | \
|
||||
# fiber | fiber LoRa
|
||||
# / | \
|
||||
# n02 n03 n04
|
||||
# | | \ |
|
||||
# fiber fiber \ fiber
|
||||
# | | wifi |
|
||||
# n05 n06 n07
|
||||
#
|
||||
# Cross-links: n03-n05 (fiber), n04-n06 (wifi)
|
||||
#
|
||||
# Test subjects:
|
||||
# - n06 has edges to n03 (fiber) and n04 (wifi) — should prefer n03
|
||||
# - n04's link to root is LoRa, so n04 has high-cost parent link
|
||||
# - n07 connects only to n04 (no choice, stuck with LoRa upstream)
|
||||
#
|
||||
# Validation: tree snapshot shows n06's parent is n03 (not n04).
|
||||
|
||||
scenario:
|
||||
name: "cost-mixed-7node"
|
||||
seed: 42
|
||||
duration_secs: 180
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 7
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n01, n04]
|
||||
- [n02, n05]
|
||||
- [n03, n06]
|
||||
- [n04, n07]
|
||||
- [n03, n05]
|
||||
- [n04, n06]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# LoRa link from n01 to n04
|
||||
- edges: ["n01-n04"]
|
||||
policy:
|
||||
delay_ms: [400, 600]
|
||||
jitter_ms: [50, 100]
|
||||
loss_pct: [5, 15]
|
||||
# WiFi link from n04 to n06
|
||||
- edges: ["n04-n06"]
|
||||
policy:
|
||||
delay_ms: [5, 20]
|
||||
jitter_ms: [2, 5]
|
||||
loss_pct: [1, 3]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 3
|
||||
interval_secs: {min: 10, max: 25}
|
||||
duration_secs: {min: 5, max: 15}
|
||||
parallel_streams: 4
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,86 @@
|
||||
# Periodic Cost Re-evaluation Test
|
||||
#
|
||||
# Topology (explicit 4-node diamond):
|
||||
#
|
||||
# n01 (root)
|
||||
# / \
|
||||
# fiber fiber
|
||||
# / \
|
||||
# n02 n03
|
||||
# \ /
|
||||
# fiber fiber
|
||||
# \ /
|
||||
# n04 (test subject)
|
||||
#
|
||||
# Initial state: All links are fiber. n04 has two candidate parents at
|
||||
# depth 1: n02 and n03. Both have identical costs (~1.01), so n04 picks
|
||||
# n02 (smaller NodeAddr, tiebreak rule).
|
||||
#
|
||||
# Mutation: Stochastic netem mutation with a single "degraded" policy
|
||||
# (LoRa-like: 400-600ms delay, 5-15% loss). With fraction=0.5, on
|
||||
# average 2 of 4 edges degrade each round. Over 12 mutation rounds
|
||||
# (180s / 15s interval), n02-n04 will be degraded in some rounds.
|
||||
#
|
||||
# Expected behavior: When n02-n04 is degraded and n03-n04 stays fiber
|
||||
# (or vice versa), periodic re-evaluation detects the cost asymmetry
|
||||
# and switches parents. Look for "trigger = periodic" in logs.
|
||||
#
|
||||
# FIPS overrides: reeval_interval_secs=15 (vs default 60) to increase
|
||||
# the chance of catching a cost asymmetry within the mutation window.
|
||||
#
|
||||
# Validation: grep n04 logs for "Parent switched via periodic cost
|
||||
# re-evaluation" (trigger=periodic). If mutation never creates enough
|
||||
# asymmetry in a particular seed, the test still validates that periodic
|
||||
# re-eval runs without interference.
|
||||
|
||||
scenario:
|
||||
name: "cost-reeval"
|
||||
seed: 42
|
||||
duration_secs: 180
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n02, n04]
|
||||
- [n03, n04]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like baseline
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
mutation:
|
||||
interval_secs: {min: 12, max: 18}
|
||||
fraction: 0.5
|
||||
policies:
|
||||
degraded:
|
||||
delay_ms: [400, 600]
|
||||
jitter_ms: [50, 100]
|
||||
loss_pct: [5, 15]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 1
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 10}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
|
||||
fips_overrides:
|
||||
node:
|
||||
tree:
|
||||
reeval_interval_secs: 15
|
||||
@@ -0,0 +1,72 @@
|
||||
# Cost-Based Parent Selection: Hysteresis Stability Test
|
||||
#
|
||||
# Topology (explicit 4-node diamond, symmetric):
|
||||
#
|
||||
# n01 (root)
|
||||
# / \
|
||||
# wifi wifi
|
||||
# / \
|
||||
# n02 n03
|
||||
# \ /
|
||||
# wifi wifi
|
||||
# \ /
|
||||
# n04 (test subject)
|
||||
#
|
||||
# All links are WiFi-like with similar characteristics. Aggressive
|
||||
# netem mutation shifts link qualities every 10-20s, but the changes
|
||||
# stay within the 20% hysteresis band. n04 should pick one parent
|
||||
# and mostly stick with it — flapping indicates insufficient hysteresis.
|
||||
#
|
||||
# Validation: count "Parent switched" in n04 logs, expect <= 5 switches
|
||||
# over the full 180s duration.
|
||||
|
||||
scenario:
|
||||
name: "cost-stability"
|
||||
seed: 42
|
||||
duration_secs: 180
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n02, n04]
|
||||
- [n03, n04]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# WiFi baseline
|
||||
delay_ms: [5, 20]
|
||||
jitter_ms: [2, 5]
|
||||
loss_pct: [1, 3]
|
||||
mutation:
|
||||
interval_secs: {min: 10, max: 20}
|
||||
fraction: 1.0
|
||||
policies:
|
||||
slightly_better:
|
||||
delay_ms: [3, 8]
|
||||
jitter_ms: [1, 3]
|
||||
loss_pct: [0, 1]
|
||||
slightly_worse:
|
||||
delay_ms: [15, 25]
|
||||
jitter_ms: [3, 8]
|
||||
loss_pct: [2, 4]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 2
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 15}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,73 @@
|
||||
# Cost-Based Parent Selection: Deeper Fiber Beats Shallow LoRa
|
||||
#
|
||||
# Topology (explicit 4-node):
|
||||
#
|
||||
# n01 (root)
|
||||
# / \
|
||||
# fiber LoRa
|
||||
# / \
|
||||
# n02 n04 (test subject)
|
||||
# | /
|
||||
# fiber fiber
|
||||
# | /
|
||||
# n03
|
||||
#
|
||||
# n04 has two candidate parents:
|
||||
# - n01 (root, depth 0) via LoRa: effective_depth = 0 + ~6.0 = ~6.0
|
||||
# - n03 (depth 2) via fiber: effective_depth = 2 + ~1.01 = ~3.01
|
||||
#
|
||||
# Without cost-based selection, n04 would pick n01 (depth 0 < depth 2).
|
||||
# With cost-based selection, n04 should pick n03 (lower effective_depth
|
||||
# despite being 2 levels deeper).
|
||||
#
|
||||
# This tests the core depth-vs-cost tradeoff: a much deeper fiber path
|
||||
# should beat a direct LoRa link to root when the cost difference is
|
||||
# large enough.
|
||||
#
|
||||
# Validation: tree snapshot shows n04's parent is n03, n04's depth is 3.
|
||||
|
||||
scenario:
|
||||
name: "depth-vs-cost"
|
||||
seed: 42
|
||||
duration_secs: 120
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n02, n03]
|
||||
- [n03, n04]
|
||||
- [n01, n04]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# LoRa link from n01 to n04
|
||||
- edges: ["n01-n04"]
|
||||
policy:
|
||||
delay_ms: [400, 600]
|
||||
jitter_ms: [50, 100]
|
||||
loss_pct: [5, 15]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 1
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 10}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,103 @@
|
||||
# Mixed Technology: 10-node heterogeneous network
|
||||
#
|
||||
# Explicit topology with LoRa, WiFi, and fiber links. Tests that
|
||||
# cost-based parent selection produces a tree favoring low-cost paths
|
||||
# when multiple link technologies coexist.
|
||||
#
|
||||
# Topology:
|
||||
#
|
||||
# n01 (root)
|
||||
# / | \
|
||||
# f f f
|
||||
# / | \
|
||||
# n02 n03 n04
|
||||
# | \ | \ | \
|
||||
# f LoRa f f LoRa f
|
||||
# | \ | | \ |
|
||||
# n05 n06 n07 n08 n09
|
||||
# \ /
|
||||
# wifi---wifi
|
||||
# n10
|
||||
#
|
||||
# Edges and link types:
|
||||
# Fiber: n01-n02, n01-n03, n01-n04, n02-n05, n03-n07, n04-n09
|
||||
# LoRa: n02-n06, n04-n08
|
||||
# WiFi: n03-n06, n08-n10
|
||||
# Fiber: n03-n08, n06-n10
|
||||
#
|
||||
# Test subjects:
|
||||
# - n06 has fiber (n03) and LoRa (n02) parents — should pick n03
|
||||
# - n08 has fiber (n03) and LoRa (n04) parents — should pick n03
|
||||
#
|
||||
# Netem mutation shifts fiber-only links between normal and degraded.
|
||||
|
||||
scenario:
|
||||
name: "mixed-technology"
|
||||
seed: 42
|
||||
duration_secs: 180
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 10
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n01, n04]
|
||||
- [n02, n05]
|
||||
- [n02, n06]
|
||||
- [n03, n06]
|
||||
- [n03, n07]
|
||||
- [n03, n08]
|
||||
- [n04, n08]
|
||||
- [n04, n09]
|
||||
- [n06, n10]
|
||||
- [n08, n10]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like defaults
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# LoRa links (high latency, moderate loss)
|
||||
- edges: ["n02-n06", "n04-n08"]
|
||||
policy:
|
||||
delay_ms: [400, 600]
|
||||
jitter_ms: [50, 100]
|
||||
loss_pct: [5, 15]
|
||||
# WiFi links (moderate latency, low loss)
|
||||
- edges: ["n03-n06", "n08-n10"]
|
||||
policy:
|
||||
delay_ms: [5, 20]
|
||||
jitter_ms: [2, 5]
|
||||
loss_pct: [1, 3]
|
||||
mutation:
|
||||
interval_secs: {min: 30, max: 60}
|
||||
fraction: 0.2
|
||||
policies:
|
||||
normal:
|
||||
delay_ms: [1, 10]
|
||||
loss_pct: [0, 1]
|
||||
degraded:
|
||||
delay_ms: [50, 100]
|
||||
jitter_ms: [10, 30]
|
||||
loss_pct: [3, 8]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 3
|
||||
interval_secs: {min: 10, max: 30}
|
||||
duration_secs: {min: 5, max: 15}
|
||||
parallel_streams: 4
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -3,9 +3,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from copy import deepcopy
|
||||
|
||||
import yaml
|
||||
|
||||
from .topology import SimTopology
|
||||
|
||||
|
||||
def _deep_merge(base: dict, override: dict) -> dict:
|
||||
"""Recursively merge override into base (override wins on conflicts)."""
|
||||
result = deepcopy(base)
|
||||
for key, value in override.items():
|
||||
if key in result and isinstance(result[key], dict) and isinstance(value, dict):
|
||||
result[key] = _deep_merge(result[key], value)
|
||||
else:
|
||||
result[key] = deepcopy(value)
|
||||
return result
|
||||
|
||||
# Path to the shared node config template
|
||||
_TEMPLATE_PATH = os.path.join(
|
||||
os.path.dirname(__file__), "..", "configs", "node.template.yaml"
|
||||
@@ -41,7 +55,10 @@ def generate_peers_block(
|
||||
|
||||
|
||||
def generate_node_config(
|
||||
topology: SimTopology, node_id: str, outbound_peers: list[str]
|
||||
topology: SimTopology,
|
||||
node_id: str,
|
||||
outbound_peers: list[str],
|
||||
fips_overrides: dict | None = None,
|
||||
) -> str:
|
||||
"""Generate a complete FIPS config YAML for one node."""
|
||||
template = _load_template()
|
||||
@@ -54,6 +71,12 @@ def generate_node_config(
|
||||
config = config.replace("{{NPUB}}", node.npub)
|
||||
config = config.replace("{{NSEC}}", node.nsec)
|
||||
config = config.replace("{{PEERS}}", peers_yaml)
|
||||
|
||||
if fips_overrides:
|
||||
parsed = yaml.safe_load(config)
|
||||
merged = _deep_merge(parsed, fips_overrides)
|
||||
config = yaml.dump(merged, default_flow_style=False, sort_keys=False)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
@@ -67,13 +90,19 @@ def generate_npubs_env(topology: SimTopology) -> str:
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def write_configs(topology: SimTopology, output_dir: str):
|
||||
def write_configs(
|
||||
topology: SimTopology,
|
||||
output_dir: str,
|
||||
fips_overrides: dict | None = None,
|
||||
):
|
||||
"""Write all node configs and npubs.env to the output directory."""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
outbound = topology.directed_outbound()
|
||||
for node_id in topology.nodes:
|
||||
config = generate_node_config(topology, node_id, outbound[node_id])
|
||||
config = generate_node_config(
|
||||
topology, node_id, outbound[node_id], fips_overrides
|
||||
)
|
||||
path = os.path.join(output_dir, f"{node_id}.yaml")
|
||||
with open(path, "w") as f:
|
||||
f.write(config)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Control socket querying via docker exec.
|
||||
|
||||
Queries FIPS nodes' control sockets to observe runtime state (tree
|
||||
structure, MMP metrics, peers) without requiring fipsctl in the
|
||||
container. Uses a Python one-liner inside docker exec since python3
|
||||
is available in the Docker image.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from .docker_exec import docker_exec, docker_exec_quiet
|
||||
from .topology import SimTopology
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Default control socket path inside containers (XDG_RUNTIME_DIR is
|
||||
# typically unset in Docker, so fips falls back to /tmp).
|
||||
CONTROL_SOCKET = "/tmp/fips-control.sock"
|
||||
|
||||
|
||||
def query_node(container: str, command: str, timeout: int = 10) -> dict | None:
|
||||
"""Send a command to a node's control socket, return the data dict.
|
||||
|
||||
Returns None if the query fails (node down, socket not ready, etc.).
|
||||
"""
|
||||
# Python one-liner that connects to the Unix socket, sends the JSON
|
||||
# command, and prints the response. Runs inside the container.
|
||||
script = (
|
||||
"import socket,json,sys; "
|
||||
"s=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM); "
|
||||
f"s.connect('{CONTROL_SOCKET}'); "
|
||||
f"s.sendall(json.dumps({{'command':'{command}'}}).encode()+b'\\n'); "
|
||||
"s.shutdown(socket.SHUT_WR); "
|
||||
"chunks=[]; "
|
||||
"[chunks.append(d) for d in iter(lambda:s.recv(65536),b'')]; "
|
||||
"print(b''.join(chunks).decode())"
|
||||
)
|
||||
stdout = docker_exec_quiet(container, f"python3 -c \"{script}\"", timeout=timeout)
|
||||
if stdout is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
response = json.loads(stdout.strip())
|
||||
except json.JSONDecodeError as e:
|
||||
log.warning("Invalid JSON from %s: %s", container, e)
|
||||
return None
|
||||
|
||||
if response.get("status") != "ok":
|
||||
msg = response.get("message", "unknown error")
|
||||
log.warning("Control query %s on %s failed: %s", command, container, msg)
|
||||
return None
|
||||
|
||||
return response.get("data", {})
|
||||
|
||||
|
||||
def query_tree(container: str) -> dict | None:
|
||||
"""Query a node's spanning tree state."""
|
||||
return query_node(container, "show_tree")
|
||||
|
||||
|
||||
def query_mmp(container: str) -> dict | None:
|
||||
"""Query a node's MMP metrics."""
|
||||
return query_node(container, "show_mmp")
|
||||
|
||||
|
||||
def query_peers(container: str) -> dict | None:
|
||||
"""Query a node's peer list with MMP metrics."""
|
||||
return query_node(container, "show_peers")
|
||||
|
||||
|
||||
def snapshot_all_trees(topology: SimTopology) -> dict[str, dict]:
|
||||
"""Query show_tree on all nodes, return {node_id: tree_data}.
|
||||
|
||||
Nodes that fail to respond are omitted from the result.
|
||||
"""
|
||||
result = {}
|
||||
for node_id in sorted(topology.nodes):
|
||||
container = topology.container_name(node_id)
|
||||
data = query_tree(container)
|
||||
if data is not None:
|
||||
result[node_id] = data
|
||||
else:
|
||||
log.warning("No tree data from %s", node_id)
|
||||
return result
|
||||
|
||||
|
||||
def snapshot_all_mmp(topology: SimTopology) -> dict[str, dict]:
|
||||
"""Query show_mmp on all nodes, return {node_id: mmp_data}.
|
||||
|
||||
Nodes that fail to respond are omitted from the result.
|
||||
"""
|
||||
result = {}
|
||||
for node_id in sorted(topology.nodes):
|
||||
container = topology.container_name(node_id)
|
||||
data = query_mmp(container)
|
||||
if data is not None:
|
||||
result[node_id] = data
|
||||
else:
|
||||
log.warning("No MMP data from %s", node_id)
|
||||
return result
|
||||
@@ -17,7 +17,7 @@ import random
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .docker_exec import docker_exec_quiet, is_container_running
|
||||
from .scenario import BandwidthConfig, NetemConfig, NetemPolicy
|
||||
from .scenario import BandwidthConfig, LinkPolicyOverride, NetemConfig, NetemPolicy
|
||||
from .topology import SimTopology
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -91,12 +91,41 @@ class NetemManager:
|
||||
rate = rng.choice(bandwidth.tiers_mbps)
|
||||
self._edge_rates[(a, b)] = rate
|
||||
self._edge_rates[(b, a)] = rate
|
||||
# Per-edge policy overrides: canonical "nXX-nYY" -> NetemPolicy
|
||||
# Build a set of canonical edge strings for validation
|
||||
topo_edge_strs = {"-".join(sorted([a, b])) for a, b in topology.edges}
|
||||
self._edge_overrides: dict[str, NetemPolicy] = {}
|
||||
for override in config.link_policies:
|
||||
policy = override.policy
|
||||
if policy is None and override.policy_name:
|
||||
policy = config.mutation.policies.get(override.policy_name)
|
||||
if policy is None:
|
||||
continue
|
||||
for edge_str in override.edges:
|
||||
if edge_str not in topo_edge_strs:
|
||||
log.warning(
|
||||
"link_policy edge %s not in topology — override ignored",
|
||||
edge_str,
|
||||
)
|
||||
self._edge_overrides[edge_str] = policy
|
||||
if self._edge_overrides:
|
||||
log.info(
|
||||
"Per-link policy overrides: %d edges",
|
||||
len(self._edge_overrides),
|
||||
)
|
||||
|
||||
def _htb_rate(self, node_id: str, peer_id: str) -> str:
|
||||
"""Return the HTB rate string for a link direction."""
|
||||
rate = self._edge_rates.get((node_id, peer_id), 0)
|
||||
return f"{rate}mbit" if rate > 0 else "1gbit"
|
||||
|
||||
def _policy_for_edge(self, node_a: str, node_b: str) -> NetemPolicy:
|
||||
"""Return the netem policy for an edge, checking overrides first."""
|
||||
canonical = "-".join(sorted([node_a, node_b]))
|
||||
if canonical in self._edge_overrides:
|
||||
return self._edge_overrides[canonical]
|
||||
return self.config.default_policy
|
||||
|
||||
def setup_initial(self):
|
||||
"""Set up HTB qdiscs and initial netem on all containers."""
|
||||
if self._edge_rates:
|
||||
@@ -128,8 +157,9 @@ class NetemManager:
|
||||
class_id = f"1:{idx}"
|
||||
netem_handle = f"{idx + 10}:"
|
||||
|
||||
# Sample initial params from default policy
|
||||
params = self._sample_policy(self.config.default_policy)
|
||||
# Sample initial params (per-edge override or default policy)
|
||||
policy = self._policy_for_edge(node_id, peer_id)
|
||||
params = self._sample_policy(policy)
|
||||
|
||||
rate = self._htb_rate(node_id, peer_id)
|
||||
rate_mbit = self._edge_rates.get((node_id, peer_id), 0)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
@@ -11,6 +12,7 @@ import time
|
||||
|
||||
from .compose import generate_compose
|
||||
from .config_gen import write_configs
|
||||
from .control import snapshot_all_mmp, snapshot_all_trees
|
||||
from .docker_exec import docker_compose
|
||||
from .links import LinkManager
|
||||
from .logs import AnalysisResult, analyze_logs, collect_logs, write_sim_metadata
|
||||
@@ -108,7 +110,7 @@ class SimRunner:
|
||||
config_dir = os.path.normpath(
|
||||
os.path.join(docker_network_dir, "generated-configs", "sim")
|
||||
)
|
||||
write_configs(self.topology, config_dir)
|
||||
write_configs(self.topology, config_dir, self.scenario.fips_overrides)
|
||||
log.info("Wrote node configs to %s", config_dir)
|
||||
|
||||
# 3. Generate docker-compose.yml
|
||||
@@ -153,6 +155,7 @@ class SimRunner:
|
||||
wait = max(10, n) # Heuristic: ~1s per node, minimum 10s
|
||||
log.info("Waiting %ds for mesh convergence...", wait)
|
||||
self._sleep(wait)
|
||||
self._take_snapshot("warmup")
|
||||
|
||||
def _simulation_loop(self):
|
||||
"""Main event loop driving stochastic behavior."""
|
||||
@@ -233,11 +236,14 @@ class SimRunner:
|
||||
log.info("Restoring downed links...")
|
||||
self.link_mgr.restore_all()
|
||||
|
||||
# Restore stopped nodes (needed for log collection)
|
||||
# Restore stopped nodes (needed for snapshots and log collection)
|
||||
if self.node_mgr:
|
||||
log.info("Restoring stopped nodes...")
|
||||
self.node_mgr.restore_all()
|
||||
|
||||
# Take final tree snapshot while nodes are still running
|
||||
self._take_snapshot("final")
|
||||
|
||||
# Collect logs before stopping containers
|
||||
container_names = [
|
||||
self.topology.container_name(nid) for nid in sorted(self.topology.nodes)
|
||||
@@ -273,6 +279,30 @@ class SimRunner:
|
||||
|
||||
return result
|
||||
|
||||
def _take_snapshot(self, label: str):
|
||||
"""Query all nodes via control socket and save tree/MMP snapshots."""
|
||||
if not self.topology:
|
||||
return
|
||||
log.info("Taking %s snapshot...", label)
|
||||
tree_snap = snapshot_all_trees(self.topology)
|
||||
mmp_snap = snapshot_all_mmp(self.topology)
|
||||
|
||||
tree_path = os.path.join(self.output_dir, f"tree-snapshot-{label}.json")
|
||||
mmp_path = os.path.join(self.output_dir, f"mmp-snapshot-{label}.json")
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
with open(tree_path, "w") as f:
|
||||
json.dump(tree_snap, f, indent=2)
|
||||
with open(mmp_path, "w") as f:
|
||||
json.dump(mmp_snap, f, indent=2)
|
||||
log.info(
|
||||
"Snapshot %s: %d/%d tree, %d/%d mmp responses",
|
||||
label,
|
||||
len(tree_snap),
|
||||
len(self.topology.nodes),
|
||||
len(mmp_snap),
|
||||
len(self.topology.nodes),
|
||||
)
|
||||
|
||||
def _schedule_next(self, now: float, interval) -> float:
|
||||
"""Schedule the next event using a Range interval."""
|
||||
return now + self.rng.uniform(interval.min, interval.max)
|
||||
|
||||
@@ -49,10 +49,25 @@ class NetemMutationConfig:
|
||||
policies: dict[str, NetemPolicy] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LinkPolicyOverride:
|
||||
"""Per-edge netem policy override.
|
||||
|
||||
Edges are specified as "nXX-nYY" strings in canonical form (sorted
|
||||
alphabetically). Either ``policy`` (inline) or ``policy_name``
|
||||
(reference to a named mutation policy) must be set, not both.
|
||||
"""
|
||||
|
||||
edges: list[str] = field(default_factory=list)
|
||||
policy: NetemPolicy | None = None
|
||||
policy_name: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class NetemConfig:
|
||||
enabled: bool = False
|
||||
default_policy: NetemPolicy = field(default_factory=NetemPolicy)
|
||||
link_policies: list[LinkPolicyOverride] = field(default_factory=list)
|
||||
mutation: NetemMutationConfig = field(default_factory=NetemMutationConfig)
|
||||
|
||||
|
||||
@@ -115,6 +130,10 @@ class Scenario:
|
||||
node_churn: NodeChurnConfig = field(default_factory=NodeChurnConfig)
|
||||
bandwidth: BandwidthConfig = field(default_factory=BandwidthConfig)
|
||||
logging: LoggingConfig = field(default_factory=LoggingConfig)
|
||||
# Raw YAML dict appended to each generated FIPS node config.
|
||||
# Allows scenarios to override any FIPS config parameter
|
||||
# (e.g., node.tree.reeval_interval_secs).
|
||||
fips_overrides: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def _parse_range(data, name: str) -> Range:
|
||||
@@ -173,6 +192,16 @@ def load_scenario(path: str) -> Scenario:
|
||||
s.netem.enabled = nc.get("enabled", False)
|
||||
if "default_policy" in nc:
|
||||
s.netem.default_policy = _parse_netem_policy(nc["default_policy"])
|
||||
if "link_policies" in nc:
|
||||
for lp_data in nc["link_policies"]:
|
||||
override = LinkPolicyOverride(
|
||||
edges=lp_data.get("edges", []),
|
||||
)
|
||||
if "policy" in lp_data:
|
||||
override.policy = _parse_netem_policy(lp_data["policy"])
|
||||
if "policy_name" in lp_data:
|
||||
override.policy_name = lp_data["policy_name"]
|
||||
s.netem.link_policies.append(override)
|
||||
if "mutation" in nc:
|
||||
mc = nc["mutation"]
|
||||
s.netem.mutation.interval_secs = _parse_range(
|
||||
@@ -233,6 +262,9 @@ def load_scenario(path: str) -> Scenario:
|
||||
s.logging.rust_log = lg.get("rust_log", "info")
|
||||
s.logging.output_dir = lg.get("output_dir", "./sim-results")
|
||||
|
||||
# FIPS config overrides (raw YAML dict appended to node configs)
|
||||
s.fips_overrides = raw.get("fips_overrides", {})
|
||||
|
||||
# Validation
|
||||
_validate(s)
|
||||
|
||||
@@ -245,11 +277,45 @@ def _validate(s: Scenario):
|
||||
raise ValueError("topology.num_nodes must be >= 2")
|
||||
if s.topology.num_nodes > 250:
|
||||
raise ValueError("topology.num_nodes must be <= 250 (subnet limit)")
|
||||
if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain"):
|
||||
if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain", "explicit"):
|
||||
raise ValueError(f"Unknown topology algorithm: {s.topology.algorithm}")
|
||||
if s.topology.algorithm == "explicit":
|
||||
adj = s.topology.params.get("adjacency")
|
||||
if not adj or not isinstance(adj, list):
|
||||
raise ValueError("explicit topology requires params.adjacency list")
|
||||
node_ids = set()
|
||||
for i, pair in enumerate(adj):
|
||||
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
|
||||
raise ValueError(
|
||||
f"explicit adjacency[{i}]: expected [nodeA, nodeB], got {pair}"
|
||||
)
|
||||
node_ids.update(str(p) for p in pair)
|
||||
if len(node_ids) != s.topology.num_nodes:
|
||||
raise ValueError(
|
||||
f"explicit adjacency references {len(node_ids)} nodes "
|
||||
f"but num_nodes is {s.topology.num_nodes}"
|
||||
)
|
||||
if s.duration_secs < 1:
|
||||
raise ValueError("duration_secs must be >= 1")
|
||||
|
||||
# Validate link_policies
|
||||
for i, lp in enumerate(s.netem.link_policies):
|
||||
if not lp.edges:
|
||||
raise ValueError(f"netem.link_policies[{i}]: edges list is empty")
|
||||
if lp.policy is not None and lp.policy_name is not None:
|
||||
raise ValueError(
|
||||
f"netem.link_policies[{i}]: specify policy or policy_name, not both"
|
||||
)
|
||||
if lp.policy is None and lp.policy_name is None:
|
||||
raise ValueError(
|
||||
f"netem.link_policies[{i}]: must specify policy or policy_name"
|
||||
)
|
||||
if lp.policy_name and lp.policy_name not in s.netem.mutation.policies:
|
||||
raise ValueError(
|
||||
f"netem.link_policies[{i}]: policy_name '{lp.policy_name}' "
|
||||
f"not found in mutation.policies"
|
||||
)
|
||||
|
||||
# Validate ranges
|
||||
if s.netem.enabled and s.netem.mutation.policies:
|
||||
s.netem.mutation.interval_secs.validate("netem.mutation.interval_secs")
|
||||
|
||||
@@ -130,6 +130,17 @@ def generate_topology(
|
||||
elif config.algorithm == "erdos_renyi":
|
||||
p = config.params.get("p", 0.3)
|
||||
edges = _generate_erdos_renyi(node_ids, p, rng)
|
||||
elif config.algorithm == "explicit":
|
||||
adjacency = config.params.get("adjacency")
|
||||
if not adjacency:
|
||||
raise ValueError("explicit topology requires params.adjacency")
|
||||
edges = _generate_explicit(adjacency)
|
||||
# Validate all referenced nodes exist
|
||||
for a, b in edges:
|
||||
if a not in nodes:
|
||||
raise ValueError(f"explicit adjacency references unknown node {a}")
|
||||
if b not in nodes:
|
||||
raise ValueError(f"explicit adjacency references unknown node {b}")
|
||||
else:
|
||||
raise ValueError(f"Unknown algorithm: {config.algorithm}")
|
||||
|
||||
@@ -212,6 +223,21 @@ def _generate_erdos_renyi(
|
||||
return edges
|
||||
|
||||
|
||||
def _generate_explicit(adjacency: list) -> set[tuple[str, str]]:
|
||||
"""Build edges from an explicit adjacency list.
|
||||
|
||||
Each entry should be a 2-element list like ["n01", "n02"].
|
||||
"""
|
||||
edges = set()
|
||||
for i, pair in enumerate(adjacency):
|
||||
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
|
||||
raise ValueError(
|
||||
f"explicit adjacency[{i}]: expected [nodeA, nodeB], got {pair}"
|
||||
)
|
||||
edges.add(_make_edge(str(pair[0]), str(pair[1])))
|
||||
return edges
|
||||
|
||||
|
||||
def _make_edge(a: str, b: str) -> tuple[str, str]:
|
||||
"""Canonical edge representation (sorted)."""
|
||||
return (min(a, b), max(a, b))
|
||||
|
||||
Reference in New Issue
Block a user