From 0d93a19e07c71dc65918b3185b3119beb03a60d7 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 23 Feb 2026 17:15:20 +0000 Subject: [PATCH] Implement cost-based parent selection with periodic re-evaluation Cost-based parent selection: - Replace depth-only parent selection with effective_depth = depth + link_cost - link_cost computed from locally measured MMP metrics: etx * (1.0 + srtt_ms / 100.0) - Prevents bottleneck subtrees in heterogeneous networks where a LoRa link at depth 1 would otherwise always beat fiber at depth 2 - Configurable hysteresis (default 0.2) prevents marginal parent switches - Configurable hold-down timer (default 30s) suppresses re-evaluation after parent switch - Mandatory switches (parent lost, root change) bypass both safeguards - Link costs passed as HashMap parameter to keep TreeState pure Periodic re-evaluation: - evaluate_parent() was only called on TreeAnnounce receipt or parent loss; after tree stabilization, link degradation went undetected - Added timer-based re-evaluation (reeval_interval_secs, default 60s) that calls evaluate_parent() from the tick handler with current MMP link costs - Respects existing hold-down and hysteresis safeguards - Short-circuits when disabled or <2 peers Design documentation: - Update 7 design docs to reflect cost-based parent selection - Replace depth-only algorithm descriptions with effective_depth model - Replace rejected cumulative path cost spec with local-only design rationale - Rewrite Example 2 (heterogeneous links) for local-only cost model - Update config docs: parent_switch_threshold replaced by parent_hysteresis, hold_down_secs, reeval_interval_secs Chaos simulation enhancements: - fips_overrides with deep merge for per-scenario FIPS config customization - Explicit topology algorithm for deterministic test graphs - Control socket querying via fipsctl for tree/MMP snapshot collection - Edge existence validation in netem manager - Per-link netem policy overrides - 9 new chaos scenarios covering cost avoidance, depth-vs-cost tradeoffs, stability, mixed topologies, periodic re-evaluation, and bottleneck parent 12 new unit tests, 667 total passing, clippy clean. --- docs/design/fips-configuration.md | 8 +- docs/design/fips-intro.md | 12 +- docs/design/fips-mesh-layer.md | 9 +- docs/design/fips-mesh-operation.md | 12 +- docs/design/fips-spanning-tree.md | 51 +- docs/design/fips-transport-layer.md | 9 +- docs/design/spanning-tree-dynamics.md | 234 +++++----- src/config/node.rs | 31 +- src/node/mod.rs | 12 +- src/node/tree.rs | 80 +++- src/peer/active.rs | 19 +- src/tree/state.rs | 89 +++- src/tree/tests.rs | 438 +++++++++++++++++- .../chaos/scenarios/bottleneck-parent.yaml | 91 ++++ testing/chaos/scenarios/cost-avoidance.yaml | 66 +++ testing/chaos/scenarios/cost-mixed-7node.yaml | 78 ++++ testing/chaos/scenarios/cost-reeval.yaml | 86 ++++ testing/chaos/scenarios/cost-stability.yaml | 72 +++ testing/chaos/scenarios/depth-vs-cost.yaml | 73 +++ testing/chaos/scenarios/mixed-technology.yaml | 103 ++++ testing/chaos/sim/config_gen.py | 35 +- testing/chaos/sim/control.py | 103 ++++ testing/chaos/sim/netem.py | 36 +- testing/chaos/sim/runner.py | 34 +- testing/chaos/sim/scenario.py | 68 ++- testing/chaos/sim/topology.py | 26 ++ 26 files changed, 1675 insertions(+), 200 deletions(-) create mode 100644 testing/chaos/scenarios/bottleneck-parent.yaml create mode 100644 testing/chaos/scenarios/cost-avoidance.yaml create mode 100644 testing/chaos/scenarios/cost-mixed-7node.yaml create mode 100644 testing/chaos/scenarios/cost-reeval.yaml create mode 100644 testing/chaos/scenarios/cost-stability.yaml create mode 100644 testing/chaos/scenarios/depth-vs-cost.yaml create mode 100644 testing/chaos/scenarios/mixed-technology.yaml create mode 100644 testing/chaos/sim/control.py diff --git a/docs/design/fips-configuration.md b/docs/design/fips-configuration.md index e53c0b9..3e36389 100644 --- a/docs/design/fips-configuration.md +++ b/docs/design/fips-configuration.md @@ -148,7 +148,9 @@ Controls tree construction and parent selection. | Parameter | Type | Default | Description | |----------------------------------------|-------|---------|--------------------------------------------------| | `node.tree.announce_min_interval_ms` | u64 | `500` | Per-peer TreeAnnounce rate limit | -| `node.tree.parent_switch_threshold` | usize | `1` | Min depth improvement required to switch parents | +| `node.tree.parent_hysteresis` | f64 | `0.2` | Cost improvement fraction required for same-root parent switch (0.0–1.0) | +| `node.tree.hold_down_secs` | u64 | `30` | Suppress non-mandatory re-evaluation after parent switch | +| `node.tree.reeval_interval_secs` | u64 | `60` | Periodic cost-based parent re-evaluation interval (0 = disabled) | ### Bloom Filter (`node.bloom.*`) @@ -346,7 +348,9 @@ node: recent_expiry_secs: 10 tree: announce_min_interval_ms: 500 - parent_switch_threshold: 1 + parent_hysteresis: 0.2 # cost improvement fraction for parent switch + hold_down_secs: 30 # suppress re-evaluation after switch + reeval_interval_secs: 60 # periodic cost-based re-evaluation (0 = disabled) bloom: update_debounce_ms: 500 session: diff --git a/docs/design/fips-intro.md b/docs/design/fips-intro.md index 7f5c5a2..50bc326 100644 --- a/docs/design/fips-intro.md +++ b/docs/design/fips-intro.md @@ -460,11 +460,13 @@ low-latency links converge quickly. Each metric carries both short-term and long-term exponentially weighted moving averages, enabling detection of quality changes against a stable baseline. -MMP's primary role today is operator visibility — periodic log lines report -per-link RTT, loss, jitter, and goodput. The metrics also provide the -foundation for quality-aware routing: MMP computes an Expected Transmission -Count (ETX) from bidirectional delivery ratios, which will feed into -candidate ranking once link-cost routing is enabled. +MMP serves dual roles: operator visibility and cost-based parent selection. +Periodic log lines report per-link RTT, loss, jitter, and goodput. MMP +computes an Expected Transmission Count (ETX) from bidirectional delivery +ratios, which feeds into cost-based parent selection where each node +evaluates `effective_depth = depth + link_cost` using +`link_cost = etx * (1.0 + srtt_ms / 100.0)`. ETX is not yet used in +`find_next_hop()` candidate ranking for data forwarding. See [fips-mesh-layer.md](fips-mesh-layer.md) for MMP operating modes, report scheduling, and the spin bit design. diff --git a/docs/design/fips-mesh-layer.md b/docs/design/fips-mesh-layer.md index bff1d35..7e64637 100644 --- a/docs/design/fips-mesh-layer.md +++ b/docs/design/fips-mesh-layer.md @@ -405,8 +405,8 @@ work together to build and maintain the mesh, and ## Metrics Measurement Protocol (MMP) Each active peer link runs an instance of the Metrics Measurement Protocol, -providing per-link quality metrics to the operator and (in future) to the -routing layer for link-cost decisions. +providing per-link quality metrics to the operator and to the spanning tree +layer for cost-based parent selection. ### Metrics Tracked @@ -422,8 +422,9 @@ fields in the FMP wire format: - **OWD trend** — One-way delay trend (µs/s, signed). Indicates congestion buildup before loss occurs. - **ETX** — Expected Transmission Count, computed from bidirectional delivery - ratios. Not yet wired into routing decisions (link cost is currently - constant). + ratios. Used in cost-based parent selection via + `link_cost = etx * (1.0 + srtt_ms / 100.0)`; not yet used in + `find_next_hop()` candidate ranking. - **Dual EWMA trends** — Short-term (α=1/4) and long-term (α=1/32) trend indicators for both RTT and loss, enabling change detection. diff --git a/docs/design/fips-mesh-operation.md b/docs/design/fips-mesh-operation.md index c16b08c..7ca60b1 100644 --- a/docs/design/fips-mesh-operation.md +++ b/docs/design/fips-mesh-operation.md @@ -52,8 +52,8 @@ Nodes self-organize into a spanning tree through distributed parent selection: No election protocol — this is a consequence of each node independently preferring lower-addressed roots. 2. **Parent selection**: Each node selects a single parent from among its - direct peers based on which offers the best path to root (considering - depth improvement threshold). + direct peers based on which offers the lowest effective depth (tree depth + weighted by local link cost). 3. **Coordinate computation**: Once a node has a parent, its coordinate is computed from its ancestry path. @@ -190,8 +190,9 @@ When bloom filters identify multiple candidate peers, they are ranked by a composite key: 1. **link_cost** — Per-link quality metric. ETX is computed from bidirectional - delivery ratios in MMP metrics but is not yet wired into `find_next_hop()` - candidate ranking; link cost remains constant in the current implementation. + delivery ratios in MMP metrics and is used in cost-based parent selection + via `effective_depth = depth + link_cost`, but is not yet wired into + `find_next_hop()` candidate ranking. 2. **tree_distance** — Coordinate-based distance to destination through this peer 3. **node_addr** — Deterministic tie-breaker @@ -621,7 +622,8 @@ recovery). | Discovery reverse-path routing | **Implemented** | | Error signal rate limiting | **Implemented** | | Leaf-only operation | Future direction | -| Link cost metrics (ETX) | Future direction | +| Link cost in parent selection (ETX) | **Implemented** | +| Link cost in candidate ranking | Future direction | | Discovery path accumulation | Future direction | ## References diff --git a/docs/design/fips-spanning-tree.md b/docs/design/fips-spanning-tree.md index ba1126b..ef7ebeb 100644 --- a/docs/design/fips-spanning-tree.md +++ b/docs/design/fips-spanning-tree.md @@ -35,28 +35,46 @@ reconverge to a single tree. ## Parent Selection Each node selects a single parent from among its direct peers. Parent -selection follows these rules: +selection uses cost-weighted depth to balance tree depth against link +quality. ### Selection Criteria 1. **Find the smallest root** visible across all peers' TreeAnnounce messages -2. Among peers that can reach that root, prefer the one offering the - **shallowest depth** (shortest path to root) -3. Apply the **depth improvement threshold**: switching parents requires the - proposed parent to offer a path at least 1 hop shallower than the current - parent (when the root is the same) +2. Compute **effective depth** for each candidate peer: + `effective_depth = peer.depth + link_cost`, where + `link_cost = etx * (1.0 + srtt_ms / 100.0)` using locally measured MMP + metrics. When MMP metrics have not yet converged, `link_cost` defaults to + 1.0, preserving pure depth-based behavior as a graceful fallback. +3. Apply **hysteresis**: switch parents only when the best candidate's + effective depth is significantly better than the current parent's: + `best_eff_depth < current_eff_depth * (1.0 - parent_hysteresis)` + (default `parent_hysteresis = 0.2`, requiring 20% improvement) -### Immediate Switch Triggers +### Mandatory Switch Triggers -Three conditions bypass the depth threshold and trigger immediate parent -reselection: +Two conditions bypass both hysteresis and the hold-down timer, triggering +immediate parent reselection: 1. **Parent loss**: Current parent is no longer in the peer set (link broken, peer disconnected) 2. **Better root**: A peer advertises a smaller root than the current - tree's root — always switch regardless of depth -3. **Depth improvement**: Same root, but the proposed parent offers depth - at least `PARENT_SWITCH_THRESHOLD` (1 hop) better than the current parent + tree's root — always switch regardless of effective depth + +### Stability Mechanisms + +- **Hold-down timer** (`hold_down_secs`, default 30s): After any parent + switch, non-mandatory re-evaluation is suppressed to allow MMP metrics + to stabilize on the new link. Mandatory switches (parent loss, root + change) bypass the hold-down. +- **Periodic re-evaluation** (`reeval_interval_secs`, default 60s): + Re-evaluates parent selection using current MMP link costs, independent + of TreeAnnounce traffic. This catches link degradation after the tree + has stabilized and TreeAnnounce gossip has stopped. +- **Local-only metrics**: Link costs use only locally measured MMP data + (ETX and SRTT). No cumulative path costs are propagated and no wire + format changes are required. This avoids the trust problems inherent + in self-reported cost metrics in a permissionless network. ### After Parent Change @@ -228,7 +246,9 @@ Example: In a 1000-node network with depth 10 and 5 peers, a node stores | Parameter | Default | Description | | --------- | ------- | ----------- | -| PARENT_SWITCH_THRESHOLD | 1 hop | Minimum depth improvement for same-root switch | +| PARENT_HYSTERESIS | 0.2 (20%) | Fractional improvement in effective depth required for same-root switch | +| HOLD_DOWN_SECS | 30s | Suppress non-mandatory re-evaluation after parent switch | +| REEVAL_INTERVAL_SECS | 60s | Periodic cost-based parent re-evaluation interval | | ANNOUNCE_MIN_INTERVAL | 500ms | Minimum between announcements to same peer | | ROOT_TIMEOUT | 60 min | Root declaration considered stale (not yet enforced; heartbeat cascading covers common case) | | TREE_ENTRY_TTL | 5–10 min | Individual entry expiration | @@ -238,7 +258,9 @@ Example: In a 1000-node network with depth 10 and 5 peers, a node stores | Feature | Status | | ------- | ------ | | Root election (smallest node_addr) | **Implemented** | -| Parent selection with depth threshold | **Implemented** | +| Cost-based parent selection with hysteresis | **Implemented** | +| Hold-down timer after parent change | **Implemented** | +| Periodic cost-based parent re-evaluation | **Implemented** | | Coordinate computation | **Implemented** | | TreeAnnounce gossip | **Implemented** | | Signature verification (outer) | **Implemented** | @@ -247,7 +269,6 @@ Example: In a 1000-node network with depth 10 and 5 peers, a node stores | Coord cache flush on parent change | **Implemented** | | Root timeout enforcement | Planned | | Tree entry TTL enforcement | Planned | -| Hold-down timer after parent change | Planned | | Per-ancestry-entry signatures | Future direction | ## References diff --git a/docs/design/fips-transport-layer.md b/docs/design/fips-transport-layer.md index 41196cd..7db8d75 100644 --- a/docs/design/fips-transport-layer.md +++ b/docs/design/fips-transport-layer.md @@ -410,9 +410,12 @@ MTU, and its own liveness tracking. ### Transport Quality and Path Selection Transport characteristics (latency, bandwidth, reliability) affect path -quality but are not currently factored into routing decisions. The spanning -tree parent selection uses a depth improvement threshold but does not -consider transport quality. This is a potential area for future optimization. +quality. The spanning tree parent selection factors in link quality through +cost-based effective depth (`effective_depth = depth + link_cost`), where +`link_cost` is derived from locally measured MMP metrics (ETX and SRTT). +This allows the tree to prefer lower-latency, lower-loss links when the +quality difference is significant. Link cost is not yet used in +`find_next_hop()` candidate ranking for data forwarding. ## References diff --git a/docs/design/spanning-tree-dynamics.md b/docs/design/spanning-tree-dynamics.md index d6ff2ff..9e7b2a2 100644 --- a/docs/design/spanning-tree-dynamics.md +++ b/docs/design/spanning-tree-dynamics.md @@ -24,7 +24,7 @@ The protocol is based on Yggdrasil v0.5's CRDT gossip design. 8. [Parent Selection](#8-parent-selection) 9. [Steady State Behavior](#9-steady-state-behavior) 10. [Worked Examples](#10-worked-examples) -11. [Known Limitations (v1 Implementation)](#known-limitations-v1-implementation) +11. [Known Limitations](#known-limitations) --- @@ -380,8 +380,10 @@ Link B ←→ C fails: ### Reconvergence Dynamics **Stability threshold**: To prevent flapping, a node only changes parent when -the improvement exceeds a threshold. In v1, this is a depth difference of at -least `PARENT_SWITCH_THRESHOLD` (1 hop). See §8 for details. +the improvement exceeds cost-based hysteresis (`parent_hysteresis`, default +0.2 = 20% improvement required). A hold-down timer (`hold_down_secs`, +default 30s) further suppresses non-mandatory re-evaluation after a switch. +See §8 for details. **Sequence number advancement**: Each parent change increments the sequence number. Nodes observing rapid sequence increases can detect instability and @@ -421,7 +423,7 @@ Nodes detect they're partitioned when: 2. **Root unreachable**: No peer has path to current root **Detection via gossip staleness** (not currently implemented — see -[Known Limitations](#known-limitations-v1-implementation)): +[Known Limitations](#known-limitations)): In principle, nodes would also detect partitions through root entry staleness: if no fresh root announcements arrive within a timeout, the root is presumed @@ -579,100 +581,114 @@ see traffic to consider the link alive. Parent selection determines tree structure and routing efficiency. -### v1 Implementation: Depth-Only Selection +### Cost-Based Selection with Effective Depth -The current implementation uses tree depth as the sole selection metric, with a -threshold to prevent thrashing between equivalent-depth paths. +The implementation uses cost-weighted depth to balance tree depth against link +quality. Each candidate parent is evaluated by its **effective depth** — the +tree depth plus a local link cost penalty derived from MMP metrics. -**Algorithm** (`TreeState::evaluate_parent()` in `tree.rs`): +**Algorithm** (`TreeState::evaluate_parent()` in `tree/state.rs`): ``` -evaluate_parent(): +evaluate_parent(peer_costs: HashMap): // 1. Find smallest root reachable through any peer smallest_root = min(peer.root for peer in peers_with_coords) if self == smallest_root and is_root: return None // Already root, no change - // 2. Among peers reaching smallest_root, find shallowest - best_peer = min( - [p for p in peers if p.root == smallest_root], - key=lambda p: p.depth - ) - proposed_depth = best_peer.depth + 1 + // 2. Compute effective depth for each candidate + for each peer with peer.root == smallest_root: + link_cost = peer_costs.get(peer) or 1.0 // default optimistic + peer.effective_depth = peer.depth + link_cost + + best_peer = min(candidates, key=effective_depth, tiebreak=node_addr) if best_peer == current_parent: return None // Already using best - // 3. Always switch if parent is gone or root is changing + // 3. Mandatory switches — bypass hysteresis and hold-down if current_parent not in peers: - return best_peer // Path broken + return best_peer // Parent lost if current_root != smallest_root: return best_peer // Better root found - // 4. For same root: require depth improvement ≥ threshold - current_depth = my_coords.depth() - if current_depth >= proposed_depth + PARENT_SWITCH_THRESHOLD: + // 4. Hold-down check — suppress non-mandatory switches + if last_parent_switch + hold_down > now: + return None // Too soon after last switch + + // 5. Hysteresis — require significant improvement + current_parent_eff = current_parent.depth + peer_costs.get(current_parent) + if best_eff_depth < current_parent_eff * (1.0 - parent_hysteresis): return best_peer return None // Not enough improvement ``` -**Constants**: +**Parameters**: ``` -PARENT_SWITCH_THRESHOLD = 1 // Minimum depth improvement to switch parents +parent_hysteresis = 0.2 // 20% improvement required for same-root switch +hold_down_secs = 30 // Suppress re-evaluation after parent switch +reeval_interval_secs = 60 // Periodic re-evaluation independent of TreeAnnounce ``` -This means a proposed parent must offer a path at least 1 hop shallower than -the current parent (under the same root) to trigger a switch. Root changes -always trigger a switch regardless of depth. - -**What this means for tree structure**: The v1 algorithm produces minimum-depth -trees, which minimizes coordinate path length and hop count for coordinate-based -greedy routing. However, it does not account for link quality—a high-latency or -lossy link at depth 1 is preferred over a fast link at depth 2. - -### v2 Planned: Cost Metrics - -The following cost-based parent selection is planned but not yet implemented. - -**Cost components**: - -- **Latency** (primary): `cost_latency = round_trip_time_ms` -- **Packet loss** (reliability): `cost_loss = 1 / (1 - loss_rate)` — - transforms loss rate into multiplicative cost (10% loss → 1.11, 50% → 2.0) -- **Bandwidth** (capacity): `cost_bandwidth = reference_bandwidth / actual_bandwidth` - -**Combined cost**: Weighted combination with application-tunable weights: +**Link cost formula** (`ActivePeer::link_cost()` in `peer/active.rs`): ``` -effective_cost = w_latency * cost_latency - + w_loss * cost_loss - + w_bandwidth * cost_bandwidth +link_cost = etx * (1.0 + srtt_ms / 100.0) ``` -**Path cost to root**: Recursive — each node advertises its cumulative cost, -allowing neighbors to compute total path cost: +Where ETX (Expected Transmission Count) comes from bidirectional MMP delivery +ratios and SRTT (Smoothed Round-Trip Time) from MMP timestamp-echo. When MMP +metrics have not yet converged, `link_cost` defaults to 1.0, preserving +depth-only behavior as a graceful fallback. -``` -path_cost(peer) = link_cost(self, peer) + peer.path_cost_to_root -``` +**What this means for tree structure**: The algorithm can prefer a deeper parent +with a better link over a shallower parent with a poor link, when the effective +depth difference is significant enough to overcome hysteresis. For example, a +fiber link at depth 2 (effective depth ≈ 3.01) beats a LoRa link at depth 1 +(effective depth ≈ 7.32 with 500ms RTT and 5% loss). In homogeneous networks +where all links have similar quality, effective depth tracks tree depth closely +and the algorithm produces minimum-depth trees as before. -**Stability threshold**: Hysteresis with both absolute and relative components: +**Periodic re-evaluation**: `evaluate_parent()` is event-driven — called on +TreeAnnounce receipt or parent loss. After the tree stabilizes and TreeAnnounce +traffic stops, link degradation goes undetected. The periodic re-evaluation +timer (`reeval_interval_secs`) calls `evaluate_parent()` from the tick handler +with current MMP link costs, independent of TreeAnnounce traffic. -``` -stability_threshold = base_threshold + current_cost * relative_threshold -``` +### Design Rationale: Local-Only Cost Metrics -**Cost measurement**: Active probing (periodic RTT measurement), passive -observation (inferred from protocol message timing), and exponential smoothing -(`alpha = 0.1–0.3`) to balance responsiveness with stability. +The original design considered cumulative path costs (OSPF-style, where each +hop adds its link cost and the total is advertised in TreeAnnounce). This +approach was rejected for three independent reasons: -**Implementation prerequisites**: The cost-based algorithm requires changes to -the TreeAnnounce wire format to carry path cost values, and a measurement -subsystem for link quality metrics. See Section 10, Example 2 for how -cost-based selection would affect tree structure in heterogeneous networks. +1. **Unverifiable self-reporting**: In a permissionless network, a node can + claim any path cost. There is no mechanism for neighbors to verify that + the reported cumulative cost is truthful. A malicious node advertising + zero cost would attract traffic as a transit node. + +2. **No shared metric semantics**: Different links measure different things. + A LoRa link's 500ms RTT and a fiber link's 1ms RTT are both "round-trip + time" but represent fundamentally different physical constraints. + Accumulating them into a single path cost obscures per-hop information + that is more useful when evaluated locally. + +3. **Accumulation amplifies error**: Small measurement noise at each hop + compounds across the path. A 5-hop path accumulates 5x the measurement + error of a single hop, while providing no more actionable information + than the local link cost to each candidate parent. + +The local-only approach uses `link_cost = etx * (1.0 + srtt_ms / 100.0)`, +where both components are locally measured via MMP. The RTT weighting +addresses a blind spot in ETX alone: a clean-but-slow link (LoRa with 0% +loss) gets ETX = 1.0, identical to fiber. The SRTT factor distinguishes them +— a 500ms LoRa link gets cost ≈ 6.0 versus fiber at ≈ 1.01. + +No wire format changes are required. TreeAnnounce messages continue to carry +depth (not cost), and each node independently evaluates its direct links +using trusted local measurements. --- @@ -874,48 +890,48 @@ Physical topology: node_addr ordering: B < A < D < E < C ``` -**Cost calculation** (using bandwidth as primary): +**Local link costs** (using `link_cost = etx * (1.0 + srtt_ms / 100.0)`): ``` -Link costs (normalized to 1 Gbps = 1): -A ═ B: cost = 1 -B ═ D: cost = 1 -D ═ E: cost = 1 -C — D: cost = 1000 (1 Mbps) -A ~ C: cost = 100000 (9600 bps) +Assumed MMP measurements after convergence: +A ═ B: fiber, 1ms RTT, 0% loss → link_cost = 1.0 * (1 + 1/100) ≈ 1.01 +B ═ D: fiber, 1ms RTT, 0% loss → link_cost = 1.0 * (1 + 1/100) ≈ 1.01 +D ═ E: fiber, 1ms RTT, 0% loss → link_cost = 1.0 * (1 + 1/100) ≈ 1.01 +C — D: DSL, 20ms RTT, 2% loss → link_cost = 1.04 * (1 + 20/100) ≈ 1.25 +A ~ C: radio, 500ms RTT, 5% loss→ link_cost = 1.11 * (1 + 500/100) ≈ 6.66 ``` -**Tree formation with costs**: +**Tree formation with effective depth**: ``` -Root = B (smallest node_addr) +Root = B (smallest node_addr, depth 0) -Parent selection: -├── A: peers are B (cost 1), C (cost 100000) -│ └── Selects B (much lower cost) +Parent selection (each node evaluates effective_depth = peer.depth + link_cost): +├── A: peers are B (depth 0, cost 1.01 → eff 1.01), C (depth ?, cost 6.66) +│ └── Selects B (lowest effective depth) │ -├── D: peers are B (cost 1), C (cost 1000), E (cost 1) -│ └── Selects B (direct, cost 1) +├── D: peers are B (depth 0, cost 1.01 → eff 1.01), +│ C (depth ?, cost 1.25), E (depth ?, cost 1.01) +│ └── Selects B (direct, lowest effective depth) │ -├── E: peer is D -│ └── Path to B: E → D → B, cost = 1 + 1 = 2 -│ └── Selects D +├── E: peer is D (depth 1, cost 1.01 → eff 2.01) +│ └── Selects D (only candidate) │ -└── C: peers are A (cost 100000), D (cost 1000) - └── Path through A: 100000 + 1 = 100001 - └── Path through D: 1000 + 1 = 1001 - └── Selects D (much lower cost despite higher local cost) +└── C: peers are A (depth 1, cost 6.66 → eff 7.66), + D (depth 1, cost 1.25 → eff 2.25) + └── Selects D (eff 2.25 vs 7.66 — much lower) Resulting tree: - B (root) + B (root, depth 0) / \ - A D + A D (both depth 1) |\ - E C + E C (both depth 2) ``` -**Note**: C chooses D despite A being "closer" in hops, because total path -cost through D is lower. +**Note**: C chooses D despite both being at depth 1 — the DSL link to D +(eff 2.25) far beats the radio link to A (eff 7.66). With local-only costs, +each node evaluates only its direct link quality, not cumulative path cost. **Radio link failure**: @@ -925,11 +941,10 @@ If A ~ C radio fails: └── C loses a potential backup path, but current tree unchanged If D — C DSL fails: -├── C loses parent +├── C loses parent (mandatory switch — bypasses hysteresis and hold-down) ├── C's only remaining peer is A (radio) -├── C selects A as parent -├── C's path to root: C → A → B (cost 100001) -└── Tree reconverges with C as child of A +├── C selects A as parent (eff depth = 1 + 6.66 = 7.66) +└── Tree reconverges with C as child of A at depth 2 ``` ### Example 3: Network Partition and Healing @@ -1019,7 +1034,7 @@ initial exchange). --- -## Known Limitations (v1 Implementation) +## Known Limitations The following limitations exist in the current implementation relative to the design described in this document. They are documented here to guide future @@ -1079,26 +1094,27 @@ expires and has no peer with a fresher root declaration is partitioned. It becomes its own root and announces, allowing the partition to converge independently. -### Known Limitation: Limited Stability Mechanisms +### Known Limitation: Remaining Stability Gaps -The implementation includes basic hysteresis (`PARENT_SWITCH_THRESHOLD = 1` -depth difference required to switch parents), but the temporal stability -mechanisms described in the design are not implemented: +The primary stability mechanisms are implemented: + +- **Cost-based hysteresis** (`parent_hysteresis = 0.2`): requires 20% + effective depth improvement to switch parents under the same root +- **Hold-down timer** (`hold_down_secs = 30`): suppresses non-mandatory + re-evaluation after a parent switch, allowing MMP metrics to stabilize +- **Periodic re-evaluation** (`reeval_interval_secs = 60`): catches link + degradation after tree stabilization independent of TreeAnnounce traffic + +Remaining gaps: -- No hold timer on parent changes (minimum time before next switch) - No sequence number advancement rate limiting -- No announcement suppression during transient topology changes - No minimum stable state duration before re-announcing -**Impact**: Rapid topology changes (e.g., a flapping link) could cause -excessive announcement traffic and repeated coordinate recomputation. -Currently mitigated by per-peer rate limiting on TreeAnnounce sends, but -the source node is not throttled. - -**Proposed fix**: Add a hold-down timer (e.g., 5-10s) after each parent -change during which further parent switches are suppressed unless the current -parent is lost entirely. Track announcement rate and suppress if exceeding a -threshold. +**Impact**: A rapidly flapping link could still cause moderate announcement +traffic. The hold-down timer limits the rate of parent switches (at most +one non-mandatory switch per 30s), and per-peer rate limiting (500ms) +bounds announcement frequency, but the source node's announcement rate is +not independently throttled. ### Known Limitation: Integration Test Gaps @@ -1123,7 +1139,7 @@ The gossip-based spanning tree protocol achieves distributed coordination through: 1. **Deterministic root election** - Smallest node_addr, no negotiation needed -2. **Local parent selection** - Each node independently chooses best path to root +2. **Cost-aware parent selection** - Each node independently chooses lowest effective depth to root using local link metrics 3. **CRDT merge semantics** - Conflicts resolved by sequence number, then timestamp 4. **Bounded state** - O(peers × depth) entries per node, not O(network size) 5. **Depth-proportional convergence** - Scales with tree height, not node count diff --git a/src/config/node.rs b/src/config/node.rs index 42e4da0..bf26de5 100644 --- a/src/config/node.rs +++ b/src/config/node.rs @@ -190,23 +190,44 @@ pub struct TreeConfig { /// Per-peer TreeAnnounce rate limit in ms (`node.tree.announce_min_interval_ms`). #[serde(default = "TreeConfig::default_announce_min_interval_ms")] pub announce_min_interval_ms: u64, - /// Min depth improvement to switch parents (`node.tree.parent_switch_threshold`). - #[serde(default = "TreeConfig::default_parent_switch_threshold")] - pub parent_switch_threshold: usize, + /// Hysteresis factor for cost-based parent re-selection (`node.tree.parent_hysteresis`). + /// + /// Only switch parents when the candidate's effective_depth is better than + /// `current_effective_depth * (1.0 - parent_hysteresis)`. Range: 0.0-1.0. + /// Set to 0.0 to disable hysteresis (switch on any improvement). + #[serde(default = "TreeConfig::default_parent_hysteresis")] + pub parent_hysteresis: f64, + /// Hold-down period after parent switch in seconds (`node.tree.hold_down_secs`). + /// + /// After switching parents, suppress re-evaluation for this duration to allow + /// MMP metrics to stabilize on the new link. Set to 0 to disable. + #[serde(default = "TreeConfig::default_hold_down_secs")] + pub hold_down_secs: u64, + /// Periodic parent re-evaluation interval in seconds (`node.tree.reeval_interval_secs`). + /// + /// How often to re-evaluate parent selection based on current MMP link costs, + /// independent of TreeAnnounce traffic. Catches link degradation after the + /// tree has stabilized. Set to 0 to disable. + #[serde(default = "TreeConfig::default_reeval_interval_secs")] + pub reeval_interval_secs: u64, } impl Default for TreeConfig { fn default() -> Self { Self { announce_min_interval_ms: 500, - parent_switch_threshold: 1, + parent_hysteresis: 0.2, + hold_down_secs: 30, + reeval_interval_secs: 60, } } } impl TreeConfig { fn default_announce_min_interval_ms() -> u64 { 500 } - fn default_parent_switch_threshold() -> usize { 1 } + fn default_parent_hysteresis() -> f64 { 0.2 } + fn default_hold_down_secs() -> u64 { 30 } + fn default_reeval_interval_secs() -> u64 { 60 } } /// Bloom filter (`node.bloom.*`). diff --git a/src/node/mod.rs b/src/node/mod.rs index ad3b6c7..9ef4e0b 100644 --- a/src/node/mod.rs +++ b/src/node/mod.rs @@ -337,6 +337,10 @@ pub struct Node { /// are exhausted. retry_pending: HashMap, + // === Periodic Parent Re-evaluation === + /// Timestamp of last periodic parent re-evaluation (for pacing). + last_parent_reeval: Option, + // === Display Names === /// Human-readable names for configured peers (alias or short npub). /// Populated at startup from peer config. @@ -368,7 +372,8 @@ impl Node { // Initialize tree state with signed self-declaration let mut tree_state = TreeState::new(node_addr); - tree_state.set_parent_switch_threshold(config.node.tree.parent_switch_threshold); + tree_state.set_parent_hysteresis(config.node.tree.parent_hysteresis); + tree_state.set_hold_down(config.node.tree.hold_down_secs); tree_state .sign_declaration(&identity) .expect("signing own declaration should never fail"); @@ -432,6 +437,7 @@ impl Node { std::time::Duration::from_millis(coords_response_interval_ms), ), retry_pending: HashMap::new(), + last_parent_reeval: None, peer_aliases: HashMap::new(), }) } @@ -451,7 +457,8 @@ impl Node { // Initialize tree state with signed self-declaration let mut tree_state = TreeState::new(node_addr); - tree_state.set_parent_switch_threshold(config.node.tree.parent_switch_threshold); + tree_state.set_parent_hysteresis(config.node.tree.parent_hysteresis); + tree_state.set_hold_down(config.node.tree.hold_down_secs); tree_state .sign_declaration(&identity) .expect("signing own declaration should never fail"); @@ -518,6 +525,7 @@ impl Node { std::time::Duration::from_millis(coords_response_interval_ms), ), retry_pending: HashMap::new(), + last_parent_reeval: None, peer_aliases: HashMap::new(), } } diff --git a/src/node/tree.rs b/src/node/tree.rs index 148b4ab..04f23f1 100644 --- a/src/node/tree.rs +++ b/src/node/tree.rs @@ -3,6 +3,8 @@ //! Handles building, sending, and receiving TreeAnnounce messages, //! including periodic root refresh and rate-limited propagation. +use std::collections::HashMap; + use crate::protocol::TreeAnnounce; use crate::NodeAddr; @@ -194,8 +196,11 @@ impl Node { self.bloom_state.mark_update_needed(*from); } - // Re-evaluate parent selection - if let Some(new_parent) = self.tree_state.evaluate_parent() { + // Re-evaluate parent selection with current link costs + let peer_costs: HashMap = self.peers.iter() + .map(|(addr, peer)| (*addr, peer.link_cost())) + .collect(); + if let Some(new_parent) = self.tree_state.evaluate_parent(&peer_costs) { let new_seq = self.tree_state.my_declaration().sequence() + 1; let timestamp = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -268,9 +273,73 @@ impl Node { /// Periodic tree maintenance, called from the tick handler. /// - /// Sends pending rate-limited announces. + /// Sends pending rate-limited announces and checks for periodic + /// parent re-evaluation based on current MMP link costs. pub(super) async fn check_tree_state(&mut self) { self.send_pending_tree_announces().await; + self.check_periodic_parent_reeval().await; + } + + /// Periodic parent re-evaluation based on current MMP link costs. + /// + /// Self-paces using `last_parent_reeval` and the configured + /// `reeval_interval_secs`. When a better parent is found, follows + /// the same switch flow as TreeAnnounce-triggered switches. + async fn check_periodic_parent_reeval(&mut self) { + let interval_secs = self.config.node.tree.reeval_interval_secs; + if interval_secs == 0 { + return; + } + + // Need at least 2 peers for a meaningful comparison + if self.peers.len() < 2 { + return; + } + + let now = std::time::Instant::now(); + let interval = std::time::Duration::from_secs(interval_secs); + + if let Some(last) = self.last_parent_reeval { + if now.duration_since(last) < interval { + return; + } + } + + self.last_parent_reeval = Some(now); + + let peer_costs: HashMap = self.peers.iter() + .map(|(addr, peer)| (*addr, peer.link_cost())) + .collect(); + + if let Some(new_parent) = self.tree_state.evaluate_parent(&peer_costs) { + let new_seq = self.tree_state.my_declaration().sequence() + 1; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + self.tree_state.set_parent(new_parent, new_seq, timestamp); + if let Err(e) = self.tree_state.sign_declaration(&self.identity) { + warn!(error = %e, "Failed to sign declaration after periodic parent re-eval"); + return; + } + self.tree_state.recompute_coords(); + self.coord_cache.clear(); + + info!( + new_parent = %self.peer_display_name(&new_parent), + new_seq = new_seq, + new_root = %self.tree_state.root(), + depth = self.tree_state.my_coords().depth(), + trigger = "periodic", + "Parent switched via periodic cost re-evaluation" + ); + + self.send_tree_announce_to_all().await; + + let all_peers: Vec = self.peers.keys().copied().collect(); + self.bloom_state.mark_all_updates_needed(all_peers); + } } /// Handle tree state cleanup when a peer is removed. @@ -286,7 +355,10 @@ impl Node { self.tree_state.remove_peer(node_addr); if was_parent { - let changed = self.tree_state.handle_parent_lost(); + let peer_costs: HashMap = self.peers.iter() + .map(|(addr, peer)| (*addr, peer.link_cost())) + .collect(); + let changed = self.tree_state.handle_parent_lost(&peer_costs); if changed { // Re-sign the new declaration if let Err(e) = self.tree_state.sign_declaration(&self.identity) { diff --git a/src/peer/active.rs b/src/peer/active.rs index fde635b..dd70d7e 100644 --- a/src/peer/active.rs +++ b/src/peer/active.rs @@ -470,11 +470,22 @@ impl ActivePeer { /// Link cost for routing decisions. /// - /// Returns a scalar cost where lower is better. Currently returns a - /// constant (all links equal). Future versions will compute from RTT, - /// loss rate, and throughput measurements. + /// Returns a scalar cost where lower is better (1.0 = ideal). + /// Computed as RTT-weighted ETX: `etx * (1.0 + srtt_ms / 100.0)`. + /// + /// Returns 1.0 (optimistic default) when MMP metrics are not yet + /// available, matching depth-only parent selection behavior. pub fn link_cost(&self) -> f64 { - 1.0 + match self.mmp() { + Some(mmp) => { + let etx = mmp.metrics.etx; + match mmp.metrics.srtt_ms() { + Some(srtt_ms) => etx * (1.0 + srtt_ms / 100.0), + None => 1.0, + } + } + None => 1.0, + } } /// When this peer was authenticated. diff --git a/src/tree/state.rs b/src/tree/state.rs index cf0ddbb..a629145 100644 --- a/src/tree/state.rs +++ b/src/tree/state.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::fmt; +use std::time::{Duration, Instant}; use super::{CoordEntry, ParentDeclaration, TreeCoordinate, TreeError}; use crate::{Identity, NodeAddr}; @@ -24,8 +25,12 @@ pub struct TreeState { peer_declarations: HashMap, /// Each peer's full ancestry to root. peer_ancestry: HashMap, - /// Minimum depth improvement required to switch parents (same root). - parent_switch_threshold: usize, + /// Hysteresis factor for cost-based parent re-selection (0.0-1.0). + parent_hysteresis: f64, + /// Hold-down period after parent switch (0 = disabled). + hold_down: Duration, + /// Timestamp of last parent switch (for hold-down enforcement). + last_parent_switch: Option, } impl TreeState { @@ -48,7 +53,9 @@ impl TreeState { root: my_node_addr, peer_declarations: HashMap::new(), peer_ancestry: HashMap::new(), - parent_switch_threshold: 1, + parent_hysteresis: 0.0, + hold_down: Duration::ZERO, + last_parent_switch: None, } } @@ -130,6 +137,7 @@ impl TreeState { /// Call this when switching parents. Updates the declaration and coordinates. pub fn set_parent(&mut self, parent_id: NodeAddr, sequence: u64, timestamp: u64) { self.my_declaration = ParentDeclaration::new(self.my_node_addr, parent_id, sequence, timestamp); + self.last_parent_switch = Some(Instant::now()); // Coordinates will be recomputed when ancestry is available } @@ -209,18 +217,25 @@ impl TreeState { } } - /// Set the parent switch threshold. - pub fn set_parent_switch_threshold(&mut self, threshold: usize) { - self.parent_switch_threshold = threshold; + /// Set the parent hysteresis factor (0.0-1.0). + pub fn set_parent_hysteresis(&mut self, hysteresis: f64) { + self.parent_hysteresis = hysteresis.clamp(0.0, 1.0); + } + + /// Set the hold-down duration after parent switches. + pub fn set_hold_down(&mut self, secs: u64) { + self.hold_down = Duration::from_secs(secs); } /// Evaluate whether to switch parents based on current peer tree state. /// - /// v1 algorithm: depth-based, no latency/loss metrics. + /// Uses effective_depth (depth + link_cost) for parent comparison. + /// `peer_costs` maps each peer's NodeAddr to its link cost (from local + /// MMP measurements). Missing entries default to 1.0 (optimistic). /// /// Returns `Some(peer_node_addr)` if a parent switch is recommended, /// or `None` if the current parent is adequate. - pub fn evaluate_parent(&self) -> Option { + pub fn evaluate_parent(&self, peer_costs: &HashMap) -> Option { if self.peer_ancestry.is_empty() { return None; } @@ -248,30 +263,36 @@ impl TreeState { return None; } - // Among peers that reach the smallest root, find the shallowest - let mut best_peer: Option<(NodeAddr, usize)> = None; // (peer_addr, depth) + // Among peers that reach the smallest root, find the lowest effective_depth. + // effective_depth(peer) = peer.depth + link_cost_to_peer + let mut best_peer: Option<(NodeAddr, f64)> = None; // (peer_addr, effective_depth) for (peer_id, coords) in &self.peer_ancestry { if *coords.root_id() != smallest_root { continue; } - let depth = coords.depth(); + let cost = peer_costs.get(peer_id).copied().unwrap_or(1.0); + let eff_depth = coords.depth() as f64 + cost; match &best_peer { - None => best_peer = Some((*peer_id, depth)), - Some((_, best_depth)) => { - if depth < *best_depth { - best_peer = Some((*peer_id, depth)); + None => best_peer = Some((*peer_id, eff_depth)), + Some((best_id, best_eff)) => { + if eff_depth < *best_eff + || (eff_depth == *best_eff && peer_id < best_id) + { + best_peer = Some((*peer_id, eff_depth)); } } } } - let (best_peer_id, best_depth) = best_peer?; + let (best_peer_id, best_eff_depth) = best_peer?; // If already using this peer as parent, no switch needed if *self.my_declaration.parent_id() == best_peer_id && !self.is_root() { return None; } + // --- Mandatory switches (bypass hold-down and hysteresis) --- + // If our current parent is gone from peer_ancestry, our path is broken — always switch if !self.is_root() && !self.peer_ancestry.contains_key(self.my_declaration.parent_id()) { return Some(best_peer_id); @@ -282,18 +303,36 @@ impl TreeState { return Some(best_peer_id); } - // Same root: require depth improvement ≥ threshold + // We're root but shouldn't be (peers have a smaller root) — always switch if self.is_root() { - // We're root but shouldn't be (peers have a smaller root) — always switch return Some(best_peer_id); } - // Compare depth: our current depth vs what we'd get through best_peer - // Our new depth would be best_depth + 1 - let current_depth = self.my_coords.depth(); - let proposed_depth = best_depth + 1; + // --- Hold-down: suppress non-mandatory re-evaluation after recent switch --- - if current_depth >= proposed_depth + self.parent_switch_threshold { + if !self.hold_down.is_zero() + && self + .last_parent_switch + .is_some_and(|last| last.elapsed() < self.hold_down) + { + return None; + } + + // --- Same root, cost-aware comparison with hysteresis --- + + // Current parent's effective_depth + let current_parent_cost = peer_costs + .get(self.my_declaration.parent_id()) + .copied() + .unwrap_or(1.0); + let current_parent_coords = self.peer_ancestry.get(self.my_declaration.parent_id()); + let current_parent_eff = match current_parent_coords { + Some(coords) => coords.depth() as f64 + current_parent_cost, + None => return Some(best_peer_id), // Parent has no coords — treat as lost + }; + + // Apply hysteresis: only switch if candidate is significantly better + if best_eff_depth < current_parent_eff * (1.0 - self.parent_hysteresis) { return Some(best_peer_id); } @@ -306,9 +345,9 @@ impl TreeState { /// If none available, becomes its own root (increments sequence). /// /// Returns `true` if the tree state changed (caller should re-announce). - pub fn handle_parent_lost(&mut self) -> bool { + pub fn handle_parent_lost(&mut self, peer_costs: &HashMap) -> bool { // Try to find an alternative parent - if let Some(new_parent) = self.evaluate_parent() { + if let Some(new_parent) = self.evaluate_parent(peer_costs) { let timestamp = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) diff --git a/src/tree/tests.rs b/src/tree/tests.rs index f057b86..39bcf64 100644 --- a/src/tree/tests.rs +++ b/src/tree/tests.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use super::*; fn make_node_addr(val: u8) -> NodeAddr { @@ -378,7 +380,7 @@ fn test_evaluate_parent_picks_smallest_root() { make_coords(&[7, 2]), ); - let result = state.evaluate_parent(); + let result = state.evaluate_parent(&HashMap::new()); assert_eq!(result, Some(peer3)); } @@ -404,7 +406,7 @@ fn test_evaluate_parent_prefers_shallowest_depth() { make_coords(&[2, 3, 4, 0]), ); - let result = state.evaluate_parent(); + let result = state.evaluate_parent(&HashMap::new()); assert_eq!(result, Some(peer1)); } @@ -421,7 +423,7 @@ fn test_evaluate_parent_stays_root_when_smallest() { make_coords(&[1, 0]), ); - assert_eq!(state.evaluate_parent(), None); + assert_eq!(state.evaluate_parent(&HashMap::new()), None); } #[test] @@ -443,7 +445,7 @@ fn test_evaluate_parent_no_switch_when_already_best() { state.recompute_coords(); // Now evaluate — should return None since peer1 is already our parent - assert_eq!(state.evaluate_parent(), None); + assert_eq!(state.evaluate_parent(&HashMap::new()), None); } #[test] @@ -451,7 +453,7 @@ fn test_evaluate_parent_no_peers() { let my_node = make_node_addr(5); let state = TreeState::new(my_node); - assert_eq!(state.evaluate_parent(), None); + assert_eq!(state.evaluate_parent(&HashMap::new()), None); } #[test] @@ -484,7 +486,7 @@ fn test_evaluate_parent_depth_threshold() { make_coords(&[3, 0]), ); - let result = state.evaluate_parent(); + let result = state.evaluate_parent(&HashMap::new()); assert_eq!(result, Some(peer3)); } @@ -512,7 +514,7 @@ fn test_handle_parent_lost_finds_alternative() { // Remove peer1 (parent lost) state.remove_peer(&peer1); - let changed = state.handle_parent_lost(); + let changed = state.handle_parent_lost(&HashMap::new()); assert!(changed); // Should have switched to peer2 @@ -540,7 +542,7 @@ fn test_handle_parent_lost_becomes_root() { // Remove peer1 (only parent) state.remove_peer(&peer1); - let changed = state.handle_parent_lost(); + let changed = state.handle_parent_lost(&HashMap::new()); assert!(changed); assert!(state.is_root()); @@ -685,3 +687,423 @@ fn test_find_next_hop_best_of_multiple() { let dest = make_coords(&[7, 3, 1, 0]); assert_eq!(state.find_next_hop(&dest), Some(make_node_addr(3))); } + +// === Cost-based parent selection tests === + +/// Build a peer_costs map from (addr_byte, cost) pairs. +fn make_costs(entries: &[(u8, f64)]) -> HashMap { + entries + .iter() + .map(|&(addr, cost)| (make_node_addr(addr), cost)) + .collect() +} + +#[test] +fn test_effective_depth_selects_lower_cost_deeper_peer() { + // Peer A at depth 1 with high cost (LoRa), peer B at depth 2 with low cost (fiber). + // effective_depth(A) = 1 + 6.0 = 7.0 + // effective_depth(B) = 2 + 1.01 = 3.01 + // Should select B despite being deeper. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + + let peer_a = make_node_addr(1); + let peer_b = make_node_addr(2); + let root = make_node_addr(0); + + // Peer A: depth 1 + state.update_peer( + ParentDeclaration::new(peer_a, root, 1, 1000), + make_coords(&[1, 0]), + ); + // Peer B: depth 2 + state.update_peer( + ParentDeclaration::new(peer_b, make_node_addr(3), 1, 1000), + make_coords(&[2, 3, 0]), + ); + + let costs = make_costs(&[(1, 6.0), (2, 1.01)]); + let result = state.evaluate_parent(&costs); + assert_eq!(result, Some(peer_b)); +} + +#[test] +fn test_effective_depth_equal_cost_degenerates_to_depth() { + // Both peers at cost 1.0 (default). Should pick shallowest, same as v1. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + + let peer1 = make_node_addr(1); + let peer2 = make_node_addr(2); + let root = make_node_addr(0); + + // Peer 1: depth 1 + state.update_peer( + ParentDeclaration::new(peer1, root, 1, 1000), + make_coords(&[1, 0]), + ); + // Peer 2: depth 3 + state.update_peer( + ParentDeclaration::new(peer2, make_node_addr(3), 1, 1000), + make_coords(&[2, 3, 4, 0]), + ); + + let costs = make_costs(&[(1, 1.0), (2, 1.0)]); + let result = state.evaluate_parent(&costs); + assert_eq!(result, Some(peer1)); +} + +#[test] +fn test_effective_depth_tiebreak_by_node_addr() { + // Two peers with identical effective_depth. Smaller NodeAddr wins. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + + let peer1 = make_node_addr(1); + let peer2 = make_node_addr(2); + let root = make_node_addr(0); + + // Both at depth 1, cost 1.0 → effective_depth 2.0 + state.update_peer( + ParentDeclaration::new(peer1, root, 1, 1000), + make_coords(&[1, 0]), + ); + state.update_peer( + ParentDeclaration::new(peer2, root, 1, 1000), + make_coords(&[2, 0]), + ); + + let costs = make_costs(&[(1, 1.0), (2, 1.0)]); + let result = state.evaluate_parent(&costs); + assert_eq!(result, Some(peer1)); // smaller NodeAddr +} + +#[test] +fn test_hysteresis_prevents_marginal_switch() { + // Current parent eff_depth 3.5, candidate 3.2. + // With 20% hysteresis, threshold = 3.5 * 0.8 = 2.8. + // 3.2 > 2.8, so no switch. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + state.set_parent_hysteresis(0.2); + + let peer_a = make_node_addr(1); // current parent + let peer_b = make_node_addr(2); // candidate + let root = make_node_addr(0); + + // Peer A: depth 1, cost 2.5 → eff 3.5 + state.update_peer( + ParentDeclaration::new(peer_a, root, 1, 1000), + make_coords(&[1, 0]), + ); + // Peer B: depth 1, cost 2.2 → eff 3.2 + state.update_peer( + ParentDeclaration::new(peer_b, root, 1, 1000), + make_coords(&[2, 0]), + ); + + // Set peer_a as current parent + state.set_parent(peer_a, 1, 1000); + state.recompute_coords(); + + let costs = make_costs(&[(1, 2.5), (2, 2.2)]); + let result = state.evaluate_parent(&costs); + assert_eq!(result, None); // marginal improvement blocked by hysteresis +} + +#[test] +fn test_hysteresis_allows_significant_switch() { + // Current parent eff_depth 7.0, candidate 3.01. + // With 20% hysteresis, threshold = 7.0 * 0.8 = 5.6. + // 3.01 < 5.6, so switch occurs. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + state.set_parent_hysteresis(0.2); + + let peer_a = make_node_addr(1); // current parent (LoRa) + let peer_b = make_node_addr(2); // candidate (fiber) + let root = make_node_addr(0); + + // Peer A: depth 1, cost 6.0 → eff 7.0 + state.update_peer( + ParentDeclaration::new(peer_a, root, 1, 1000), + make_coords(&[1, 0]), + ); + // Peer B: depth 2, cost 1.01 → eff 3.01 + state.update_peer( + ParentDeclaration::new(peer_b, make_node_addr(3), 1, 1000), + make_coords(&[2, 3, 0]), + ); + + // Set peer_a as current parent + state.set_parent(peer_a, 1, 1000); + state.recompute_coords(); + + let costs = make_costs(&[(1, 6.0), (2, 1.01)]); + let result = state.evaluate_parent(&costs); + assert_eq!(result, Some(peer_b)); +} + +#[test] +fn test_cold_start_default_cost() { + // Peer with no cost entry in map gets default 1.0. + // This degenerates to depth-only selection. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + + let peer1 = make_node_addr(1); + let peer2 = make_node_addr(2); + let root = make_node_addr(0); + + // Peer 1: depth 1, peer 2: depth 3 + state.update_peer( + ParentDeclaration::new(peer1, root, 1, 1000), + make_coords(&[1, 0]), + ); + state.update_peer( + ParentDeclaration::new(peer2, make_node_addr(3), 1, 1000), + make_coords(&[2, 3, 4, 0]), + ); + + // Empty cost map — all peers get default 1.0 + let result = state.evaluate_parent(&HashMap::new()); + assert_eq!(result, Some(peer1)); // shallowest wins +} + +#[test] +fn test_hold_down_suppresses_reeval() { + // After a parent switch, re-evaluation returns None during hold-down. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + state.set_hold_down(60); // 60s hold-down + + let peer_a = make_node_addr(1); + let peer_b = make_node_addr(2); + let root = make_node_addr(0); + + state.update_peer( + ParentDeclaration::new(peer_a, root, 1, 1000), + make_coords(&[1, 0]), + ); + state.update_peer( + ParentDeclaration::new(peer_b, root, 1, 1000), + make_coords(&[2, 0]), + ); + + // Switch to peer_a (sets last_parent_switch) + state.set_parent(peer_a, 1, 1000); + state.recompute_coords(); + + // Peer_b now offers better cost, but hold-down suppresses + let costs = make_costs(&[(1, 5.0), (2, 1.0)]); + state.set_parent_hysteresis(0.0); // no hysteresis, only hold-down + let result = state.evaluate_parent(&costs); + assert_eq!(result, None); // suppressed by hold-down +} + +#[test] +fn test_mandatory_switch_bypasses_hold_down() { + // Parent loss during hold-down still triggers switch. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + state.set_hold_down(60); // 60s hold-down + + let peer_a = make_node_addr(1); + let peer_b = make_node_addr(2); + let root = make_node_addr(0); + + state.update_peer( + ParentDeclaration::new(peer_a, root, 1, 1000), + make_coords(&[1, 0]), + ); + state.update_peer( + ParentDeclaration::new(peer_b, root, 1, 1000), + make_coords(&[2, 0]), + ); + + // Switch to peer_a + state.set_parent(peer_a, 1, 1000); + state.recompute_coords(); + + // Remove peer_a (parent lost) — should bypass hold-down + state.remove_peer(&peer_a); + let result = state.evaluate_parent(&HashMap::new()); + assert_eq!(result, Some(peer_b)); // mandatory switch +} + +#[test] +fn test_heterogeneous_7node_avoids_bottleneck() { + // 7-node topology simulating a mixed fiber/LoRa network: + // + // 0 (root) + // ├── 1 (fiber, cost 1.01) — depth 1 + // │ ├── 3 (fiber, cost 1.01) — depth 2 + // │ └── 4 (fiber, cost 1.01) — depth 2 + // ├── 2 (LoRa, cost 6.0) — depth 1 + // │ └── 5 (fiber, cost 1.01) — depth 2 (inherits LoRa bottleneck!) + // └── 6 (wifi, cost 1.07) — depth 1 + // + // Node 5 is connected to both node 2 (LoRa parent, depth 1) and + // node 1 (fiber, depth 1). Without cost-awareness, node 5 could + // pick node 2 as parent (both at depth 1, tiebreak by addr). + // With cost-awareness, node 5 should pick node 1 (eff 2.01) over + // node 2 (eff 7.0). + + let root = make_node_addr(0); + + // Test from node 5's perspective + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + + let peer1 = make_node_addr(1); // fiber peer at depth 1 + let peer2 = make_node_addr(2); // LoRa peer at depth 1 + + // Both peers reach root 0 at depth 1 + state.update_peer( + ParentDeclaration::new(peer1, root, 1, 1000), + make_coords(&[1, 0]), + ); + state.update_peer( + ParentDeclaration::new(peer2, root, 1, 1000), + make_coords(&[2, 0]), + ); + + // Without costs (all 1.0): picks peer 1 (smaller addr) — correct by luck + let result_no_cost = state.evaluate_parent(&HashMap::new()); + assert_eq!(result_no_cost, Some(peer1)); + + // With costs: fiber (1.01) vs LoRa (6.0) — fiber wins definitively + let costs = make_costs(&[(1, 1.01), (2, 6.0)]); + let result_with_cost = state.evaluate_parent(&costs); + assert_eq!(result_with_cost, Some(peer1)); + + // Now test the critical case: node 5 currently has LoRa parent (peer 2). + // Even without hysteresis, it should want to switch to fiber (peer 1). + state.set_parent(peer2, 1, 1000); + state.recompute_coords(); + assert_eq!(state.my_coords().depth(), 2); // depth 2 through LoRa peer + + let result_switch = state.evaluate_parent(&costs); + assert_eq!(result_switch, Some(peer1)); // switches away from LoRa bottleneck + + // With hysteresis enabled, still switches because the cost difference is large + state.set_parent_hysteresis(0.2); + // current_parent_eff = 1 + 6.0 = 7.0, best_eff = 1 + 1.01 = 2.01 + // threshold = 7.0 * 0.8 = 5.6, 2.01 < 5.6 → switch + let result_hyst = state.evaluate_parent(&costs); + assert_eq!(result_hyst, Some(peer1)); +} + +// ===================================================================== +// Cost degradation tests (periodic re-evaluation scenarios) +// ===================================================================== +// +// These test evaluate_parent() with changing cost maps, validating the +// scenarios that periodic re-evaluation is designed to catch: link +// quality changes after the tree has stabilized. + +#[test] +fn test_cost_degradation_triggers_switch() { + // Node 5 has two peers at depth 1. Initially both have similar costs + // (both fiber). After stabilization, peer A's link degrades (becomes + // LoRa-like). Re-evaluation with updated costs should trigger a switch. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + state.set_parent_hysteresis(0.2); + + let peer_a = make_node_addr(1); + let peer_b = make_node_addr(2); + let root = make_node_addr(0); + + state.update_peer( + ParentDeclaration::new(peer_a, root, 1, 1000), + make_coords(&[1, 0]), + ); + state.update_peer( + ParentDeclaration::new(peer_b, root, 1, 1000), + make_coords(&[2, 0]), + ); + + // Initial: both fiber-like costs. Node picks peer_a (smaller addr). + let initial_costs = make_costs(&[(1, 1.05), (2, 1.08)]); + let result = state.evaluate_parent(&initial_costs); + assert_eq!(result, Some(peer_a)); + + state.set_parent(peer_a, 1, 1000); + state.recompute_coords(); + + // Verify stable: no switch with same costs + let result = state.evaluate_parent(&initial_costs); + assert_eq!(result, None); + + // Peer A's link degrades significantly (LoRa-like latency + loss) + // current_parent_eff = 1 + 6.0 = 7.0 + // best_eff = 1 + 1.08 = 2.08 + // threshold = 7.0 * 0.8 = 5.6, 2.08 < 5.6 → switch + let degraded_costs = make_costs(&[(1, 6.0), (2, 1.08)]); + let result = state.evaluate_parent(°raded_costs); + assert_eq!(result, Some(peer_b)); +} + +#[test] +fn test_cost_improvement_within_hysteresis_no_switch() { + // Node 5 has parent peer_a. Peer_b's cost improves slightly but + // stays within the hysteresis band. Re-evaluation should not switch. + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + state.set_parent_hysteresis(0.2); + + let peer_a = make_node_addr(1); + let peer_b = make_node_addr(2); + let root = make_node_addr(0); + + state.update_peer( + ParentDeclaration::new(peer_a, root, 1, 1000), + make_coords(&[1, 0]), + ); + state.update_peer( + ParentDeclaration::new(peer_b, root, 1, 1000), + make_coords(&[2, 0]), + ); + + state.set_parent(peer_a, 1, 1000); + state.recompute_coords(); + + // Peer B slightly better: cost 1.5 vs peer A cost 2.0 + // current_parent_eff = 1 + 2.0 = 3.0 + // best_eff = 1 + 1.5 = 2.5 + // threshold = 3.0 * 0.8 = 2.4, 2.5 > 2.4 → no switch + let costs = make_costs(&[(1, 2.0), (2, 1.5)]); + let result = state.evaluate_parent(&costs); + assert_eq!(result, None); +} + +#[test] +fn test_single_peer_no_reeval_benefit() { + // With only one peer, evaluate_parent should select it initially, + // but once it's our parent, re-evaluation returns None regardless + // of cost changes (no alternative exists). + let my_node = make_node_addr(5); + let mut state = TreeState::new(my_node); + + let peer_a = make_node_addr(1); + let root = make_node_addr(0); + + state.update_peer( + ParentDeclaration::new(peer_a, root, 1, 1000), + make_coords(&[1, 0]), + ); + + // Initial selection: picks the only peer + let costs = make_costs(&[(1, 1.05)]); + let result = state.evaluate_parent(&costs); + assert_eq!(result, Some(peer_a)); + + state.set_parent(peer_a, 1, 1000); + state.recompute_coords(); + + // Even with terrible cost, no switch (no alternative) + let bad_costs = make_costs(&[(1, 50.0)]); + let result = state.evaluate_parent(&bad_costs); + assert_eq!(result, None); +} diff --git a/testing/chaos/scenarios/bottleneck-parent.yaml b/testing/chaos/scenarios/bottleneck-parent.yaml new file mode 100644 index 0000000..c5f4c66 --- /dev/null +++ b/testing/chaos/scenarios/bottleneck-parent.yaml @@ -0,0 +1,91 @@ +# Bottleneck Parent: 10-node focused LoRa bottleneck test +# +# Explicit topology with two LoRa links that create bottleneck parent +# candidates. Tests that nodes avoid choosing LoRa parents when fiber +# alternatives exist at the same or slightly greater depth. +# +# Topology: +# +# n01 (root) +# / | \ \ +# f f f f +# / | \ \ +# n02 n03 n04 n05 +# | / | \ | +# LoRa f f LoRa +# | / | \ | +# n06 n07 n08 n09 +# | +# f +# | +# n10 +# +# Edges and link types: +# Fiber: n01-n02, n01-n03, n01-n04, n01-n05, +# n03-n06, n03-n07, n04-n08, n08-n10 +# LoRa: n02-n06, n05-n09 +# +# Cross-links: n03-n08 (fiber) — gives n08 a fiber alternative to n04 +# +# Test subjects: +# - n06 has LoRa (n02, depth 1) and fiber (n03, depth 1) — should pick n03 +# - n09 has only LoRa (n05) — no alternative, stuck with LoRa parent + +scenario: + name: "bottleneck-parent" + seed: 42 + duration_secs: 120 + +topology: + algorithm: explicit + num_nodes: 10 + params: + adjacency: + - [n01, n02] + - [n01, n03] + - [n01, n04] + - [n01, n05] + - [n02, n06] + - [n03, n06] + - [n03, n07] + - [n03, n08] + - [n04, n08] + - [n05, n09] + - [n08, n10] + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + delay_ms: [1, 5] + jitter_ms: [0, 1] + loss_pct: [0, 0.5] + link_policies: + # LoRa links (high delay, moderate loss) + - edges: ["n02-n06", "n05-n09"] + policy: + delay_ms: [400, 600] + jitter_ms: [50, 100] + loss_pct: [5, 15] + mutation: + interval_secs: {min: 30, max: 60} + fraction: 0.2 + policies: + normal: + delay_ms: [1, 5] + loss_pct: [0, 0.5] + +link_flaps: + enabled: false + +traffic: + enabled: true + max_concurrent: 2 + interval_secs: {min: 15, max: 30} + duration_secs: {min: 5, max: 10} + parallel_streams: 2 + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/cost-avoidance.yaml b/testing/chaos/scenarios/cost-avoidance.yaml new file mode 100644 index 0000000..296a50d --- /dev/null +++ b/testing/chaos/scenarios/cost-avoidance.yaml @@ -0,0 +1,66 @@ +# Cost-Based Parent Selection: Bottleneck Avoidance Test +# +# Topology (explicit 4-node diamond): +# +# n01 (root — smallest addr) +# / \ +# fiber fiber +# / \ +# n02 n03 +# \ / +# LoRa fiber +# \ / +# n04 (test subject) +# +# n04 has two candidate parents at depth 1: n02 (via LoRa) and n03 (via +# fiber). Cost-based selection should pick n03 because: +# effective_depth(n02) = 1 + ~6.0 (LoRa) = ~7.0 +# effective_depth(n03) = 1 + ~1.01 (fiber) = ~2.01 +# +# Validation: tree snapshot shows n04's parent is n03. + +scenario: + name: "cost-avoidance" + seed: 42 + duration_secs: 120 + +topology: + algorithm: explicit + num_nodes: 4 + params: + adjacency: + - [n01, n02] + - [n01, n03] + - [n02, n04] + - [n03, n04] + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + # Fiber-like + delay_ms: [1, 5] + jitter_ms: [0, 1] + loss_pct: [0, 0.5] + link_policies: + # LoRa link from n02 to n04 + - edges: ["n02-n04"] + policy: + delay_ms: [400, 600] + jitter_ms: [50, 100] + loss_pct: [5, 15] + +link_flaps: + enabled: false + +traffic: + enabled: true + max_concurrent: 2 + interval_secs: {min: 15, max: 30} + duration_secs: {min: 5, max: 10} + parallel_streams: 2 + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/cost-mixed-7node.yaml b/testing/chaos/scenarios/cost-mixed-7node.yaml new file mode 100644 index 0000000..f0d377b --- /dev/null +++ b/testing/chaos/scenarios/cost-mixed-7node.yaml @@ -0,0 +1,78 @@ +# Cost-Based Parent Selection: Mixed Technology 7-Node Test +# +# Topology (explicit, multiple link types): +# +# n01 (root) +# / | \ +# fiber | fiber LoRa +# / | \ +# n02 n03 n04 +# | | \ | +# fiber fiber \ fiber +# | | wifi | +# n05 n06 n07 +# +# Cross-links: n03-n05 (fiber), n04-n06 (wifi) +# +# Test subjects: +# - n06 has edges to n03 (fiber) and n04 (wifi) — should prefer n03 +# - n04's link to root is LoRa, so n04 has high-cost parent link +# - n07 connects only to n04 (no choice, stuck with LoRa upstream) +# +# Validation: tree snapshot shows n06's parent is n03 (not n04). + +scenario: + name: "cost-mixed-7node" + seed: 42 + duration_secs: 180 + +topology: + algorithm: explicit + num_nodes: 7 + params: + adjacency: + - [n01, n02] + - [n01, n03] + - [n01, n04] + - [n02, n05] + - [n03, n06] + - [n04, n07] + - [n03, n05] + - [n04, n06] + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + # Fiber-like + delay_ms: [1, 5] + jitter_ms: [0, 1] + loss_pct: [0, 0.5] + link_policies: + # LoRa link from n01 to n04 + - edges: ["n01-n04"] + policy: + delay_ms: [400, 600] + jitter_ms: [50, 100] + loss_pct: [5, 15] + # WiFi link from n04 to n06 + - edges: ["n04-n06"] + policy: + delay_ms: [5, 20] + jitter_ms: [2, 5] + loss_pct: [1, 3] + +link_flaps: + enabled: false + +traffic: + enabled: true + max_concurrent: 3 + interval_secs: {min: 10, max: 25} + duration_secs: {min: 5, max: 15} + parallel_streams: 4 + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/cost-reeval.yaml b/testing/chaos/scenarios/cost-reeval.yaml new file mode 100644 index 0000000..afa48af --- /dev/null +++ b/testing/chaos/scenarios/cost-reeval.yaml @@ -0,0 +1,86 @@ +# Periodic Cost Re-evaluation Test +# +# Topology (explicit 4-node diamond): +# +# n01 (root) +# / \ +# fiber fiber +# / \ +# n02 n03 +# \ / +# fiber fiber +# \ / +# n04 (test subject) +# +# Initial state: All links are fiber. n04 has two candidate parents at +# depth 1: n02 and n03. Both have identical costs (~1.01), so n04 picks +# n02 (smaller NodeAddr, tiebreak rule). +# +# Mutation: Stochastic netem mutation with a single "degraded" policy +# (LoRa-like: 400-600ms delay, 5-15% loss). With fraction=0.5, on +# average 2 of 4 edges degrade each round. Over 12 mutation rounds +# (180s / 15s interval), n02-n04 will be degraded in some rounds. +# +# Expected behavior: When n02-n04 is degraded and n03-n04 stays fiber +# (or vice versa), periodic re-evaluation detects the cost asymmetry +# and switches parents. Look for "trigger = periodic" in logs. +# +# FIPS overrides: reeval_interval_secs=15 (vs default 60) to increase +# the chance of catching a cost asymmetry within the mutation window. +# +# Validation: grep n04 logs for "Parent switched via periodic cost +# re-evaluation" (trigger=periodic). If mutation never creates enough +# asymmetry in a particular seed, the test still validates that periodic +# re-eval runs without interference. + +scenario: + name: "cost-reeval" + seed: 42 + duration_secs: 180 + +topology: + algorithm: explicit + num_nodes: 4 + params: + adjacency: + - [n01, n02] + - [n01, n03] + - [n02, n04] + - [n03, n04] + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + # Fiber-like baseline + delay_ms: [1, 5] + jitter_ms: [0, 1] + loss_pct: [0, 0.5] + mutation: + interval_secs: {min: 12, max: 18} + fraction: 0.5 + policies: + degraded: + delay_ms: [400, 600] + jitter_ms: [50, 100] + loss_pct: [5, 15] + +link_flaps: + enabled: false + +traffic: + enabled: true + max_concurrent: 1 + interval_secs: {min: 15, max: 30} + duration_secs: {min: 5, max: 10} + parallel_streams: 2 + +logging: + rust_log: "info" + output_dir: "./sim-results" + +fips_overrides: + node: + tree: + reeval_interval_secs: 15 diff --git a/testing/chaos/scenarios/cost-stability.yaml b/testing/chaos/scenarios/cost-stability.yaml new file mode 100644 index 0000000..0f80e56 --- /dev/null +++ b/testing/chaos/scenarios/cost-stability.yaml @@ -0,0 +1,72 @@ +# Cost-Based Parent Selection: Hysteresis Stability Test +# +# Topology (explicit 4-node diamond, symmetric): +# +# n01 (root) +# / \ +# wifi wifi +# / \ +# n02 n03 +# \ / +# wifi wifi +# \ / +# n04 (test subject) +# +# All links are WiFi-like with similar characteristics. Aggressive +# netem mutation shifts link qualities every 10-20s, but the changes +# stay within the 20% hysteresis band. n04 should pick one parent +# and mostly stick with it — flapping indicates insufficient hysteresis. +# +# Validation: count "Parent switched" in n04 logs, expect <= 5 switches +# over the full 180s duration. + +scenario: + name: "cost-stability" + seed: 42 + duration_secs: 180 + +topology: + algorithm: explicit + num_nodes: 4 + params: + adjacency: + - [n01, n02] + - [n01, n03] + - [n02, n04] + - [n03, n04] + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + # WiFi baseline + delay_ms: [5, 20] + jitter_ms: [2, 5] + loss_pct: [1, 3] + mutation: + interval_secs: {min: 10, max: 20} + fraction: 1.0 + policies: + slightly_better: + delay_ms: [3, 8] + jitter_ms: [1, 3] + loss_pct: [0, 1] + slightly_worse: + delay_ms: [15, 25] + jitter_ms: [3, 8] + loss_pct: [2, 4] + +link_flaps: + enabled: false + +traffic: + enabled: true + max_concurrent: 2 + interval_secs: {min: 15, max: 30} + duration_secs: {min: 5, max: 15} + parallel_streams: 2 + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/depth-vs-cost.yaml b/testing/chaos/scenarios/depth-vs-cost.yaml new file mode 100644 index 0000000..41c997e --- /dev/null +++ b/testing/chaos/scenarios/depth-vs-cost.yaml @@ -0,0 +1,73 @@ +# Cost-Based Parent Selection: Deeper Fiber Beats Shallow LoRa +# +# Topology (explicit 4-node): +# +# n01 (root) +# / \ +# fiber LoRa +# / \ +# n02 n04 (test subject) +# | / +# fiber fiber +# | / +# n03 +# +# n04 has two candidate parents: +# - n01 (root, depth 0) via LoRa: effective_depth = 0 + ~6.0 = ~6.0 +# - n03 (depth 2) via fiber: effective_depth = 2 + ~1.01 = ~3.01 +# +# Without cost-based selection, n04 would pick n01 (depth 0 < depth 2). +# With cost-based selection, n04 should pick n03 (lower effective_depth +# despite being 2 levels deeper). +# +# This tests the core depth-vs-cost tradeoff: a much deeper fiber path +# should beat a direct LoRa link to root when the cost difference is +# large enough. +# +# Validation: tree snapshot shows n04's parent is n03, n04's depth is 3. + +scenario: + name: "depth-vs-cost" + seed: 42 + duration_secs: 120 + +topology: + algorithm: explicit + num_nodes: 4 + params: + adjacency: + - [n01, n02] + - [n02, n03] + - [n03, n04] + - [n01, n04] + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + # Fiber-like + delay_ms: [1, 5] + jitter_ms: [0, 1] + loss_pct: [0, 0.5] + link_policies: + # LoRa link from n01 to n04 + - edges: ["n01-n04"] + policy: + delay_ms: [400, 600] + jitter_ms: [50, 100] + loss_pct: [5, 15] + +link_flaps: + enabled: false + +traffic: + enabled: true + max_concurrent: 1 + interval_secs: {min: 15, max: 30} + duration_secs: {min: 5, max: 10} + parallel_streams: 2 + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/scenarios/mixed-technology.yaml b/testing/chaos/scenarios/mixed-technology.yaml new file mode 100644 index 0000000..8d18942 --- /dev/null +++ b/testing/chaos/scenarios/mixed-technology.yaml @@ -0,0 +1,103 @@ +# Mixed Technology: 10-node heterogeneous network +# +# Explicit topology with LoRa, WiFi, and fiber links. Tests that +# cost-based parent selection produces a tree favoring low-cost paths +# when multiple link technologies coexist. +# +# Topology: +# +# n01 (root) +# / | \ +# f f f +# / | \ +# n02 n03 n04 +# | \ | \ | \ +# f LoRa f f LoRa f +# | \ | | \ | +# n05 n06 n07 n08 n09 +# \ / +# wifi---wifi +# n10 +# +# Edges and link types: +# Fiber: n01-n02, n01-n03, n01-n04, n02-n05, n03-n07, n04-n09 +# LoRa: n02-n06, n04-n08 +# WiFi: n03-n06, n08-n10 +# Fiber: n03-n08, n06-n10 +# +# Test subjects: +# - n06 has fiber (n03) and LoRa (n02) parents — should pick n03 +# - n08 has fiber (n03) and LoRa (n04) parents — should pick n03 +# +# Netem mutation shifts fiber-only links between normal and degraded. + +scenario: + name: "mixed-technology" + seed: 42 + duration_secs: 180 + +topology: + algorithm: explicit + num_nodes: 10 + params: + adjacency: + - [n01, n02] + - [n01, n03] + - [n01, n04] + - [n02, n05] + - [n02, n06] + - [n03, n06] + - [n03, n07] + - [n03, n08] + - [n04, n08] + - [n04, n09] + - [n06, n10] + - [n08, n10] + subnet: "172.20.0.0/24" + ip_start: 10 + +netem: + enabled: true + default_policy: + # Fiber-like defaults + delay_ms: [1, 5] + jitter_ms: [0, 1] + loss_pct: [0, 0.5] + link_policies: + # LoRa links (high latency, moderate loss) + - edges: ["n02-n06", "n04-n08"] + policy: + delay_ms: [400, 600] + jitter_ms: [50, 100] + loss_pct: [5, 15] + # WiFi links (moderate latency, low loss) + - edges: ["n03-n06", "n08-n10"] + policy: + delay_ms: [5, 20] + jitter_ms: [2, 5] + loss_pct: [1, 3] + mutation: + interval_secs: {min: 30, max: 60} + fraction: 0.2 + policies: + normal: + delay_ms: [1, 10] + loss_pct: [0, 1] + degraded: + delay_ms: [50, 100] + jitter_ms: [10, 30] + loss_pct: [3, 8] + +link_flaps: + enabled: false + +traffic: + enabled: true + max_concurrent: 3 + interval_secs: {min: 10, max: 30} + duration_secs: {min: 5, max: 15} + parallel_streams: 4 + +logging: + rust_log: "info" + output_dir: "./sim-results" diff --git a/testing/chaos/sim/config_gen.py b/testing/chaos/sim/config_gen.py index ca8bc9b..b42ae08 100644 --- a/testing/chaos/sim/config_gen.py +++ b/testing/chaos/sim/config_gen.py @@ -3,9 +3,23 @@ from __future__ import annotations import os +from copy import deepcopy + +import yaml from .topology import SimTopology + +def _deep_merge(base: dict, override: dict) -> dict: + """Recursively merge override into base (override wins on conflicts).""" + result = deepcopy(base) + for key, value in override.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + result[key] = _deep_merge(result[key], value) + else: + result[key] = deepcopy(value) + return result + # Path to the shared node config template _TEMPLATE_PATH = os.path.join( os.path.dirname(__file__), "..", "configs", "node.template.yaml" @@ -41,7 +55,10 @@ def generate_peers_block( def generate_node_config( - topology: SimTopology, node_id: str, outbound_peers: list[str] + topology: SimTopology, + node_id: str, + outbound_peers: list[str], + fips_overrides: dict | None = None, ) -> str: """Generate a complete FIPS config YAML for one node.""" template = _load_template() @@ -54,6 +71,12 @@ def generate_node_config( config = config.replace("{{NPUB}}", node.npub) config = config.replace("{{NSEC}}", node.nsec) config = config.replace("{{PEERS}}", peers_yaml) + + if fips_overrides: + parsed = yaml.safe_load(config) + merged = _deep_merge(parsed, fips_overrides) + config = yaml.dump(merged, default_flow_style=False, sort_keys=False) + return config @@ -67,13 +90,19 @@ def generate_npubs_env(topology: SimTopology) -> str: return "\n".join(lines) + "\n" -def write_configs(topology: SimTopology, output_dir: str): +def write_configs( + topology: SimTopology, + output_dir: str, + fips_overrides: dict | None = None, +): """Write all node configs and npubs.env to the output directory.""" os.makedirs(output_dir, exist_ok=True) outbound = topology.directed_outbound() for node_id in topology.nodes: - config = generate_node_config(topology, node_id, outbound[node_id]) + config = generate_node_config( + topology, node_id, outbound[node_id], fips_overrides + ) path = os.path.join(output_dir, f"{node_id}.yaml") with open(path, "w") as f: f.write(config) diff --git a/testing/chaos/sim/control.py b/testing/chaos/sim/control.py new file mode 100644 index 0000000..c1628a8 --- /dev/null +++ b/testing/chaos/sim/control.py @@ -0,0 +1,103 @@ +"""Control socket querying via docker exec. + +Queries FIPS nodes' control sockets to observe runtime state (tree +structure, MMP metrics, peers) without requiring fipsctl in the +container. Uses a Python one-liner inside docker exec since python3 +is available in the Docker image. +""" + +from __future__ import annotations + +import json +import logging + +from .docker_exec import docker_exec, docker_exec_quiet +from .topology import SimTopology + +log = logging.getLogger(__name__) + +# Default control socket path inside containers (XDG_RUNTIME_DIR is +# typically unset in Docker, so fips falls back to /tmp). +CONTROL_SOCKET = "/tmp/fips-control.sock" + + +def query_node(container: str, command: str, timeout: int = 10) -> dict | None: + """Send a command to a node's control socket, return the data dict. + + Returns None if the query fails (node down, socket not ready, etc.). + """ + # Python one-liner that connects to the Unix socket, sends the JSON + # command, and prints the response. Runs inside the container. + script = ( + "import socket,json,sys; " + "s=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM); " + f"s.connect('{CONTROL_SOCKET}'); " + f"s.sendall(json.dumps({{'command':'{command}'}}).encode()+b'\\n'); " + "s.shutdown(socket.SHUT_WR); " + "chunks=[]; " + "[chunks.append(d) for d in iter(lambda:s.recv(65536),b'')]; " + "print(b''.join(chunks).decode())" + ) + stdout = docker_exec_quiet(container, f"python3 -c \"{script}\"", timeout=timeout) + if stdout is None: + return None + + try: + response = json.loads(stdout.strip()) + except json.JSONDecodeError as e: + log.warning("Invalid JSON from %s: %s", container, e) + return None + + if response.get("status") != "ok": + msg = response.get("message", "unknown error") + log.warning("Control query %s on %s failed: %s", command, container, msg) + return None + + return response.get("data", {}) + + +def query_tree(container: str) -> dict | None: + """Query a node's spanning tree state.""" + return query_node(container, "show_tree") + + +def query_mmp(container: str) -> dict | None: + """Query a node's MMP metrics.""" + return query_node(container, "show_mmp") + + +def query_peers(container: str) -> dict | None: + """Query a node's peer list with MMP metrics.""" + return query_node(container, "show_peers") + + +def snapshot_all_trees(topology: SimTopology) -> dict[str, dict]: + """Query show_tree on all nodes, return {node_id: tree_data}. + + Nodes that fail to respond are omitted from the result. + """ + result = {} + for node_id in sorted(topology.nodes): + container = topology.container_name(node_id) + data = query_tree(container) + if data is not None: + result[node_id] = data + else: + log.warning("No tree data from %s", node_id) + return result + + +def snapshot_all_mmp(topology: SimTopology) -> dict[str, dict]: + """Query show_mmp on all nodes, return {node_id: mmp_data}. + + Nodes that fail to respond are omitted from the result. + """ + result = {} + for node_id in sorted(topology.nodes): + container = topology.container_name(node_id) + data = query_mmp(container) + if data is not None: + result[node_id] = data + else: + log.warning("No MMP data from %s", node_id) + return result diff --git a/testing/chaos/sim/netem.py b/testing/chaos/sim/netem.py index c73a209..59e9f1a 100644 --- a/testing/chaos/sim/netem.py +++ b/testing/chaos/sim/netem.py @@ -17,7 +17,7 @@ import random from dataclasses import dataclass, field from .docker_exec import docker_exec_quiet, is_container_running -from .scenario import BandwidthConfig, NetemConfig, NetemPolicy +from .scenario import BandwidthConfig, LinkPolicyOverride, NetemConfig, NetemPolicy from .topology import SimTopology log = logging.getLogger(__name__) @@ -91,12 +91,41 @@ class NetemManager: rate = rng.choice(bandwidth.tiers_mbps) self._edge_rates[(a, b)] = rate self._edge_rates[(b, a)] = rate + # Per-edge policy overrides: canonical "nXX-nYY" -> NetemPolicy + # Build a set of canonical edge strings for validation + topo_edge_strs = {"-".join(sorted([a, b])) for a, b in topology.edges} + self._edge_overrides: dict[str, NetemPolicy] = {} + for override in config.link_policies: + policy = override.policy + if policy is None and override.policy_name: + policy = config.mutation.policies.get(override.policy_name) + if policy is None: + continue + for edge_str in override.edges: + if edge_str not in topo_edge_strs: + log.warning( + "link_policy edge %s not in topology — override ignored", + edge_str, + ) + self._edge_overrides[edge_str] = policy + if self._edge_overrides: + log.info( + "Per-link policy overrides: %d edges", + len(self._edge_overrides), + ) def _htb_rate(self, node_id: str, peer_id: str) -> str: """Return the HTB rate string for a link direction.""" rate = self._edge_rates.get((node_id, peer_id), 0) return f"{rate}mbit" if rate > 0 else "1gbit" + def _policy_for_edge(self, node_a: str, node_b: str) -> NetemPolicy: + """Return the netem policy for an edge, checking overrides first.""" + canonical = "-".join(sorted([node_a, node_b])) + if canonical in self._edge_overrides: + return self._edge_overrides[canonical] + return self.config.default_policy + def setup_initial(self): """Set up HTB qdiscs and initial netem on all containers.""" if self._edge_rates: @@ -128,8 +157,9 @@ class NetemManager: class_id = f"1:{idx}" netem_handle = f"{idx + 10}:" - # Sample initial params from default policy - params = self._sample_policy(self.config.default_policy) + # Sample initial params (per-edge override or default policy) + policy = self._policy_for_edge(node_id, peer_id) + params = self._sample_policy(policy) rate = self._htb_rate(node_id, peer_id) rate_mbit = self._edge_rates.get((node_id, peer_id), 0) diff --git a/testing/chaos/sim/runner.py b/testing/chaos/sim/runner.py index 8ed566a..9ce648a 100644 --- a/testing/chaos/sim/runner.py +++ b/testing/chaos/sim/runner.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import logging import os import random @@ -11,6 +12,7 @@ import time from .compose import generate_compose from .config_gen import write_configs +from .control import snapshot_all_mmp, snapshot_all_trees from .docker_exec import docker_compose from .links import LinkManager from .logs import AnalysisResult, analyze_logs, collect_logs, write_sim_metadata @@ -108,7 +110,7 @@ class SimRunner: config_dir = os.path.normpath( os.path.join(docker_network_dir, "generated-configs", "sim") ) - write_configs(self.topology, config_dir) + write_configs(self.topology, config_dir, self.scenario.fips_overrides) log.info("Wrote node configs to %s", config_dir) # 3. Generate docker-compose.yml @@ -153,6 +155,7 @@ class SimRunner: wait = max(10, n) # Heuristic: ~1s per node, minimum 10s log.info("Waiting %ds for mesh convergence...", wait) self._sleep(wait) + self._take_snapshot("warmup") def _simulation_loop(self): """Main event loop driving stochastic behavior.""" @@ -233,11 +236,14 @@ class SimRunner: log.info("Restoring downed links...") self.link_mgr.restore_all() - # Restore stopped nodes (needed for log collection) + # Restore stopped nodes (needed for snapshots and log collection) if self.node_mgr: log.info("Restoring stopped nodes...") self.node_mgr.restore_all() + # Take final tree snapshot while nodes are still running + self._take_snapshot("final") + # Collect logs before stopping containers container_names = [ self.topology.container_name(nid) for nid in sorted(self.topology.nodes) @@ -273,6 +279,30 @@ class SimRunner: return result + def _take_snapshot(self, label: str): + """Query all nodes via control socket and save tree/MMP snapshots.""" + if not self.topology: + return + log.info("Taking %s snapshot...", label) + tree_snap = snapshot_all_trees(self.topology) + mmp_snap = snapshot_all_mmp(self.topology) + + tree_path = os.path.join(self.output_dir, f"tree-snapshot-{label}.json") + mmp_path = os.path.join(self.output_dir, f"mmp-snapshot-{label}.json") + os.makedirs(self.output_dir, exist_ok=True) + with open(tree_path, "w") as f: + json.dump(tree_snap, f, indent=2) + with open(mmp_path, "w") as f: + json.dump(mmp_snap, f, indent=2) + log.info( + "Snapshot %s: %d/%d tree, %d/%d mmp responses", + label, + len(tree_snap), + len(self.topology.nodes), + len(mmp_snap), + len(self.topology.nodes), + ) + def _schedule_next(self, now: float, interval) -> float: """Schedule the next event using a Range interval.""" return now + self.rng.uniform(interval.min, interval.max) diff --git a/testing/chaos/sim/scenario.py b/testing/chaos/sim/scenario.py index 46113d5..409ba82 100644 --- a/testing/chaos/sim/scenario.py +++ b/testing/chaos/sim/scenario.py @@ -49,10 +49,25 @@ class NetemMutationConfig: policies: dict[str, NetemPolicy] = field(default_factory=dict) +@dataclass +class LinkPolicyOverride: + """Per-edge netem policy override. + + Edges are specified as "nXX-nYY" strings in canonical form (sorted + alphabetically). Either ``policy`` (inline) or ``policy_name`` + (reference to a named mutation policy) must be set, not both. + """ + + edges: list[str] = field(default_factory=list) + policy: NetemPolicy | None = None + policy_name: str | None = None + + @dataclass class NetemConfig: enabled: bool = False default_policy: NetemPolicy = field(default_factory=NetemPolicy) + link_policies: list[LinkPolicyOverride] = field(default_factory=list) mutation: NetemMutationConfig = field(default_factory=NetemMutationConfig) @@ -115,6 +130,10 @@ class Scenario: node_churn: NodeChurnConfig = field(default_factory=NodeChurnConfig) bandwidth: BandwidthConfig = field(default_factory=BandwidthConfig) logging: LoggingConfig = field(default_factory=LoggingConfig) + # Raw YAML dict appended to each generated FIPS node config. + # Allows scenarios to override any FIPS config parameter + # (e.g., node.tree.reeval_interval_secs). + fips_overrides: dict = field(default_factory=dict) def _parse_range(data, name: str) -> Range: @@ -173,6 +192,16 @@ def load_scenario(path: str) -> Scenario: s.netem.enabled = nc.get("enabled", False) if "default_policy" in nc: s.netem.default_policy = _parse_netem_policy(nc["default_policy"]) + if "link_policies" in nc: + for lp_data in nc["link_policies"]: + override = LinkPolicyOverride( + edges=lp_data.get("edges", []), + ) + if "policy" in lp_data: + override.policy = _parse_netem_policy(lp_data["policy"]) + if "policy_name" in lp_data: + override.policy_name = lp_data["policy_name"] + s.netem.link_policies.append(override) if "mutation" in nc: mc = nc["mutation"] s.netem.mutation.interval_secs = _parse_range( @@ -233,6 +262,9 @@ def load_scenario(path: str) -> Scenario: s.logging.rust_log = lg.get("rust_log", "info") s.logging.output_dir = lg.get("output_dir", "./sim-results") + # FIPS config overrides (raw YAML dict appended to node configs) + s.fips_overrides = raw.get("fips_overrides", {}) + # Validation _validate(s) @@ -245,11 +277,45 @@ def _validate(s: Scenario): raise ValueError("topology.num_nodes must be >= 2") if s.topology.num_nodes > 250: raise ValueError("topology.num_nodes must be <= 250 (subnet limit)") - if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain"): + if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain", "explicit"): raise ValueError(f"Unknown topology algorithm: {s.topology.algorithm}") + if s.topology.algorithm == "explicit": + adj = s.topology.params.get("adjacency") + if not adj or not isinstance(adj, list): + raise ValueError("explicit topology requires params.adjacency list") + node_ids = set() + for i, pair in enumerate(adj): + if not isinstance(pair, (list, tuple)) or len(pair) != 2: + raise ValueError( + f"explicit adjacency[{i}]: expected [nodeA, nodeB], got {pair}" + ) + node_ids.update(str(p) for p in pair) + if len(node_ids) != s.topology.num_nodes: + raise ValueError( + f"explicit adjacency references {len(node_ids)} nodes " + f"but num_nodes is {s.topology.num_nodes}" + ) if s.duration_secs < 1: raise ValueError("duration_secs must be >= 1") + # Validate link_policies + for i, lp in enumerate(s.netem.link_policies): + if not lp.edges: + raise ValueError(f"netem.link_policies[{i}]: edges list is empty") + if lp.policy is not None and lp.policy_name is not None: + raise ValueError( + f"netem.link_policies[{i}]: specify policy or policy_name, not both" + ) + if lp.policy is None and lp.policy_name is None: + raise ValueError( + f"netem.link_policies[{i}]: must specify policy or policy_name" + ) + if lp.policy_name and lp.policy_name not in s.netem.mutation.policies: + raise ValueError( + f"netem.link_policies[{i}]: policy_name '{lp.policy_name}' " + f"not found in mutation.policies" + ) + # Validate ranges if s.netem.enabled and s.netem.mutation.policies: s.netem.mutation.interval_secs.validate("netem.mutation.interval_secs") diff --git a/testing/chaos/sim/topology.py b/testing/chaos/sim/topology.py index d2470a2..29db6ff 100644 --- a/testing/chaos/sim/topology.py +++ b/testing/chaos/sim/topology.py @@ -130,6 +130,17 @@ def generate_topology( elif config.algorithm == "erdos_renyi": p = config.params.get("p", 0.3) edges = _generate_erdos_renyi(node_ids, p, rng) + elif config.algorithm == "explicit": + adjacency = config.params.get("adjacency") + if not adjacency: + raise ValueError("explicit topology requires params.adjacency") + edges = _generate_explicit(adjacency) + # Validate all referenced nodes exist + for a, b in edges: + if a not in nodes: + raise ValueError(f"explicit adjacency references unknown node {a}") + if b not in nodes: + raise ValueError(f"explicit adjacency references unknown node {b}") else: raise ValueError(f"Unknown algorithm: {config.algorithm}") @@ -212,6 +223,21 @@ def _generate_erdos_renyi( return edges +def _generate_explicit(adjacency: list) -> set[tuple[str, str]]: + """Build edges from an explicit adjacency list. + + Each entry should be a 2-element list like ["n01", "n02"]. + """ + edges = set() + for i, pair in enumerate(adjacency): + if not isinstance(pair, (list, tuple)) or len(pair) != 2: + raise ValueError( + f"explicit adjacency[{i}]: expected [nodeA, nodeB], got {pair}" + ) + edges.add(_make_edge(str(pair[0]), str(pair[1]))) + return edges + + def _make_edge(a: str, b: str) -> tuple[str, str]: """Canonical edge representation (sorted).""" return (min(a, b), max(a, b))