mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 00:04:54 +00:00
Test cost-based parent selection and kernel-drop detection as unit tests
The cost-selection chaos scenarios (cost-reeval, cost-avoidance, cost-stability, depth-vs-cost, mixed-technology, bottleneck-parent) tested TreeState::evaluate_parent's decision logic through a Docker mesh that could not exercise it reliably: the tree roots at whichever node holds the smallest NodeAddr, MMP link costs take several measurement windows to settle, and the parent hold-down plus hysteresis timing all confound the outcome. A deterministic link-cost flap still produced zero periodic parent switches in a full run. Replace those six scenarios with deterministic unit tests in src/tree/tests.rs that drive evaluate_parent directly: cheaper-link selection at equal depth, switch-on-cost-change, hysteresis suppressing a marginal change while allowing a significant one, and the depth-versus-cost effective-depth tradeoff. Each is constructed so that breaking the cost or hysteresis logic makes it fail. The congestion kernel-drop signal (SO_RXQ_OVFL) cannot be provoked deterministically in Docker: a fresh daemon reader keeps up with container-speed traffic, so the socket receive queue never overflows (an unshaped run with a 4 KB buffer and heavy traffic recorded zero drops on every node). Extract the drop-detection edge -- read the cumulative counter, fire an event only on the transition into a new drop burst -- into TransportDropState::observe_drops and unit-test it directly. congestion-stress keeps its ECN and MMP congestion-signal assertions, which do need the real shaped bottleneck queue. Remove the retired scenarios from both CI runners and update the chaos README.
This commit is contained in:
@@ -417,13 +417,10 @@ impl Node {
|
||||
for (&tid, transport) in &self.transports {
|
||||
let congestion = transport.congestion();
|
||||
let state = self.transport_drops.entry(tid).or_default();
|
||||
if let Some(current) = congestion.recv_drops {
|
||||
let new_drops = current > state.prev_drops;
|
||||
if new_drops && !state.dropping {
|
||||
new_drop_events.push(tid);
|
||||
}
|
||||
state.dropping = new_drops;
|
||||
state.prev_drops = current;
|
||||
if let Some(current) = congestion.recv_drops
|
||||
&& state.observe_drops(current)
|
||||
{
|
||||
new_drop_events.push(tid);
|
||||
}
|
||||
}
|
||||
for tid in new_drop_events {
|
||||
|
||||
@@ -266,6 +266,32 @@ struct TransportDropState {
|
||||
dropping: bool,
|
||||
}
|
||||
|
||||
impl TransportDropState {
|
||||
/// Fold a new cumulative `recv_drops` sample into the state and report
|
||||
/// whether it marks the *transition* into a dropping condition.
|
||||
///
|
||||
/// Returns true only on the edge where the cumulative `SO_RXQ_OVFL`
|
||||
/// counter rose since the previous sample **and** the transport was not
|
||||
/// already flagged as dropping. That edge is what `kernel_drop_events`
|
||||
/// counts: a first observation of a new drop burst, not every sample in
|
||||
/// which the counter happens to be non-zero. A sample with no rise
|
||||
/// clears the flag, so a later rise counts as a fresh event.
|
||||
///
|
||||
/// Pure and sans-IO by design: the tick handler reads the kernel
|
||||
/// counter from the socket and does the logging, but the detection
|
||||
/// decision lives here so it can be tested without a socket, a
|
||||
/// transport, or a running node — which is the only way it can be
|
||||
/// tested at all, since the kernel drop itself cannot be provoked
|
||||
/// deterministically.
|
||||
fn observe_drops(&mut self, current: u64) -> bool {
|
||||
let rose = current > self.prev_drops;
|
||||
let new_event = rose && !self.dropping;
|
||||
self.dropping = rose;
|
||||
self.prev_drops = current;
|
||||
new_event
|
||||
}
|
||||
}
|
||||
|
||||
/// State for a link waiting for transport-level connection establishment.
|
||||
///
|
||||
/// For connection-oriented transports (TCP, Tor), the transport connect runs
|
||||
|
||||
@@ -1995,3 +1995,40 @@ async fn handle_msg1_admits_existing_peer_at_cap() {
|
||||
"rate limiter must rebalance after the (bypass-admitted) handler returns"
|
||||
);
|
||||
}
|
||||
|
||||
// ===== Transport kernel-drop detection (sans-IO) =====
|
||||
//
|
||||
// The drop-detection edge-detector, tested directly. It replaces the
|
||||
// congestion-drops docker scenario, which could not provoke SO_RXQ_OVFL
|
||||
// deterministically (a fresh daemon reader keeps up with container-speed
|
||||
// traffic, so the kernel never overflows the socket queue). The kernel
|
||||
// dropping datagrams is not FIPS behaviour to test; the FIPS behaviour is
|
||||
// reading the SO_RXQ_OVFL counter and firing kernel_drop_events on the
|
||||
// transition into a new drop burst, which is exactly this decision.
|
||||
|
||||
#[test]
|
||||
fn test_transport_drop_state_fires_on_edge_and_rearms() {
|
||||
let mut s = TransportDropState::default();
|
||||
// Cumulative counter still 0: no rise, no event.
|
||||
assert!(!s.observe_drops(0));
|
||||
// First rise (0 -> 5): a new drop burst is observed, so it fires.
|
||||
assert!(s.observe_drops(5));
|
||||
// Counter keeps rising (5 -> 9) but we are already dropping: this is
|
||||
// the "first observed" contract, so it must NOT fire again.
|
||||
assert!(!s.observe_drops(9));
|
||||
// A sample with no further rise clears the dropping flag (no event).
|
||||
assert!(!s.observe_drops(9));
|
||||
// A later rise (9 -> 12) is a fresh burst and fires again.
|
||||
assert!(s.observe_drops(12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transport_drop_state_steady_counter_fires_once() {
|
||||
let mut s = TransportDropState::default();
|
||||
// A cumulative counter that jumps once and then holds steady must
|
||||
// register exactly one event, not one per sample — otherwise a single
|
||||
// historical drop burst would report congestion forever.
|
||||
assert!(s.observe_drops(7));
|
||||
assert!(!s.observe_drops(7));
|
||||
assert!(!s.observe_drops(7));
|
||||
}
|
||||
|
||||
@@ -536,6 +536,172 @@ fn test_evaluate_parent_picks_loop_free_over_loopy() {
|
||||
assert_eq!(result, Some(peer2));
|
||||
}
|
||||
|
||||
// ===== Cost-based parent selection =====
|
||||
//
|
||||
// These exercise evaluate_parent's MMP-cost path directly, as a sans-IO
|
||||
// unit test of the exact decision. They replace six Docker chaos
|
||||
// scenarios — cost-reeval, cost-avoidance, cost-stability, depth-vs-cost,
|
||||
// mixed-technology and bottleneck-parent — whose subject was this
|
||||
// decision but which could not test it reliably: the mesh's root is
|
||||
// whichever node holds the smallest NodeAddr, MMP costs take several
|
||||
// measurement windows to settle, and hold-down plus hysteresis timing all
|
||||
// confounded the assertion. Here the peer ancestry, depths and costs are
|
||||
// constructed directly, so the decision is deterministic and each check
|
||||
// can fail on a real regression.
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_cost_prefers_cheaper_link_at_equal_depth() {
|
||||
// mixed-technology / cost-avoidance subject: two candidate parents at
|
||||
// the SAME depth, one over a cheap (fiber) link and one over an
|
||||
// expensive (Bluetooth) link. The cheaper link must win.
|
||||
//
|
||||
// The cheap peer is given the LARGER NodeAddr on purpose: with cost
|
||||
// ignored the two candidates tie on depth and the NodeAddr tiebreak
|
||||
// would pick the expensive, smaller-addr peer. Only a cost-aware
|
||||
// decision picks the cheaper, larger-addr one, so the assertion
|
||||
// discriminates.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
|
||||
let expensive = make_node_addr(2); // smaller addr, high cost (Bluetooth)
|
||||
let cheap = make_node_addr(3); // larger addr, low cost (fiber)
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(expensive, root, 1, 1000),
|
||||
make_coords(&[2, 0]), // depth 1
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(cheap, root, 1, 1000),
|
||||
make_coords(&[3, 0]), // depth 1
|
||||
);
|
||||
|
||||
// eff_depth(expensive) = 1 + 4.0 = 5.0; eff_depth(cheap) = 1 + 1.0 = 2.0
|
||||
let costs = HashMap::from([(expensive, 4.0_f64), (cheap, 1.0_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), Some(cheap));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_cost_switches_when_link_to_parent_degrades() {
|
||||
// cost-reeval subject: the node is parented to A over a cheap link;
|
||||
// that link then degrades so the alternative B is strictly cheaper.
|
||||
// Re-evaluation must switch to B. This is the periodic-reeval decision,
|
||||
// taken here without any timer, netem or MMP-measurement latency.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
|
||||
let peer_a = make_node_addr(2);
|
||||
let peer_b = make_node_addr(3);
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_a, root, 1, 1000),
|
||||
make_coords(&[2, 0]), // depth 1
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_b, root, 1, 1000),
|
||||
make_coords(&[3, 0]), // depth 1
|
||||
);
|
||||
|
||||
// Adopt A as parent (both links cheap at first).
|
||||
state.set_parent(peer_a, 1, 1000);
|
||||
state.recompute_coords();
|
||||
assert!(!state.is_root());
|
||||
|
||||
// A's link degrades: eff(A) = 1 + 5.0 = 6.0, eff(B) = 1 + 1.0 = 2.0.
|
||||
// Default hysteresis is zero, so the strictly-cheaper B wins.
|
||||
let costs = HashMap::from([(peer_a, 5.0_f64), (peer_b, 1.0_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), Some(peer_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_hysteresis_suppresses_marginal_cost_change() {
|
||||
// cost-stability subject: a cost change smaller than the hysteresis
|
||||
// band must NOT trigger a reparent. This is the property the scenario
|
||||
// was named for and could only approximate with a switch-count ceiling.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
state.set_parent_hysteresis(0.2);
|
||||
|
||||
let peer_a = make_node_addr(2);
|
||||
let peer_b = make_node_addr(3);
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_a, root, 1, 1000),
|
||||
make_coords(&[2, 0]),
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_b, root, 1, 1000),
|
||||
make_coords(&[3, 0]),
|
||||
);
|
||||
state.set_parent(peer_a, 1, 1000);
|
||||
state.recompute_coords();
|
||||
|
||||
// eff(A) = 1 + 1.0 = 2.0; eff(B) = 1 + 0.9 = 1.9. B is cheaper, but
|
||||
// 1.9 is not below 2.0 * (1 - 0.2) = 1.6, so hysteresis holds the parent.
|
||||
let costs = HashMap::from([(peer_a, 1.0_f64), (peer_b, 0.9_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_hysteresis_allows_significant_cost_change() {
|
||||
// cost-stability healthy-path companion: a change LARGER than the band
|
||||
// must still switch, so the hysteresis test above is not passing merely
|
||||
// because the node never reparents.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
state.set_parent_hysteresis(0.2);
|
||||
|
||||
let peer_a = make_node_addr(2);
|
||||
let peer_b = make_node_addr(3);
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_a, root, 1, 1000),
|
||||
make_coords(&[2, 0]),
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_b, root, 1, 1000),
|
||||
make_coords(&[3, 0]),
|
||||
);
|
||||
state.set_parent(peer_a, 1, 1000);
|
||||
state.recompute_coords();
|
||||
|
||||
// eff(A) = 2.0; eff(B) = 1 + 0.3 = 1.3 < 1.6 threshold → switch to B.
|
||||
let costs = HashMap::from([(peer_a, 1.0_f64), (peer_b, 0.3_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), Some(peer_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_effective_depth_weighs_depth_against_cost() {
|
||||
// depth-vs-cost / bottleneck-parent subject: a shallow parent reached
|
||||
// over an expensive (bottleneck) link versus a deeper parent over a
|
||||
// cheap link. effective_depth = depth + link_cost decides, and here the
|
||||
// deeper-but-cheaper path wins — the outcome the depth-vs-cost scenario
|
||||
// named no falsifiable answer for.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
|
||||
let shallow = make_node_addr(2); // depth 1, bottleneck link
|
||||
let deep = make_node_addr(3); // depth 3, cheap link
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(shallow, root, 1, 1000),
|
||||
make_coords(&[2, 0]), // depth 1
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(deep, make_node_addr(6), 1, 1000),
|
||||
make_coords(&[3, 6, 7, 0]), // depth 3
|
||||
);
|
||||
|
||||
// eff(shallow) = 1 + 3.0 = 4.0; eff(deep) = 3 + 0.5 = 3.5 → pick deep.
|
||||
// A depth-only decision would take the shallow bottleneck instead.
|
||||
let costs = HashMap::from([(shallow, 3.0_f64), (deep, 0.5_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), Some(deep));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_handle_parent_lost_finds_alternative() {
|
||||
let my_node = make_node_addr(5);
|
||||
|
||||
Reference in New Issue
Block a user