laantungir/fips
1
0
Fork 0
mirror of https://github.com/jmcorgan/fips.git synced 2026-08-09 16:24:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge master into next (MMP stale-report rejection; convergence-gate near-budget hold)

Browse Source
This commit is contained in:
Johnathan Corgan
2026-06-08 00:19:37 +00:00
parent 9ce4f2e908 42011a9a2f
commit aed64f6d80
9 changed files with 838 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+7
View File
@@ -99,6 +99,13 @@ with v0.3.x peers.
## [Unreleased]
### Fixed
- MMP sender metrics now ignore duplicate or regressed receiver reports
before updating RTT, loss, goodput, or ETX. Receiver reports also
suppress timestamp echo when dwell time overflows, so stale reports
cannot inflate SRTT.
### Added
- Typed `RejectReason` classification for receive-path silent-rejection
docs/design/fips-mmp.md
+5
View File
@@ -95,6 +95,11 @@ with dwell-time compensation, applied via the Jacobson/Karels
algorithm (RFC 6298, α = 1/8). This is the sole SRTT source at both
layers.
Duplicate or regressed ReceiverReports are ignored before any RTT, loss,
goodput, or ETX update. If receiver-side dwell time exceeds the wire
field, the report keeps its counters but sends a zero timestamp echo so
the sender cannot form an invalid RTT sample.
## ECN Congestion Signaling
The CE (Congestion Experienced) flag (bit 1 in the FMP flags byte)
src/mmp/metrics.rs
+118 -15
View File
@@ -118,25 +118,64 @@ impl MmpMetrics {
) -> bool {
let had_srtt = self.srtt.initialized();
if self.has_prev_rr {
let counters_regressed = rr.highest_counter < self.prev_rr_highest_counter
|| rr.cumulative_packets_recv < self.prev_rr_cum_packets
|| rr.cumulative_bytes_recv < self.prev_rr_cum_bytes
|| rr.ecn_ce_count < self.prev_rr_ecn_ce
|| rr.cumulative_reorder_count < self.prev_rr_reorder;
let duplicate_counters = rr.highest_counter == self.prev_rr_highest_counter
&& rr.cumulative_packets_recv == self.prev_rr_cum_packets
&& rr.cumulative_bytes_recv == self.prev_rr_cum_bytes
&& rr.ecn_ce_count == self.prev_rr_ecn_ce
&& rr.cumulative_reorder_count == self.prev_rr_reorder;
// Safe to drop: reports are only built after interval data, so
// a fresh report always advances at least one cumulative counter.
if counters_regressed || duplicate_counters {
trace!(
highest_counter = rr.highest_counter,
prev_highest_counter = self.prev_rr_highest_counter,
cumulative_packets_recv = rr.cumulative_packets_recv,
prev_cumulative_packets_recv = self.prev_rr_cum_packets,
cumulative_bytes_recv = rr.cumulative_bytes_recv,
prev_cumulative_bytes_recv = self.prev_rr_cum_bytes,
"Ignoring stale MMP ReceiverReport"
);
return false;
}
}
// --- RTT from timestamp echo ---
// RTT = now - echoed_timestamp - dwell_time
if rr.timestamp_echo > 0 {
let echo_ms = rr.timestamp_echo;
let dwell_ms = rr.dwell_time as u32;
// Guard against timestamp wrap or bogus values
if our_timestamp_ms > echo_ms + dwell_ms {
let rtt_ms = our_timestamp_ms - echo_ms - dwell_ms;
let rtt_us = (rtt_ms as i64) * 1000;
trace!(
our_ts = our_timestamp_ms,
echo = echo_ms,
dwell = dwell_ms,
rtt_ms = rtt_ms,
srtt_ms = self.srtt.srtt_us() as f64 / 1000.0,
"RTT sample from timestamp echo"
);
self.srtt.update(rtt_us);
self.rtt_trend.update(rtt_us as f64);
let dwell_ms = u32::from(rr.dwell_time);
let rtt_sample_ms = echo_ms
.checked_add(dwell_ms)
.and_then(|send_done_ms| our_timestamp_ms.checked_sub(send_done_ms));
match rtt_sample_ms {
Some(rtt_ms) if rtt_ms > 0 => {
let rtt_us = (rtt_ms as i64) * 1000;
trace!(
our_ts = our_timestamp_ms,
echo = echo_ms,
dwell = dwell_ms,
rtt_ms = rtt_ms,
srtt_ms = self.srtt.srtt_us() as f64 / 1000.0,
"RTT sample from timestamp echo"
);
self.srtt.update(rtt_us);
self.rtt_trend.update(rtt_us as f64);
}
_ => {
trace!(
our_ts = our_timestamp_ms,
echo = echo_ms,
dwell = dwell_ms,
"Ignoring invalid MMP RTT sample"
);
}
}
}
@@ -314,6 +353,70 @@ mod tests {
assert!((srtt_ms - 45.0).abs() < 1.0, "srtt={srtt_ms}, expected ~45");
}
#[test]
fn test_ignores_duplicate_receiver_report_after_valid_sample() {
let mut m = MmpMetrics::new();
let t0 = Instant::now();
let rr1 = make_rr(10, 10, 5_000, 1_000, 5, 0);
m.process_receiver_report(&rr1, 1_050, t0);
let rr2 = make_rr(20, 18, 14_000, 1_100, 5, 0);
m.process_receiver_report(&rr2, 1_150, t0 + Duration::from_secs(1));
let baseline_srtt_ms = m.srtt_ms().unwrap();
let baseline_loss = m.loss_rate();
let baseline_goodput = m.goodput_bps();
assert!(baseline_loss > 0.0);
assert!(baseline_goodput > 0.0);
// A duplicate of the same counters arriving later would be a 4.895s
// RTT sample if accepted. It is stale and must not move metrics.
m.process_receiver_report(&rr2, 6_000, t0 + Duration::from_secs(5));
assert_eq!(m.srtt_ms().unwrap(), baseline_srtt_ms);
assert_eq!(m.loss_rate(), baseline_loss);
assert_eq!(m.goodput_bps(), baseline_goodput);
}
#[test]
fn test_ignores_out_of_order_receiver_report_after_valid_sample() {
let mut m = MmpMetrics::new();
let now = Instant::now();
let valid_rr = make_rr(20, 20, 10000, 1000, 5, 0);
m.process_receiver_report(&valid_rr, 1050, now);
let baseline_srtt_ms = m.srtt_ms().unwrap();
let old_rr = make_rr(10, 10, 5000, 1000, 0, 0);
m.process_receiver_report(&old_rr, 6000, now + Duration::from_secs(5));
let srtt_ms = m.srtt_ms().unwrap();
assert_eq!(srtt_ms, baseline_srtt_ms);
}
#[test]
fn test_ignores_wrapped_rtt_sample() {
let mut m = MmpMetrics::new();
let now = Instant::now();
let wrapped_rr = make_rr(10, 10, 5000, u32::MAX - 10, 20, 0);
m.process_receiver_report(&wrapped_rr, 15, now);
assert!(m.srtt_ms().is_none());
}
#[test]
fn test_ignores_future_rtt_sample() {
let mut m = MmpMetrics::new();
let now = Instant::now();
let future_rr = make_rr(10, 10, 5_000, 2_000, 5, 0);
m.process_receiver_report(&future_rr, 1_000, now);
assert!(m.srtt_ms().is_none());
}
#[test]
fn test_loss_rate_computation() {
let mut m = MmpMetrics::new();
src/mmp/receiver.rs
+29 -5
View File
@@ -323,16 +323,25 @@ impl ReceiverState {
return None;
}
// Dwell time: ms between last frame reception and report generation
let dwell_time = self
// Dwell time: ms between last frame reception and report generation.
// If it no longer fits on the wire, the timestamp echo cannot produce
// a valid RTT sample. Preserve the counters but suppress the echo.
let (timestamp_echo, dwell_time) = self
.last_recv_time
.map(|t| now.duration_since(t).as_millis() as u16)
.unwrap_or(0);
.map(|t| {
let dwell_ms = now.duration_since(t).as_millis();
if dwell_ms > u128::from(u16::MAX) {
(0, u16::MAX)
} else {
(self.last_sender_timestamp, dwell_ms as u16)
}
})
.unwrap_or((0, 0));
let (burst_count, _max_burst, _mean_burst) = self.gap_tracker.take_interval_stats();
let report = ReceiverReport {
timestamp_echo: self.last_sender_timestamp,
timestamp_echo,
dwell_time,
highest_counter: self.highest_counter,
cumulative_packets_recv: self.cumulative_packets_recv,
@@ -499,6 +508,21 @@ mod tests {
assert_eq!(report.timestamp_echo, 200); // last sender timestamp
}
#[test]
fn test_build_report_suppresses_rtt_echo_when_dwell_overflows() {
let mut r = ReceiverState::new(32);
let t0 = Instant::now();
r.record_recv(1, 100, 500, false, t0);
let report = r
.build_report(t0 + Duration::from_millis(u64::from(u16::MAX) + 1))
.unwrap();
assert_eq!(report.timestamp_echo, 0);
assert_eq!(report.dwell_time, u16::MAX);
assert_eq!(report.cumulative_packets_recv, 1);
}
#[test]
fn test_build_report_resets_interval() {
let mut r = ReceiverState::new(32);
testing/interop/generate-configs.sh
+103 -5
View File
@@ -18,7 +18,9 @@
# - IPv4 = 172.30.0.1<index>, index = 0-based position in the spec
# (10, 11, 12, 13, ...).
#
# Topology is a FULL MESH: every node auto-connects to every other node.
# Topology is a FULL MESH by default: every node auto-connects to every
# other node. Set FIPS_INTEROP_EDGES (see Environment) to an explicit
# undirected edge list for a multi-hop / leaf topology instead.
# Node identities are derived deterministically via the shared
# derive_keys.py helper, keyed by a unique per-node string so that two
# nodes of the same slot (a1, a2) still get DISTINCT identities.
@@ -41,6 +43,13 @@
# Environment:
# REKEY_AFTER_SECS FSP/FMP rekey interval (default 35, matching
# the static rekey suite).
# FIPS_INTEROP_EDGES Optional undirected edge list (space/comma
# separated `nid-nid` tokens, e.g.
# "a1-b1 a1-c1 b1-a2"). Unset ⇒ full mesh. When
# set, each node peers only with its listed
# neighbors; the graph must be connected with no
# isolated node. Enables multi-hop forwarding /
# leaf-node topologies.
# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/,
# .stress-runs/, generated-configs/). When
# unset, falls back to in-tree paths under
@@ -124,6 +133,79 @@ for slot in "${SPEC[@]}"; do
index=$(( index + 1 ))
done
# ── Topology: adjacency (full mesh by default, or explicit edges) ─────
#
# FIPS_INTEROP_EDGES, when set, is a space/comma list of `nid-nid` tokens
# describing UNDIRECTED edges (direct FMP peer links), e.g.
# "a1-b1 a1-c1 b1-a2". Empty ⇒ FULL MESH (every node peers with every
# other, the historical behavior). Edges are symmetric: `x-y` makes x and
# y each other's auto_connect peer.
declare -A ADJ # ADJ[nid] = space-separated neighbor ids
declare -A _ADJ_SEEN # "x|y" dedup guard
TOPOLOGY="full-mesh"
EDGES_LIST=()
add_edge() {
local x="$1" y="$2"
if [ "$x" = "$y" ]; then
echo "ERROR: self-edge '$x-$x' not allowed" >&2; exit 1
fi
if [ -z "${NODE_SLOT[$x]:-}" ]; then
echo "ERROR: edge references unknown node '$x'" >&2; exit 1
fi
if [ -z "${NODE_SLOT[$y]:-}" ]; then
echo "ERROR: edge references unknown node '$y'" >&2; exit 1
fi
if [ -z "${_ADJ_SEEN[$x|$y]:-}" ]; then
ADJ[$x]="${ADJ[$x]:+${ADJ[$x]} }$y"; _ADJ_SEEN[$x|$y]=1
fi
if [ -z "${_ADJ_SEEN[$y|$x]:-}" ]; then
ADJ[$y]="${ADJ[$y]:+${ADJ[$y]} }$x"; _ADJ_SEEN[$y|$x]=1
fi
}
if [ -n "${FIPS_INTEROP_EDGES:-}" ]; then
TOPOLOGY="custom"
for tok in ${FIPS_INTEROP_EDGES//,/ }; do
x="${tok%%-*}"; y="${tok#*-}"
if [ "$x" = "$tok" ] || [ -z "$x" ] || [ -z "$y" ] || [[ "$y" == *-* ]]; then
echo "ERROR: malformed edge token '$tok' (want nid-nid)" >&2; exit 1
fi
add_edge "$x" "$y"
EDGES_LIST+=("$x-$y")
done
# Validate: no isolated node, graph connected (BFS from node 0).
for nid in "${NODE_IDS[@]}"; do
if [ -z "${ADJ[$nid]:-}" ]; then
echo "ERROR: node '$nid' has no edges (isolated)" >&2; exit 1
fi
done
declare -A _BFS_SEEN=()
bfs_queue=("${NODE_IDS[0]}"); _BFS_SEEN[${NODE_IDS[0]}]=1; bfs_reached=1
while [ "${#bfs_queue[@]}" -gt 0 ]; do
cur="${bfs_queue[0]}"; bfs_queue=("${bfs_queue[@]:1}")
for nb in ${ADJ[$cur]}; do
if [ -z "${_BFS_SEEN[$nb]:-}" ]; then
_BFS_SEEN[$nb]=1; bfs_reached=$(( bfs_reached + 1 ))
bfs_queue+=("$nb")
fi
done
done
if [ "$bfs_reached" -ne "${#NODE_IDS[@]}" ]; then
echo "ERROR: topology is disconnected ($bfs_reached/${#NODE_IDS[@]} reachable from ${NODE_IDS[0]})" >&2
exit 1
fi
else
# Full mesh: every node adjacent to every other, in spec order.
for a in "${NODE_IDS[@]}"; do
for b in "${NODE_IDS[@]}"; do
[ "$a" = "$b" ] && continue
ADJ[$a]="${ADJ[$a]:+${ADJ[$a]} }$b"
done
done
fi
mkdir -p "$OUT_DIR"
# Clear stale per-node configs from a previous (differently-shaped) spec
@@ -186,8 +268,7 @@ for nid in "${NODE_IDS[@]}"; do
echo " mtu: 1472"
echo ""
echo "peers:"
for pid in "${NODE_IDS[@]}"; do
[ "$pid" = "$nid" ] && continue
for pid in ${ADJ[$nid]}; do
emit_peer_block "$pid"
done
} > "$cfg"
@@ -259,6 +340,19 @@ MANIFEST="$OUT_DIR/nodes.env"
echo "# Node-spec: ${SPEC[*]}"
echo "INTEROP_SPEC=\"${SPEC[*]}\""
echo "INTEROP_NODE_IDS=\"${NODE_IDS[*]}\""
echo "INTEROP_TOPOLOGY=\"$TOPOLOGY\""
echo "INTEROP_TOPOLOGY_EDGES=\"${EDGES_LIST[*]:-}\""
# Per-node expected DIRECT-peer count (adjacency degree). The driver
# uses this for the per-node peer-convergence check instead of N-1.
{
printf 'INTEROP_NODE_DEGREE="'
for nid in "${NODE_IDS[@]}"; do
d=0
for _nb in ${ADJ[$nid]}; do d=$(( d + 1 )); done
printf '%s:%s ' "$nid" "$d"
done
printf '"\n'
}
# Per-node maps, one var each, space-separated 'nodeid:value' tokens.
{
printf 'INTEROP_NODE_SLOTS="'
@@ -297,7 +391,11 @@ ENV_FILE="$OUT_DIR/npubs.env"
echo " generated $ENV_FILE"
echo ""
echo "Mesh configs ready (spec='${SPEC[*]}', rekey.after_secs=$REKEY_AFTER_SECS):"
echo "Mesh configs ready (spec='${SPEC[*]}', topology=$TOPOLOGY, rekey.after_secs=$REKEY_AFTER_SECS):"
for nid in "${NODE_IDS[@]}"; do
echo " $nid slot ${NODE_SLOT[$nid]} ${NODE_IP[$nid]} ${NPUB[$nid]}"
d=0; for _nb in ${ADJ[$nid]}; do d=$(( d + 1 )); done
echo " $nid slot ${NODE_SLOT[$nid]} ${NODE_IP[$nid]} deg=$d ${NPUB[$nid]}"
done
if [ "$TOPOLOGY" = "custom" ]; then
echo " edges: ${EDGES_LIST[*]}"
fi
testing/interop/interop-stress.sh
+31 -5
View File
@@ -26,9 +26,13 @@
# fixed Docker network, so two reps must never overlap.
#
# Usage:
# ./interop-stress.sh [--reps N] [node-spec...]
# ./interop-stress.sh [--reps N] [--topology <name> | node-spec...]
#
# --reps N repetitions (default 10).
# --topology built-in multi-hop topology forwarded to interop-test.sh
# (e.g. multihop-3v-cycle). Mutually exclusive with a
# positional node-spec. Adds multi-hop forwarding,
# data-plane continuity, and mesh-size checks to each rep.
# node-spec slot letters (a/b/c), default `a a b c` (the control
# topology — one same-version pair + five mixed pairs).
#
@@ -81,8 +85,17 @@ RUNS_BASE="$INTEROP_RUNS_BASE/.stress-runs"
REPS=10
SPEC=()
TOPOLOGY_ARG=""
while [ "$#" -gt 0 ]; do
case "$1" in
--topology)
TOPOLOGY_ARG="${2:-}"
shift 2
;;
--topology=*)
TOPOLOGY_ARG="${1#--topology=}"
shift
;;
--reps)
REPS="${2:-}"
if ! [[ "$REPS" =~ ^[0-9]+$ ]] || [ "$REPS" -lt 1 ]; then
@@ -119,10 +132,23 @@ while [ "$#" -gt 0 ]; do
esac
done
if [ "${#SPEC[@]}" -eq 0 ]; then
SPEC=(a a b c)
# A --topology selects spec + edges inside the driver; otherwise use the
# positional node-spec (default the control-arm topology `a a b c`).
DRIVER_ARGS=()
if [ -n "$TOPOLOGY_ARG" ]; then
if [ "${#SPEC[@]}" -gt 0 ]; then
echo "ERROR: pass either --topology or a node-spec, not both" >&2
exit 1
fi
DRIVER_ARGS=(--topology "$TOPOLOGY_ARG")
SPEC_STR="(topology: $TOPOLOGY_ARG)"
else
if [ "${#SPEC[@]}" -eq 0 ]; then
SPEC=(a a b c)
fi
DRIVER_ARGS=("${SPEC[@]}")
SPEC_STR="${SPEC[*]}"
fi
SPEC_STR="${SPEC[*]}"
# ── Preflight ────────────────────────────────────────────────────────
@@ -175,7 +201,7 @@ for ((rep = 1; rep <= REPS; rep++)); do
# Run the driver, capturing full output and exit code. Netem is
# passed through the environment; interop-test.sh applies it.
FIPS_INTEROP_NETEM="${FIPS_INTEROP_NETEM:-}" \
bash "$DRIVER" "${SPEC[@]}" >"$driver_log" 2>&1
bash "$DRIVER" "${DRIVER_ARGS[@]}" >"$driver_log" 2>&1
rc=$?
if [ "$rc" -eq 0 ]; then
testing/interop/interop-test.sh
+330 -23
View File
@@ -24,18 +24,39 @@
# 2. configs are (re)generated automatically below for the given spec.
#
# Usage:
# ./interop-test.sh [node-spec...]
# ./interop-test.sh [--topology <name>] [node-spec...]
#
# --topology built-in multi-hop topology selecting a node-spec + edge
# set + default data-plane streams. Known: multihop-3v-cycle
# (6 nodes, 2 of each version, one cycle, two leaves —
# exercises cross-version forwarding + mesh-size). Without
# it the mesh is a full mesh from the positional node-spec.
# node-spec space-separated slot letters (a/b/c), default `a b c`.
# e.g. `a a b c` (4-node, one same-version control pair),
# `a a a` (3-node same-version flake rig).
#
# Beyond FMP/FSP rekey survival, a --topology run also verifies multi-hop
# FORWARDING (all-pairs ping over non-adjacent pairs), DATA-PLANE
# CONTINUITY across rekey (control-differential ping-loss stream,
# Phase 5b), and MESH-SIZE estimate convergence across versions
# (Phase 7).
#
# Environment:
# FIPS_INTEROP_NETEM tc-netem arg string applied to every container's
# eth0, e.g. "delay 10ms 5ms 25% loss 1%". Unset =
# no impairment.
# FIPS_INTEROP_EDGES explicit undirected edge list (overrides the
# full mesh) — see generate-configs.sh. Usually
# set for you by --topology.
# FIPS_INTEROP_STREAMS data-plane stream pairs (`nid-nid` tokens, both
# directions streamed). Enables Phase 1b/5b. Set
# by --topology; empty = streams off.
# STREAM_LOSS_MARGIN_PCT rekey-vs-control loss margin (default 1).
# CONTROL_STREAM_SECS quiet control-window length (default 12).
# MESH_SIZE_TIMEOUT Phase 7 convergence poll budget (default 180).
# FIPS_INTEROP_KEEP_UP 1 = leave containers running after the test (debug).
# REKEY_AFTER_SECS rekey interval to generate configs with (default 35).
# REKEY_AFTER_SECS rekey interval to generate configs with (default
# 35; multihop-3v-cycle defaults it to 50).
# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/,
# .stress-runs/, generated-configs/). When
# unset, falls back to in-tree paths under
@@ -74,6 +95,48 @@ COMPOSE_FILE="$GEN_DIR/docker-compose.generated.yml"
NODES_ENV="$GEN_DIR/nodes.env"
REFS_ENV="$RUNS_BASE/.build/refs.env"
# ── Built-in topology selection ──────────────────────────────────────
#
# --topology <name> selects a node-spec + an explicit edge set (a
# multi-hop, mixed-version, leaf-bearing graph) + a default data-plane
# stream set, exercising forwarding / routing / mesh-size convergence
# across versions. Without it, positional args are the node-spec and the
# mesh is a full mesh (historical behavior).
TOPOLOGY_NAME=""
_ARGS=()
while [ "$#" -gt 0 ]; do
case "$1" in
--topology) TOPOLOGY_NAME="${2:-}"; shift 2 ;;
--topology=*) TOPOLOGY_NAME="${1#--topology=}"; shift ;;
*) _ARGS+=("$1"); shift ;;
esac
done
set -- ${_ARGS[@]+"${_ARGS[@]}"}
if [ -n "$TOPOLOGY_NAME" ]; then
case "$TOPOLOGY_NAME" in
multihop-3v-cycle)
# 6 nodes (2 of each version), one cycle, two leaves.
# a1
# / \ leaves: a2, c2
# b1 c1 cycle: b1-a1-c1-b2-b1
# / \ \
# a2 b2------c2-edge(b2-c1)
set -- a a b b c c
export FIPS_INTEROP_EDGES="a1-b1 a1-c1 b1-a2 b1-b2 c1-c2 b2-c1"
: "${FIPS_INTEROP_STREAMS:=a2-c2 a1-b1}"
export FIPS_INTEROP_STREAMS
# Multi-hop convergence + a clean pre-rekey control window
# want more headroom than the 35s full-mesh default.
: "${REKEY_AFTER_SECS:=50}"
;;
*)
echo "ERROR: unknown --topology '$TOPOLOGY_NAME' (known: multihop-3v-cycle)" >&2
exit 2
;;
esac
fi
REKEY_AFTER_SECS="${REKEY_AFTER_SECS:-35}"
# ── Node-spec ────────────────────────────────────────────────────────
@@ -86,7 +149,7 @@ SPEC_STR="${SPEC[*]}"
# ── Timing ───────────────────────────────────────────────────────────
CONVERGENCE_TIMEOUT=60 # wait for full mesh + clean ping baseline
CONVERGENCE_TIMEOUT="${CONVERGENCE_TIMEOUT:-90}" # detector settle budget (env-overridable; larger meshes under loss converge slower)
PING_TIMEOUT=5
MAX_PING_ATTEMPTS=4 # strict-ping retry (mirrors rekey-test.sh)
PING_RETRY_DELAY=1
@@ -101,6 +164,19 @@ REKEY_SETTLE=12 # FSP-cutover settle budget (Phase 6)
POST_REKEY_TIMEOUT=45
LOG_POLL_INTERVAL=2
# Data-plane continuity stream (control-differential). Streams run a
# sustained ping6 over the overlay across the rekey window vs a quiet
# control window; loss is compared to prove rekey is hitless.
STREAM_RATE_HZ=5 # ping6 -i 0.2
CONTROL_STREAM_SECS="${CONTROL_STREAM_SECS:-12}" # quiet pre-rekey window
STREAM_LOSS_MARGIN_PCT="${STREAM_LOSS_MARGIN_PCT:-1}"
# The rekey-window stream must span Phases 2-5 (both cutovers + reconverge).
REKEY_STREAM_SECS=$(( FIRST_REKEY_TIMEOUT + SECOND_REKEY_WAIT + POST_REKEY_TIMEOUT + 15 ))
# Mesh-size estimate convergence (strict ±25% of true N). Generous poll
# budget — the bloom-union estimate converges over minutes.
MESH_SIZE_TIMEOUT="${MESH_SIZE_TIMEOUT:-180}"
# ── Counters ─────────────────────────────────────────────────────────
PASSED=0
@@ -140,7 +216,9 @@ done
need_regen=1
if [ -f "$NODES_ENV" ]; then
existing_spec="$(sed -n 's/^INTEROP_SPEC="\(.*\)"$/\1/p' "$NODES_ENV")"
if [ "$existing_spec" = "$SPEC_STR" ]; then
existing_edges="$(sed -n 's/^INTEROP_TOPOLOGY_EDGES="\(.*\)"$/\1/p' "$NODES_ENV")"
if [ "$existing_spec" = "$SPEC_STR" ] \
&& [ "$existing_edges" = "${FIPS_INTEROP_EDGES:-}" ]; then
need_regen=0
fi
fi
@@ -179,6 +257,41 @@ parse_map CONTAINER "$INTEROP_NODE_CONTAINERS"
parse_map NODE_IP_OF "$INTEROP_NODE_IPS"
parse_map NPUB_OF "$INTEROP_NODE_NPUBS"
# Topology metadata: per-node expected direct-peer degree, and the
# undirected direct-edge set (for direct-vs-routed pair labeling).
TOPOLOGY_KIND="${INTEROP_TOPOLOGY:-full-mesh}"
declare -A DEGREE_OF
parse_map DEGREE_OF "${INTEROP_NODE_DEGREE:-}"
declare -A IS_DIRECT_EDGE
if [ -n "${INTEROP_TOPOLOGY_EDGES:-}" ]; then
for _e in $INTEROP_TOPOLOGY_EDGES; do
_x="${_e%%-*}"; _y="${_e#*-}"
IS_DIRECT_EDGE["$_x|$_y"]=1
IS_DIRECT_EDGE["$_y|$_x"]=1
done
else
# Full mesh: every ordered pair is a direct edge.
for _a in "${NODES[@]}"; do
for _b in "${NODES[@]}"; do
[ "$_a" = "$_b" ] && continue
IS_DIRECT_EDGE["$_a|$_b"]=1
done
done
fi
pair_is_direct() { [ -n "${IS_DIRECT_EDGE[$1|$2]:-}" ]; }
hop_label() { if pair_is_direct "$1" "$2"; then echo "direct"; else echo "routed"; fi; }
# Data-plane stream directed pairs (both directions of each token).
STREAM_PAIRS=()
if [ -n "${FIPS_INTEROP_STREAMS:-}" ]; then
for _tok in ${FIPS_INTEROP_STREAMS//,/ }; do
_x="${_tok%%-*}"; _y="${_tok#*-}"
STREAM_PAIRS+=("$_x $_y" "$_y $_x")
done
fi
# Unordered pairs of the mesh.
PAIRS=()
for ((i = 0; i < ${#NODES[@]}; i++)); do
@@ -309,9 +422,12 @@ ping_all_pairs() {
# every rep that takes a moment to converge. Only the
# strict assertion sweeps (baseline / post-rekey-*) record.
if [ "$context" != "convergence" ]; then
local kind
local kind hop
kind="$(pair_kind "$i" "$j")"
INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j"): ping $i->$j FAILED")
hop="$(hop_label "$i" "$j")"
# NOTE: keep "$kind pair" contiguous — interop-stress.sh
# greps "MIXED pair"/"same pair". The [hop] tag is additive.
INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j") [$hop]: ping $i->$j FAILED")
fi
fi
done
@@ -376,6 +492,66 @@ wait_for_log_pattern_count() {
[ "$(count_log_pattern "$pattern")" -ge "$min_count" ]
}
# ── Data-plane continuity streams ────────────────────────────────────
#
# A sustained ping6 stream over the overlay (<npub>.fips) is data-plane
# traffic: it traverses TUN → FSP session → FMP link → forwarding, the
# same path application packets take. We run streams across the rekey
# window and across a quiet control window, then compare loss — a rekey
# that drops data shows up as rekey-window loss above the control.
declare -A STREAM_TX STREAM_RX
_STREAM_PIDS=()
_STREAM_FILES=() # "label:from->to:resultfile"
_STREAM_TMP=""
# One directed ping6 stream of <dur>s at STREAM_RATE_HZ; writes "tx rx".
_stream_one() {
local from="$1" to="$2" dur="$3" outfile="$4"
local from_ctr="${CONTAINER[$from]}" to_npub="${NPUB_OF[$to]}"
local count=$(( dur * STREAM_RATE_HZ ))
local out tx rx
out=$(docker exec "$from_ctr" ping6 -i 0.2 -c "$count" -W "$PING_TIMEOUT" \
"${to_npub}.fips" 2>&1)
tx=$(echo "$out" | grep -oE '[0-9]+ packets transmitted' | grep -oE '^[0-9]+')
rx=$(echo "$out" | grep -oE '[0-9]+ received' | grep -oE '^[0-9]+')
echo "${tx:-0} ${rx:-0}" > "$outfile"
}
# Launch all stream pairs in the background for <label> over <dur>s.
launch_streams() {
local label="$1" dur="$2"
_STREAM_TMP="$(mktemp -d)"
_STREAM_PIDS=(); _STREAM_FILES=()
local sp from to
for sp in "${STREAM_PAIRS[@]}"; do
read -r from to <<< "$sp"
_stream_one "$from" "$to" "$dur" "$_STREAM_TMP/$label-$from-$to" &
_STREAM_PIDS+=("$!")
_STREAM_FILES+=("$label:$from->$to:$_STREAM_TMP/$label-$from-$to")
done
}
# Wait for the launched streams and read tx/rx into STREAM_TX/STREAM_RX.
collect_streams() {
[ "${#_STREAM_PIDS[@]}" -gt 0 ] && wait "${_STREAM_PIDS[@]}" 2>/dev/null || true
local entry label rest key f tx rx
for entry in "${_STREAM_FILES[@]}"; do
label="${entry%%:*}"; rest="${entry#*:}"; key="${rest%%:*}"; f="${rest#*:}"
if read -r tx rx < "$f" 2>/dev/null; then :; else tx=0; rx=0; fi
STREAM_TX["$label:$key"]="${tx:-0}"
STREAM_RX["$label:$key"]="${rx:-0}"
done
[ -n "$_STREAM_TMP" ] && rm -rf "$_STREAM_TMP"
_STREAM_TMP=""
}
# loss% from tx/rx (one decimal), or "NA" when tx==0.
_loss_pct() {
awk -v t="${1:-0}" -v r="${2:-0}" \
'BEGIN{ if (t>0) printf "%.1f", (t-r)*100/t; else print "NA" }'
}
dump_diagnostics() {
echo ""
echo "=== Peer / link snapshot ==="
@@ -402,6 +578,8 @@ compose() {
}
cleanup() {
# Reap any in-flight stream tmpdir (e.g. on interrupt mid-window).
[ -n "${_STREAM_TMP:-}" ] && rm -rf "$_STREAM_TMP" 2>/dev/null
if [ "${FIPS_INTEROP_KEEP_UP:-}" = "1" ]; then
echo ""
echo "FIPS_INTEROP_KEEP_UP=1 — leaving mesh running. Tear down with:"
@@ -422,19 +600,31 @@ echo " FIPS Mixed-Version Interop Test"
echo "=============================================================="
echo ""
echo "Node-spec: $SPEC_STR ($NUM_NODES nodes, $NUM_PAIRS pairs, $NUM_DIRECTED directed)"
echo "Topology : $TOPOLOGY_KIND${TOPOLOGY_NAME:+ ($TOPOLOGY_NAME)}"
if [ "$TOPOLOGY_KIND" != "full-mesh" ]; then
echo " edges: ${INTEROP_TOPOLOGY_EDGES:-}"
fi
echo ""
echo "Mesh nodes:"
for n in "${NODES[@]}"; do
s="${SLOT_OF[$n]}"
u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
echo " $n slot $u ${SLOT_REF[$s]} @ ${SLOT_SHA[$s]} (${NODE_IP_OF[$n]})"
echo " $n slot $u ${SLOT_REF[$s]} @ ${SLOT_SHA[$s]} deg=${DEGREE_OF[$n]:-?} (${NODE_IP_OF[$n]})"
done
echo ""
echo "Mesh pairs:"
for p in "${PAIRS[@]}"; do
read -r n1 n2 <<< "$p"
echo " $(pair_kind "$n1" "$n2") $(pair_label "$n1" "$n2")"
echo " $(pair_kind "$n1" "$n2") $(pair_label "$n1" "$n2") [$(hop_label "$n1" "$n2")]"
done
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
echo ""
echo "Data-plane streams (continuity across rekey):"
for sp in "${STREAM_PAIRS[@]}"; do
read -r sf st <<< "$sp"
echo " $sf -> $st [$(hop_label "$sf" "$st")]"
done
fi
echo ""
if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then
echo "Netem impairment: $FIPS_INTEROP_NETEM"
@@ -474,24 +664,32 @@ echo ""
echo "Phase 1: Link/session establishment + connectivity baseline"
PASSED=0; FAILED=0
# Every node must reach N-1 authenticated peers. A peer in `show peers`
# is an authenticated FMP-link peer.
# Every node must reach its DIRECT-peer degree of authenticated peers (a
# peer in `show peers` is an authenticated FMP-link peer). In a full mesh
# that degree is N-1; in a multi-hop topology it is the node's adjacency.
for n in "${NODES[@]}"; do
if ! wait_for_peers "${CONTAINER[$n]}" "$PEERS_EXPECTED" "$CONVERGENCE_TIMEOUT"; then
echo " $n did not reach $PEERS_EXPECTED authenticated peers"
exp="${DEGREE_OF[$n]:-$PEERS_EXPECTED}"
if ! wait_for_peers "${CONTAINER[$n]}" "$exp" "$CONVERGENCE_TIMEOUT"; then
echo " $n did not reach $exp authenticated direct peer(s)"
fi
done
# The all-pairs ping over fips0 is the definitive FSP-session check: in
# a full mesh every pair is a direct neighbor, so a successful ping
# proves that pair's FSP session carries traffic in that direction.
if wait_for_full_baseline "$CONVERGENCE_TIMEOUT"; then
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
else
echo " Best baseline before timeout: $PASSED/$((PASSED + FAILED))"
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
# The all-pairs ping over fips0 is the definitive reachability check.
# Direct-neighbor pairs prove their FSP session; NON-adjacent pairs in a
# multi-hop topology can only succeed via forwarding, so all-pairs ping
# is also the cross-version FORWARDING test (we keep pinging every pair).
# The convergence detector is an ADVISORY settle-wait, not the assertion.
# It needs one fully-clean, no-retry sweep of every directed pair; under
# packet loss that is statistically unlikely on a large mesh even when the
# mesh is perfectly healthy (e.g. 30 pairs at 2% loss => ~55% of sweeps
# are clean), so its timeout must NOT be fatal. The strict, retrying ping
# below is the real baseline assertion — it decides pass/fail.
if ! wait_for_full_baseline "$CONVERGENCE_TIMEOUT"; then
echo " detector saw no fully-clean sweep within ${CONVERGENCE_TIMEOUT}s (best $PASSED/$NUM_DIRECTED); strict re-ping decides"
fi
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
if [ "$FAILED" -ne 0 ]; then
dump_diagnostics
echo ""
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
@@ -500,6 +698,34 @@ else
fi
echo ""
# ── Phase 1b: data-plane control window (quiet, pre-rekey) ───────────
#
# Measure stream loss over a window with NO rekey cutover, as the control
# baseline for the differential. Validated cutover-free by confirming the
# FMP cutover count did not advance during the window. Then launch the
# rekey-window streams, which run in the background across Phases 2-5 and
# are collected/asserted in Phase 5b.
control_contaminated=0
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
echo "Phase 1b: Data-plane control stream (${CONTROL_STREAM_SECS}s quiet window)"
pre_cut="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"
launch_streams CONTROL "$CONTROL_STREAM_SECS"
collect_streams
post_cut="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"
if [ "$post_cut" -ne "$pre_cut" ]; then
control_contaminated=1
echo " WARN a rekey cutover occurred during the control window — control loss may be contaminated"
fi
for sp in "${STREAM_PAIRS[@]}"; do
read -r sf st <<< "$sp"
key="$sf->$st"
echo " control $key: tx=${STREAM_TX[CONTROL:$key]:-0} rx=${STREAM_RX[CONTROL:$key]:-0} loss=$(_loss_pct "${STREAM_TX[CONTROL:$key]:-0}" "${STREAM_RX[CONTROL:$key]:-0}")%"
done
echo " Launching rekey-window streams (${REKEY_STREAM_SECS}s, spanning Phases 2-5)"
launch_streams REKEY "$REKEY_STREAM_SECS"
echo ""
fi
# ── Phase 2: first rekey cycle ───────────────────────────────────────
echo "Phase 2: First rekey cycle (waiting up to ${FIRST_REKEY_TIMEOUT}s)"
@@ -544,6 +770,41 @@ ping_all_pairs "" "$MAX_PING_ATTEMPTS" "post-rekey-2"
phase_result "Post-second-rekey (all $NUM_DIRECTED directed pairs)"
echo ""
# ── Phase 5b: data-plane continuity across rekey (control-differential)
#
# Collect the rekey-window streams launched in Phase 1b and compare their
# loss to the control window. A hitless rekey keeps rekey-window loss at
# or below the control (plus a small margin); excess loss is data the
# rekey dropped — the failure the cutover fixes (4af3730/6e5cb89) and the
# pipelined wire layout are supposed to prevent.
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
echo "Phase 5b: Data-plane continuity across rekey (control-differential, margin ${STREAM_LOSS_MARGIN_PCT}%)"
PASSED=0; FAILED=0
collect_streams
printf ' %-16s %11s %11s %8s %s\n' "stream" "ctrl-loss%" "rekey-loss%" "delta" "verdict"
for sp in "${STREAM_PAIRS[@]}"; do
read -r sf st <<< "$sp"
key="$sf->$st"
cpct="$(_loss_pct "${STREAM_TX[CONTROL:$key]:-0}" "${STREAM_RX[CONTROL:$key]:-0}")"
rpct="$(_loss_pct "${STREAM_TX[REKEY:$key]:-0}" "${STREAM_RX[REKEY:$key]:-0}")"
verdict="$(awk -v c="$cpct" -v k="$rpct" -v m="$STREAM_LOSS_MARGIN_PCT" 'BEGIN{
if (c=="NA" || k=="NA") { print "NODATA"; exit }
if (k <= c + m) print "PASS"; else print "FAIL"
}')"
delta="$(awk -v c="$cpct" -v k="$rpct" 'BEGIN{ if(c=="NA"||k=="NA") print "NA"; else printf "%+.1f", k-c }')"
printf ' %-16s %11s %11s %8s %s\n' "$key" "$cpct" "$rpct" "$delta" "$verdict"
if [ "$verdict" = "PASS" ]; then
PASSED=$((PASSED + 1))
else
FAILED=$((FAILED + 1))
INTEROP_FAILURES+=("[stream] $key ($(hop_label "$sf" "$st")): rekey-window loss ${rpct}% vs control ${cpct}% (+${STREAM_LOSS_MARGIN_PCT}% margin) -> $verdict")
fi
done
[ "${control_contaminated:-0}" -eq 1 ] && echo " NOTE: control window saw a cutover; differential may understate rekey loss."
phase_result "Data-plane continuity across rekey"
echo ""
fi
# ── Phase 6: per-node, per-pair interop log analysis ─────────────────
#
# This is the interop-specific analysis. For each unordered pair it
@@ -627,7 +888,7 @@ echo " -- Per-pair interop summary --"
for p in "${PAIRS[@]}"; do
read -r n1 n2 <<< "$p"
kind="$(pair_kind "$n1" "$n2")"
label="$(pair_label "$n1" "$n2")"
label="$(pair_label "$n1" "$n2") [$(hop_label "$n1" "$n2")]"
pair_failed=0
if [ "${#INTEROP_FAILURES[@]}" -gt 0 ]; then
for f in "${INTEROP_FAILURES[@]}"; do
@@ -649,6 +910,52 @@ done
phase_result "Interop log analysis"
echo ""
# ── Phase 7: mesh-size estimate convergence (strict ±25%) ────────────
#
# Each node's bloom-union mesh-size estimate (fipsctl show status
# .estimated_mesh_size) should converge to the true node count across
# versions. A mixed-version bloom/tree-encoding divergence shows up as a
# node that never produces an in-band estimate (or returns null). Strict
# band = [0.75N, 1.25N]; polled up to MESH_SIZE_TIMEOUT (the estimate
# converges over minutes — see ISSUE-2026-0046 on its transient jitter).
echo "Phase 7: Mesh-size estimate convergence (strict ±25% of true N=$NUM_NODES)"
PASSED=0; FAILED=0
ms_lo="$(awk -v n="$NUM_NODES" 'BEGIN{printf "%.2f", 0.75*n}')"
ms_hi="$(awk -v n="$NUM_NODES" 'BEGIN{printf "%.2f", 1.25*n}')"
echo " band [$ms_lo, $ms_hi], poll up to ${MESH_SIZE_TIMEOUT}s"
declare -A MS_EST MS_OK
ms_deadline=$(( SECONDS + MESH_SIZE_TIMEOUT ))
while :; do
all_ok=1
for n in "${NODES[@]}"; do
[ "${MS_OK[$n]:-0}" = "1" ] && continue
est="$(docker exec "${CONTAINER[$n]}" fipsctl show status 2>/dev/null \
| python3 -c "import sys,json; v=json.load(sys.stdin).get('estimated_mesh_size'); print(v if v is not None else 'null')" 2>/dev/null || echo null)"
MS_EST[$n]="$est"
if [ "$est" != "null" ] && awk "BEGIN{exit !($est>=$ms_lo && $est<=$ms_hi)}"; then
MS_OK[$n]=1
else
all_ok=0
fi
done
[ "$all_ok" = "1" ] && break
[ "$SECONDS" -ge "$ms_deadline" ] && break
sleep 3
done
for n in "${NODES[@]}"; do
s="${SLOT_OF[$n]}"; u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
if [ "${MS_OK[$n]:-0}" = "1" ]; then
echo " PASS $n [$u]: estimated_mesh_size=${MS_EST[$n]}"
PASSED=$((PASSED + 1))
else
echo " FAIL $n [$u]: estimated_mesh_size=${MS_EST[$n]:-null} (outside [$ms_lo,$ms_hi] after ${MESH_SIZE_TIMEOUT}s)"
FAILED=$((FAILED + 1))
INTEROP_FAILURES+=("[mesh-size] node $n ($u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}): estimate=${MS_EST[$n]:-null} outside [$ms_lo,$ms_hi]")
fi
done
phase_result "Mesh-size estimate convergence"
echo ""
# ── Summary ──────────────────────────────────────────────────────────
echo "=============================================================="
testing/lib/wait-converge-test.sh
Executable
+191
View File
@@ -0,0 +1,191 @@
#!/bin/bash
# Unit tests for wait_until_connected() in wait-converge.sh.
#
# These tests drive the convergence gate with synthetic connectivity
# checks (ping_fn stand-ins) that report scripted PASSED/FAILED counts
# keyed off the same SECONDS clock the gate uses. No containers or
# network are involved, so the whole suite runs in a few seconds and is
# safe to run in CI.
#
# Run:
# ./wait-converge-test.sh
# Exits 0 only if every case passes; non-zero if any case fails.
set -u
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/wait-converge.sh"
RESULTS=()
FAILURES=0
# Record a single assertion result.
check() {
local name="$1"
local ok="$2" # 0 = pass, anything else = fail
local detail="${3:-}"
if [ "$ok" -eq 0 ]; then
RESULTS+=("PASS $name")
echo "PASS $name${detail:+ ($detail)}"
else
RESULTS+=("FAIL $name")
echo "FAIL $name${detail:+ ($detail)}"
FAILURES=$((FAILURES + 1))
fi
}
# Each ping_fn records its own start on first call so its schedule is
# measured from when the gate began polling it, regardless of the wall
# clock at suite start. PT (ping-elapsed time) is computed inline in each
# ping_fn — NOT via a subshell — so the PING_START assignment persists.
PING_START=-1
reset_ping() {
PING_START=-1
}
# --- Synthetic connectivity checks ------------------------------------
# Sets global PT to ping-elapsed seconds. Must be called (not subshelled)
# at the top of each ping_fn so the first-call timestamp persists.
PT=0
set_pt() {
if (( PING_START < 0 )); then
PING_START=$SECONDS
fi
PT=$(( SECONDS - PING_START ))
}
# Case 1 trace: improves (16 reachable) early, climbs to 18, then holds
# at FAILED=1 (within slack) until late, then fully converges. Mirrors a
# deep node whose last pair clears only after stacked discovery backoff.
ping_near_converged_hold() {
set_pt; local t=$PT
if (( t < 3 )); then
PASSED=16; FAILED=4
elif (( t < 6 )); then
PASSED=18; FAILED=2
elif (( t < 12 )); then
# Stuck within slack: one straggling pair pending.
PASSED=19; FAILED=1
else
PASSED=20; FAILED=0
fi
}
# Case 2 trace: climbs a little, then wedges far from convergence with
# FAILED well above the slack. Should fast-bail on stall.
ping_far_stall() {
set_pt; local t=$PT
if (( t < 3 )); then
PASSED=8; FAILED=12
else
# Stuck for good, many pairs still pending (> slack).
PASSED=10; FAILED=10
fi
}
# Case 3 trace: never converges; always one pair pending and never makes
# further progress after the first reading. Should hit the hard cap.
# FAILED stays at exactly 1 here so it is within the default slack=2 and
# the near-converged hold keeps polling all the way to max_secs.
ping_never_converges() {
PASSED=19; FAILED=1
}
# Case 4 trace: backward-compat. Same shape as case 1 (near-converged
# straggler) but driven with only 4 args so the default slack applies.
ping_backcompat_hold() {
set_pt; local t=$PT
if (( t < 3 )); then
PASSED=16; FAILED=4
elif (( t < 6 )); then
PASSED=18; FAILED=2
elif (( t < 12 )); then
PASSED=19; FAILED=1
else
PASSED=20; FAILED=0
fi
}
HOLD_MSG="holding for full budget"
STUCK_MSG="STUCK"
# --- Case 1: near-converged hold --------------------------------------
echo
echo "== Case 1: near-converged hold (slack saves it) =="
reset_ping
out=$(wait_until_connected ping_near_converged_hold 20 4 1 2); rc=$?
echo "$out"
c1_rc_ok=1; [ "$rc" -eq 0 ] && c1_rc_ok=0
check "case1: returns 0 (eventually converges)" "$c1_rc_ok" "rc=$rc"
c1_hold_ok=1; echo "$out" | grep -q "$HOLD_MSG" && c1_hold_ok=0
check "case1: near-converged hold branch taken" "$c1_hold_ok" "expected '$HOLD_MSG' in output"
# Same trace with slack=0 must fail (old behavior) — proves slack matters.
echo "-- Case 1b: same trace, slack=0 (old behavior) must bail --"
reset_ping
out0=$(wait_until_connected ping_near_converged_hold 20 4 1 0); rc0=$?
echo "$out0"
c1b_rc_ok=1; [ "$rc0" -ne 0 ] && c1b_rc_ok=0
check "case1b: returns 1 with slack=0" "$c1b_rc_ok" "rc=$rc0"
c1b_stuck_ok=1; echo "$out0" | grep -q "$STUCK_MSG" && c1b_stuck_ok=0
check "case1b: bailed via STUCK (not hold)" "$c1b_stuck_ok" "expected '$STUCK_MSG' in output"
# --- Case 2: far-from-converged stall (fast-bail) ---------------------
echo
echo "== Case 2: far-from-converged stall (fast-bail) =="
reset_ping
start=$SECONDS
out=$(wait_until_connected ping_far_stall 30 4 1 2); rc=$?
elapsed=$((SECONDS - start))
echo "$out"
c2_rc_ok=1; [ "$rc" -ne 0 ] && c2_rc_ok=0
check "case2: returns 1 (stall bail)" "$c2_rc_ok" "rc=$rc"
c2_stuck_ok=1; echo "$out" | grep -q "$STUCK_MSG" && c2_stuck_ok=0
check "case2: bailed via STUCK message" "$c2_stuck_ok"
# Fast-bail: must finish well before the 30s hard cap.
c2_fast_ok=1; [ "$elapsed" -lt 20 ] && c2_fast_ok=0
check "case2: fast-bail (before hard cap)" "$c2_fast_ok" "elapsed=${elapsed}s < 20s"
c2_nocap_ok=1; echo "$out" | grep -q "TIMEOUT" || c2_nocap_ok=0
check "case2: did NOT hit hard-cap TIMEOUT" "$c2_nocap_ok"
# --- Case 3: never converges (hard cap) -------------------------------
echo
echo "== Case 3: never converges (hard cap) =="
reset_ping
start=$SECONDS
out=$(wait_until_connected ping_never_converges 6 3 1 2); rc=$?
elapsed=$((SECONDS - start))
echo "$out"
c3_rc_ok=1; [ "$rc" -ne 0 ] && c3_rc_ok=0
check "case3: returns 1 (never converges)" "$c3_rc_ok" "rc=$rc"
c3_cap_ok=1; echo "$out" | grep -q "TIMEOUT" && c3_cap_ok=0
check "case3: hit hard-cap TIMEOUT message" "$c3_cap_ok"
# Hard cap: must run roughly to max_secs (6s), not bail early.
c3_dur_ok=1; [ "$elapsed" -ge 6 ] && c3_dur_ok=0
check "case3: ran to hard cap (~max_secs)" "$c3_dur_ok" "elapsed=${elapsed}s >= 6s"
# --- Case 4: backward-compat (4 args, default slack=2) ----------------
echo
echo "== Case 4: backward-compat, no slack arg (default=2) =="
reset_ping
out=$(wait_until_connected ping_backcompat_hold 20 4 1); rc=$?
echo "$out"
c4_rc_ok=1; [ "$rc" -eq 0 ] && c4_rc_ok=0
check "case4: returns 0 with 4 args" "$c4_rc_ok" "rc=$rc"
c4_hold_ok=1; echo "$out" | grep -q "$HOLD_MSG" && c4_hold_ok=0
check "case4: default slack triggered near-converged hold" "$c4_hold_ok"
# --- Summary ----------------------------------------------------------
echo
echo "=============================================="
for r in "${RESULTS[@]}"; do
echo " $r"
done
echo "=============================================="
if [ "$FAILURES" -ne 0 ]; then
echo "RESULT: $FAILURES assertion(s) FAILED"
exit 1
fi
echo "RESULT: all assertions passed"
exit 0
testing/lib/wait-converge.sh
+24 -5
View File
@@ -8,7 +8,8 @@
# source "$(dirname "$0")/../../lib/wait-converge.sh"
# wait_for_links <container> <min_links> [timeout_secs]
# wait_for_peers <container> <min_peers> [timeout_secs]
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs]
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs] \
# [near_converged_slack]
# Wait until a container has at least min_links active links.
# Returns 0 on success, 1 on timeout.
@@ -55,7 +56,8 @@ wait_for_peers() {
# Wait until a connectivity check reports every pair reachable, using a
# progress-aware deadline instead of a fixed one.
#
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs]
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs] \
# [near_converged_slack]
#
# <ping_fn> is the name of a function that runs the suite's own
# connectivity check and sets two globals each call:
@@ -69,7 +71,16 @@ wait_for_peers() {
# clock and keep waiting (slow-but-improving is not a failure, so it
# does not false-time-out under CI load).
# - stuck: PASSED has not improved for stall_secs -> return 1 (fail
# fast rather than burn the whole budget on a genuinely wedged pair).
# fast rather than burn the whole budget on a genuinely wedged pair),
# BUT only when FAILED > near_converged_slack (default 2). A mesh
# that is genuinely far from convergence still bails fast on stall.
# - near-converged hold: when FAILED <= near_converged_slack and the
# stall window has elapsed, do NOT bail. A handful of straggling
# pairs (e.g. a deep node whose last pair clears only after stacked
# discovery backoff + late bloom propagation) is a rare timing event,
# not a routing defect, so the gate keeps polling toward max_secs
# rather than emitting a false RED with budget still unspent. A
# genuinely never-converging single pair still hits the hard cap.
# - hard cap: max_secs elapsed -> return 1 (never runs unbounded).
#
# Returns 0 once fully connected, 1 on stall or timeout.
@@ -78,10 +89,12 @@ wait_until_connected() {
local max_secs="$2"
local stall_secs="$3"
local poll_secs="${4:-1}"
local near_converged_slack="${5:-2}"
local start_secs=$SECONDS
local best=-1
local last_progress=$SECONDS
local held_for_budget=0
while (( SECONDS - start_secs < max_secs )); do
"$ping_fn"
@@ -94,8 +107,14 @@ wait_until_connected() {
last_progress=$SECONDS
echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s"
elif (( SECONDS - last_progress >= stall_secs )); then
echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)"
return 1
if (( FAILED > near_converged_slack )); then
echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)"
return 1
fi
if (( held_for_budget == 0 )); then
held_for_budget=1
echo " converge: near-converged ($PASSED reachable / $FAILED pending <= slack=$near_converged_slack) — holding for full budget, not bailing (after $((SECONDS - start_secs))s)"
fi
fi
sleep "$poll_secs"
done