From 1b7528ce89be22c118ccf807f82afc5201f8e79b Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 7 Jun 2026 20:31:53 +0000 Subject: [PATCH] testing: hold convergence gate for full budget when near-converged wait_until_connected fail-fasted on stall even when the mesh was all but converged, abandoning the run with budget still unspent because one hard pair straggled to come up. On the rekey-outbound-only topology this turned a rare deep-node timing straggle (stacked discovery backoff + late bloom propagation, which clears well inside the budget) into a false baseline RED. Add a near_converged_slack threshold (default 2): when the number of still-failing pairs is at or below the slack and the stall window elapses, keep polling toward max_secs instead of bailing. A mesh genuinely far from convergence still fast-bails on stall, and a never-converging pair still hits the hard cap, so a real regression is never masked. Add a self-contained behavioral test (wait-converge-test.sh) covering the four contract cases: near-converged hold (with a slack=0 contrast that proves the slack is what saves the run), far-from-converged fast-bail, never-converging hard cap, and four-argument backward compatibility. --- testing/lib/wait-converge-test.sh | 191 ++++++++++++++++++++++++++++++ testing/lib/wait-converge.sh | 29 ++++- 2 files changed, 215 insertions(+), 5 deletions(-) create mode 100755 testing/lib/wait-converge-test.sh diff --git a/testing/lib/wait-converge-test.sh b/testing/lib/wait-converge-test.sh new file mode 100755 index 0000000..3abd2de --- /dev/null +++ b/testing/lib/wait-converge-test.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# Unit tests for wait_until_connected() in wait-converge.sh. +# +# These tests drive the convergence gate with synthetic connectivity +# checks (ping_fn stand-ins) that report scripted PASSED/FAILED counts +# keyed off the same SECONDS clock the gate uses. No containers or +# network are involved, so the whole suite runs in a few seconds and is +# safe to run in CI. +# +# Run: +# ./wait-converge-test.sh +# Exits 0 only if every case passes; non-zero if any case fails. + +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/wait-converge.sh" + +RESULTS=() +FAILURES=0 + +# Record a single assertion result. +check() { + local name="$1" + local ok="$2" # 0 = pass, anything else = fail + local detail="${3:-}" + if [ "$ok" -eq 0 ]; then + RESULTS+=("PASS $name") + echo "PASS $name${detail:+ ($detail)}" + else + RESULTS+=("FAIL $name") + echo "FAIL $name${detail:+ ($detail)}" + FAILURES=$((FAILURES + 1)) + fi +} + +# Each ping_fn records its own start on first call so its schedule is +# measured from when the gate began polling it, regardless of the wall +# clock at suite start. PT (ping-elapsed time) is computed inline in each +# ping_fn — NOT via a subshell — so the PING_START assignment persists. +PING_START=-1 +reset_ping() { + PING_START=-1 +} + +# --- Synthetic connectivity checks ------------------------------------ + +# Sets global PT to ping-elapsed seconds. Must be called (not subshelled) +# at the top of each ping_fn so the first-call timestamp persists. +PT=0 +set_pt() { + if (( PING_START < 0 )); then + PING_START=$SECONDS + fi + PT=$(( SECONDS - PING_START )) +} + +# Case 1 trace: improves (16 reachable) early, climbs to 18, then holds +# at FAILED=1 (within slack) until late, then fully converges. Mirrors a +# deep node whose last pair clears only after stacked discovery backoff. +ping_near_converged_hold() { + set_pt; local t=$PT + if (( t < 3 )); then + PASSED=16; FAILED=4 + elif (( t < 6 )); then + PASSED=18; FAILED=2 + elif (( t < 12 )); then + # Stuck within slack: one straggling pair pending. + PASSED=19; FAILED=1 + else + PASSED=20; FAILED=0 + fi +} + +# Case 2 trace: climbs a little, then wedges far from convergence with +# FAILED well above the slack. Should fast-bail on stall. +ping_far_stall() { + set_pt; local t=$PT + if (( t < 3 )); then + PASSED=8; FAILED=12 + else + # Stuck for good, many pairs still pending (> slack). + PASSED=10; FAILED=10 + fi +} + +# Case 3 trace: never converges; always one pair pending and never makes +# further progress after the first reading. Should hit the hard cap. +# FAILED stays at exactly 1 here so it is within the default slack=2 and +# the near-converged hold keeps polling all the way to max_secs. +ping_never_converges() { + PASSED=19; FAILED=1 +} + +# Case 4 trace: backward-compat. Same shape as case 1 (near-converged +# straggler) but driven with only 4 args so the default slack applies. +ping_backcompat_hold() { + set_pt; local t=$PT + if (( t < 3 )); then + PASSED=16; FAILED=4 + elif (( t < 6 )); then + PASSED=18; FAILED=2 + elif (( t < 12 )); then + PASSED=19; FAILED=1 + else + PASSED=20; FAILED=0 + fi +} + +HOLD_MSG="holding for full budget" +STUCK_MSG="STUCK" + +# --- Case 1: near-converged hold -------------------------------------- +echo +echo "== Case 1: near-converged hold (slack saves it) ==" +reset_ping +out=$(wait_until_connected ping_near_converged_hold 20 4 1 2); rc=$? +echo "$out" +c1_rc_ok=1; [ "$rc" -eq 0 ] && c1_rc_ok=0 +check "case1: returns 0 (eventually converges)" "$c1_rc_ok" "rc=$rc" +c1_hold_ok=1; echo "$out" | grep -q "$HOLD_MSG" && c1_hold_ok=0 +check "case1: near-converged hold branch taken" "$c1_hold_ok" "expected '$HOLD_MSG' in output" + +# Same trace with slack=0 must fail (old behavior) — proves slack matters. +echo "-- Case 1b: same trace, slack=0 (old behavior) must bail --" +reset_ping +out0=$(wait_until_connected ping_near_converged_hold 20 4 1 0); rc0=$? +echo "$out0" +c1b_rc_ok=1; [ "$rc0" -ne 0 ] && c1b_rc_ok=0 +check "case1b: returns 1 with slack=0" "$c1b_rc_ok" "rc=$rc0" +c1b_stuck_ok=1; echo "$out0" | grep -q "$STUCK_MSG" && c1b_stuck_ok=0 +check "case1b: bailed via STUCK (not hold)" "$c1b_stuck_ok" "expected '$STUCK_MSG' in output" + +# --- Case 2: far-from-converged stall (fast-bail) --------------------- +echo +echo "== Case 2: far-from-converged stall (fast-bail) ==" +reset_ping +start=$SECONDS +out=$(wait_until_connected ping_far_stall 30 4 1 2); rc=$? +elapsed=$((SECONDS - start)) +echo "$out" +c2_rc_ok=1; [ "$rc" -ne 0 ] && c2_rc_ok=0 +check "case2: returns 1 (stall bail)" "$c2_rc_ok" "rc=$rc" +c2_stuck_ok=1; echo "$out" | grep -q "$STUCK_MSG" && c2_stuck_ok=0 +check "case2: bailed via STUCK message" "$c2_stuck_ok" +# Fast-bail: must finish well before the 30s hard cap. +c2_fast_ok=1; [ "$elapsed" -lt 20 ] && c2_fast_ok=0 +check "case2: fast-bail (before hard cap)" "$c2_fast_ok" "elapsed=${elapsed}s < 20s" +c2_nocap_ok=1; echo "$out" | grep -q "TIMEOUT" || c2_nocap_ok=0 +check "case2: did NOT hit hard-cap TIMEOUT" "$c2_nocap_ok" + +# --- Case 3: never converges (hard cap) ------------------------------- +echo +echo "== Case 3: never converges (hard cap) ==" +reset_ping +start=$SECONDS +out=$(wait_until_connected ping_never_converges 6 3 1 2); rc=$? +elapsed=$((SECONDS - start)) +echo "$out" +c3_rc_ok=1; [ "$rc" -ne 0 ] && c3_rc_ok=0 +check "case3: returns 1 (never converges)" "$c3_rc_ok" "rc=$rc" +c3_cap_ok=1; echo "$out" | grep -q "TIMEOUT" && c3_cap_ok=0 +check "case3: hit hard-cap TIMEOUT message" "$c3_cap_ok" +# Hard cap: must run roughly to max_secs (6s), not bail early. +c3_dur_ok=1; [ "$elapsed" -ge 6 ] && c3_dur_ok=0 +check "case3: ran to hard cap (~max_secs)" "$c3_dur_ok" "elapsed=${elapsed}s >= 6s" + +# --- Case 4: backward-compat (4 args, default slack=2) ---------------- +echo +echo "== Case 4: backward-compat, no slack arg (default=2) ==" +reset_ping +out=$(wait_until_connected ping_backcompat_hold 20 4 1); rc=$? +echo "$out" +c4_rc_ok=1; [ "$rc" -eq 0 ] && c4_rc_ok=0 +check "case4: returns 0 with 4 args" "$c4_rc_ok" "rc=$rc" +c4_hold_ok=1; echo "$out" | grep -q "$HOLD_MSG" && c4_hold_ok=0 +check "case4: default slack triggered near-converged hold" "$c4_hold_ok" + +# --- Summary ---------------------------------------------------------- +echo +echo "==============================================" +for r in "${RESULTS[@]}"; do + echo " $r" +done +echo "==============================================" +if [ "$FAILURES" -ne 0 ]; then + echo "RESULT: $FAILURES assertion(s) FAILED" + exit 1 +fi +echo "RESULT: all assertions passed" +exit 0 diff --git a/testing/lib/wait-converge.sh b/testing/lib/wait-converge.sh index c203eca..8832b67 100644 --- a/testing/lib/wait-converge.sh +++ b/testing/lib/wait-converge.sh @@ -8,7 +8,8 @@ # source "$(dirname "$0")/../../lib/wait-converge.sh" # wait_for_links [timeout_secs] # wait_for_peers [timeout_secs] -# wait_until_connected [poll_secs] +# wait_until_connected [poll_secs] \ +# [near_converged_slack] # Wait until a container has at least min_links active links. # Returns 0 on success, 1 on timeout. @@ -55,7 +56,8 @@ wait_for_peers() { # Wait until a connectivity check reports every pair reachable, using a # progress-aware deadline instead of a fixed one. # -# wait_until_connected [poll_secs] +# wait_until_connected [poll_secs] \ +# [near_converged_slack] # # is the name of a function that runs the suite's own # connectivity check and sets two globals each call: @@ -69,7 +71,16 @@ wait_for_peers() { # clock and keep waiting (slow-but-improving is not a failure, so it # does not false-time-out under CI load). # - stuck: PASSED has not improved for stall_secs -> return 1 (fail -# fast rather than burn the whole budget on a genuinely wedged pair). +# fast rather than burn the whole budget on a genuinely wedged pair), +# BUT only when FAILED > near_converged_slack (default 2). A mesh +# that is genuinely far from convergence still bails fast on stall. +# - near-converged hold: when FAILED <= near_converged_slack and the +# stall window has elapsed, do NOT bail. A handful of straggling +# pairs (e.g. a deep node whose last pair clears only after stacked +# discovery backoff + late bloom propagation) is a rare timing event, +# not a routing defect, so the gate keeps polling toward max_secs +# rather than emitting a false RED with budget still unspent. A +# genuinely never-converging single pair still hits the hard cap. # - hard cap: max_secs elapsed -> return 1 (never runs unbounded). # # Returns 0 once fully connected, 1 on stall or timeout. @@ -78,10 +89,12 @@ wait_until_connected() { local max_secs="$2" local stall_secs="$3" local poll_secs="${4:-1}" + local near_converged_slack="${5:-2}" local start_secs=$SECONDS local best=-1 local last_progress=$SECONDS + local held_for_budget=0 while (( SECONDS - start_secs < max_secs )); do "$ping_fn" @@ -94,8 +107,14 @@ wait_until_connected() { last_progress=$SECONDS echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s" elif (( SECONDS - last_progress >= stall_secs )); then - echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)" - return 1 + if (( FAILED > near_converged_slack )); then + echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)" + return 1 + fi + if (( held_for_budget == 0 )); then + held_for_budget=1 + echo " converge: near-converged ($PASSED reachable / $FAILED pending <= slack=$near_converged_slack) — holding for full budget, not bailing (after $((SECONDS - start_secs))s)" + fi fi sleep "$poll_secs" done