diff --git a/testing/lib/wait-converge-test.sh b/testing/lib/wait-converge-test.sh index 48763531..a99974b1 100755 --- a/testing/lib/wait-converge-test.sh +++ b/testing/lib/wait-converge-test.sh @@ -7,7 +7,8 @@ # network are involved, so the suite is hermetic and safe to run in CI. # # It is not fast, though: it drives real timeouts against the real clock -# and takes about 45 seconds, measured 2026-07-23. The header claimed "a +# and took about 45 seconds when measured 2026-07-23, before case 7 added +# roughly another minute. The header claimed "a # few seconds" from the day it was written until then, which nothing had # contradicted because no runner had ever invoked it. # @@ -122,6 +123,59 @@ ping_quick_converge() { fi } +# Case 7 traces: the near-converged acceptance window. Each is keyed off +# PT like the traces above, and each is shaped so its expected verdict holds +# with at least a second of margin on either side of the accept window. + +# 7d: still progressing until t=4, so the hold that follows (armed at t=6 +# with stall_secs=2) has lasted only ~2s when the cap falls at 8s, short of +# a 4s accept window. +ping_late_progress() { + set_pt; local t=$PT + if (( t < 4 )); then + PASSED=$(( 15 + t )); FAILED=$(( 5 - t )) + else + PASSED=19; FAILED=1 + fi +} + +# 7f: holds at 19/1 from the start and converges at t=6, before a 10s cap. +# A gate that accepted as soon as the accept window elapsed (t~4) would +# report near_converged here instead of converged. +ping_converges_after_window() { + set_pt; local t=$PT + if (( t < 6 )); then + PASSED=19; FAILED=1 + else + PASSED=20; FAILED=0 + fi +} + +# 7g: enters the hold at 18/2 (armed at t=2), makes progress to 19/1 at +# t=4, then holds there long enough (re-armed at t=6, cap 11) to exceed a +# 3s accept window. Reds under a gate that arms the hold once and never +# re-arms it after progress. +ping_rearm_hold() { + set_pt; local t=$PT + if (( t < 4 )); then + PASSED=18; FAILED=2 + else + PASSED=19; FAILED=1 + fi +} + +# 7h: the same shape, but the progress to 19/1 comes at t=8, so the second +# hold (re-armed at t=10) is ~1s old when the cap falls at 11s. Reds under a +# gate that keeps the first hold's start across the progress. +ping_late_rearm() { + set_pt; local t=$PT + if (( t < 8 )); then + PASSED=18; FAILED=2 + else + PASSED=19; FAILED=1 + fi +} + HOLD_MSG="holding for full budget" STUCK_MSG="STUCK" NOCONV_MSG="tree did not converge" @@ -292,6 +346,111 @@ check "case6c: verdict carries a clean tree" "$c6c_cnt_ok" \ c6c_quiet_ok=0; grep -q "$NOCONV_MSG" "$VERDICT_OUT" && c6c_quiet_ok=1 check "case6c: no non-convergence message on a clean run" "$c6c_quiet_ok" +# --- Case 7: near-converged acceptance at the hard cap ----------------- +# +# The sixth argument lets a caller hand a mesh that has held within slack for +# at least that many seconds to its own strict assertion instead of failing +# the gate. It acts only at the hard cap, which is reachable only in states +# that are red without it, so it can never turn a run that would have +# converged by the cap into a red. With the argument absent or 0 the gate +# must behave exactly as before. +echo +echo "== Case 7: near-converged acceptance at the hard cap ==" + +echo "-- Case 7a: held within slack past the accept window, accepted at the cap --" +run_gate ping_never_converges 6 3 1 2 2; rc=$? +cat "$VERDICT_OUT" +c7a_rc_ok=1; [ "$rc" -eq 0 ] && c7a_rc_ok=0 +check "case7a: near-converged hold past the window returns 0" "$c7a_rc_ok" "rc=$rc" +c7a_out_ok=1; [ "$CONVERGE_OUTCOME" = "near_converged" ] && c7a_out_ok=0 +check "case7a: verdict is near_converged" "$c7a_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" +c7a_cnt_ok=1 +[ "$CONVERGE_REACHED" -eq 19 ] && [ "$CONVERGE_PENDING" -eq 1 ] && c7a_cnt_ok=0 +check "case7a: verdict carries the shortfall" "$c7a_cnt_ok" \ + "reached=$CONVERGE_REACHED pending=$CONVERGE_PENDING" + +echo "-- Case 7b: the same trace with five arguments, then with an explicit 0 --" +run_gate ping_never_converges 6 3 1 2; rc=$? +cat "$VERDICT_OUT" +c7b_rc_ok=1; [ "$rc" -eq 1 ] && c7b_rc_ok=0 +check "case7b: five arguments still time out" "$c7b_rc_ok" "rc=$rc" +c7b_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7b_out_ok=0 +check "case7b: five arguments give verdict timeout" "$c7b_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" +run_gate ping_never_converges 6 3 1 2 0; rc=$? +cat "$VERDICT_OUT" +c7b0_rc_ok=1; [ "$rc" -eq 1 ] && c7b0_rc_ok=0 +check "case7b: an explicit 0 still times out" "$c7b0_rc_ok" "rc=$rc" +c7b0_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7b0_out_ok=0 +check "case7b: an explicit 0 gives verdict timeout" "$c7b0_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" + +echo "-- Case 7c: wedged far from convergence, acceptance does not rescue it --" +run_gate ping_far_stall 30 4 1 2 2; rc=$? +cat "$VERDICT_OUT" +c7c_rc_ok=1; [ "$rc" -eq 1 ] && c7c_rc_ok=0 +check "case7c: a stall beyond slack still reds with acceptance enabled" "$c7c_rc_ok" "rc=$rc" +c7c_out_ok=1; [ "$CONVERGE_OUTCOME" = "stalled" ] && c7c_out_ok=0 +check "case7c: verdict is stalled" "$c7c_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" + +echo "-- Case 7d: hold shorter than the accept window at the cap --" +run_gate ping_late_progress 8 2 1 2 4; rc=$? +cat "$VERDICT_OUT" +c7d_rc_ok=1; [ "$rc" -eq 1 ] && c7d_rc_ok=0 +check "case7d: a hold shorter than the window still times out" "$c7d_rc_ok" "rc=$rc" +c7d_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7d_out_ok=0 +check "case7d: verdict is timeout" "$c7d_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" + +echo "-- Case 7e: converged tree with acceptance enabled --" +run_gate ping_quick_converge 20 4 1 2 2; rc=$? +cat "$VERDICT_OUT" +c7e_rc_ok=1; [ "$rc" -eq 0 ] && c7e_rc_ok=0 +check "case7e: converged tree passes with acceptance enabled" "$c7e_rc_ok" "rc=$rc" +c7e_out_ok=1; [ "$CONVERGE_OUTCOME" = "converged" ] && c7e_out_ok=0 +check "case7e: verdict is converged" "$c7e_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" + +echo "-- Case 7f: converges after the window would have elapsed, before the cap --" +run_gate ping_converges_after_window 10 2 1 2 2; rc=$? +cat "$VERDICT_OUT" +c7f_rc_ok=1; [ "$rc" -eq 0 ] && c7f_rc_ok=0 +check "case7f: late convergence before the cap passes" "$c7f_rc_ok" "rc=$rc" +c7f_out_ok=1; [ "$CONVERGE_OUTCOME" = "converged" ] && c7f_out_ok=0 +check "case7f: acceptance waits for the cap, verdict is converged" "$c7f_out_ok" \ + "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" + +echo "-- Case 7g: hold, progress, then a second hold past the window --" +run_gate ping_rearm_hold 11 2 1 2 3; rc=$? +cat "$VERDICT_OUT" +c7g_rc_ok=1; [ "$rc" -eq 0 ] && c7g_rc_ok=0 +check "case7g: a re-armed hold past the window returns 0" "$c7g_rc_ok" "rc=$rc" +c7g_out_ok=1; [ "$CONVERGE_OUTCOME" = "near_converged" ] && c7g_out_ok=0 +check "case7g: verdict is near_converged" "$c7g_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" +c7g_cnt_ok=1 +[ "$CONVERGE_REACHED" -eq 19 ] && [ "$CONVERGE_PENDING" -eq 1 ] && c7g_cnt_ok=0 +check "case7g: verdict carries the shortfall" "$c7g_cnt_ok" \ + "reached=$CONVERGE_REACHED pending=$CONVERGE_PENDING" + +echo "-- Case 7h: hold, late progress, second hold shorter than the window --" +run_gate ping_late_rearm 11 2 1 2 3; rc=$? +cat "$VERDICT_OUT" +c7h_rc_ok=1; [ "$rc" -eq 1 ] && c7h_rc_ok=0 +check "case7h: progress resets the hold, so a short second hold times out" "$c7h_rc_ok" "rc=$rc" +c7h_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7h_out_ok=0 +check "case7h: verdict is timeout" "$c7h_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" + +echo "-- Case 7i: hold armed, but shorter than the window when the cap falls --" +# The hold arms at t~3 (stall_secs after the first reading) and the cap falls +# at 6, so it is ~3s old against a 10s window: seven seconds of margin, far +# above SECONDS' one-second granularity. This is the case that carries the +# duration condition; 7d and 7h pass even with it removed if their hold +# happens not to arm. +run_gate ping_never_converges 6 3 1 2 10; rc=$? +cat "$VERDICT_OUT" +c7i_rc_ok=1; [ "$rc" -eq 1 ] && c7i_rc_ok=0 +check "case7i: an armed hold shorter than the window still times out" "$c7i_rc_ok" "rc=$rc" +c7i_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7i_out_ok=0 +check "case7i: verdict is timeout" "$c7i_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" +c7i_hold_ok=1; grep -q "$HOLD_MSG" "$VERDICT_OUT" && c7i_hold_ok=0 +check "case7i: the hold was armed" "$c7i_hold_ok" "expected '$HOLD_MSG' in output" + rm -f "$VERDICT_OUT" # --- Summary ---------------------------------------------------------- diff --git a/testing/lib/wait-converge.sh b/testing/lib/wait-converge.sh index a9588b15..dd7f42b7 100644 --- a/testing/lib/wait-converge.sh +++ b/testing/lib/wait-converge.sh @@ -7,7 +7,7 @@ # source "$(dirname "$0")/../../lib/wait-converge.sh" # wait_for_peers [timeout_secs] # wait_until_connected [poll_secs] \ -# [near_converged_slack] +# [near_converged_slack] [near_converged_accept_secs] # # wait_until_connected also sets CONVERGE_OUTCOME / CONVERGE_REACHED / # CONVERGE_PENDING; see the block above it. @@ -61,7 +61,7 @@ wait_for_peers() { # Verdict of the most recent wait_until_connected() call, so a caller can # report WHICH condition failed rather than only that one did: -# CONVERGE_OUTCOME converged | stalled | timeout +# CONVERGE_OUTCOME converged | near_converged | stalled | timeout # CONVERGE_REACHED reachable pairs at the moment of the verdict # CONVERGE_PENDING unreachable pairs at that moment # @@ -70,6 +70,10 @@ wait_for_peers() { # the strict all-pairs assertion 20/20. Without them the caller's summary # line reads "20 passed, 0 failed" on a non-convergence exit, which a # reader cannot tell from a connectivity failure. +# +# near_converged is a success return (0): the hard cap fell while the mesh +# had held within slack for at least near_converged_accept_secs, and the +# caller asked for that state to be handed to its own strict assertion. CONVERGE_OUTCOME="" CONVERGE_REACHED=0 CONVERGE_PENDING=0 @@ -87,7 +91,7 @@ _converge_verdict() { # progress-aware deadline instead of a fixed one. # # wait_until_connected [poll_secs] \ -# [near_converged_slack] +# [near_converged_slack] [near_converged_accept_secs] # # is the name of a function that runs the suite's own # connectivity check and sets two globals each call: @@ -112,19 +116,34 @@ _converge_verdict() { # rather than emitting a false RED with budget still unspent. A # genuinely never-converging single pair still hits the hard cap. # - hard cap: max_secs elapsed -> return 1 (never runs unbounded). +# - near-converged acceptance, off unless near_converged_accept_secs is +# non-zero: at the hard cap, if the current near-converged hold has +# lasted at least that long and the last poll was still within slack, +# return 0 with verdict near_converged instead. It acts only at the cap, +# a state that is red without it, so it can never turn a run that would +# have converged by the cap into a red one. Acceptance means "hand the +# mesh to the caller's strict assertion", never "skip that assertion". +# Progress disarms the hold, so only the hold that ends at the cap counts. # -# Returns 0 once fully connected, 1 on stall or timeout. +# Returns 0 once fully connected or accepted as near-converged, 1 on stall +# or timeout. wait_until_connected() { local ping_fn="$1" local max_secs="$2" local stall_secs="$3" local poll_secs="${4:-1}" local near_converged_slack="${5:-2}" + local near_converged_accept_secs="${6:-0}" local start_secs=$SECONDS local best=-1 local last_progress=$SECONDS local held_for_budget=0 + # Start of the current near-converged hold, or -1 when disarmed. Kept + # apart from held_for_budget, which is set once and never reset and so + # cannot measure the hold that ends at the cap. -1 rather than 0 because + # SECONDS can legitimately be 0. + local hold_start=-1 while (( SECONDS - start_secs < max_secs )); do "$ping_fn" @@ -136,6 +155,7 @@ wait_until_connected() { if (( PASSED > best )); then best=$PASSED last_progress=$SECONDS + hold_start=-1 echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s" elif (( SECONDS - last_progress >= stall_secs )); then if (( FAILED > near_converged_slack )); then @@ -143,6 +163,9 @@ wait_until_connected() { echo " converge: STUCK — tree did not converge: $PASSED reachable / $FAILED pending, no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)" return 1 fi + if (( hold_start < 0 )); then + hold_start=$SECONDS + fi if (( held_for_budget == 0 )); then held_for_budget=1 echo " converge: near-converged ($PASSED reachable / $FAILED pending <= slack=$near_converged_slack) — holding for full budget, not bailing (after $((SECONDS - start_secs))s)" @@ -151,6 +174,17 @@ wait_until_connected() { sleep "$poll_secs" done + # The slack test repeats what the loop already guarantees (once the hold + # is armed, any poll beyond slack exits as stalled), so that acceptance + # stays tied to slack if the loop's exits are ever reordered. + if (( near_converged_accept_secs > 0 && hold_start >= 0 \ + && SECONDS - hold_start >= near_converged_accept_secs \ + && FAILED <= near_converged_slack )); then + _converge_verdict near_converged + echo " converge: near-converged at the cap — $PASSED reachable / $FAILED pending, held $((SECONDS - hold_start))s >= ${near_converged_accept_secs}s; handing to the strict assertion (after ${max_secs}s)" + return 0 + fi + _converge_verdict timeout echo " converge: TIMEOUT — tree did not converge: $PASSED reachable / $FAILED pending after ${max_secs}s" return 1 diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index 1e774c40..2ca46a75 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -154,6 +154,12 @@ trap 'echo ""; echo "Test interrupted"; exit 130' INT # wait early-exits on PASS, so successful reps are unaffected by the # extra headroom. BASELINE_CONVERGENCE_TIMEOUT=65 +# A mesh that has held within the gate's slack (two pairs) for this long when +# BASELINE_CONVERGENCE_TIMEOUT falls is handed to the strict all-pairs +# assertion instead of failing the gate. The strict assertion still decides. +# 10s accepts a straggler that held for most of the window and refuses a +# pair that only fell into the hold in the last few seconds. +BASELINE_NEAR_CONVERGED_ACCEPT=10 REKEY_SETTLE=12 # > DRAIN_WINDOW_SECS (10) so post-rekey samples are off the old session # First FMP rekey should follow shortly after the 35s interval once the mesh is # fully converged. Keep this bounded to preserve a meaningful scheduling check @@ -421,7 +427,13 @@ echo "" # it no longer false-times-out under concurrent CI load. The strict # ping_all below is the actual assertion, run only after convergence. echo "Phase 1: Pre-rekey connectivity (waiting for convergence)" -if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20; then +if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20 1 2 \ + "$BASELINE_NEAR_CONVERGED_ACCEPT"; then + if [ "$CONVERGE_OUTCOME" = "near_converged" ]; then + echo " Gate accepted a near-converged mesh" \ + "($CONVERGE_REACHED/$((CONVERGE_REACHED + CONVERGE_PENDING)) reachable);" \ + "the strict assertion below decides" + fi ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS" phase_result "Pre-rekey baseline (all 20 pairs)" if [ "$FAILED" -ne 0 ]; then