laantungir/fips
1
0
Fork 0
mirror of https://github.com/jmcorgan/fips.git synced 2026-10-05 11:08:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: let the rekey convergence gate hand a near-converged mesh to the strict assertion

Browse Source 
The rekey suite's Phase-1 gate fails the whole run when one pair is still
unreachable at the 65s cap, even though the strict all-pairs assertion that
follows is the real test. On slower runners a mesh that sat at 19/20 for
most of the window was reported as a convergence failure rather than
reaching the assertion that would say what was wrong.

wait_until_connected gains an optional sixth argument,
near_converged_accept_secs, default 0. It acts only at the hard cap: if the
current near-converged hold (within slack, no progress for stall_secs) has
lasted at least that long, the gate returns 0 with the new verdict
near_converged instead of timing out. Progress disarms the hold, so only the
hold that ends at the cap counts. Acceptance is reachable only in states that
were red before, so it cannot turn a run that would have converged by the cap
into a failure, and the strict assertion still decides.

Only the rekey Phase-1 gate passes the argument (10s). Every other caller
passes five or fewer arguments and is unchanged. The hermetic gate test gains
case 7 covering acceptance, the default of 0, stalls beyond slack, holds
shorter than the window, late convergence before the cap, and the hold's
re-arm and reset on progress.
This commit is contained in:
Johnathan Corgan
2026-09-19 03:25:45 +00:00
parent f3e3a4ed9e
commit e8f8dbf2de
3 changed files with 211 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
testing/lib/wait-converge-test.sh
+160 -1
View File
@@ -7,7 +7,8 @@
# network are involved, so the suite is hermetic and safe to run in CI.
#
# It is not fast, though: it drives real timeouts against the real clock
# and takes about 45 seconds, measured 2026-07-23. The header claimed "a
# and took about 45 seconds when measured 2026-07-23, before case 7 added
# roughly another minute. The header claimed "a
# few seconds" from the day it was written until then, which nothing had
# contradicted because no runner had ever invoked it.
#
@@ -122,6 +123,59 @@ ping_quick_converge() {
fi
}
# Case 7 traces: the near-converged acceptance window. Each is keyed off
# PT like the traces above, and each is shaped so its expected verdict holds
# with at least a second of margin on either side of the accept window.
# 7d: still progressing until t=4, so the hold that follows (armed at t=6
# with stall_secs=2) has lasted only ~2s when the cap falls at 8s, short of
# a 4s accept window.
ping_late_progress() {
set_pt; local t=$PT
if (( t < 4 )); then
PASSED=$(( 15 + t )); FAILED=$(( 5 - t ))
else
PASSED=19; FAILED=1
fi
}
# 7f: holds at 19/1 from the start and converges at t=6, before a 10s cap.
# A gate that accepted as soon as the accept window elapsed (t~4) would
# report near_converged here instead of converged.
ping_converges_after_window() {
set_pt; local t=$PT
if (( t < 6 )); then
PASSED=19; FAILED=1
else
PASSED=20; FAILED=0
fi
}
# 7g: enters the hold at 18/2 (armed at t=2), makes progress to 19/1 at
# t=4, then holds there long enough (re-armed at t=6, cap 11) to exceed a
# 3s accept window. Reds under a gate that arms the hold once and never
# re-arms it after progress.
ping_rearm_hold() {
set_pt; local t=$PT
if (( t < 4 )); then
PASSED=18; FAILED=2
else
PASSED=19; FAILED=1
fi
}
# 7h: the same shape, but the progress to 19/1 comes at t=8, so the second
# hold (re-armed at t=10) is ~1s old when the cap falls at 11s. Reds under a
# gate that keeps the first hold's start across the progress.
ping_late_rearm() {
set_pt; local t=$PT
if (( t < 8 )); then
PASSED=18; FAILED=2
else
PASSED=19; FAILED=1
fi
}
HOLD_MSG="holding for full budget"
STUCK_MSG="STUCK"
NOCONV_MSG="tree did not converge"
@@ -292,6 +346,111 @@ check "case6c: verdict carries a clean tree" "$c6c_cnt_ok" \
c6c_quiet_ok=0; grep -q "$NOCONV_MSG" "$VERDICT_OUT" && c6c_quiet_ok=1
check "case6c: no non-convergence message on a clean run" "$c6c_quiet_ok"
# --- Case 7: near-converged acceptance at the hard cap -----------------
#
# The sixth argument lets a caller hand a mesh that has held within slack for
# at least that many seconds to its own strict assertion instead of failing
# the gate. It acts only at the hard cap, which is reachable only in states
# that are red without it, so it can never turn a run that would have
# converged by the cap into a red. With the argument absent or 0 the gate
# must behave exactly as before.
echo
echo "== Case 7: near-converged acceptance at the hard cap =="
echo "-- Case 7a: held within slack past the accept window, accepted at the cap --"
run_gate ping_never_converges 6 3 1 2 2; rc=$?
cat "$VERDICT_OUT"
c7a_rc_ok=1; [ "$rc" -eq 0 ] && c7a_rc_ok=0
check "case7a: near-converged hold past the window returns 0" "$c7a_rc_ok" "rc=$rc"
c7a_out_ok=1; [ "$CONVERGE_OUTCOME" = "near_converged" ] && c7a_out_ok=0
check "case7a: verdict is near_converged" "$c7a_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
c7a_cnt_ok=1
[ "$CONVERGE_REACHED" -eq 19 ] && [ "$CONVERGE_PENDING" -eq 1 ] && c7a_cnt_ok=0
check "case7a: verdict carries the shortfall" "$c7a_cnt_ok" \
"reached=$CONVERGE_REACHED pending=$CONVERGE_PENDING"
echo "-- Case 7b: the same trace with five arguments, then with an explicit 0 --"
run_gate ping_never_converges 6 3 1 2; rc=$?
cat "$VERDICT_OUT"
c7b_rc_ok=1; [ "$rc" -eq 1 ] && c7b_rc_ok=0
check "case7b: five arguments still time out" "$c7b_rc_ok" "rc=$rc"
c7b_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7b_out_ok=0
check "case7b: five arguments give verdict timeout" "$c7b_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
run_gate ping_never_converges 6 3 1 2 0; rc=$?
cat "$VERDICT_OUT"
c7b0_rc_ok=1; [ "$rc" -eq 1 ] && c7b0_rc_ok=0
check "case7b: an explicit 0 still times out" "$c7b0_rc_ok" "rc=$rc"
c7b0_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7b0_out_ok=0
check "case7b: an explicit 0 gives verdict timeout" "$c7b0_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
echo "-- Case 7c: wedged far from convergence, acceptance does not rescue it --"
run_gate ping_far_stall 30 4 1 2 2; rc=$?
cat "$VERDICT_OUT"
c7c_rc_ok=1; [ "$rc" -eq 1 ] && c7c_rc_ok=0
check "case7c: a stall beyond slack still reds with acceptance enabled" "$c7c_rc_ok" "rc=$rc"
c7c_out_ok=1; [ "$CONVERGE_OUTCOME" = "stalled" ] && c7c_out_ok=0
check "case7c: verdict is stalled" "$c7c_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
echo "-- Case 7d: hold shorter than the accept window at the cap --"
run_gate ping_late_progress 8 2 1 2 4; rc=$?
cat "$VERDICT_OUT"
c7d_rc_ok=1; [ "$rc" -eq 1 ] && c7d_rc_ok=0
check "case7d: a hold shorter than the window still times out" "$c7d_rc_ok" "rc=$rc"
c7d_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7d_out_ok=0
check "case7d: verdict is timeout" "$c7d_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
echo "-- Case 7e: converged tree with acceptance enabled --"
run_gate ping_quick_converge 20 4 1 2 2; rc=$?
cat "$VERDICT_OUT"
c7e_rc_ok=1; [ "$rc" -eq 0 ] && c7e_rc_ok=0
check "case7e: converged tree passes with acceptance enabled" "$c7e_rc_ok" "rc=$rc"
c7e_out_ok=1; [ "$CONVERGE_OUTCOME" = "converged" ] && c7e_out_ok=0
check "case7e: verdict is converged" "$c7e_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
echo "-- Case 7f: converges after the window would have elapsed, before the cap --"
run_gate ping_converges_after_window 10 2 1 2 2; rc=$?
cat "$VERDICT_OUT"
c7f_rc_ok=1; [ "$rc" -eq 0 ] && c7f_rc_ok=0
check "case7f: late convergence before the cap passes" "$c7f_rc_ok" "rc=$rc"
c7f_out_ok=1; [ "$CONVERGE_OUTCOME" = "converged" ] && c7f_out_ok=0
check "case7f: acceptance waits for the cap, verdict is converged" "$c7f_out_ok" \
"CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
echo "-- Case 7g: hold, progress, then a second hold past the window --"
run_gate ping_rearm_hold 11 2 1 2 3; rc=$?
cat "$VERDICT_OUT"
c7g_rc_ok=1; [ "$rc" -eq 0 ] && c7g_rc_ok=0
check "case7g: a re-armed hold past the window returns 0" "$c7g_rc_ok" "rc=$rc"
c7g_out_ok=1; [ "$CONVERGE_OUTCOME" = "near_converged" ] && c7g_out_ok=0
check "case7g: verdict is near_converged" "$c7g_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
c7g_cnt_ok=1
[ "$CONVERGE_REACHED" -eq 19 ] && [ "$CONVERGE_PENDING" -eq 1 ] && c7g_cnt_ok=0
check "case7g: verdict carries the shortfall" "$c7g_cnt_ok" \
"reached=$CONVERGE_REACHED pending=$CONVERGE_PENDING"
echo "-- Case 7h: hold, late progress, second hold shorter than the window --"
run_gate ping_late_rearm 11 2 1 2 3; rc=$?
cat "$VERDICT_OUT"
c7h_rc_ok=1; [ "$rc" -eq 1 ] && c7h_rc_ok=0
check "case7h: progress resets the hold, so a short second hold times out" "$c7h_rc_ok" "rc=$rc"
c7h_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7h_out_ok=0
check "case7h: verdict is timeout" "$c7h_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
echo "-- Case 7i: hold armed, but shorter than the window when the cap falls --"
# The hold arms at t~3 (stall_secs after the first reading) and the cap falls
# at 6, so it is ~3s old against a 10s window: seven seconds of margin, far
# above SECONDS' one-second granularity. This is the case that carries the
# duration condition; 7d and 7h pass even with it removed if their hold
# happens not to arm.
run_gate ping_never_converges 6 3 1 2 10; rc=$?
cat "$VERDICT_OUT"
c7i_rc_ok=1; [ "$rc" -eq 1 ] && c7i_rc_ok=0
check "case7i: an armed hold shorter than the window still times out" "$c7i_rc_ok" "rc=$rc"
c7i_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c7i_out_ok=0
check "case7i: verdict is timeout" "$c7i_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
c7i_hold_ok=1; grep -q "$HOLD_MSG" "$VERDICT_OUT" && c7i_hold_ok=0
check "case7i: the hold was armed" "$c7i_hold_ok" "expected '$HOLD_MSG' in output"
rm -f "$VERDICT_OUT"
# --- Summary ----------------------------------------------------------
testing/lib/wait-converge.sh
+38 -4
View File
@@ -7,7 +7,7 @@
# source "$(dirname "$0")/../../lib/wait-converge.sh"
# wait_for_peers <container> <min_peers> [timeout_secs]
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs] \
# [near_converged_slack]
# [near_converged_slack] [near_converged_accept_secs]
#
# wait_until_connected also sets CONVERGE_OUTCOME / CONVERGE_REACHED /
# CONVERGE_PENDING; see the block above it.
@@ -61,7 +61,7 @@ wait_for_peers() {
# Verdict of the most recent wait_until_connected() call, so a caller can
# report WHICH condition failed rather than only that one did:
# CONVERGE_OUTCOME converged | stalled | timeout
# CONVERGE_OUTCOME converged | near_converged | stalled | timeout
# CONVERGE_REACHED reachable pairs at the moment of the verdict
# CONVERGE_PENDING unreachable pairs at that moment
#
@@ -70,6 +70,10 @@ wait_for_peers() {
# the strict all-pairs assertion 20/20. Without them the caller's summary
# line reads "20 passed, 0 failed" on a non-convergence exit, which a
# reader cannot tell from a connectivity failure.
#
# near_converged is a success return (0): the hard cap fell while the mesh
# had held within slack for at least near_converged_accept_secs, and the
# caller asked for that state to be handed to its own strict assertion.
CONVERGE_OUTCOME=""
CONVERGE_REACHED=0
CONVERGE_PENDING=0
@@ -87,7 +91,7 @@ _converge_verdict() {
# progress-aware deadline instead of a fixed one.
#
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs] \
# [near_converged_slack]
# [near_converged_slack] [near_converged_accept_secs]
#
# <ping_fn> is the name of a function that runs the suite's own
# connectivity check and sets two globals each call:
@@ -112,19 +116,34 @@ _converge_verdict() {
# rather than emitting a false RED with budget still unspent. A
# genuinely never-converging single pair still hits the hard cap.
# - hard cap: max_secs elapsed -> return 1 (never runs unbounded).
# - near-converged acceptance, off unless near_converged_accept_secs is
# non-zero: at the hard cap, if the current near-converged hold has
# lasted at least that long and the last poll was still within slack,
# return 0 with verdict near_converged instead. It acts only at the cap,
# a state that is red without it, so it can never turn a run that would
# have converged by the cap into a red one. Acceptance means "hand the
# mesh to the caller's strict assertion", never "skip that assertion".
# Progress disarms the hold, so only the hold that ends at the cap counts.
#
# Returns 0 once fully connected, 1 on stall or timeout.
# Returns 0 once fully connected or accepted as near-converged, 1 on stall
# or timeout.
wait_until_connected() {
local ping_fn="$1"
local max_secs="$2"
local stall_secs="$3"
local poll_secs="${4:-1}"
local near_converged_slack="${5:-2}"
local near_converged_accept_secs="${6:-0}"
local start_secs=$SECONDS
local best=-1
local last_progress=$SECONDS
local held_for_budget=0
# Start of the current near-converged hold, or -1 when disarmed. Kept
# apart from held_for_budget, which is set once and never reset and so
# cannot measure the hold that ends at the cap. -1 rather than 0 because
# SECONDS can legitimately be 0.
local hold_start=-1
while (( SECONDS - start_secs < max_secs )); do
"$ping_fn"
@@ -136,6 +155,7 @@ wait_until_connected() {
if (( PASSED > best )); then
best=$PASSED
last_progress=$SECONDS
hold_start=-1
echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s"
elif (( SECONDS - last_progress >= stall_secs )); then
if (( FAILED > near_converged_slack )); then
@@ -143,6 +163,9 @@ wait_until_connected() {
echo " converge: STUCK — tree did not converge: $PASSED reachable / $FAILED pending, no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)"
return 1
fi
if (( hold_start < 0 )); then
hold_start=$SECONDS
fi
if (( held_for_budget == 0 )); then
held_for_budget=1
echo " converge: near-converged ($PASSED reachable / $FAILED pending <= slack=$near_converged_slack) — holding for full budget, not bailing (after $((SECONDS - start_secs))s)"
@@ -151,6 +174,17 @@ wait_until_connected() {
sleep "$poll_secs"
done
# The slack test repeats what the loop already guarantees (once the hold
# is armed, any poll beyond slack exits as stalled), so that acceptance
# stays tied to slack if the loop's exits are ever reordered.
if (( near_converged_accept_secs > 0 && hold_start >= 0 \
&& SECONDS - hold_start >= near_converged_accept_secs \
&& FAILED <= near_converged_slack )); then
_converge_verdict near_converged
echo " converge: near-converged at the cap — $PASSED reachable / $FAILED pending, held $((SECONDS - hold_start))s >= ${near_converged_accept_secs}s; handing to the strict assertion (after ${max_secs}s)"
return 0
fi
_converge_verdict timeout
echo " converge: TIMEOUT — tree did not converge: $PASSED reachable / $FAILED pending after ${max_secs}s"
return 1
testing/static/scripts/rekey-test.sh
+13 -1
View File
@@ -154,6 +154,12 @@ trap 'echo ""; echo "Test interrupted"; exit 130' INT
# wait early-exits on PASS, so successful reps are unaffected by the
# extra headroom.
BASELINE_CONVERGENCE_TIMEOUT=65
# A mesh that has held within the gate's slack (two pairs) for this long when
# BASELINE_CONVERGENCE_TIMEOUT falls is handed to the strict all-pairs
# assertion instead of failing the gate. The strict assertion still decides.
# 10s accepts a straggler that held for most of the window and refuses a
# pair that only fell into the hold in the last few seconds.
BASELINE_NEAR_CONVERGED_ACCEPT=10
REKEY_SETTLE=12 # > DRAIN_WINDOW_SECS (10) so post-rekey samples are off the old session
# First FMP rekey should follow shortly after the 35s interval once the mesh is
# fully converged. Keep this bounded to preserve a meaningful scheduling check
@@ -421,7 +427,13 @@ echo ""
# it no longer false-times-out under concurrent CI load. The strict
# ping_all below is the actual assertion, run only after convergence.
echo "Phase 1: Pre-rekey connectivity (waiting for convergence)"
if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20; then
if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20 1 2 \
"$BASELINE_NEAR_CONVERGED_ACCEPT"; then
if [ "$CONVERGE_OUTCOME" = "near_converged" ]; then
echo " Gate accepted a near-converged mesh" \
"($CONVERGE_REACHED/$((CONVERGE_REACHED + CONVERGE_PENDING)) reachable);" \
"the strict assertion below decides"
fi
ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS"
phase_result "Pre-rekey baseline (all 20 pairs)"
if [ "$FAILED" -ne 0 ]; then