mirror of
https://github.com/jmcorgan/fips.git
synced 2026-10-05 19:18:25 +00:00
Make the convergence gate say which condition failed
The gate could not distinguish a tree that did not converge from connectivity that failed. The recorded run exits 1 while reporting "20 passed, 0 failed", because all twenty connectivity pairs passed and the tree reached only 18 of 20 relationships inside the budget. A reader of that line cannot tell the two apart. wait_until_connected now records an outcome, the count reached and the count pending, and its failure messages name the condition. The rekey suite's results line carries the non-convergence clause when that is what happened, and is unchanged otherwise. No timing, threshold or control-flow change. The budget is deliberately untouched: the recorded run reached 18 of 20 at eleven seconds and did not move for the remaining fifty-four, which is not the shape of a budget that is too short. Widening it until the flake stops reproducing would produce a gate that cannot red, which is the failure this change exists to make visible.
This commit is contained in:
Johnathan Corgan
@@ -111,8 +111,31 @@ ping_backcompat_hold() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Case 6 trace: converges quickly, so the verdict case that needs a
|
||||
# converged run does not spend the near-converged hold's twelve seconds.
|
||||
ping_quick_converge() {
|
||||
set_pt; local t=$PT
|
||||
if (( t < 2 )); then
|
||||
PASSED=18; FAILED=2
|
||||
else
|
||||
PASSED=20; FAILED=0
|
||||
fi
|
||||
}
|
||||
|
||||
HOLD_MSG="holding for full budget"
|
||||
STUCK_MSG="STUCK"
|
||||
NOCONV_MSG="tree did not converge"
|
||||
|
||||
# Run the gate in THIS shell (not a subshell) so the CONVERGE_* verdict
|
||||
# globals survive, capturing its output to a file instead. `$(...)` runs the
|
||||
# gate in a fork, which discards those assignments — that is why cases 1-4
|
||||
# can only assert on text.
|
||||
VERDICT_OUT=$(mktemp)
|
||||
run_gate() {
|
||||
reset_ping
|
||||
CONVERGE_OUTCOME=""; CONVERGE_REACHED=-1; CONVERGE_PENDING=-1
|
||||
wait_until_connected "$@" >"$VERDICT_OUT" 2>&1
|
||||
}
|
||||
|
||||
# --- Case 1: near-converged hold --------------------------------------
|
||||
echo
|
||||
@@ -221,6 +244,56 @@ check "case5: floor of 1 polled its full budget" "$c5_polled_ok" "elapsed=${elap
|
||||
|
||||
unset -f docker
|
||||
|
||||
# --- Case 6: the verdict discriminates non-convergence from connectivity --
|
||||
#
|
||||
# This is the break-what-it-guards check for ISSUE-2026-0069. The recorded
|
||||
# failure exited 1 while reporting "20 passed, 0 failed": every connectivity
|
||||
# pair passed and only the tree fell short, and nothing in the summary told
|
||||
# the two apart. The gate now names its verdict, so drive it into each
|
||||
# outcome and assert the verdict is the one that outcome deserves.
|
||||
echo
|
||||
echo "== Case 6: verdict names which condition failed =="
|
||||
|
||||
echo "-- Case 6a: genuinely unconverged tree, hard cap --"
|
||||
run_gate ping_never_converges 6 3 1 2; rc=$?
|
||||
cat "$VERDICT_OUT"
|
||||
c6a_rc_ok=1; [ "$rc" -ne 0 ] && c6a_rc_ok=0
|
||||
check "case6a: unconverged tree still reds" "$c6a_rc_ok" "rc=$rc"
|
||||
c6a_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c6a_out_ok=0
|
||||
check "case6a: verdict is timeout" "$c6a_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
|
||||
c6a_cnt_ok=1
|
||||
[ "$CONVERGE_REACHED" -eq 19 ] && [ "$CONVERGE_PENDING" -eq 1 ] && c6a_cnt_ok=0
|
||||
check "case6a: verdict carries the shortfall" "$c6a_cnt_ok" \
|
||||
"reached=$CONVERGE_REACHED pending=$CONVERGE_PENDING"
|
||||
c6a_msg_ok=1; grep -q "$NOCONV_MSG" "$VERDICT_OUT" && c6a_msg_ok=0
|
||||
check "case6a: message says the tree did not converge" "$c6a_msg_ok"
|
||||
|
||||
echo "-- Case 6b: wedged far from convergence, stall bail --"
|
||||
run_gate ping_far_stall 30 4 1 2; rc=$?
|
||||
cat "$VERDICT_OUT"
|
||||
c6b_rc_ok=1; [ "$rc" -ne 0 ] && c6b_rc_ok=0
|
||||
check "case6b: wedged tree still reds" "$c6b_rc_ok" "rc=$rc"
|
||||
c6b_out_ok=1; [ "$CONVERGE_OUTCOME" = "stalled" ] && c6b_out_ok=0
|
||||
check "case6b: verdict is stalled" "$c6b_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
|
||||
c6b_msg_ok=1; grep -q "$NOCONV_MSG" "$VERDICT_OUT" && c6b_msg_ok=0
|
||||
check "case6b: message says the tree did not converge" "$c6b_msg_ok"
|
||||
|
||||
echo "-- Case 6c: converged tree, and the verdict does not cry non-convergence --"
|
||||
run_gate ping_quick_converge 20 4 1 2; rc=$?
|
||||
cat "$VERDICT_OUT"
|
||||
c6c_rc_ok=1; [ "$rc" -eq 0 ] && c6c_rc_ok=0
|
||||
check "case6c: converged tree still passes" "$c6c_rc_ok" "rc=$rc"
|
||||
c6c_out_ok=1; [ "$CONVERGE_OUTCOME" = "converged" ] && c6c_out_ok=0
|
||||
check "case6c: verdict is converged" "$c6c_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME"
|
||||
c6c_cnt_ok=1
|
||||
[ "$CONVERGE_REACHED" -eq 20 ] && [ "$CONVERGE_PENDING" -eq 0 ] && c6c_cnt_ok=0
|
||||
check "case6c: verdict carries a clean tree" "$c6c_cnt_ok" \
|
||||
"reached=$CONVERGE_REACHED pending=$CONVERGE_PENDING"
|
||||
c6c_quiet_ok=0; grep -q "$NOCONV_MSG" "$VERDICT_OUT" && c6c_quiet_ok=1
|
||||
check "case6c: no non-convergence message on a clean run" "$c6c_quiet_ok"
|
||||
|
||||
rm -f "$VERDICT_OUT"
|
||||
|
||||
# --- Summary ----------------------------------------------------------
|
||||
echo
|
||||
echo "=============================================="
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs] \
|
||||
# [near_converged_slack]
|
||||
#
|
||||
# wait_until_connected also sets CONVERGE_OUTCOME / CONVERGE_REACHED /
|
||||
# CONVERGE_PENDING; see the block above it.
|
||||
#
|
||||
# There was a wait_for_links() here. It was removed rather than kept for
|
||||
# symmetry: it had no caller anywhere in the tree on any branch, and its
|
||||
# reader carried the same failure-to-zero fallback wait_for_peers does. An
|
||||
@@ -56,6 +59,30 @@ wait_for_peers() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# Verdict of the most recent wait_until_connected() call, so a caller can
|
||||
# report WHICH condition failed rather than only that one did:
|
||||
# CONVERGE_OUTCOME converged | stalled | timeout
|
||||
# CONVERGE_REACHED reachable pairs at the moment of the verdict
|
||||
# CONVERGE_PENDING unreachable pairs at that moment
|
||||
#
|
||||
# These exist because the gate's own probe is strictly harsher than the
|
||||
# assertion it guards, so a run can fail the gate at 18/20 and then pass
|
||||
# the strict all-pairs assertion 20/20. Without them the caller's summary
|
||||
# line reads "20 passed, 0 failed" on a non-convergence exit, which a
|
||||
# reader cannot tell from a connectivity failure.
|
||||
CONVERGE_OUTCOME=""
|
||||
CONVERGE_REACHED=0
|
||||
CONVERGE_PENDING=0
|
||||
|
||||
# Record the verdict of a wait_until_connected() return.
|
||||
#
|
||||
# shellcheck disable=SC2034 # read by sourcing suites, not within this file
|
||||
_converge_verdict() {
|
||||
CONVERGE_OUTCOME="$1"
|
||||
CONVERGE_REACHED="$PASSED"
|
||||
CONVERGE_PENDING="$FAILED"
|
||||
}
|
||||
|
||||
# Wait until a connectivity check reports every pair reachable, using a
|
||||
# progress-aware deadline instead of a fixed one.
|
||||
#
|
||||
@@ -102,6 +129,7 @@ wait_until_connected() {
|
||||
while (( SECONDS - start_secs < max_secs )); do
|
||||
"$ping_fn"
|
||||
if (( FAILED == 0 )); then
|
||||
_converge_verdict converged
|
||||
echo " converge: all $PASSED pair(s) reachable after $((SECONDS - start_secs))s"
|
||||
return 0
|
||||
fi
|
||||
@@ -111,7 +139,8 @@ wait_until_connected() {
|
||||
echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s"
|
||||
elif (( SECONDS - last_progress >= stall_secs )); then
|
||||
if (( FAILED > near_converged_slack )); then
|
||||
echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)"
|
||||
_converge_verdict stalled
|
||||
echo " converge: STUCK — tree did not converge: $PASSED reachable / $FAILED pending, no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)"
|
||||
return 1
|
||||
fi
|
||||
if (( held_for_budget == 0 )); then
|
||||
@@ -122,6 +151,7 @@ wait_until_connected() {
|
||||
sleep "$poll_secs"
|
||||
done
|
||||
|
||||
echo " converge: TIMEOUT at $PASSED reachable / $FAILED pending after ${max_secs}s"
|
||||
_converge_verdict timeout
|
||||
echo " converge: TIMEOUT — tree did not converge: $PASSED reachable / $FAILED pending after ${max_secs}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -196,6 +196,11 @@ PASSED=0
|
||||
FAILED=0
|
||||
TOTAL_PASSED=0
|
||||
TOTAL_FAILED=0
|
||||
# Counted separately from TOTAL_FAILED on purpose: a convergence-gate
|
||||
# failure and a connectivity failure are different outcomes, and folding
|
||||
# the first into the second is what made the recorded transcript report
|
||||
# "20 passed, 0 failed" on an exit-1 run.
|
||||
TOTAL_UNCONVERGED=0
|
||||
|
||||
# Node identities
|
||||
ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env"
|
||||
@@ -274,6 +279,23 @@ _baseline_ping() {
|
||||
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
|
||||
}
|
||||
|
||||
# Emit the one-line run summary.
|
||||
#
|
||||
# When the convergence gate is what failed, the line says so and names the
|
||||
# shortfall. The gate's probe is strictly harsher than the strict all-pairs
|
||||
# assertion it guards, so a run can fail the gate at 18/20 relationships and
|
||||
# still pass the assertion 20/20 — which is exactly the recorded failure this
|
||||
# discriminator exists for. Without it both outcomes print the same counts.
|
||||
results_line() {
|
||||
local line="=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed"
|
||||
if [ "$TOTAL_UNCONVERGED" -ne 0 ]; then
|
||||
line+=", tree did not converge"
|
||||
line+=" ($CONVERGE_REACHED/$((CONVERGE_REACHED + CONVERGE_PENDING))"
|
||||
line+=" relationships, $CONVERGE_OUTCOME)"
|
||||
fi
|
||||
echo "$line ==="
|
||||
}
|
||||
|
||||
phase_result() {
|
||||
local phase="$1"
|
||||
TOTAL_PASSED=$((TOTAL_PASSED + PASSED))
|
||||
@@ -404,16 +426,19 @@ if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20; then
|
||||
if [ "$FAILED" -ne 0 ]; then
|
||||
echo ""
|
||||
dump_peer_connectivity
|
||||
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
||||
results_line
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo " Mesh did not reach a converged tree before timeout"
|
||||
TOTAL_UNCONVERGED=$((TOTAL_UNCONVERGED + 1))
|
||||
echo " Mesh did not reach a converged tree before timeout" \
|
||||
"($CONVERGE_OUTCOME at $CONVERGE_REACHED reachable /" \
|
||||
"$CONVERGE_PENDING pending)"
|
||||
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
|
||||
phase_result "Pre-rekey baseline (all 20 pairs)"
|
||||
echo ""
|
||||
dump_peer_connectivity
|
||||
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
||||
results_line
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
@@ -570,9 +595,9 @@ phase_result "Log analysis"
|
||||
echo ""
|
||||
|
||||
# ── Summary ────────────────────────────────────────────────────────────
|
||||
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
||||
results_line
|
||||
|
||||
if [ "$TOTAL_FAILED" -eq 0 ]; then
|
||||
if [ "$TOTAL_FAILED" -eq 0 ] && [ "$TOTAL_UNCONVERGED" -eq 0 ]; then
|
||||
exit 0
|
||||
else
|
||||
# Dump logs on failure for diagnostics.
|
||||
|
||||
Reference in New Issue
Block a user