diff --git a/testing/lib/wait-converge-test.sh b/testing/lib/wait-converge-test.sh index 7b7c0ab9..5ad50dfb 100755 --- a/testing/lib/wait-converge-test.sh +++ b/testing/lib/wait-converge-test.sh @@ -111,8 +111,31 @@ ping_backcompat_hold() { fi } +# Case 6 trace: converges quickly, so the verdict case that needs a +# converged run does not spend the near-converged hold's twelve seconds. +ping_quick_converge() { + set_pt; local t=$PT + if (( t < 2 )); then + PASSED=18; FAILED=2 + else + PASSED=20; FAILED=0 + fi +} + HOLD_MSG="holding for full budget" STUCK_MSG="STUCK" +NOCONV_MSG="tree did not converge" + +# Run the gate in THIS shell (not a subshell) so the CONVERGE_* verdict +# globals survive, capturing its output to a file instead. `$(...)` runs the +# gate in a fork, which discards those assignments — that is why cases 1-4 +# can only assert on text. +VERDICT_OUT=$(mktemp) +run_gate() { + reset_ping + CONVERGE_OUTCOME=""; CONVERGE_REACHED=-1; CONVERGE_PENDING=-1 + wait_until_connected "$@" >"$VERDICT_OUT" 2>&1 +} # --- Case 1: near-converged hold -------------------------------------- echo @@ -221,6 +244,56 @@ check "case5: floor of 1 polled its full budget" "$c5_polled_ok" "elapsed=${elap unset -f docker +# --- Case 6: the verdict discriminates non-convergence from connectivity -- +# +# This is the break-what-it-guards check for ISSUE-2026-0069. The recorded +# failure exited 1 while reporting "20 passed, 0 failed": every connectivity +# pair passed and only the tree fell short, and nothing in the summary told +# the two apart. The gate now names its verdict, so drive it into each +# outcome and assert the verdict is the one that outcome deserves. +echo +echo "== Case 6: verdict names which condition failed ==" + +echo "-- Case 6a: genuinely unconverged tree, hard cap --" +run_gate ping_never_converges 6 3 1 2; rc=$? +cat "$VERDICT_OUT" +c6a_rc_ok=1; [ "$rc" -ne 0 ] && c6a_rc_ok=0 +check "case6a: unconverged tree still reds" "$c6a_rc_ok" "rc=$rc" +c6a_out_ok=1; [ "$CONVERGE_OUTCOME" = "timeout" ] && c6a_out_ok=0 +check "case6a: verdict is timeout" "$c6a_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" +c6a_cnt_ok=1 +[ "$CONVERGE_REACHED" -eq 19 ] && [ "$CONVERGE_PENDING" -eq 1 ] && c6a_cnt_ok=0 +check "case6a: verdict carries the shortfall" "$c6a_cnt_ok" \ + "reached=$CONVERGE_REACHED pending=$CONVERGE_PENDING" +c6a_msg_ok=1; grep -q "$NOCONV_MSG" "$VERDICT_OUT" && c6a_msg_ok=0 +check "case6a: message says the tree did not converge" "$c6a_msg_ok" + +echo "-- Case 6b: wedged far from convergence, stall bail --" +run_gate ping_far_stall 30 4 1 2; rc=$? +cat "$VERDICT_OUT" +c6b_rc_ok=1; [ "$rc" -ne 0 ] && c6b_rc_ok=0 +check "case6b: wedged tree still reds" "$c6b_rc_ok" "rc=$rc" +c6b_out_ok=1; [ "$CONVERGE_OUTCOME" = "stalled" ] && c6b_out_ok=0 +check "case6b: verdict is stalled" "$c6b_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" +c6b_msg_ok=1; grep -q "$NOCONV_MSG" "$VERDICT_OUT" && c6b_msg_ok=0 +check "case6b: message says the tree did not converge" "$c6b_msg_ok" + +echo "-- Case 6c: converged tree, and the verdict does not cry non-convergence --" +run_gate ping_quick_converge 20 4 1 2; rc=$? +cat "$VERDICT_OUT" +c6c_rc_ok=1; [ "$rc" -eq 0 ] && c6c_rc_ok=0 +check "case6c: converged tree still passes" "$c6c_rc_ok" "rc=$rc" +c6c_out_ok=1; [ "$CONVERGE_OUTCOME" = "converged" ] && c6c_out_ok=0 +check "case6c: verdict is converged" "$c6c_out_ok" "CONVERGE_OUTCOME=$CONVERGE_OUTCOME" +c6c_cnt_ok=1 +[ "$CONVERGE_REACHED" -eq 20 ] && [ "$CONVERGE_PENDING" -eq 0 ] && c6c_cnt_ok=0 +check "case6c: verdict carries a clean tree" "$c6c_cnt_ok" \ + "reached=$CONVERGE_REACHED pending=$CONVERGE_PENDING" +c6c_quiet_ok=0; grep -q "$NOCONV_MSG" "$VERDICT_OUT" && c6c_quiet_ok=1 +check "case6c: no non-convergence message on a clean run" "$c6c_quiet_ok" + +rm -f "$VERDICT_OUT" + # --- Summary ---------------------------------------------------------- echo echo "==============================================" diff --git a/testing/lib/wait-converge.sh b/testing/lib/wait-converge.sh index 17933a5c..a9588b15 100644 --- a/testing/lib/wait-converge.sh +++ b/testing/lib/wait-converge.sh @@ -9,6 +9,9 @@ # wait_until_connected [poll_secs] \ # [near_converged_slack] # +# wait_until_connected also sets CONVERGE_OUTCOME / CONVERGE_REACHED / +# CONVERGE_PENDING; see the block above it. +# # There was a wait_for_links() here. It was removed rather than kept for # symmetry: it had no caller anywhere in the tree on any branch, and its # reader carried the same failure-to-zero fallback wait_for_peers does. An @@ -56,6 +59,30 @@ wait_for_peers() { return 1 } +# Verdict of the most recent wait_until_connected() call, so a caller can +# report WHICH condition failed rather than only that one did: +# CONVERGE_OUTCOME converged | stalled | timeout +# CONVERGE_REACHED reachable pairs at the moment of the verdict +# CONVERGE_PENDING unreachable pairs at that moment +# +# These exist because the gate's own probe is strictly harsher than the +# assertion it guards, so a run can fail the gate at 18/20 and then pass +# the strict all-pairs assertion 20/20. Without them the caller's summary +# line reads "20 passed, 0 failed" on a non-convergence exit, which a +# reader cannot tell from a connectivity failure. +CONVERGE_OUTCOME="" +CONVERGE_REACHED=0 +CONVERGE_PENDING=0 + +# Record the verdict of a wait_until_connected() return. +# +# shellcheck disable=SC2034 # read by sourcing suites, not within this file +_converge_verdict() { + CONVERGE_OUTCOME="$1" + CONVERGE_REACHED="$PASSED" + CONVERGE_PENDING="$FAILED" +} + # Wait until a connectivity check reports every pair reachable, using a # progress-aware deadline instead of a fixed one. # @@ -102,6 +129,7 @@ wait_until_connected() { while (( SECONDS - start_secs < max_secs )); do "$ping_fn" if (( FAILED == 0 )); then + _converge_verdict converged echo " converge: all $PASSED pair(s) reachable after $((SECONDS - start_secs))s" return 0 fi @@ -111,7 +139,8 @@ wait_until_connected() { echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s" elif (( SECONDS - last_progress >= stall_secs )); then if (( FAILED > near_converged_slack )); then - echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)" + _converge_verdict stalled + echo " converge: STUCK — tree did not converge: $PASSED reachable / $FAILED pending, no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)" return 1 fi if (( held_for_budget == 0 )); then @@ -122,6 +151,7 @@ wait_until_connected() { sleep "$poll_secs" done - echo " converge: TIMEOUT at $PASSED reachable / $FAILED pending after ${max_secs}s" + _converge_verdict timeout + echo " converge: TIMEOUT — tree did not converge: $PASSED reachable / $FAILED pending after ${max_secs}s" return 1 } diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index 4a4d39c2..b999482b 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -196,6 +196,11 @@ PASSED=0 FAILED=0 TOTAL_PASSED=0 TOTAL_FAILED=0 +# Counted separately from TOTAL_FAILED on purpose: a convergence-gate +# failure and a connectivity failure are different outcomes, and folding +# the first into the second is what made the recorded transcript report +# "20 passed, 0 failed" on an exit-1 run. +TOTAL_UNCONVERGED=0 # Node identities ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env" @@ -274,6 +279,23 @@ _baseline_ping() { ping_all quiet "$CONVERGENCE_PING_TIMEOUT" } +# Emit the one-line run summary. +# +# When the convergence gate is what failed, the line says so and names the +# shortfall. The gate's probe is strictly harsher than the strict all-pairs +# assertion it guards, so a run can fail the gate at 18/20 relationships and +# still pass the assertion 20/20 — which is exactly the recorded failure this +# discriminator exists for. Without it both outcomes print the same counts. +results_line() { + local line="=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed" + if [ "$TOTAL_UNCONVERGED" -ne 0 ]; then + line+=", tree did not converge" + line+=" ($CONVERGE_REACHED/$((CONVERGE_REACHED + CONVERGE_PENDING))" + line+=" relationships, $CONVERGE_OUTCOME)" + fi + echo "$line ===" +} + phase_result() { local phase="$1" TOTAL_PASSED=$((TOTAL_PASSED + PASSED)) @@ -404,16 +426,19 @@ if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20; then if [ "$FAILED" -ne 0 ]; then echo "" dump_peer_connectivity - echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ===" + results_line exit 1 fi else - echo " Mesh did not reach a converged tree before timeout" + TOTAL_UNCONVERGED=$((TOTAL_UNCONVERGED + 1)) + echo " Mesh did not reach a converged tree before timeout" \ + "($CONVERGE_OUTCOME at $CONVERGE_REACHED reachable /" \ + "$CONVERGE_PENDING pending)" ping_all quiet "$CONVERGENCE_PING_TIMEOUT" phase_result "Pre-rekey baseline (all 20 pairs)" echo "" dump_peer_connectivity - echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ===" + results_line exit 1 fi echo "" @@ -570,9 +595,9 @@ phase_result "Log analysis" echo "" # ── Summary ──────────────────────────────────────────────────────────── -echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ===" +results_line -if [ "$TOTAL_FAILED" -eq 0 ]; then +if [ "$TOTAL_FAILED" -eq 0 ] && [ "$TOTAL_UNCONVERGED" -eq 0 ]; then exit 0 else # Dump logs on failure for diagnostics.