From 306e455513593c34eb151a9319cb7f9a19af3e67 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 17 May 2026 17:35:04 +0000 Subject: [PATCH] rekey-test: retry strict-ping asserts on failure The Phase 1, Phase 3, and Phase 5 strict asserts each fire a single ping per directed pair. Under low-level packet loss (e.g. 1% i.i.d. per-direction loss from a CI runner under pressure), a single-shot round-trip fails at ~2% per pair, so a 20-pair strict assert misses with probability 1 - (0.98)^20 = ~33% per phase from ICMP noise alone, well above the routing-state signal the asserts are meant to catch. ping_one gains a max_attempts parameter (default 1, preserving existing call sites). On failure it retries up to MAX_PING_ATTEMPTS-1 additional times with PING_RETRY_DELAY seconds between attempts. Per-pair worst case under the defaults (4 attempts, 1 s spacing, 5 s ping6 -W timeout) is 4*5 + 3 = 23 s; per-rep worst case scales with the failing-pair count. Successful retries log "OK (RTT, attempt N)"; exhausted retries log "FAIL (after N attempts)". The retry budget is wired into all three strict asserts: - Phase 1 final ping_all (after wait_for_full_baseline converges) - Phase 3 ping_all (post-first-rekey) - Phase 5 ping_all (post-second-rekey) The wait_for_full_baseline convergence loop itself stays single-shot. Its job is to detect when the mesh first sees a fully clean 20-pair batch, and retries inside the loop would conflate transient ping loss with still-converging routing state. No daemon code changes. --- testing/static/scripts/rekey-test.sh | 59 ++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index 6465b39..28ca83c 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -157,6 +157,23 @@ LOG_EVENT_POLL_INTERVAL=1 TIMEOUT=5 CONVERGENCE_PING_TIMEOUT=1 +# Strict-ping retry policy for the per-phase ping_all asserts. Under 1% +# i.i.d. packet loss (the lab condition surfaced by ISSUE-2026-0028) a +# single-shot ping fails at roughly 2% per directed pair, so a 20-pair +# strict assert misses with probability ~1 - (0.98)^20 ≈ 33%, which is +# below the per-pair loss-math floor but well above the routing-state +# signal we want to measure. Retrying each failing pair up to +# MAX_PING_ATTEMPTS-1 additional times with ~PING_RETRY_DELAY between +# attempts drops the loss-math floor to ~(0.02)^MAX_PING_ATTEMPTS per +# pair, making any residual failure attributable to a non-loss +# mechanism rather than ICMP noise. Applied to Phase 1 (final strict +# ping_all after wait_for_full_baseline converges) and Phase 3 / Phase +# 5 (post-rekey strict asserts). The wait_for_full_baseline convergence +# loop itself stays single-shot — its job is to detect when the mesh +# first sees a fully clean 20-pair batch, and retries inside the loop +# would conflate "transient ping loss" with "still converging." +MAX_PING_ATTEMPTS=4 +PING_RETRY_DELAY=1 PASSED=0 FAILED=0 TOTAL_PASSED=0 @@ -181,25 +198,43 @@ ping_one() { local label="$3" local quiet="${4:-}" local ping_timeout="${5:-$TIMEOUT}" + local max_attempts="${6:-1}" - if output=$(docker exec "fips-$from" ping6 -c 1 -W "$ping_timeout" "${to_npub}.fips" 2>&1); then - local rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2) - if [ -z "$quiet" ]; then - echo " $label ... OK (${rtt:-?}ms)" + local attempt=1 + local output rtt + while (( attempt <= max_attempts )); do + if (( attempt > 1 )); then + sleep "$PING_RETRY_DELAY" fi - PASSED=$((PASSED + 1)) - else - if [ -z "$quiet" ]; then + if output=$(docker exec "fips-$from" ping6 -c 1 -W "$ping_timeout" "${to_npub}.fips" 2>&1); then + rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2) + if [ -z "$quiet" ]; then + if (( attempt == 1 )); then + echo " $label ... OK (${rtt:-?}ms)" + else + echo " $label ... OK (${rtt:-?}ms, attempt $attempt)" + fi + fi + PASSED=$((PASSED + 1)) + return + fi + attempt=$((attempt + 1)) + done + if [ -z "$quiet" ]; then + if (( max_attempts > 1 )); then + echo " $label ... FAIL (after $max_attempts attempts)" + else echo " $label ... FAIL" fi - FAILED=$((FAILED + 1)) fi + FAILED=$((FAILED + 1)) } # Run all 20 directed pairs ping_all() { local quiet="${1:-}" local ping_timeout="${2:-$TIMEOUT}" + local max_attempts="${3:-1}" PASSED=0 FAILED=0 for i in 0 1 2 3 4; do @@ -209,7 +244,7 @@ ping_all() { for j in 0 1 2 3 4; do [ "$i" -eq "$j" ] && continue ping_one "node-${LABELS[$i],,}" "${NPUBS[$j]}" \ - "${LABELS[$i]} → ${LABELS[$j]}" "$quiet" "$ping_timeout" + "${LABELS[$i]} → ${LABELS[$j]}" "$quiet" "$ping_timeout" "$max_attempts" done done } @@ -332,7 +367,7 @@ echo "" echo "Phase 1: Pre-rekey connectivity (waiting for convergence)" wait_for_peers fips-node-a 2 "$BASELINE_CONVERGENCE_TIMEOUT" || true if wait_for_full_baseline "$BASELINE_CONVERGENCE_TIMEOUT"; then - ping_all + ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS" phase_result "Pre-rekey baseline (all 20 pairs)" else echo " Best observed baseline before timeout: $PASSED/$((PASSED + FAILED)) passed" @@ -359,7 +394,7 @@ echo "" # Verify connectivity after first rekey (strict — no failures allowed) echo "Phase 3: Post-rekey connectivity (settling ${REKEY_SETTLE}s)" sleep "$REKEY_SETTLE" -ping_all +ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS" phase_result "Post-first-rekey (all 20 pairs)" echo "" @@ -370,7 +405,7 @@ sleep "$SECOND_REKEY_WAIT" # Verify connectivity after second rekey (back-to-back) echo "Phase 5: Post-second-rekey connectivity (settling ${REKEY_SETTLE}s)" sleep "$REKEY_SETTLE" -ping_all +ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS" phase_result "Post-second-rekey (all 20 pairs)" echo ""