mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
test: replace fixed convergence timeouts with progress-aware connectivity wait
Add wait_until_connected to the shared convergence helpers: it polls a suite's own pairwise pings (the signal it actually asserts on), returns as soon as every pair is reachable, extends its deadline while the reachable-pair count is still climbing, and gives up only when progress stalls. Use it in the rekey, static-mesh, and sidecar suites in place of the fixed wall-clock baseline timeout and the blind sleep, which timed out under concurrent CI load while the mesh was still converging.
This commit is contained in:
Johnathan Corgan
@@ -71,23 +71,6 @@ ping_all_quiet() {
|
||||
done
|
||||
}
|
||||
|
||||
# Wait until all ping pairs succeed or timeout.
|
||||
wait_for_full_connectivity() {
|
||||
local timeout="${1:-30}"
|
||||
local start_secs=$SECONDS
|
||||
|
||||
while (( SECONDS - start_secs < timeout )); do
|
||||
ping_all_quiet
|
||||
if [ "$FAILED" -eq 0 ]; then
|
||||
echo " All $PASSED pairs reachable after $((SECONDS - start_secs))s"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo " TIMEOUT: $PASSED passed, $FAILED failed after ${timeout}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "=== FIPS Ping Test ($PROFILE topology) ==="
|
||||
echo ""
|
||||
|
||||
@@ -108,8 +91,12 @@ elif [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
|
||||
wait_for_peers fips-node-d 3 20 || true
|
||||
wait_for_peers fips-node-e 3 20 || true
|
||||
fi
|
||||
# Wait for FSP-level connectivity (discovery + session establishment)
|
||||
wait_for_full_connectivity 30 || true
|
||||
# Wait for full pairwise connectivity, progress-aware: the actual pings
|
||||
# are the convergence signal, the deadline extends while more pairs come
|
||||
# up, and it only gives up if progress stalls — so it does not
|
||||
# false-time-out under load while routing is still converging. The
|
||||
# directed-pair ping assertions below remain the actual test.
|
||||
wait_until_connected ping_all_quiet 45 15 || true
|
||||
|
||||
# Reset counters for the actual test
|
||||
PASSED=0
|
||||
|
||||
@@ -149,8 +149,8 @@ trap 'echo ""; echo "Test interrupted"; exit 130' INT
|
||||
# BASELINE_CONVERGENCE_TIMEOUT must cover one full daemon
|
||||
# node.tree.reeval_interval_secs (default 60) plus a small margin
|
||||
# so any partition that only heals via the periodic TreeAnnounce
|
||||
# re-broadcast lands inside the convergence window. wait_for_full_baseline
|
||||
# early-exits on PASS, so successful reps are unaffected by the
|
||||
# re-broadcast lands inside the convergence window. The Phase-1 baseline
|
||||
# wait early-exits on PASS, so successful reps are unaffected by the
|
||||
# extra headroom.
|
||||
BASELINE_CONVERGENCE_TIMEOUT=65
|
||||
REKEY_SETTLE=12 # > DRAIN_WINDOW_SECS (10) so post-rekey samples are off the old session
|
||||
@@ -173,8 +173,8 @@ CONVERGENCE_PING_TIMEOUT=1
|
||||
# attempts drops the loss-math floor to ~(0.02)^MAX_PING_ATTEMPTS per
|
||||
# pair, making any residual failure attributable to a non-loss
|
||||
# mechanism rather than ICMP noise. Applied to Phase 1 (final strict
|
||||
# ping_all after wait_for_full_baseline converges) and Phase 3 / Phase
|
||||
# 5 (post-rekey strict asserts). The wait_for_full_baseline convergence
|
||||
# ping_all after the Phase-1 baseline wait converges) and Phase 3 / Phase
|
||||
# 5 (post-rekey strict asserts). The Phase-1 baseline wait's convergence
|
||||
# loop itself stays single-shot — its job is to detect when the mesh
|
||||
# first sees a fully clean 20-pair batch, and retries inside the loop
|
||||
# would conflate "transient ping loss" with "still converging."
|
||||
@@ -255,27 +255,11 @@ ping_all() {
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_full_baseline() {
|
||||
local timeout="${1:-30}"
|
||||
local start_secs=$SECONDS
|
||||
local best_passed=0
|
||||
local best_failed=20
|
||||
|
||||
while (( SECONDS - start_secs < timeout )); do
|
||||
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
|
||||
if [ "$PASSED" -gt "$best_passed" ]; then
|
||||
best_passed="$PASSED"
|
||||
best_failed="$FAILED"
|
||||
fi
|
||||
if [ "$FAILED" -eq 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
PASSED="$best_passed"
|
||||
FAILED="$best_failed"
|
||||
return 1
|
||||
# Connectivity probe for the progress-aware baseline wait: one full
|
||||
# all-pairs ping sweep, setting PASSED/FAILED (consumed by
|
||||
# wait_until_connected).
|
||||
_baseline_ping() {
|
||||
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
|
||||
}
|
||||
|
||||
phase_result() {
|
||||
@@ -370,13 +354,24 @@ echo "Config: rekey.after_secs=$REKEY_AFTER_SECS"
|
||||
echo ""
|
||||
|
||||
# ── Phase 1: Pre-rekey baseline ───────────────────────────────────────
|
||||
# Wait for full pre-rekey connectivity with a progress-aware deadline:
|
||||
# the all-pairs ping sweep is the convergence signal, the window extends
|
||||
# while more pairs come up, and it gives up only if progress stalls — so
|
||||
# it no longer false-times-out under concurrent CI load. The strict
|
||||
# ping_all below is the actual assertion, run only after convergence.
|
||||
echo "Phase 1: Pre-rekey connectivity (waiting for convergence)"
|
||||
wait_for_peers fips-node-a 2 "$BASELINE_CONVERGENCE_TIMEOUT" || true
|
||||
if wait_for_full_baseline "$BASELINE_CONVERGENCE_TIMEOUT"; then
|
||||
if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20; then
|
||||
ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS"
|
||||
phase_result "Pre-rekey baseline (all 20 pairs)"
|
||||
if [ "$FAILED" -ne 0 ]; then
|
||||
echo ""
|
||||
dump_peer_connectivity
|
||||
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo " Best observed baseline before timeout: $PASSED/$((PASSED + FAILED)) passed"
|
||||
echo " Mesh did not reach a converged tree before timeout"
|
||||
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
|
||||
phase_result "Pre-rekey baseline (all 20 pairs)"
|
||||
echo ""
|
||||
dump_peer_connectivity
|
||||
@@ -503,14 +498,30 @@ echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
||||
if [ "$TOTAL_FAILED" -eq 0 ]; then
|
||||
exit 0
|
||||
else
|
||||
# Dump logs on failure for diagnostics
|
||||
# Dump logs on failure for diagnostics.
|
||||
#
|
||||
# Wider pattern + larger line cap than the original head -30 (which
|
||||
# truncated before the actual ping-failure timestamp on Phase 5
|
||||
# fires, making the post-cutover convergence window invisible to
|
||||
# offline triage). Two passes per node:
|
||||
# 1) rekey-related events across the whole run (cap 200 lines).
|
||||
# 2) Last 80 lines unfiltered, to surface forwarding decisions,
|
||||
# route lookups, congestion warnings, decrypt errors, and any
|
||||
# other surrounding context near the failure point.
|
||||
echo ""
|
||||
echo "=== Node logs (rekey-related) ==="
|
||||
echo "=== Node logs (rekey-related, head -200) ==="
|
||||
for node in $NODES; do
|
||||
echo "--- node-$node ---"
|
||||
docker logs "fips-node-$node" 2>&1 | \
|
||||
grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit)" | \
|
||||
head -30
|
||||
grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit|no route|next hop|TTL exhausted|MTU exceeded|Congestion|decrypt|Decrypt|AEAD|Notify|drain|Drain|promot)" | \
|
||||
head -200
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "=== Node logs (last 80 lines, unfiltered) ==="
|
||||
for node in $NODES; do
|
||||
echo "--- node-$node ---"
|
||||
docker logs "fips-node-$node" 2>&1 | tail -80
|
||||
echo ""
|
||||
done
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user