diff --git a/testing/lib/wait-converge.sh b/testing/lib/wait-converge.sh index 3f64803..c203eca 100644 --- a/testing/lib/wait-converge.sh +++ b/testing/lib/wait-converge.sh @@ -1,12 +1,14 @@ #!/bin/bash # Shared convergence wait helpers for FIPS integration tests. # -# Source this file to get wait_for_links() and wait_for_peers(). +# Source this file to get wait_for_links(), wait_for_peers(), and +# wait_until_connected(). # # Usage: # source "$(dirname "$0")/../../lib/wait-converge.sh" # wait_for_links [timeout_secs] # wait_for_peers [timeout_secs] +# wait_until_connected [poll_secs] # Wait until a container has at least min_links active links. # Returns 0 on success, 1 on timeout. @@ -49,3 +51,55 @@ wait_for_peers() { echo " $container: TIMEOUT waiting for $min_peers peer(s) after ${timeout}s" return 1 } + +# Wait until a connectivity check reports every pair reachable, using a +# progress-aware deadline instead of a fixed one. +# +# wait_until_connected [poll_secs] +# +# is the name of a function that runs the suite's own +# connectivity check and sets two globals each call: +# PASSED number of reachable pairs this round +# FAILED number of unreachable pairs this round +# +# The convergence signal is the suite's real pings (the same signal it +# asserts on), not a structural proxy. Behaviour: +# - converged: FAILED == 0 -> return 0. +# - progressing: PASSED climbed past the best seen -> reset the stall +# clock and keep waiting (slow-but-improving is not a failure, so it +# does not false-time-out under CI load). +# - stuck: PASSED has not improved for stall_secs -> return 1 (fail +# fast rather than burn the whole budget on a genuinely wedged pair). +# - hard cap: max_secs elapsed -> return 1 (never runs unbounded). +# +# Returns 0 once fully connected, 1 on stall or timeout. +wait_until_connected() { + local ping_fn="$1" + local max_secs="$2" + local stall_secs="$3" + local poll_secs="${4:-1}" + + local start_secs=$SECONDS + local best=-1 + local last_progress=$SECONDS + + while (( SECONDS - start_secs < max_secs )); do + "$ping_fn" + if (( FAILED == 0 )); then + echo " converge: all $PASSED pair(s) reachable after $((SECONDS - start_secs))s" + return 0 + fi + if (( PASSED > best )); then + best=$PASSED + last_progress=$SECONDS + echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s" + elif (( SECONDS - last_progress >= stall_secs )); then + echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)" + return 1 + fi + sleep "$poll_secs" + done + + echo " converge: TIMEOUT at $PASSED reachable / $FAILED pending after ${max_secs}s" + return 1 +} diff --git a/testing/sidecar/scripts/test-sidecar.sh b/testing/sidecar/scripts/test-sidecar.sh index 2087c41..7952c2d 100755 --- a/testing/sidecar/scripts/test-sidecar.sh +++ b/testing/sidecar/scripts/test-sidecar.sh @@ -14,6 +14,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SIDECAR_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../../lib/wait-converge.sh +source "$SCRIPT_DIR/../../lib/wait-converge.sh" # Deterministic keys derived from: derive-keys.py sidecar-test node-{a,b,c} NODE_A_NSEC="9e688d0879fa9cd025fea0487ac23495080e3de626070fdb9b78dc1f619dd453" @@ -127,8 +129,17 @@ else exit 1 fi -# Allow a few more seconds for tree convergence and coordinate propagation -sleep 3 +# Wait for end-to-end multi-hop connectivity (the same app-to-app pings +# the test asserts on) with a progress-aware deadline, instead of a +# blind fixed sleep that can fire before coordinates propagate across the +# chain. The directed pings below remain the actual assertions. +_sidecar_converged() { + PASSED=0; FAILED=0 + if docker exec sidecar-b-app-1 ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi + if docker exec sidecar-c-app-1 ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi + if docker exec sidecar-a-app-1 ping6 -c1 -W2 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi +} +wait_until_connected _sidecar_converged "$CONVERGE_TIMEOUT" 10 || true # ── Link verification ───────────────────────────────────────────────────── diff --git a/testing/static/scripts/ping-test.sh b/testing/static/scripts/ping-test.sh index de0ede1..54690b8 100755 --- a/testing/static/scripts/ping-test.sh +++ b/testing/static/scripts/ping-test.sh @@ -71,23 +71,6 @@ ping_all_quiet() { done } -# Wait until all ping pairs succeed or timeout. -wait_for_full_connectivity() { - local timeout="${1:-30}" - local start_secs=$SECONDS - - while (( SECONDS - start_secs < timeout )); do - ping_all_quiet - if [ "$FAILED" -eq 0 ]; then - echo " All $PASSED pairs reachable after $((SECONDS - start_secs))s" - return 0 - fi - sleep 1 - done - echo " TIMEOUT: $PASSED passed, $FAILED failed after ${timeout}s" - return 1 -} - echo "=== FIPS Ping Test ($PROFILE topology) ===" echo "" @@ -108,8 +91,12 @@ elif [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then wait_for_peers fips-node-d 3 20 || true wait_for_peers fips-node-e 3 20 || true fi -# Wait for FSP-level connectivity (discovery + session establishment) -wait_for_full_connectivity 30 || true +# Wait for full pairwise connectivity, progress-aware: the actual pings +# are the convergence signal, the deadline extends while more pairs come +# up, and it only gives up if progress stalls — so it does not +# false-time-out under load while routing is still converging. The +# directed-pair ping assertions below remain the actual test. +wait_until_connected ping_all_quiet 45 15 || true # Reset counters for the actual test PASSED=0 diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index 9b1507f..a81b49b 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -149,8 +149,8 @@ trap 'echo ""; echo "Test interrupted"; exit 130' INT # BASELINE_CONVERGENCE_TIMEOUT must cover one full daemon # node.tree.reeval_interval_secs (default 60) plus a small margin # so any partition that only heals via the periodic TreeAnnounce -# re-broadcast lands inside the convergence window. wait_for_full_baseline -# early-exits on PASS, so successful reps are unaffected by the +# re-broadcast lands inside the convergence window. The Phase-1 baseline +# wait early-exits on PASS, so successful reps are unaffected by the # extra headroom. BASELINE_CONVERGENCE_TIMEOUT=65 REKEY_SETTLE=12 # > DRAIN_WINDOW_SECS (10) so post-rekey samples are off the old session @@ -173,8 +173,8 @@ CONVERGENCE_PING_TIMEOUT=1 # attempts drops the loss-math floor to ~(0.02)^MAX_PING_ATTEMPTS per # pair, making any residual failure attributable to a non-loss # mechanism rather than ICMP noise. Applied to Phase 1 (final strict -# ping_all after wait_for_full_baseline converges) and Phase 3 / Phase -# 5 (post-rekey strict asserts). The wait_for_full_baseline convergence +# ping_all after the Phase-1 baseline wait converges) and Phase 3 / Phase +# 5 (post-rekey strict asserts). The Phase-1 baseline wait's convergence # loop itself stays single-shot — its job is to detect when the mesh # first sees a fully clean 20-pair batch, and retries inside the loop # would conflate "transient ping loss" with "still converging." @@ -255,27 +255,11 @@ ping_all() { done } -wait_for_full_baseline() { - local timeout="${1:-30}" - local start_secs=$SECONDS - local best_passed=0 - local best_failed=20 - - while (( SECONDS - start_secs < timeout )); do - ping_all quiet "$CONVERGENCE_PING_TIMEOUT" - if [ "$PASSED" -gt "$best_passed" ]; then - best_passed="$PASSED" - best_failed="$FAILED" - fi - if [ "$FAILED" -eq 0 ]; then - return 0 - fi - sleep 1 - done - - PASSED="$best_passed" - FAILED="$best_failed" - return 1 +# Connectivity probe for the progress-aware baseline wait: one full +# all-pairs ping sweep, setting PASSED/FAILED (consumed by +# wait_until_connected). +_baseline_ping() { + ping_all quiet "$CONVERGENCE_PING_TIMEOUT" } phase_result() { @@ -370,13 +354,24 @@ echo "Config: rekey.after_secs=$REKEY_AFTER_SECS" echo "" # ── Phase 1: Pre-rekey baseline ─────────────────────────────────────── +# Wait for full pre-rekey connectivity with a progress-aware deadline: +# the all-pairs ping sweep is the convergence signal, the window extends +# while more pairs come up, and it gives up only if progress stalls — so +# it no longer false-times-out under concurrent CI load. The strict +# ping_all below is the actual assertion, run only after convergence. echo "Phase 1: Pre-rekey connectivity (waiting for convergence)" -wait_for_peers fips-node-a 2 "$BASELINE_CONVERGENCE_TIMEOUT" || true -if wait_for_full_baseline "$BASELINE_CONVERGENCE_TIMEOUT"; then +if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20; then ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS" phase_result "Pre-rekey baseline (all 20 pairs)" + if [ "$FAILED" -ne 0 ]; then + echo "" + dump_peer_connectivity + echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ===" + exit 1 + fi else - echo " Best observed baseline before timeout: $PASSED/$((PASSED + FAILED)) passed" + echo " Mesh did not reach a converged tree before timeout" + ping_all quiet "$CONVERGENCE_PING_TIMEOUT" phase_result "Pre-rekey baseline (all 20 pairs)" echo "" dump_peer_connectivity @@ -503,14 +498,30 @@ echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ===" if [ "$TOTAL_FAILED" -eq 0 ]; then exit 0 else - # Dump logs on failure for diagnostics + # Dump logs on failure for diagnostics. + # + # Wider pattern + larger line cap than the original head -30 (which + # truncated before the actual ping-failure timestamp on Phase 5 + # fires, making the post-cutover convergence window invisible to + # offline triage). Two passes per node: + # 1) rekey-related events across the whole run (cap 200 lines). + # 2) Last 80 lines unfiltered, to surface forwarding decisions, + # route lookups, congestion warnings, decrypt errors, and any + # other surrounding context near the failure point. echo "" - echo "=== Node logs (rekey-related) ===" + echo "=== Node logs (rekey-related, head -200) ===" for node in $NODES; do echo "--- node-$node ---" docker logs "fips-node-$node" 2>&1 | \ - grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit)" | \ - head -30 + grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit|no route|next hop|TTL exhausted|MTU exceeded|Congestion|decrypt|Decrypt|AEAD|Notify|drain|Drain|promot)" | \ + head -200 + echo "" + done + + echo "=== Node logs (last 80 lines, unfiltered) ===" + for node in $NODES; do + echo "--- node-$node ---" + docker logs "fips-node-$node" 2>&1 | tail -80 echo "" done exit 1