laantungir/fips
1
0
Fork 0
mirror of https://github.com/jmcorgan/fips.git synced 2026-08-09 00:04:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into next

Browse Source
This commit is contained in:
Johnathan Corgan
2026-06-05 04:47:00 +00:00
parent 97f496afc4 e7349202b5
commit 2bc3a505f6
16 changed files with 400 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
testing/lib/wait-converge.sh
+55 -1
View File
@@ -1,12 +1,14 @@
#!/bin/bash
# Shared convergence wait helpers for FIPS integration tests.
#
# Source this file to get wait_for_links() and wait_for_peers().
# Source this file to get wait_for_links(), wait_for_peers(), and
# wait_until_connected().
#
# Usage:
# source "$(dirname "$0")/../../lib/wait-converge.sh"
# wait_for_links <container> <min_links> [timeout_secs]
# wait_for_peers <container> <min_peers> [timeout_secs]
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs]
# Wait until a container has at least min_links active links.
# Returns 0 on success, 1 on timeout.
@@ -49,3 +51,55 @@ wait_for_peers() {
echo " $container: TIMEOUT waiting for $min_peers peer(s) after ${timeout}s"
return 1
}
# Wait until a connectivity check reports every pair reachable, using a
# progress-aware deadline instead of a fixed one.
#
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs]
#
# <ping_fn> is the name of a function that runs the suite's own
# connectivity check and sets two globals each call:
# PASSED number of reachable pairs this round
# FAILED number of unreachable pairs this round
#
# The convergence signal is the suite's real pings (the same signal it
# asserts on), not a structural proxy. Behaviour:
# - converged: FAILED == 0 -> return 0.
# - progressing: PASSED climbed past the best seen -> reset the stall
# clock and keep waiting (slow-but-improving is not a failure, so it
# does not false-time-out under CI load).
# - stuck: PASSED has not improved for stall_secs -> return 1 (fail
# fast rather than burn the whole budget on a genuinely wedged pair).
# - hard cap: max_secs elapsed -> return 1 (never runs unbounded).
#
# Returns 0 once fully connected, 1 on stall or timeout.
wait_until_connected() {
local ping_fn="$1"
local max_secs="$2"
local stall_secs="$3"
local poll_secs="${4:-1}"
local start_secs=$SECONDS
local best=-1
local last_progress=$SECONDS
while (( SECONDS - start_secs < max_secs )); do
"$ping_fn"
if (( FAILED == 0 )); then
echo " converge: all $PASSED pair(s) reachable after $((SECONDS - start_secs))s"
return 0
fi
if (( PASSED > best )); then
best=$PASSED
last_progress=$SECONDS
echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s"
elif (( SECONDS - last_progress >= stall_secs )); then
echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)"
return 1
fi
sleep "$poll_secs"
done
echo " converge: TIMEOUT at $PASSED reachable / $FAILED pending after ${max_secs}s"
return 1
}
testing/sidecar/scripts/test-sidecar.sh
+13 -2
View File
@@ -14,6 +14,8 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SIDECAR_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
# shellcheck source=../../lib/wait-converge.sh
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
# Deterministic keys derived from: derive-keys.py sidecar-test node-{a,b,c}
NODE_A_NSEC="9e688d0879fa9cd025fea0487ac23495080e3de626070fdb9b78dc1f619dd453"
@@ -127,8 +129,17 @@ else
exit 1
fi
# Allow a few more seconds for tree convergence and coordinate propagation
sleep 3
# Wait for end-to-end multi-hop connectivity (the same app-to-app pings
# the test asserts on) with a progress-aware deadline, instead of a
# blind fixed sleep that can fire before coordinates propagate across the
# chain. The directed pings below remain the actual assertions.
_sidecar_converged() {
PASSED=0; FAILED=0
if docker exec sidecar-b-app-1 ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
if docker exec sidecar-c-app-1 ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
if docker exec sidecar-a-app-1 ping6 -c1 -W2 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
}
wait_until_connected _sidecar_converged "$CONVERGE_TIMEOUT" 10 || true
# ── Link verification ─────────────────────────────────────────────────────
testing/static/scripts/ping-test.sh
+6 -19
View File
@@ -71,23 +71,6 @@ ping_all_quiet() {
done
}
# Wait until all ping pairs succeed or timeout.
wait_for_full_connectivity() {
local timeout="${1:-30}"
local start_secs=$SECONDS
while (( SECONDS - start_secs < timeout )); do
ping_all_quiet
if [ "$FAILED" -eq 0 ]; then
echo " All $PASSED pairs reachable after $((SECONDS - start_secs))s"
return 0
fi
sleep 1
done
echo " TIMEOUT: $PASSED passed, $FAILED failed after ${timeout}s"
return 1
}
echo "=== FIPS Ping Test ($PROFILE topology) ==="
echo ""
@@ -108,8 +91,12 @@ elif [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
wait_for_peers fips-node-d 3 20 || true
wait_for_peers fips-node-e 3 20 || true
fi
# Wait for FSP-level connectivity (discovery + session establishment)
wait_for_full_connectivity 45 || true
# Wait for full pairwise connectivity, progress-aware: the actual pings
# are the convergence signal, the deadline extends while more pairs come
# up, and it only gives up if progress stalls — so it does not
# false-time-out under load while routing is still converging. The
# directed-pair ping assertions below remain the actual test.
wait_until_connected ping_all_quiet 45 15 || true
# Reset counters for the actual test
PASSED=0
testing/static/scripts/rekey-test.sh
+23 -28
View File
@@ -149,8 +149,8 @@ trap 'echo ""; echo "Test interrupted"; exit 130' INT
# BASELINE_CONVERGENCE_TIMEOUT must cover one full daemon
# node.tree.reeval_interval_secs (default 60) plus a small margin
# so any partition that only heals via the periodic TreeAnnounce
# re-broadcast lands inside the convergence window. wait_for_full_baseline
# early-exits on PASS, so successful reps are unaffected by the
# re-broadcast lands inside the convergence window. The Phase-1 baseline
# wait early-exits on PASS, so successful reps are unaffected by the
# extra headroom.
BASELINE_CONVERGENCE_TIMEOUT=65
REKEY_SETTLE=12 # > DRAIN_WINDOW_SECS (10) so post-rekey samples are off the old session
@@ -173,8 +173,8 @@ CONVERGENCE_PING_TIMEOUT=1
# attempts drops the loss-math floor to ~(0.02)^MAX_PING_ATTEMPTS per
# pair, making any residual failure attributable to a non-loss
# mechanism rather than ICMP noise. Applied to Phase 1 (final strict
# ping_all after wait_for_full_baseline converges) and Phase 3 / Phase
# 5 (post-rekey strict asserts). The wait_for_full_baseline convergence
# ping_all after the Phase-1 baseline wait converges) and Phase 3 / Phase
# 5 (post-rekey strict asserts). The Phase-1 baseline wait's convergence
# loop itself stays single-shot — its job is to detect when the mesh
# first sees a fully clean 20-pair batch, and retries inside the loop
# would conflate "transient ping loss" with "still converging."
@@ -255,27 +255,11 @@ ping_all() {
done
}
wait_for_full_baseline() {
local timeout="${1:-30}"
local start_secs=$SECONDS
local best_passed=0
local best_failed=20
while (( SECONDS - start_secs < timeout )); do
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
if [ "$PASSED" -gt "$best_passed" ]; then
best_passed="$PASSED"
best_failed="$FAILED"
fi
if [ "$FAILED" -eq 0 ]; then
return 0
fi
sleep 1
done
PASSED="$best_passed"
FAILED="$best_failed"
return 1
# Connectivity probe for the progress-aware baseline wait: one full
# all-pairs ping sweep, setting PASSED/FAILED (consumed by
# wait_until_connected).
_baseline_ping() {
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
}
phase_result() {
@@ -370,13 +354,24 @@ echo "Config: rekey.after_secs=$REKEY_AFTER_SECS"
echo ""
# ── Phase 1: Pre-rekey baseline ───────────────────────────────────────
# Wait for full pre-rekey connectivity with a progress-aware deadline:
# the all-pairs ping sweep is the convergence signal, the window extends
# while more pairs come up, and it gives up only if progress stalls — so
# it no longer false-times-out under concurrent CI load. The strict
# ping_all below is the actual assertion, run only after convergence.
echo "Phase 1: Pre-rekey connectivity (waiting for convergence)"
wait_for_peers fips-node-a 2 "$BASELINE_CONVERGENCE_TIMEOUT" || true
if wait_for_full_baseline "$BASELINE_CONVERGENCE_TIMEOUT"; then
if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20; then
ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS"
phase_result "Pre-rekey baseline (all 20 pairs)"
if [ "$FAILED" -ne 0 ]; then
echo ""
dump_peer_connectivity
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
exit 1
fi
else
echo " Best observed baseline before timeout: $PASSED/$((PASSED + FAILED)) passed"
echo " Mesh did not reach a converged tree before timeout"
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
phase_result "Pre-rekey baseline (all 20 pairs)"
echo ""
dump_peer_connectivity