mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
test: replace fixed convergence timeouts with progress-aware connectivity wait
Add wait_until_connected to the shared convergence helpers: it polls a suite's own pairwise pings (the signal it actually asserts on), returns as soon as every pair is reachable, extends its deadline while the reachable-pair count is still climbing, and gives up only when progress stalls. Use it in the rekey, static-mesh, and sidecar suites in place of the fixed wall-clock baseline timeout and the blind sleep, which timed out under concurrent CI load while the mesh was still converging.
This commit is contained in:
@@ -1,12 +1,14 @@
|
||||
#!/bin/bash
|
||||
# Shared convergence wait helpers for FIPS integration tests.
|
||||
#
|
||||
# Source this file to get wait_for_links() and wait_for_peers().
|
||||
# Source this file to get wait_for_links(), wait_for_peers(), and
|
||||
# wait_until_connected().
|
||||
#
|
||||
# Usage:
|
||||
# source "$(dirname "$0")/../../lib/wait-converge.sh"
|
||||
# wait_for_links <container> <min_links> [timeout_secs]
|
||||
# wait_for_peers <container> <min_peers> [timeout_secs]
|
||||
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs]
|
||||
|
||||
# Wait until a container has at least min_links active links.
|
||||
# Returns 0 on success, 1 on timeout.
|
||||
@@ -49,3 +51,55 @@ wait_for_peers() {
|
||||
echo " $container: TIMEOUT waiting for $min_peers peer(s) after ${timeout}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Wait until a connectivity check reports every pair reachable, using a
|
||||
# progress-aware deadline instead of a fixed one.
|
||||
#
|
||||
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs]
|
||||
#
|
||||
# <ping_fn> is the name of a function that runs the suite's own
|
||||
# connectivity check and sets two globals each call:
|
||||
# PASSED number of reachable pairs this round
|
||||
# FAILED number of unreachable pairs this round
|
||||
#
|
||||
# The convergence signal is the suite's real pings (the same signal it
|
||||
# asserts on), not a structural proxy. Behaviour:
|
||||
# - converged: FAILED == 0 -> return 0.
|
||||
# - progressing: PASSED climbed past the best seen -> reset the stall
|
||||
# clock and keep waiting (slow-but-improving is not a failure, so it
|
||||
# does not false-time-out under CI load).
|
||||
# - stuck: PASSED has not improved for stall_secs -> return 1 (fail
|
||||
# fast rather than burn the whole budget on a genuinely wedged pair).
|
||||
# - hard cap: max_secs elapsed -> return 1 (never runs unbounded).
|
||||
#
|
||||
# Returns 0 once fully connected, 1 on stall or timeout.
|
||||
wait_until_connected() {
|
||||
local ping_fn="$1"
|
||||
local max_secs="$2"
|
||||
local stall_secs="$3"
|
||||
local poll_secs="${4:-1}"
|
||||
|
||||
local start_secs=$SECONDS
|
||||
local best=-1
|
||||
local last_progress=$SECONDS
|
||||
|
||||
while (( SECONDS - start_secs < max_secs )); do
|
||||
"$ping_fn"
|
||||
if (( FAILED == 0 )); then
|
||||
echo " converge: all $PASSED pair(s) reachable after $((SECONDS - start_secs))s"
|
||||
return 0
|
||||
fi
|
||||
if (( PASSED > best )); then
|
||||
best=$PASSED
|
||||
last_progress=$SECONDS
|
||||
echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s"
|
||||
elif (( SECONDS - last_progress >= stall_secs )); then
|
||||
echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)"
|
||||
return 1
|
||||
fi
|
||||
sleep "$poll_secs"
|
||||
done
|
||||
|
||||
echo " converge: TIMEOUT at $PASSED reachable / $FAILED pending after ${max_secs}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user