mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Two local CI runs on one host both asked docker for 172.20.0.0/24 and the second lost its whole static family to "Pool overlaps". Docker honours a fixed subnet request verbatim, so the only robust fix is to stop making one: fips-net now requests no subnet and docker assigns from its own pool, which cannot hand the same range to two runs. That means node addresses are not known before `up`, so peers address each other by container hostname instead. The generator emits node-<id>, or the topology's docker_host where the compose hostname differs — only the gateway profile, whose services are gw-*. External peers keep the address the topology gives them, since it is not ours to assign. The resolv.conf mount stays: dnsmasq is what forwards these names to docker's resolver and .fips to the daemon, so removing it would take out every .fips assertion. generated-configs is now per-run as well. A shared directory let two runs overwrite each other's node configs, which the subnet collision had been hiding by killing runs before that window opened. The generator, the compose bind mounts and env_file, the six scripts that read it, and teardown all follow FIPS_CI_NAME_SUFFIX; unset, every path renders as before. Teardown keeps the directory after a failed run, where it is the evidence of what the failing nodes were configured with. Three things this exposed that were wrong independently: admission-cap built its tcpdump patterns from the topology file's docker_ip literals. Floating the subnet makes those match nothing, which would have left its expect-zero "no Msg2 leaked" assertion passing because it could no longer see anything at all. It now reads addresses from the running containers. Restarting the denied peers together also made them swap addresses, so each peer's counts were really the pair's total; they are restarted one at a time now, and a check fails the suite outright if two denied peers ever share an address, because per-peer attribution is impossible once they do. Attribute lookups in the generator used a fixed ten-line window and read the next node's fields when a node omitted an attribute. An external node followed by an internal one was classified as internal, which under hostname peering would emit a name that resolves nowhere. Lookups are bounded to the node's own block; generated output is byte-identical for all eight topologies. The rekey outbound-only variant used to rewrite peer addresses to hostnames to set up its scenario. The generator now does that everywhere, so the rewrite matched nothing and was silently doing no work. It asserts the premise instead, and fails if a numeric address ever reappears. Verified by running three instances of this compose at once — tcp-chain plus two independent meshes — which drew 10.128.2/3/4.0/24 with no overlap while both meshes passed ping-test 20/20 over the real .fips path. tcp-chain is run by neither CI runner, so it was checked by hand: chain peer counts 1/2/1 and multi-hop .fips reachable both directions over TCP. gateway-lan still pins its own IPv4 and fd02:: ranges and is unchanged here, so the gateway profile is not yet concurrency-safe.
163 lines
5.2 KiB
Bash
Executable File
163 lines
5.2 KiB
Bash
Executable File
#!/bin/bash
|
||
# End-to-end ping test between FIPS nodes via DNS resolution.
|
||
# Usage: ./ping-test.sh [mesh|chain]
|
||
#
|
||
# Requires containers to be running:
|
||
# docker compose --profile mesh up -d
|
||
# ./scripts/ping-test.sh mesh
|
||
set -e
|
||
|
||
# Exit entire script on Ctrl+C
|
||
trap 'echo ""; echo "Test interrupted"; exit 130' INT
|
||
|
||
PROFILE="${1:-mesh}"
|
||
COUNT=1
|
||
TIMEOUT=5
|
||
PASSED=0
|
||
FAILED=0
|
||
|
||
# Node identities (from generated env file)
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
|
||
ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env"
|
||
if [ ! -f "$ENV_FILE" ]; then
|
||
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
|
||
exit 1
|
||
fi
|
||
# shellcheck source=../generated-configs/npubs.env
|
||
source "$ENV_FILE"
|
||
|
||
NPUBS=("$NPUB_A" "$NPUB_B" "$NPUB_C" "$NPUB_D" "$NPUB_E")
|
||
LABELS=(A B C D E)
|
||
|
||
ping_test() {
|
||
local from="$1"
|
||
local to_npub="$2"
|
||
local label="$3"
|
||
|
||
echo -n " $label ... "
|
||
local output
|
||
if output=$(docker exec "fips-${from}${FIPS_CI_NAME_SUFFIX:-}" ping6 -c "$COUNT" -W "$TIMEOUT" "${to_npub}.fips" 2>&1); then
|
||
# Extract round-trip time from ping output
|
||
local rtt
|
||
rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2)
|
||
if [ -n "$rtt" ]; then
|
||
echo "OK (${rtt}ms)"
|
||
else
|
||
echo "OK"
|
||
fi
|
||
PASSED=$((PASSED + 1))
|
||
else
|
||
echo "FAIL"
|
||
FAILED=$((FAILED + 1))
|
||
fi
|
||
}
|
||
|
||
# Quietly ping all pairs to check FSP-level convergence.
|
||
ping_all_quiet() {
|
||
PASSED=0
|
||
FAILED=0
|
||
local n=${#LABELS[@]}
|
||
for ((i=0; i<n; i++)); do
|
||
for ((j=0; j<n; j++)); do
|
||
[ "$i" -eq "$j" ] && continue
|
||
if docker exec "fips-node-${LABELS[$i],,}${FIPS_CI_NAME_SUFFIX:-}" \
|
||
ping6 -c 1 -W 1 "${NPUBS[$j]}.fips" >/dev/null 2>&1; then
|
||
PASSED=$((PASSED + 1))
|
||
else
|
||
FAILED=$((FAILED + 1))
|
||
fi
|
||
done
|
||
done
|
||
}
|
||
|
||
echo "=== FIPS Ping Test ($PROFILE topology) ==="
|
||
echo ""
|
||
|
||
# Wait for nodes to converge — all nodes must reach expected peer counts.
|
||
echo "Waiting for mesh convergence..."
|
||
if [ "$PROFILE" = "chain" ]; then
|
||
# Chain: A-B-C-D-E, each interior node has 2 peers, endpoints have 1
|
||
wait_for_peers fips-node-a${FIPS_CI_NAME_SUFFIX:-} 1 20 || true
|
||
wait_for_peers fips-node-b${FIPS_CI_NAME_SUFFIX:-} 2 20 || true
|
||
wait_for_peers fips-node-c${FIPS_CI_NAME_SUFFIX:-} 2 20 || true
|
||
wait_for_peers fips-node-d${FIPS_CI_NAME_SUFFIX:-} 2 20 || true
|
||
wait_for_peers fips-node-e${FIPS_CI_NAME_SUFFIX:-} 1 20 || true
|
||
elif [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
|
||
# Mesh: check all nodes reach their configured peer counts
|
||
wait_for_peers fips-node-a${FIPS_CI_NAME_SUFFIX:-} 2 20 || true
|
||
wait_for_peers fips-node-b${FIPS_CI_NAME_SUFFIX:-} 1 20 || true
|
||
wait_for_peers fips-node-c${FIPS_CI_NAME_SUFFIX:-} 3 20 || true
|
||
wait_for_peers fips-node-d${FIPS_CI_NAME_SUFFIX:-} 3 20 || true
|
||
wait_for_peers fips-node-e${FIPS_CI_NAME_SUFFIX:-} 3 20 || true
|
||
fi
|
||
# Wait for full pairwise connectivity, progress-aware: the actual pings
|
||
# are the convergence signal, the deadline extends while more pairs come
|
||
# up, and it only gives up if progress stalls — so it does not
|
||
# false-time-out under load while routing is still converging. The
|
||
# directed-pair ping assertions below remain the actual test.
|
||
wait_until_connected ping_all_quiet 45 15 || true
|
||
|
||
# Reset counters for the actual test
|
||
PASSED=0
|
||
FAILED=0
|
||
|
||
if [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
|
||
# Sparse mesh topology: A-B, B-C, C-D, D-E, E-A, A-D
|
||
# Test all 20 directed pairs (5 nodes × 4 targets each)
|
||
echo ""
|
||
echo "From node-a:"
|
||
ping_test node-a "$NPUB_B" "A → B"
|
||
ping_test node-a "$NPUB_C" "A → C"
|
||
ping_test node-a "$NPUB_D" "A → D"
|
||
ping_test node-a "$NPUB_E" "A → E"
|
||
|
||
echo ""
|
||
echo "From node-b:"
|
||
ping_test node-b "$NPUB_A" "B → A"
|
||
ping_test node-b "$NPUB_C" "B → C"
|
||
ping_test node-b "$NPUB_D" "B → D"
|
||
ping_test node-b "$NPUB_E" "B → E"
|
||
|
||
echo ""
|
||
echo "From node-c:"
|
||
ping_test node-c "$NPUB_A" "C → A"
|
||
ping_test node-c "$NPUB_B" "C → B"
|
||
ping_test node-c "$NPUB_D" "C → D"
|
||
ping_test node-c "$NPUB_E" "C → E"
|
||
|
||
echo ""
|
||
echo "From node-d:"
|
||
ping_test node-d "$NPUB_A" "D → A"
|
||
ping_test node-d "$NPUB_B" "D → B"
|
||
ping_test node-d "$NPUB_C" "D → C"
|
||
ping_test node-d "$NPUB_E" "D → E"
|
||
|
||
echo ""
|
||
echo "From node-e:"
|
||
ping_test node-e "$NPUB_A" "E → A"
|
||
ping_test node-e "$NPUB_B" "E → B"
|
||
ping_test node-e "$NPUB_C" "E → C"
|
||
ping_test node-e "$NPUB_D" "E → D"
|
||
|
||
elif [ "$PROFILE" = "chain" ]; then
|
||
echo ""
|
||
echo "Adjacent peer tests:"
|
||
ping_test node-a "$NPUB_B" "A → B (1 hop)"
|
||
ping_test node-b "$NPUB_C" "B → C (1 hop)"
|
||
|
||
echo ""
|
||
echo "Multi-hop tests:"
|
||
ping_test node-a "$NPUB_C" "A → C (2 hops)"
|
||
ping_test node-a "$NPUB_D" "A → D (3 hops)"
|
||
ping_test node-a "$NPUB_E" "A → E (4 hops)"
|
||
|
||
echo ""
|
||
echo "Reverse multi-hop:"
|
||
ping_test node-e "$NPUB_A" "E → A (4 hops)"
|
||
fi
|
||
|
||
echo ""
|
||
echo "=== Results: $PASSED passed, $FAILED failed ==="
|
||
[ "$FAILED" -eq 0 ] && exit 0 || exit 1
|