mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Stop concurrent local CI runs from clobbering each other's containers
Two runs on one host destroyed each other's containers, producing mid-test "No such container" failures that look like real defects. The automated builder runs a full local CI on the same box every few minutes, so the machine is contended almost always and this has red-ed both a hand run and an automated gate. Two independent causes are fixed here. Container names were hardcoded, and docker names are global rather than scoped by compose project, so two runs collided on the same name; every name now takes an optional suffix that the harness sets from the run id. And the cleanup sweep matched a label shared by every run, so one run's teardown force-removed another's containers; resources now also carry a per-run label and the sweep can be narrowed to it. A third hazard turned up that was not in the original report: the sidecar suite passes explicit compose project names, which override the run-scoped project and put it outside the shared prefix entirely. Its project names, network, and derived container references are now scoped too. Both are default-off. With the suffix unset, names render exactly as they do today and a bare compose invocation is unchanged, which is what keeps the hosted CI and the documentation correct. A cleanup run with no run id still reaps everything, which is what a manual "clear the box" wants. The literal-name sweep was not sufficient: six scripts build container names dynamically from node labels, and two suites create their own containers outside compose. Those are handled at their construction sites. Verified: syntax check on all modified scripts; compose validation on every modified file with the suffix both set and unset; and a synthetic two-run reproduction that shows the old cleanup destroying a bystander run and the new one leaving it alone. Known gap: subnets are still hardcoded, so two concurrent full runs will still collide on address-pool overlap. That fix reaches into topology configs, chaos scenarios, diagrams and production source, so it is left for its own change rather than half-done here.
This commit is contained in:
Johnathan Corgan
@@ -41,7 +41,7 @@ services:
|
||||
node-a:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh"]
|
||||
container_name: fips-node-a
|
||||
container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-a
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -53,7 +53,7 @@ services:
|
||||
node-b:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh"]
|
||||
container_name: fips-node-b
|
||||
container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-b
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -65,7 +65,7 @@ services:
|
||||
node-c:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh"]
|
||||
container_name: fips-node-c
|
||||
container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-c
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -77,7 +77,7 @@ services:
|
||||
node-d:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh"]
|
||||
container_name: fips-node-d
|
||||
container_name: fips-node-d${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-d
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -89,7 +89,7 @@ services:
|
||||
node-e:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh"]
|
||||
container_name: fips-node-e
|
||||
container_name: fips-node-e${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-e
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -102,7 +102,7 @@ services:
|
||||
pub-a:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh-public"]
|
||||
container_name: fips-node-a
|
||||
container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-a
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -114,7 +114,7 @@ services:
|
||||
pub-b:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh-public"]
|
||||
container_name: fips-node-b
|
||||
container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-b
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -126,7 +126,7 @@ services:
|
||||
pub-c:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh-public"]
|
||||
container_name: fips-node-c
|
||||
container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-c
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -138,7 +138,7 @@ services:
|
||||
pub-d:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh-public"]
|
||||
container_name: fips-node-d
|
||||
container_name: fips-node-d${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-d
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -150,7 +150,7 @@ services:
|
||||
pub-e:
|
||||
<<: *fips-common
|
||||
profiles: ["mesh-public"]
|
||||
container_name: fips-node-e
|
||||
container_name: fips-node-e${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-e
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -163,7 +163,7 @@ services:
|
||||
chain-a:
|
||||
<<: *fips-common
|
||||
profiles: ["chain"]
|
||||
container_name: fips-node-a
|
||||
container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-a
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -175,7 +175,7 @@ services:
|
||||
chain-b:
|
||||
<<: *fips-common
|
||||
profiles: ["chain"]
|
||||
container_name: fips-node-b
|
||||
container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-b
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -187,7 +187,7 @@ services:
|
||||
chain-c:
|
||||
<<: *fips-common
|
||||
profiles: ["chain"]
|
||||
container_name: fips-node-c
|
||||
container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-c
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -199,7 +199,7 @@ services:
|
||||
chain-d:
|
||||
<<: *fips-common
|
||||
profiles: ["chain"]
|
||||
container_name: fips-node-d
|
||||
container_name: fips-node-d${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-d
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -211,7 +211,7 @@ services:
|
||||
chain-e:
|
||||
<<: *fips-common
|
||||
profiles: ["chain"]
|
||||
container_name: fips-node-e
|
||||
container_name: fips-node-e${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-e
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -224,7 +224,7 @@ services:
|
||||
rekey-a:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey"]
|
||||
container_name: fips-node-a
|
||||
container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-a
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug
|
||||
@@ -238,7 +238,7 @@ services:
|
||||
rekey-b:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey"]
|
||||
container_name: fips-node-b
|
||||
container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-b
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug
|
||||
@@ -252,7 +252,7 @@ services:
|
||||
rekey-c:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey"]
|
||||
container_name: fips-node-c
|
||||
container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-c
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug
|
||||
@@ -266,7 +266,7 @@ services:
|
||||
rekey-d:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey"]
|
||||
container_name: fips-node-d
|
||||
container_name: fips-node-d${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-d
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug
|
||||
@@ -280,7 +280,7 @@ services:
|
||||
rekey-e:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey"]
|
||||
container_name: fips-node-e
|
||||
container_name: fips-node-e${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-e
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug
|
||||
@@ -299,7 +299,7 @@ services:
|
||||
rekey-accept-off-a:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-accept-off"]
|
||||
container_name: fips-node-a
|
||||
container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-a
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -313,7 +313,7 @@ services:
|
||||
rekey-accept-off-b:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-accept-off"]
|
||||
container_name: fips-node-b
|
||||
container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-b
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -327,7 +327,7 @@ services:
|
||||
rekey-accept-off-c:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-accept-off"]
|
||||
container_name: fips-node-c
|
||||
container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-c
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -341,7 +341,7 @@ services:
|
||||
rekey-accept-off-d:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-accept-off"]
|
||||
container_name: fips-node-d
|
||||
container_name: fips-node-d${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-d
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -355,7 +355,7 @@ services:
|
||||
rekey-accept-off-e:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-accept-off"]
|
||||
container_name: fips-node-e
|
||||
container_name: fips-node-e${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-e
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -379,7 +379,7 @@ services:
|
||||
rekey-outbound-only-a:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-outbound-only"]
|
||||
container_name: fips-node-a
|
||||
container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-a
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -393,7 +393,7 @@ services:
|
||||
rekey-outbound-only-b:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-outbound-only"]
|
||||
container_name: fips-node-b
|
||||
container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-b
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -407,7 +407,7 @@ services:
|
||||
rekey-outbound-only-c:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-outbound-only"]
|
||||
container_name: fips-node-c
|
||||
container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-c
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -421,7 +421,7 @@ services:
|
||||
rekey-outbound-only-d:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-outbound-only"]
|
||||
container_name: fips-node-d
|
||||
container_name: fips-node-d${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-d
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -435,7 +435,7 @@ services:
|
||||
rekey-outbound-only-e:
|
||||
<<: *fips-common
|
||||
profiles: ["rekey-outbound-only"]
|
||||
container_name: fips-node-e
|
||||
container_name: fips-node-e${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-e
|
||||
environment:
|
||||
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
|
||||
@@ -450,7 +450,7 @@ services:
|
||||
tcp-a:
|
||||
<<: *fips-common
|
||||
profiles: ["tcp-chain"]
|
||||
container_name: fips-node-a
|
||||
container_name: fips-node-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-a
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -462,7 +462,7 @@ services:
|
||||
tcp-b:
|
||||
<<: *fips-common
|
||||
profiles: ["tcp-chain"]
|
||||
container_name: fips-node-b
|
||||
container_name: fips-node-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-b
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -474,7 +474,7 @@ services:
|
||||
tcp-c:
|
||||
<<: *fips-common
|
||||
profiles: ["tcp-chain"]
|
||||
container_name: fips-node-c
|
||||
container_name: fips-node-c${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: node-c
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -487,7 +487,7 @@ services:
|
||||
gw-gateway:
|
||||
<<: *fips-common
|
||||
profiles: ["gateway"]
|
||||
container_name: fips-gw-gateway
|
||||
container_name: fips-gw-gateway${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: gw-gateway
|
||||
# Privileged required: gateway must enable IPv6 on eth1 (second network,
|
||||
# attached after container start) and manage nftables NAT rules.
|
||||
@@ -513,7 +513,7 @@ services:
|
||||
gw-server:
|
||||
<<: *fips-common
|
||||
profiles: ["gateway"]
|
||||
container_name: fips-gw-server
|
||||
container_name: fips-gw-server${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: gw-server
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -528,7 +528,7 @@ services:
|
||||
gw-server-2:
|
||||
<<: *fips-common
|
||||
profiles: ["gateway"]
|
||||
container_name: fips-gw-server-2
|
||||
container_name: fips-gw-server-2${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: gw-server-2
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
@@ -540,7 +540,7 @@ services:
|
||||
gw-client:
|
||||
image: ${FIPS_TEST_APP_IMAGE:-fips-test-app:latest}
|
||||
profiles: ["gateway"]
|
||||
container_name: fips-gw-client
|
||||
container_name: fips-gw-client${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: gw-client
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
@@ -562,7 +562,7 @@ services:
|
||||
gw-client-2:
|
||||
image: ${FIPS_TEST_APP_IMAGE:-fips-test-app:latest}
|
||||
profiles: ["gateway"]
|
||||
container_name: fips-gw-client-2
|
||||
container_name: fips-gw-client-2${FIPS_CI_NAME_SUFFIX:-}
|
||||
hostname: gw-client-2
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
|
||||
@@ -92,7 +92,7 @@ info "phase 1: wait for node-$CAP_NODE peer_count to reach $MAX_PEERS (90s timeo
|
||||
deadline=$(($(date +%s) + 90))
|
||||
pc=0
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
pc=$(docker exec fips-node-$CAP_NODE fipsctl show status 2>/dev/null \
|
||||
pc=$(docker exec "fips-node-${CAP_NODE}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show status 2>/dev/null \
|
||||
| grep -m1 peer_count | sed 's/.*: *//' | tr -d ',' || echo 0)
|
||||
[ "$pc" = "$MAX_PEERS" ] && break
|
||||
sleep 2
|
||||
@@ -102,7 +102,7 @@ done
|
||||
info "node-$CAP_NODE converged: peer_count=$pc"
|
||||
|
||||
# Identify admitted vs denied peers among configured peers
|
||||
ADMITTED_NPUBS=$(docker exec fips-node-$CAP_NODE fipsctl show peers 2>/dev/null \
|
||||
ADMITTED_NPUBS=$(docker exec "fips-node-${CAP_NODE}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show peers 2>/dev/null \
|
||||
| grep -oE 'npub1[a-z0-9]+' | sort -u || true)
|
||||
DENIED=""
|
||||
ADMITTED=""
|
||||
@@ -130,8 +130,8 @@ info "denied (sustained-retry): ${DENIED:-<none>}"
|
||||
# with restarts every 15s we get ~30-50 firings across both denied peers.
|
||||
info "phase 2: capture UDP/2121 on node-$CAP_NODE for ${CAPTURE_SECS}s, with denied-peer restart loop"
|
||||
CAP_FILE=$(mktemp /tmp/admission-cap-pcap.XXXXXX.txt)
|
||||
HELPER_IMAGE=$(docker inspect -f '{{.Config.Image}}' fips-node-$CAP_NODE 2>/dev/null)
|
||||
[ -n "$HELPER_IMAGE" ] || fail "could not resolve helper image from fips-node-$CAP_NODE"
|
||||
HELPER_IMAGE=$(docker inspect -f '{{.Config.Image}}' "fips-node-${CAP_NODE}${FIPS_CI_NAME_SUFFIX:-}" 2>/dev/null)
|
||||
[ -n "$HELPER_IMAGE" ] || fail "could not resolve helper image from fips-node-${CAP_NODE}${FIPS_CI_NAME_SUFFIX:-}"
|
||||
|
||||
# Background: cycle denied peers to reset their backoff and drive load.
|
||||
(
|
||||
@@ -140,7 +140,7 @@ HELPER_IMAGE=$(docker inspect -f '{{.Config.Image}}' fips-node-$CAP_NODE 2>/dev/
|
||||
sleep 15
|
||||
elapsed=$((elapsed + 15))
|
||||
for n in $DENIED; do
|
||||
docker restart "fips-node-$n" >/dev/null 2>&1 &
|
||||
docker restart "fips-node-${n}${FIPS_CI_NAME_SUFFIX:-}" >/dev/null 2>&1 &
|
||||
done
|
||||
wait
|
||||
info " [load-driver] restarted denied peers ($DENIED) at t+${elapsed}s"
|
||||
@@ -149,7 +149,7 @@ HELPER_IMAGE=$(docker inspect -f '{{.Config.Image}}' fips-node-$CAP_NODE 2>/dev/
|
||||
LOAD_PID=$!
|
||||
|
||||
# Foreground: tcpdump capture for CAPTURE_SECS
|
||||
docker run --rm --label com.corganlabs.fips-ci=1 --net=container:fips-node-$CAP_NODE \
|
||||
docker run --rm --label com.corganlabs.fips-ci=1 --label "com.corganlabs.fips-ci.run=${FIPS_CI_RUN_ID:-manual}" --net=container:"fips-node-${CAP_NODE}${FIPS_CI_NAME_SUFFIX:-}" \
|
||||
--cap-add NET_ADMIN --cap-add NET_RAW \
|
||||
--entrypoint sh "$HELPER_IMAGE" \
|
||||
-c "timeout $CAPTURE_SECS tcpdump -nn -i any 'udp port 2121' -l 2>&1 || true" \
|
||||
@@ -186,7 +186,7 @@ for n in $DENIED; do
|
||||
done
|
||||
|
||||
# ── Phase 4: cap'd node still at exactly max_peers ───────────────────
|
||||
pc_final=$(docker exec fips-node-$CAP_NODE fipsctl show status 2>/dev/null \
|
||||
pc_final=$(docker exec "fips-node-${CAP_NODE}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show status 2>/dev/null \
|
||||
| grep -m1 peer_count | sed 's/.*: *//' | tr -d ',' || echo 0)
|
||||
info "node-$CAP_NODE final peer_count=$pc_final (expected $MAX_PEERS)"
|
||||
[ "$pc_final" = "$MAX_PEERS" ] || OVERALL=1
|
||||
|
||||
@@ -47,7 +47,7 @@ source "$ENV_FILE"
|
||||
iperf_once() {
|
||||
local client="$1" dest_npub="$2"
|
||||
local out
|
||||
if ! out=$(docker exec "fips-$client" iperf3 -c "${dest_npub}.fips" \
|
||||
if ! out=$(docker exec "fips-${client}${FIPS_CI_NAME_SUFFIX:-}" iperf3 -c "${dest_npub}.fips" \
|
||||
-t "$DURATION" -P "$PARALLEL" -J 2>&1); then
|
||||
echo FAIL
|
||||
return
|
||||
@@ -105,7 +105,7 @@ PYEOF
|
||||
ping_path() {
|
||||
local client="$1" dest_npub="$2"
|
||||
local out
|
||||
if ! out=$(docker exec "fips-$client" \
|
||||
if ! out=$(docker exec "fips-${client}${FIPS_CI_NAME_SUFFIX:-}" \
|
||||
ping -c "$PING_COUNT" -i 0.2 -w "$((PING_COUNT * 2))" \
|
||||
-q "${dest_npub}.fips" 2>&1); then
|
||||
echo FAIL
|
||||
@@ -203,7 +203,7 @@ CONVERGE_SECS="${FIPS_BENCH_CONVERGE_SECS:-15}"
|
||||
|
||||
peer_is_direct() {
|
||||
local client="$1" dest_npub="$2"
|
||||
docker exec "fips-$client" fipsctl show peers 2>/dev/null \
|
||||
docker exec "fips-${client}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show peers 2>/dev/null \
|
||||
| python3 -c '
|
||||
import json, sys
|
||||
target = sys.argv[1]
|
||||
@@ -225,7 +225,7 @@ except Exception:
|
||||
# multihop alternative).
|
||||
peer_bytes_sent_snapshot() {
|
||||
local client="$1"
|
||||
docker exec "fips-$client" fipsctl show peers 2>/dev/null \
|
||||
docker exec "fips-${client}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show peers 2>/dev/null \
|
||||
| python3 -c '
|
||||
import json, sys
|
||||
try:
|
||||
|
||||
@@ -20,11 +20,11 @@ source "$SCRIPT_DIR/../../lib/wait-converge.sh"
|
||||
GENERATED_DIR="$SCRIPT_DIR/../generated-configs"
|
||||
ENV_FILE="$GENERATED_DIR/npubs.env"
|
||||
|
||||
GATEWAY="fips-gw-gateway"
|
||||
SERVER="fips-gw-server"
|
||||
SERVER2="fips-gw-server-2"
|
||||
CLIENT="fips-gw-client"
|
||||
CLIENT2="fips-gw-client-2"
|
||||
GATEWAY="fips-gw-gateway${FIPS_CI_NAME_SUFFIX:-}"
|
||||
SERVER="fips-gw-server${FIPS_CI_NAME_SUFFIX:-}"
|
||||
SERVER2="fips-gw-server-2${FIPS_CI_NAME_SUFFIX:-}"
|
||||
CLIENT="fips-gw-client${FIPS_CI_NAME_SUFFIX:-}"
|
||||
CLIENT2="fips-gw-client-2${FIPS_CI_NAME_SUFFIX:-}"
|
||||
|
||||
# ── inject-config subcommand ─────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ iperf_test() {
|
||||
if [ "$LIVE_OUTPUT" = true ]; then
|
||||
# Show live output
|
||||
echo "Running iperf3 test (live output):"
|
||||
if docker exec "fips-$client_node" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL"; then
|
||||
if docker exec "fips-${client_node}${FIPS_CI_NAME_SUFFIX:-}" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL"; then
|
||||
PASSED=$((PASSED + 1))
|
||||
else
|
||||
echo "FAIL"
|
||||
@@ -59,7 +59,7 @@ iperf_test() {
|
||||
# Capture and summarize output
|
||||
echo -n "Running iperf3 test... "
|
||||
local output
|
||||
if output=$(docker exec "fips-$client_node" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL" 2>&1); then
|
||||
if output=$(docker exec "fips-${client_node}${FIPS_CI_NAME_SUFFIX:-}" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL" 2>&1); then
|
||||
# Check if we got valid results
|
||||
if echo "$output" | grep -q "sender"; then
|
||||
# Extract and display results (get SUM line for aggregate bandwidth)
|
||||
|
||||
@@ -180,7 +180,7 @@ do_apply() {
|
||||
echo ""
|
||||
|
||||
for node in $NODES; do
|
||||
local container="fips-node-$node"
|
||||
local container="fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}"
|
||||
echo -n " $container ... "
|
||||
if ! container_running "$container"; then
|
||||
echo "SKIP (not running)"
|
||||
@@ -199,7 +199,7 @@ do_remove() {
|
||||
echo ""
|
||||
|
||||
for node in $NODES; do
|
||||
local container="fips-node-$node"
|
||||
local container="fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}"
|
||||
echo -n " $container ... "
|
||||
if ! container_running "$container"; then
|
||||
echo "SKIP (not running)"
|
||||
@@ -219,7 +219,7 @@ do_status() {
|
||||
echo ""
|
||||
|
||||
for node in $NODES; do
|
||||
local container="fips-node-$node"
|
||||
local container="fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}"
|
||||
echo " $container:"
|
||||
if ! container_running "$container"; then
|
||||
echo " (not running)"
|
||||
|
||||
@@ -37,7 +37,7 @@ ping_test() {
|
||||
|
||||
echo -n " $label ... "
|
||||
local output
|
||||
if output=$(docker exec "fips-$from" ping6 -c "$COUNT" -W "$TIMEOUT" "${to_npub}.fips" 2>&1); then
|
||||
if output=$(docker exec "fips-${from}${FIPS_CI_NAME_SUFFIX:-}" ping6 -c "$COUNT" -W "$TIMEOUT" "${to_npub}.fips" 2>&1); then
|
||||
# Extract round-trip time from ping output
|
||||
local rtt
|
||||
rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2)
|
||||
@@ -61,7 +61,7 @@ ping_all_quiet() {
|
||||
for ((i=0; i<n; i++)); do
|
||||
for ((j=0; j<n; j++)); do
|
||||
[ "$i" -eq "$j" ] && continue
|
||||
if docker exec "fips-node-${LABELS[$i],,}" \
|
||||
if docker exec "fips-node-${LABELS[$i],,}${FIPS_CI_NAME_SUFFIX:-}" \
|
||||
ping6 -c 1 -W 1 "${NPUBS[$j]}.fips" >/dev/null 2>&1; then
|
||||
PASSED=$((PASSED + 1))
|
||||
else
|
||||
@@ -78,18 +78,18 @@ echo ""
|
||||
echo "Waiting for mesh convergence..."
|
||||
if [ "$PROFILE" = "chain" ]; then
|
||||
# Chain: A-B-C-D-E, each interior node has 2 peers, endpoints have 1
|
||||
wait_for_peers fips-node-a 1 20 || true
|
||||
wait_for_peers fips-node-b 2 20 || true
|
||||
wait_for_peers fips-node-c 2 20 || true
|
||||
wait_for_peers fips-node-d 2 20 || true
|
||||
wait_for_peers fips-node-e 1 20 || true
|
||||
wait_for_peers fips-node-a${FIPS_CI_NAME_SUFFIX:-} 1 20 || true
|
||||
wait_for_peers fips-node-b${FIPS_CI_NAME_SUFFIX:-} 2 20 || true
|
||||
wait_for_peers fips-node-c${FIPS_CI_NAME_SUFFIX:-} 2 20 || true
|
||||
wait_for_peers fips-node-d${FIPS_CI_NAME_SUFFIX:-} 2 20 || true
|
||||
wait_for_peers fips-node-e${FIPS_CI_NAME_SUFFIX:-} 1 20 || true
|
||||
elif [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
|
||||
# Mesh: check all nodes reach their configured peer counts
|
||||
wait_for_peers fips-node-a 2 20 || true
|
||||
wait_for_peers fips-node-b 1 20 || true
|
||||
wait_for_peers fips-node-c 3 20 || true
|
||||
wait_for_peers fips-node-d 3 20 || true
|
||||
wait_for_peers fips-node-e 3 20 || true
|
||||
wait_for_peers fips-node-a${FIPS_CI_NAME_SUFFIX:-} 2 20 || true
|
||||
wait_for_peers fips-node-b${FIPS_CI_NAME_SUFFIX:-} 1 20 || true
|
||||
wait_for_peers fips-node-c${FIPS_CI_NAME_SUFFIX:-} 3 20 || true
|
||||
wait_for_peers fips-node-d${FIPS_CI_NAME_SUFFIX:-} 3 20 || true
|
||||
wait_for_peers fips-node-e${FIPS_CI_NAME_SUFFIX:-} 3 20 || true
|
||||
fi
|
||||
# Wait for full pairwise connectivity, progress-aware: the actual pings
|
||||
# are the convergence signal, the deadline extends while more pairs come
|
||||
|
||||
@@ -223,7 +223,7 @@ ping_one() {
|
||||
if (( attempt > 1 )); then
|
||||
sleep "$PING_RETRY_DELAY"
|
||||
fi
|
||||
if output=$(docker exec "fips-$from" ping6 -c 1 -W "$ping_timeout" "${to_npub}.fips" 2>&1); then
|
||||
if output=$(docker exec "fips-${from}${FIPS_CI_NAME_SUFFIX:-}" ping6 -c 1 -W "$ping_timeout" "${to_npub}.fips" 2>&1); then
|
||||
rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2)
|
||||
if [ -z "$quiet" ]; then
|
||||
if (( attempt == 1 )); then
|
||||
@@ -289,7 +289,7 @@ count_log_pattern() {
|
||||
local pattern="$1"
|
||||
local total=0
|
||||
for node in $NODES; do
|
||||
local count=$(docker logs "fips-node-$node" 2>&1 | grep -c "$pattern" || true)
|
||||
local count=$(docker logs "fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" 2>&1 | grep -c "$pattern" || true)
|
||||
total=$((total + count))
|
||||
done
|
||||
echo "$total"
|
||||
@@ -352,7 +352,7 @@ dump_peer_connectivity() {
|
||||
echo "=== Peer connectivity snapshot ==="
|
||||
for node in $NODES; do
|
||||
echo "--- node-$node ---"
|
||||
docker exec "fips-node-$node" fipsctl show peers 2>/dev/null || true
|
||||
docker exec "fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show peers 2>/dev/null || true
|
||||
echo ""
|
||||
done
|
||||
}
|
||||
@@ -504,7 +504,7 @@ assert_zero_count "Session AEAD decryption failed" \
|
||||
if [ -n "$REKEY_ACCEPT_OFF_NODES" ]; then
|
||||
DUAL_INIT_THRESHOLD=10
|
||||
for off_node in ${REKEY_ACCEPT_OFF_NODES//,/ }; do
|
||||
count=$(docker logs "fips-node-$off_node" 2>&1 \
|
||||
count=$(docker logs "fips-node-${off_node}${FIPS_CI_NAME_SUFFIX:-}" 2>&1 \
|
||||
| grep -cE "Dual rekey initiation: we win" || true)
|
||||
if [ "${count:-0}" -le "$DUAL_INIT_THRESHOLD" ]; then
|
||||
echo " PASS: node-$off_node dual-init drops below threshold ($count <= $DUAL_INIT_THRESHOLD)"
|
||||
@@ -527,7 +527,7 @@ fi
|
||||
if [ -n "$REKEY_OUTBOUND_ONLY_NODES" ]; then
|
||||
DUAL_INIT_THRESHOLD=10
|
||||
for n in $NODES; do
|
||||
count=$(docker logs "fips-node-$n" 2>&1 \
|
||||
count=$(docker logs "fips-node-${n}${FIPS_CI_NAME_SUFFIX:-}" 2>&1 \
|
||||
| grep -cE "Dual rekey initiation: we win" || true)
|
||||
if [ "${count:-0}" -le "$DUAL_INIT_THRESHOLD" ]; then
|
||||
echo " PASS: node-$n dual-init drops below threshold ($count <= $DUAL_INIT_THRESHOLD)"
|
||||
@@ -562,7 +562,7 @@ else
|
||||
echo "=== Node logs (rekey-related, head -200) ==="
|
||||
for node in $NODES; do
|
||||
echo "--- node-$node ---"
|
||||
docker logs "fips-node-$node" 2>&1 | \
|
||||
docker logs "fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" 2>&1 | \
|
||||
grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit|no route|next hop|TTL exhausted|MTU exceeded|Congestion|decrypt|Decrypt|AEAD|Notify|drain|Drain|promot)" | \
|
||||
head -200
|
||||
echo ""
|
||||
@@ -571,7 +571,7 @@ else
|
||||
echo "=== Node logs (last 80 lines, unfiltered) ==="
|
||||
for node in $NODES; do
|
||||
echo "--- node-$node ---"
|
||||
docker logs "fips-node-$node" 2>&1 | tail -80
|
||||
docker logs "fips-node-${node}${FIPS_CI_NAME_SUFFIX:-}" 2>&1 | tail -80
|
||||
echo ""
|
||||
done
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user