mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 08:14:42 +00:00
testing: add admission-cap integration suite for inbound silent-drop gate
New integration scenario verifying the early-gate silent-drop behavior
of the inbound max_peers admission check at sustained scale, using the
existing 5-node mesh topology with one node's node.limits.max_peers
lowered to 1. This forces 2 of the cap'd node's 3 configured peers
into a sustained denied state, and asserts via tcpdump that no Msg2
responses go back to those denied peers across a 60s capture window.
A background load-driver restarts the denied peer containers every 15s
to reset their auto-reconnect exponential backoff (5s base / 300s cap),
producing fresh Msg1 bursts each cycle. Without this loop the gate
fires ~3-4 times per denied peer in a 60s window; with restarts the
observed rate is 15 per denied peer (~30 total firings), high enough
that any Msg2 leakage would be caught with strong statistical
confidence.
Local run on this branch: cap'd node-c converged to peer_count=1 with
node-b admitted; nodes d and e sustained-retried as denied; tcpdump
captured 30 inbound Msg1 (len 84) packets from the denied pair and 0
outbound Msg2 (len 104) packets, with final peer_count unchanged.
Files:
testing/static/scripts/admission-cap-test.sh — new test script with
inject-config subcommand (sets node.limits.max_peers) and a
3-phase test driver (converge, capture-with-load, per-peer assert)
testing/ci-local.sh — register admission-cap as a new suite category
(ADMISSION_SUITES), wire run_admission_cap function, add to
run_suite dispatch, list_suites, and the default integration sweep
Together with the existing unit-level coverage in src/node/tests/unit.rs
(handle_msg1_silent_drops_at_cap_for_new_peer with mock-transport Msg2
discriminator, and handle_msg1_admits_existing_peer_at_cap as the
bypass regression guard), the gate's silent-drop behavior is now
verified both at single-firing wire-observable resolution and at
sustained multi-firing cross-process scale.
This commit is contained in:
Johnathan Corgan
@@ -57,6 +57,7 @@ ONLY_SUITE=""
|
||||
# All integration suites matching ci.yml
|
||||
STATIC_SUITES=(static-mesh static-chain)
|
||||
REKEY_SUITES=(rekey rekey-accept-off rekey-outbound-only)
|
||||
ADMISSION_SUITES=(admission-cap)
|
||||
# Each entry: "display-name scenario [--flag value ...]"
|
||||
CHAOS_SUITES=(
|
||||
"smoke-10 smoke-10"
|
||||
@@ -111,6 +112,9 @@ list_suites() {
|
||||
echo " Rekey:"
|
||||
for s in "${REKEY_SUITES[@]}"; do echo " $s"; done
|
||||
echo ""
|
||||
echo " Admission cap:"
|
||||
for s in "${ADMISSION_SUITES[@]}"; do echo " $s"; done
|
||||
echo ""
|
||||
echo " Gateway:"
|
||||
for s in "${GATEWAY_SUITES[@]}"; do echo " $s"; done
|
||||
echo ""
|
||||
@@ -320,6 +324,34 @@ run_rekey() {
|
||||
record "rekey" $rc
|
||||
}
|
||||
|
||||
# Run the admission-cap integration test
|
||||
# Verifies the inbound max_peers early-gate silent-drops at scale by
|
||||
# lowering node.max_peers on one mesh node and asserting via tcpdump
|
||||
# that no Msg2 responses go to the sustained-retrying denied peers.
|
||||
run_admission_cap() {
|
||||
local compose="testing/static/docker-compose.yml"
|
||||
local rc=0
|
||||
|
||||
info "[admission-cap] Generating configs"
|
||||
bash testing/static/scripts/generate-configs.sh mesh || { record "admission-cap" 1; return; }
|
||||
bash testing/static/scripts/admission-cap-test.sh inject-config || { record "admission-cap" 1; return; }
|
||||
|
||||
info "[admission-cap] Starting containers (mesh profile)"
|
||||
docker compose -f "$compose" --profile mesh up -d || { record "admission-cap" 1; return; }
|
||||
|
||||
info "[admission-cap] Running admission-cap test"
|
||||
if bash testing/static/scripts/admission-cap-test.sh; then
|
||||
rc=0
|
||||
else
|
||||
rc=1
|
||||
info "[admission-cap] Collecting failure logs"
|
||||
docker compose -f "$compose" --profile mesh logs --no-color 2>&1 | tail -100
|
||||
fi
|
||||
|
||||
docker compose -f "$compose" --profile mesh down --volumes --remove-orphans 2>/dev/null
|
||||
record "admission-cap" $rc
|
||||
}
|
||||
|
||||
# Run a chaos scenario
|
||||
run_chaos() {
|
||||
local name="$1"
|
||||
@@ -568,6 +600,11 @@ run_integration() {
|
||||
run_rekey_accept_off
|
||||
run_rekey_outbound_only
|
||||
|
||||
# Admission cap (mesh profile, max_peers=1 on one node)
|
||||
for _suite in "${ADMISSION_SUITES[@]}"; do
|
||||
run_admission_cap
|
||||
done
|
||||
|
||||
# Gateway
|
||||
run_gateway
|
||||
|
||||
@@ -670,6 +707,8 @@ run_suite() {
|
||||
run_rekey_accept_off ;;
|
||||
rekey-outbound-only)
|
||||
run_rekey_outbound_only ;;
|
||||
admission-cap)
|
||||
run_admission_cap ;;
|
||||
gateway)
|
||||
run_gateway ;;
|
||||
acl-allowlist)
|
||||
|
||||
Executable
+205
@@ -0,0 +1,205 @@
|
||||
#!/bin/bash
|
||||
# Integration test for the inbound max_peers admission gate.
|
||||
#
|
||||
# Verifies the silent-drop behavior of the early-gate in handle_msg1 at
|
||||
# scale, using the mesh topology with one node's node.max_peers lowered
|
||||
# to 1. This forces 2 of node-c's 3 configured peers (b, d, e) into a
|
||||
# sustained denied state, and asserts via tcpdump that no Msg2 responses
|
||||
# go back to those denied peers across a 60s capture window.
|
||||
#
|
||||
# Tested behavior:
|
||||
# - Denied peers DO arrive at the cap'd node (inbound FMP-IK Msg1, 84 B)
|
||||
# - Cap'd node sends NO Msg2 responses (104 B) to denied peers
|
||||
# - Cap'd node maintains exactly max_peers active sessions
|
||||
# - Admitted peer's session stays healthy throughout the window
|
||||
#
|
||||
# Usage:
|
||||
# ./admission-cap-test.sh Run the test (containers must be up)
|
||||
# ./admission-cap-test.sh inject-config Inject node.max_peers into generated configs
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
CAP_NODE="${ADMISSION_CAP_NODE:-c}"
|
||||
MAX_PEERS="${ADMISSION_MAX_PEERS:-1}"
|
||||
CAPTURE_SECS="${ADMISSION_CAPTURE_SECS:-60}"
|
||||
TOPOLOGY="mesh"
|
||||
TOPO_FILE="$SCRIPT_DIR/../configs/topologies/$TOPOLOGY.yaml"
|
||||
|
||||
# ── inject-config subcommand ─────────────────────────────────────────
|
||||
# Inject node.max_peers into generated configs. Called separately by CI
|
||||
# before building Docker images.
|
||||
if [ "${1:-}" = "inject-config" ]; then
|
||||
echo "Injecting node.limits.max_peers: $MAX_PEERS into node-$CAP_NODE ($TOPOLOGY topology)..."
|
||||
cfg="$SCRIPT_DIR/../generated-configs/$TOPOLOGY/node-$CAP_NODE.yaml"
|
||||
if [ ! -f "$cfg" ]; then
|
||||
echo " Error: $cfg not found (run generate-configs.sh $TOPOLOGY first)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Insert under node.limits (the actual config path per src/config/node.rs).
|
||||
# Three cases: limits.max_peers already present (update), limits: present
|
||||
# without max_peers (append), or no limits block (insert full subtree).
|
||||
if grep -qE "^ max_peers:" "$cfg"; then
|
||||
sed -i -E "s/^ max_peers: *[0-9]+/ max_peers: $MAX_PEERS/" "$cfg"
|
||||
elif grep -qE "^ limits:" "$cfg"; then
|
||||
sed -i "/^ limits:/a\\
|
||||
max_peers: $MAX_PEERS" "$cfg"
|
||||
else
|
||||
sed -i "/^node:/a\\
|
||||
limits:\\
|
||||
max_peers: $MAX_PEERS" "$cfg"
|
||||
fi
|
||||
echo " node-$CAP_NODE limits block:"
|
||||
sed -n '/^ limits:/,/^ [a-z]/p' "$cfg" | head -5
|
||||
exit 0
|
||||
fi
|
||||
|
||||
stamp() { date '+%H:%M:%S'; }
|
||||
info() { echo "[$(stamp)] $*"; }
|
||||
fail() { echo "[$(stamp)] FAIL: $*"; exit 1; }
|
||||
pass() { echo "[$(stamp)] PASS: $*"; }
|
||||
|
||||
# Extract docker_ip for a node from the topology file
|
||||
node_ip() {
|
||||
grep -A 5 "^ $1:" "$TOPO_FILE" \
|
||||
| grep -m1 'docker_ip:' \
|
||||
| sed 's/.*: *"*\([^"]*\)".*/\1/'
|
||||
}
|
||||
|
||||
# Extract npub for a node from the topology file
|
||||
node_npub() {
|
||||
grep -A 5 "^ $1:" "$TOPO_FILE" \
|
||||
| grep -m1 'npub:' \
|
||||
| sed 's/.*: *"*\([^"]*\)".*/\1/'
|
||||
}
|
||||
|
||||
# Extract configured peers list for a node from the topology file
|
||||
node_peers() {
|
||||
grep -A 5 "^ $1:" "$TOPO_FILE" \
|
||||
| grep -m1 'peers:' \
|
||||
| sed 's/.*\[\(.*\)\].*/\1/' \
|
||||
| tr -d ' ' \
|
||||
| tr ',' ' '
|
||||
}
|
||||
|
||||
CAP_IP=$(node_ip "$CAP_NODE")
|
||||
[ -n "$CAP_IP" ] || fail "could not resolve docker_ip for node-$CAP_NODE in $TOPO_FILE"
|
||||
info "cap'd node: node-$CAP_NODE (ip $CAP_IP, max_peers=$MAX_PEERS)"
|
||||
|
||||
# ── Phase 1: wait for convergence ────────────────────────────────────
|
||||
info "phase 1: wait for node-$CAP_NODE peer_count to reach $MAX_PEERS (90s timeout)"
|
||||
deadline=$(($(date +%s) + 90))
|
||||
pc=0
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
pc=$(docker exec fips-node-$CAP_NODE fipsctl show status 2>/dev/null \
|
||||
| grep -m1 peer_count | sed 's/.*: *//' | tr -d ',' || echo 0)
|
||||
[ "$pc" = "$MAX_PEERS" ] && break
|
||||
sleep 2
|
||||
done
|
||||
[ "$pc" = "$MAX_PEERS" ] \
|
||||
|| fail "node-$CAP_NODE peer_count=$pc after 90s, expected $MAX_PEERS"
|
||||
info "node-$CAP_NODE converged: peer_count=$pc"
|
||||
|
||||
# Identify admitted vs denied peers among configured peers
|
||||
ADMITTED_NPUBS=$(docker exec fips-node-$CAP_NODE fipsctl show peers 2>/dev/null \
|
||||
| grep -oE 'npub1[a-z0-9]+' | sort -u || true)
|
||||
DENIED=""
|
||||
ADMITTED=""
|
||||
for p in $(node_peers "$CAP_NODE"); do
|
||||
npub=$(node_npub "$p")
|
||||
if echo "$ADMITTED_NPUBS" | grep -q "$npub"; then
|
||||
ADMITTED="$ADMITTED $p"
|
||||
else
|
||||
DENIED="$DENIED $p"
|
||||
fi
|
||||
done
|
||||
ADMITTED=$(echo $ADMITTED | xargs)
|
||||
DENIED=$(echo $DENIED | xargs)
|
||||
info "admitted: ${ADMITTED:-<none>}"
|
||||
info "denied (sustained-retry): ${DENIED:-<none>}"
|
||||
[ -n "$DENIED" ] \
|
||||
|| fail "no denied peers — test setup wrong (cap=$MAX_PEERS too high vs configured peers)"
|
||||
|
||||
# ── Phase 2: capture wire traffic for CAPTURE_SECS seconds ───────────
|
||||
# Drives sustained load by restarting denied peer containers on a cadence
|
||||
# during the capture window. Each restart resets the auto-reconnect
|
||||
# exponential backoff (5s base / 300s cap), producing a fresh burst of
|
||||
# Msg1s that exercises the silent-drop gate at meaningful rate. Without
|
||||
# this loop the gate fires ~3-4 times per denied peer in a 60s window;
|
||||
# with restarts every 15s we get ~30-50 firings across both denied peers.
|
||||
info "phase 2: capture UDP/2121 on node-$CAP_NODE for ${CAPTURE_SECS}s, with denied-peer restart loop"
|
||||
CAP_FILE=$(mktemp /tmp/admission-cap-pcap.XXXXXX.txt)
|
||||
HELPER_IMAGE=$(docker inspect -f '{{.Config.Image}}' fips-node-$CAP_NODE 2>/dev/null)
|
||||
[ -n "$HELPER_IMAGE" ] || fail "could not resolve helper image from fips-node-$CAP_NODE"
|
||||
|
||||
# Background: cycle denied peers to reset their backoff and drive load.
|
||||
(
|
||||
elapsed=0
|
||||
while [ $elapsed -lt $((CAPTURE_SECS - 5)) ]; do
|
||||
sleep 15
|
||||
elapsed=$((elapsed + 15))
|
||||
for n in $DENIED; do
|
||||
docker restart "fips-node-$n" >/dev/null 2>&1 &
|
||||
done
|
||||
wait
|
||||
info " [load-driver] restarted denied peers ($DENIED) at t+${elapsed}s"
|
||||
done
|
||||
) &
|
||||
LOAD_PID=$!
|
||||
|
||||
# Foreground: tcpdump capture for CAPTURE_SECS
|
||||
docker run --rm --net=container:fips-node-$CAP_NODE \
|
||||
--cap-add NET_ADMIN --cap-add NET_RAW \
|
||||
--entrypoint sh "$HELPER_IMAGE" \
|
||||
-c "timeout $CAPTURE_SECS tcpdump -nn -i any 'udp port 2121' -l 2>&1 || true" \
|
||||
> "$CAP_FILE" 2>&1
|
||||
|
||||
# Reap load-driver if it's still running (should be ~done)
|
||||
wait $LOAD_PID 2>/dev/null || true
|
||||
|
||||
captured=$(wc -l < "$CAP_FILE")
|
||||
info "captured $captured tcpdump lines → $CAP_FILE"
|
||||
|
||||
# ── Phase 3: per-denied-peer wire-level assertion ────────────────────
|
||||
info "phase 3: per-denied-peer assertion (inbound Msg1 > 0, outbound Msg2 == 0)"
|
||||
OVERALL=0
|
||||
TOTAL_MSG1_IN=0
|
||||
TOTAL_MSG2_OUT=0
|
||||
for n in $DENIED; do
|
||||
n_ip=$(node_ip "$n")
|
||||
# Inbound: src=n_ip:* → dst=cap_ip:2121; FMP-IK Msg1 wire size = 84 B
|
||||
msg1_in=$(grep -cE "IP $n_ip\.[0-9]+ > $CAP_IP\.2121: UDP, length 84" "$CAP_FILE" || true)
|
||||
# Outbound: src=cap_ip:2121 → dst=n_ip:*; FMP-IK Msg2 wire size = 104 B
|
||||
msg2_out=$(grep -cE "IP $CAP_IP\.2121 > $n_ip\.[0-9]+: UDP, length 104" "$CAP_FILE" || true)
|
||||
info " node-$n ($n_ip): inbound Msg1 (len 84) = $msg1_in, outbound Msg2 (len 104) = $msg2_out"
|
||||
TOTAL_MSG1_IN=$((TOTAL_MSG1_IN + msg1_in))
|
||||
TOTAL_MSG2_OUT=$((TOTAL_MSG2_OUT + msg2_out))
|
||||
if [ "$msg1_in" -eq 0 ]; then
|
||||
info " FAIL: expected inbound Msg1 retries from denied peer (peer not sustained-retrying?)"
|
||||
OVERALL=1
|
||||
fi
|
||||
if [ "$msg2_out" -gt 0 ]; then
|
||||
info " FAIL: silent-drop gate leaked — expected 0 outbound Msg2, got $msg2_out"
|
||||
OVERALL=1
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Phase 4: cap'd node still at exactly max_peers ───────────────────
|
||||
pc_final=$(docker exec fips-node-$CAP_NODE fipsctl show status 2>/dev/null \
|
||||
| grep -m1 peer_count | sed 's/.*: *//' | tr -d ',' || echo 0)
|
||||
info "node-$CAP_NODE final peer_count=$pc_final (expected $MAX_PEERS)"
|
||||
[ "$pc_final" = "$MAX_PEERS" ] || OVERALL=1
|
||||
|
||||
if [ "$OVERALL" -eq 0 ]; then
|
||||
pass "admission-cap: silent-drop gate verified at scale"
|
||||
pass " denied peers: $(echo $DENIED | wc -w), capture: ${CAPTURE_SECS}s"
|
||||
pass " total inbound Msg1 from denied: $TOTAL_MSG1_IN (sustained retries observed)"
|
||||
pass " total outbound Msg2 to denied: $TOTAL_MSG2_OUT (silent-drop holds)"
|
||||
rm -f "$CAP_FILE"
|
||||
exit 0
|
||||
else
|
||||
info "--- tcpdump capture tail (last 50 lines) ---"
|
||||
tail -50 "$CAP_FILE"
|
||||
fail "admission-cap: see failures above (capture preserved at $CAP_FILE)"
|
||||
fi
|
||||
Reference in New Issue
Block a user