From 43639fecb975f4348663541f997156a10260e674 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sun, 3 May 2026 18:35:36 +0000 Subject: [PATCH] Add stun-faults integration suite covering STUN failure/recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover the previously untested STUN client behavior under server unreachable, response timeout, and packet loss. The 3 NAT scenarios test happy paths only; if the STUN client mishandled a fault (panic, hang, missing log signal), it would silently degrade NAT traversal without surfacing in CI. testing/nat/scripts/stun-faults-test.sh (new, 244 lines): Phase 1 (drop, ~12s): tc prio + netem loss 100% band + u32 filter on dst 172.31.10.40 udp 3478. Falls back to iptables -j DROP if netem isn't available. Asserts daemon process alive, no panic, log line matching stun.*(timed?out|fail|fallback|unreachable|no address) within the phase window. Phase 2 (delay then clear, ~17s): tc qdisc add dev eth0 root netem delay 5000ms for 7s, then deleted. 10s settle. Asserts process alive, no panic, AND "STUN observation succeeded" log line after clear (recovery proof). Phase 3 (kill, ~12s): docker stop fips-nat-stun. Asserts process alive, no panic, fault evidence in logs. testing/nat/docker-compose.yml: stun-faults profile adds two services. stun-fault-node is fips-test:latest on shared-lan at 172.31.10.50. stun-fault-shim is fips-test:latest sharing the daemon's network namespace via network_mode: service:stun-fault- node, with cap_add NET_ADMIN, NET_RAW; entrypoint sleep infinity so the script can docker exec into it. Reuses existing stun (172.31.10.40:3478) and relay (172.31.10.30:7777) services. testing/nat/scripts/generate-configs.sh: 3-hunk update so the generator accepts the new scenario and points its peer config at the existing relay/STUN. The peer is configured for connect_peer() so the daemon retries traversal on a loop, repeatedly invoking observe_traversal_addresses() — which is the fault-injection target. testing/ci-local.sh: STUN_FAULTS_SUITES=(stun-faults) array, run_stun_faults runner, list/integration-loop/--only-dispatch hooks. .github/workflows/ci.yml: matrix row {suite: stun-faults, type: stun-faults} + 3 steps gated on matrix.type == 'stun-faults' between nostr-publish-consume and any chaos suite. Reuses fips-linux artifact + fips-test:latest image. Approach: script-driven via docker exec stun-fault-shim. Sharing network namespace means tc rules on the shim's eth0 affect daemon egress. No timing logic in the shim itself. --- .github/workflows/ci.yml | 25 ++ testing/ci-local.sh | 27 +- testing/nat/docker-compose.yml | 66 +++++ testing/nat/scripts/generate-configs.sh | 21 +- testing/nat/scripts/stun-faults-test.sh | 333 ++++++++++++++++++++++++ 5 files changed, 469 insertions(+), 3 deletions(-) create mode 100755 testing/nat/scripts/stun-faults-test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e06b2c..e5ebed1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -386,6 +386,14 @@ jobs: # without crashing). UDP transport baseline for v0.3.0. - suite: nostr-publish-consume type: nostr-publish-consume + # ── STUN fault-injection ─────────────────────────────────────── + # One FIPS daemon + a netns-sharing shim that injects tc/iptables + # faults against UDP egress to the in-lab STUN server. Three + # phases: 100% drop, ~5s delay then clear, then full STUN + # container kill. Asserts the daemon notices each fault, + # recovers from delay, and never panics. + - suite: stun-faults + type: stun-faults # ── Real-deb install across target distros ───────────────────── # Boots a privileged systemd container per distro, runs # `apt install ./fips_*.deb` with the locally-built package, @@ -660,6 +668,23 @@ jobs: docker compose -f testing/nat/docker-compose.yml \ --profile nostr-publish-consume down --volumes --remove-orphans + # ── STUN fault-injection ─────────────────────────────────────────── + - name: Run STUN fault-injection test + if: matrix.type == 'stun-faults' + run: bash testing/nat/scripts/stun-faults-test.sh + + - name: Collect logs on failure (stun-faults) + if: matrix.type == 'stun-faults' && failure() + run: | + docker compose -f testing/nat/docker-compose.yml \ + --profile stun-faults logs --no-color | tail -300 + + - name: Stop containers (stun-faults) + if: matrix.type == 'stun-faults' && always() + run: | + docker compose -f testing/nat/docker-compose.yml \ + --profile stun-faults down --volumes --remove-orphans + # ── Outbound LAN gateway integration test ────────────────────────── - name: Generate configs (gateway) if: matrix.type == 'gateway' diff --git a/testing/ci-local.sh b/testing/ci-local.sh index bcda5b9..9afb0ef 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -18,7 +18,7 @@ # static-mesh, static-chain, rekey, rekey-accept-off, # rekey-outbound-only, gateway, # acl-allowlist, firewall, nat-cone, nat-symmetric, nat-lan, -# nostr-publish-consume, +# nostr-publish-consume, stun-faults, # chaos-smoke-10, chaos-churn-mixed-10, chaos-ethernet-mesh, # chaos-ethernet-only, chaos-tcp-mesh, chaos-bottleneck-parent, # chaos-cost-avoidance, chaos-cost-reeval, chaos-cost-stability, @@ -77,6 +77,7 @@ ACL_SUITES=(acl-allowlist) FIREWALL_SUITES=(firewall) NAT_SUITES=(cone symmetric lan) NOSTR_RELAY_SUITES=(nostr-publish-consume) +STUN_FAULTS_SUITES=(stun-faults) DNS_RESOLVER_SUITES=(dns-resolver) DEB_INSTALL_SUITES=(deb-install) TOR_SUITES=(tor-socks5 tor-directory) @@ -123,6 +124,9 @@ list_suites() { echo " Nostr publish/consume:" for s in "${NOSTR_RELAY_SUITES[@]}"; do echo " $s"; done echo "" + echo " STUN fault-injection:" + for s in "${STUN_FAULTS_SUITES[@]}"; do echo " $s"; done + echo "" echo " Chaos scenarios:" for entry in "${CHAOS_SUITES[@]}"; do read -ra parts <<< "$entry" @@ -478,6 +482,20 @@ run_nostr_publish_consume() { fi } +# Run the STUN fault-injection integration test. +# One FIPS daemon + a netns-sharing shim that injects tc/iptables faults +# against UDP egress to the STUN service. Three phases: drop, delay, +# kill. Asserts the daemon detects each fault, recovers from delay, and +# never panics. +run_stun_faults() { + info "[stun-faults] Running STUN fault-injection test" + if bash testing/nat/scripts/stun-faults-test.sh 2>&1; then + record "stun-faults" 0 + else + record "stun-faults" 1 + fi +} + # Run dns-resolver harness (multi-distro + e2e scenarios) run_dns_resolver() { info "[dns-resolver] Running multi-distro test (slow — builds per-distro images)" @@ -567,6 +585,11 @@ run_integration() { run_nostr_publish_consume done + # STUN fault-injection (sequential — shares the NAT compose project) + for _suite in "${STUN_FAULTS_SUITES[@]}"; do + run_stun_faults + done + # Chaos scenarios (parallel, throttled) if [[ "$SKIP_CHAOS" != true ]]; then info "Running ${#CHAOS_SUITES[@]} chaos scenarios (max $PARALLEL_JOBS parallel)" @@ -655,6 +678,8 @@ run_suite() { run_nat "${suite#nat-}" ;; nostr-publish-consume) run_nostr_publish_consume ;; + stun-faults) + run_stun_faults ;; chaos-*) local chaos_name="${suite#chaos-}" local found=false diff --git a/testing/nat/docker-compose.yml b/testing/nat/docker-compose.yml index 3f2ea33..b7948e3 100644 --- a/testing/nat/docker-compose.yml +++ b/testing/nat/docker-compose.yml @@ -269,3 +269,69 @@ services: networks: shared-lan: ipv4_address: 172.31.10.21 + + # ── STUN fault-injection profile ─────────────────────────────────────── + # One FIPS daemon + a netns-sharing shim that injects tc/iptables faults + # against UDP egress to the STUN service. The runner script drives the + # shim via `docker exec` (Approach A) — no scripted timing inside the + # shim itself. Three phases: + # 1. drop — 100% UDP egress drop to STUN; assert daemon notices the + # observation timeout and retries. + # 2. delay — ~5s netem delay; assert daemon recovers and STUN succeeds + # again once the rule is removed. + # 3. kill — `docker stop fips-nat-stun`; assert daemon stays up and + # continues to handle "STUN unreachable" gracefully. + # The shim shares the daemon's network namespace so `tc qdisc add dev + # eth0 ...` operates on the daemon's egress path. The shim therefore + # has its own NET_ADMIN cap; the daemon already has one for TUN. + stun-fault-node: + <<: *fips-common + profiles: ["stun-faults"] + container_name: fips-nat-stun-fault-node + hostname: fips-nat-stun-fault-node + depends_on: + - relay + - stun + volumes: + - ../docker/resolv.conf:/etc/resolv.conf:ro + - ./generated-configs/stun-faults/stun-fault-node.yaml:/etc/fips/fips.yaml:ro + networks: + shared-lan: + ipv4_address: 172.31.10.50 + + # Fault-free peer that publishes a valid overlay advert, so the + # fault-node's NAT-traversal attempt actually reaches + # observe_traversal_addresses() (the STUN client). Without this peer the + # daemon would abort with "no overlay advert" and never generate the + # STUN egress that the shim's tc/iptables rules are meant to drop. + # Intentionally has NO fault shim sharing its netns; runs cleanly. + stun-fault-peer: + <<: *fips-common + profiles: ["stun-faults"] + container_name: fips-nat-stun-fault-peer + hostname: fips-nat-stun-fault-peer + depends_on: + - relay + - stun + volumes: + - ../docker/resolv.conf:/etc/resolv.conf:ro + - ./generated-configs/stun-faults/stun-fault-peer.yaml:/etc/fips/fips.yaml:ro + networks: + shared-lan: + ipv4_address: 172.31.10.51 + + stun-fault-shim: + image: fips-test:latest + profiles: ["stun-faults"] + container_name: fips-nat-stun-fault-shim + depends_on: + - stun-fault-node + cap_add: + - NET_ADMIN + - NET_RAW + network_mode: "service:stun-fault-node" + restart: "no" + entrypoint: + - /bin/sh + - -c + - "exec sleep infinity" diff --git a/testing/nat/scripts/generate-configs.sh b/testing/nat/scripts/generate-configs.sh index 1078d9b..b307d95 100755 --- a/testing/nat/scripts/generate-configs.sh +++ b/testing/nat/scripts/generate-configs.sh @@ -11,7 +11,7 @@ SCENARIO="${1:?usage: generate-configs.sh [mesh-name]}" MESH_NAME="${2:-nat-lab-$(date +%s)-$$}" case "$SCENARIO" in - cone|symmetric|lan|nostr-publish-consume) ;; + cone|symmetric|lan|nostr-publish-consume|stun-faults) ;; *) echo "Unknown scenario: $SCENARIO" >&2 exit 1 @@ -30,7 +30,8 @@ npub_b="$(echo "$keys_b" | awk -F= '/^npub=/{print $2}')" relay_addr="ws://172.31.254.30:7777" stun_addr="stun:172.31.254.40:3478" -if [ "$SCENARIO" = "lan" ] || [ "$SCENARIO" = "nostr-publish-consume" ]; then +if [ "$SCENARIO" = "lan" ] || [ "$SCENARIO" = "nostr-publish-consume" ] \ + || [ "$SCENARIO" = "stun-faults" ]; then relay_addr="ws://172.31.10.30:7777" stun_addr="stun:172.31.10.40:3478" fi @@ -125,6 +126,22 @@ EOF write_config "$OUTPUT_DIR/$SCENARIO/node-a.yaml" "$nsec_a" "$peer_block_a" write_config "$OUTPUT_DIR/$SCENARIO/node-b.yaml" "$nsec_b" "$peer_block_b" +# stun-faults runs two real FIPS daemons: +# stun-fault-node (key "a") — target of tc/iptables faults via the shim +# stun-fault-peer (key "b") — fault-free peer that publishes a valid +# overlay advert so the fault-node's +# traversal actually invokes the STUN client +# Mutual peering ensures both sides advertise; without a real advert the +# fault-node would abort at "no overlay advert" and never generate STUN +# egress. The shim's netem/iptables rules can then meaningfully drop +# the STUN UDP traffic during Phase 1. +if [ "$SCENARIO" = "stun-faults" ]; then + write_config "$OUTPUT_DIR/$SCENARIO/stun-fault-node.yaml" \ + "$nsec_a" "$peer_block_a" + write_config "$OUTPUT_DIR/$SCENARIO/stun-fault-peer.yaml" \ + "$nsec_b" "$peer_block_b" +fi + cat > "$OUTPUT_DIR/$SCENARIO/npubs.env" </dev/null || true + docker exec "$SHIM" iptables -D OUTPUT -p udp -d "$STUN_HOST" \ + --dport "$STUN_PORT" -j DROP 2>/dev/null || true + "${COMPOSE[@]}" --profile "$PROFILE" down -v --remove-orphans \ + >/dev/null 2>&1 || true +} + +trap 'echo ""; echo "stun-faults-test interrupted"; cleanup; exit 130' INT TERM + +require_docker_daemon() { + if ! docker info >/dev/null 2>&1; then + echo "Docker daemon is not reachable; cannot run stun-faults-test" >&2 + exit 1 + fi +} + +require_test_image() { + if ! docker image inspect fips-test:latest >/dev/null 2>&1; then + echo "fips-test:latest not found; building test image" + "$BUILD_SCRIPT" + fi +} + +dump_diagnostics() { + echo "" + echo "=== stun-faults diagnostics ===" + for c in "$NODE" "$PEER" "$SHIM" "$STUN_CONTAINER"; do + echo "" + echo "--- $c: logs (last 80) ---" + docker logs "$c" 2>&1 | tail -80 || true + done + echo "" + echo "--- $SHIM: tc qdisc state ---" + docker exec "$SHIM" tc qdisc show dev "$DEV" 2>&1 || true + echo "" + echo "--- $SHIM: iptables OUTPUT ---" + docker exec "$SHIM" iptables -vnL OUTPUT 2>&1 || true + echo "" + echo "--- $NODE: fipsctl show status ---" + docker exec "$NODE" fipsctl show status 2>&1 || true + echo "" + echo "--- $NODE: fipsctl show peers ---" + docker exec "$NODE" fipsctl show peers 2>&1 || true +} + +# Apply a UDP-egress drop rule to STUN. Tries tc netem first (so the +# daemon's send_to() calls themselves silently disappear); falls back to +# iptables if netem isn't available. +apply_drop() { + if docker exec "$SHIM" tc qdisc add dev "$DEV" root \ + handle 1: prio 2>/dev/null \ + && docker exec "$SHIM" tc qdisc add dev "$DEV" parent 1:3 \ + handle 30: netem loss 100% 2>/dev/null \ + && docker exec "$SHIM" tc filter add dev "$DEV" protocol ip \ + parent 1:0 prio 3 u32 match ip dst "${STUN_HOST}/32" \ + match ip protocol 17 0xff flowid 1:3 2>/dev/null; then + echo " drop: tc netem loss 100% applied to ${STUN_HOST}" + FAULT_MODE=tc + return 0 + fi + # Cleanup any partial tc state before falling back. + docker exec "$SHIM" tc qdisc del dev "$DEV" root 2>/dev/null || true + docker exec "$SHIM" iptables -I OUTPUT -p udp -d "$STUN_HOST" \ + --dport "$STUN_PORT" -j DROP + echo " drop: iptables DROP applied (tc netem unavailable)" + FAULT_MODE=iptables +} + +clear_drop() { + if [[ "${FAULT_MODE:-}" == "tc" ]]; then + docker exec "$SHIM" tc qdisc del dev "$DEV" root 2>/dev/null || true + elif [[ "${FAULT_MODE:-}" == "iptables" ]]; then + docker exec "$SHIM" iptables -D OUTPUT -p udp -d "$STUN_HOST" \ + --dport "$STUN_PORT" -j DROP 2>/dev/null || true + fi + FAULT_MODE="" + echo " drop: cleared" +} + +apply_delay() { + docker exec "$SHIM" tc qdisc add dev "$DEV" root netem delay 5000ms 2>/dev/null \ + || { echo " delay: tc netem unavailable, skipping" >&2; return 1; } + echo " delay: tc netem 5000ms applied" +} + +clear_delay() { + docker exec "$SHIM" tc qdisc del dev "$DEV" root 2>/dev/null || true + echo " delay: cleared" +} + +assert_process_alive() { + if ! docker exec "$NODE" pidof fips >/dev/null 2>&1; then + echo "fips daemon NOT running in $NODE" >&2 + return 1 + fi + echo " $NODE: fips daemon alive" +} + +assert_no_panic() { + local logs + logs="$(docker logs "$NODE" 2>&1 || true)" + if grep -Eq "panicked at|RUST_BACKTRACE|fatal runtime error" <<<"$logs"; then + echo "panic detected in $NODE logs" >&2 + return 1 + fi +} + +# Look for STUN-related fault evidence in the daemon's logs. The +# nostr/stun module emits "stun observation failed, falling back to +# LAN-only addresses" at debug when STUN times out. Also accept the +# generic bootstrap "timed out waiting for" / "no address for" / any +# log line containing both "stun" and ("timed out" | "fail" | "fallback" +# | "unreachable"). +assert_stun_fault_observed() { + local since="$1" # seconds back from now + local logs + logs="$(docker logs --since "${since}s" "$NODE" 2>&1 || true)" + if grep -Eiq 'stun.*(timed? ?out|fail|fallback|unreachable|no address)' <<<"$logs"; then + echo " $NODE: STUN fault evidence observed in logs" + return 0 + fi + echo "no STUN fault evidence in $NODE logs (last ${since}s)" >&2 + echo "--- recent log tail ---" >&2 + echo "$logs" | tail -40 >&2 + return 1 +} + +# Look for STUN observation success (debug-level) since N seconds ago. +assert_stun_success_observed() { + local since="$1" + local logs + logs="$(docker logs --since "${since}s" "$NODE" 2>&1 || true)" + if grep -Eiq 'STUN observation succeeded|STUN observed' <<<"$logs"; then + echo " $NODE: STUN success observed in logs" + return 0 + fi + echo "no STUN success evidence in $NODE logs (last ${since}s)" >&2 + return 1 +} + +# Pre-flight: with no fault injected, the fault-node must (a) discover +# the peer's overlay advert via the relay, and (b) successfully invoke +# the STUN client at least once. If either is missing, the rest of the +# test would only show the "no overlay advert" path — i.e. a setup bug, +# not a real fault-evidence miss. Polls up to `timeout_secs` for a +# "traversal: initiator STUN observed" or "STUN observation succeeded" +# log line in the fault-node. +preflight_assert_stun_active() { + local timeout_secs="${1:-45}" + local deadline=$(( SECONDS + timeout_secs )) + while (( SECONDS < deadline )); do + local logs + logs="$(docker logs "$NODE" 2>&1 || true)" + if grep -Eq 'traversal: initiator STUN observed|STUN observation succeeded' \ + <<<"$logs"; then + echo " $NODE: pre-flight STUN observation confirmed" + return 0 + fi + sleep 2 + done + echo "pre-flight FAIL: $NODE never invoked STUN within ${timeout_secs}s" >&2 + echo "(likely cause: peer advert not yet published, or peer config wrong)" >&2 + echo "--- $NODE recent log tail ---" >&2 + docker logs "$NODE" 2>&1 | tail -40 >&2 || true + echo "--- $PEER recent log tail ---" >&2 + docker logs "$PEER" 2>&1 | tail -40 >&2 || true + return 1 +} + +run_test() { + echo "=== stun-faults-test: setup ===" + cleanup + "$GENERATE_SCRIPT" "$SCENARIO" + "${COMPOSE[@]}" --profile "$PROFILE" up -d --build --force-recreate + + # Give the daemons time to come up. Both fault-node and fault-peer + # need to start, publish their adverts to the relay, and discover + # each other before the fault-node will reach the STUN client. + echo "" + echo "--- waiting for daemons to start ---" + sleep 10 + + if ! docker exec "$NODE" pidof fips >/dev/null 2>&1; then + dump_diagnostics + echo "fips daemon failed to start in $NODE" >&2 + return 1 + fi + if ! docker exec "$PEER" pidof fips >/dev/null 2>&1; then + dump_diagnostics + echo "fips daemon failed to start in $PEER" >&2 + return 1 + fi + + # Phase 0 / pre-flight: assert that with NO fault injected, the + # fault-node successfully reaches the STUN client at least once. + # Without this guard, a Phase-1 fault-evidence miss could be either + # the real bug we're testing OR a setup bug (e.g., missing advert). + echo "" + echo "=== Phase 0: pre-flight — confirm STUN baseline (no faults) ===" + if ! preflight_assert_stun_active 45; then + dump_diagnostics + return 1 + fi + + # Sanity dump: show the recent STUN-related lines for the operator. + docker logs "$NODE" 2>&1 | grep -Ei 'stun|traversal' | tail -10 || true + + # IMPORTANT: STUN observation is event-driven, not periodic. The + # daemon calls observe_traversal_addresses() once per fresh traversal + # attempt; once the resulting reflexive address is cached, the next + # observation does not happen until advert_refresh_secs (30 min by + # default). To force a fresh STUN attempt during each phase, restart + # the PEER container — fault-node sees the peer disconnect and + # retries traversal (auto_connect with backoff), which re-invokes + # observe_traversal_addresses() under the fault. + # + # Restarting fault-node itself does NOT work: the shim shares + # fault-node's network namespace (network_mode: service:...), so a + # fault-node restart wipes the tc/iptables rules the shim applied. + # Restarting the peer leaves fault-node's netns + shim faults intact. + + echo "" + echo "=== Phase 1: drop 100% UDP egress to STUN (restart peer under fault) ===" + apply_drop + docker restart "$PEER" >/dev/null + local phase_start=$SECONDS + # Wait long enough for fault-node to detect peer loss and retry. + # Auto-connect backoff is exponential 5s base; first retry ~5s after + # detection, second ~10s. Allow ~25s. + sleep 25 + local phase_elapsed=$(( SECONDS - phase_start + 4 )) + + assert_process_alive || { dump_diagnostics; return 1; } + assert_no_panic || { dump_diagnostics; return 1; } + assert_stun_fault_observed "$phase_elapsed" || { + dump_diagnostics + return 1 + } + clear_drop + + echo "" + echo "=== Phase 2: delay 5000ms then clear (peer restart for clean STUN) ===" + if apply_delay; then + docker restart "$PEER" >/dev/null + # Slow STUN should eventually succeed under 5s delay. + sleep 12 + clear_delay + sleep 10 + else + echo " Phase 2 skipped (no tc netem available); proceeding to Phase 3" + fi + + assert_process_alive || { dump_diagnostics; return 1; } + assert_no_panic || { dump_diagnostics; return 1; } + # Recovery assertion: STUN must succeed at least once after the rule + # is removed. + if ! assert_stun_success_observed 30; then + echo "Phase 2 recovery assertion failed (no STUN success after delay clear)" >&2 + dump_diagnostics + return 1 + fi + + echo "" + echo "=== Phase 3: kill STUN container, restart peer, assert survival ===" + docker stop "$STUN_CONTAINER" >/dev/null + docker restart "$PEER" >/dev/null + local p3_start=$SECONDS + sleep 25 + local p3_elapsed=$(( SECONDS - p3_start + 4 )) + + assert_process_alive || { dump_diagnostics; return 1; } + assert_no_panic || { dump_diagnostics; return 1; } + assert_stun_fault_observed "$p3_elapsed" || { + dump_diagnostics + return 1 + } + + cleanup + echo "stun-faults-test passed" +} + +main() { + require_docker_daemon + require_test_image + run_test +} + +main "$@"