diff --git a/testing/ci-cleanup.sh b/testing/ci-cleanup.sh index fa32a86..407b11b 100755 --- a/testing/ci-cleanup.sh +++ b/testing/ci-cleanup.sh @@ -174,6 +174,14 @@ VETH_NODE_ID='(0[0-9]|[1-9][0-9]+)' # which carry a different token — cannot match. The token derivation is read # from the simulation itself rather than repeated here, so widening it cannot # leave this matching the old width. Empty output means "reap nothing". +# +# Two producers, two shapes. The chaos simulation makes vh{token}{NN}{MM}{a,b}; +# the NAT lab (nat/scripts/setup-topology.sh) makes vn{a,b}{token}{0,1}, using +# the RUN-wide suffix rather than any chaos scenario's. Widening this regex is +# only half the fix: the token set is derived separately below, so a suffix +# list carrying no NAT suffix leaves the NAT half matching nothing while +# looking correct. ci-local.sh's ci_teardown therefore appends the run-wide +# suffix to --veth-suffixes. veth_pattern() { # vh{token}{NN}{MM}{a,b}: the token is 4 hex or wholly absent — never a # part of one — and the two node ids follow. Anchored and shaped this @@ -182,7 +190,8 @@ veth_pattern() { # single run, no missing or empty suffix list may widen this back out to # every run. if [[ -z "$RUN_ID" && -z "$VETH_SUFFIXES" ]]; then - printf '^vh([0-9a-f]{4})?%s%s[ab]$' "$VETH_NODE_ID" "$VETH_NODE_ID" + printf '^vh([0-9a-f]{4})?%s%s[ab]$|^vn[ab]([0-9a-f]{4})?[01]$' \ + "$VETH_NODE_ID" "$VETH_NODE_ID" return 0 fi if [[ -z "$VETH_SUFFIXES" ]]; then @@ -215,7 +224,8 @@ veth_pattern() { veth_warn "no interface tokens derived from: ${sfx[*]}" return 0 fi - printf '^vh(%s)%s%s[ab]$' "$alt" "$VETH_NODE_ID" "$VETH_NODE_ID" + printf '^vh(%s)%s%s[ab]$|^vn[ab](%s)[01]$' \ + "$alt" "$VETH_NODE_ID" "$VETH_NODE_ID" "$alt" } # ip(8) in a privileged --net=host container, matching how the simulation diff --git a/testing/ci-local.sh b/testing/ci-local.sh index b02a23b..6858e5f 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -408,6 +408,12 @@ ci_teardown() { for _entry in "${CHAOS_SUITES[@]}"; do _suffixes+=("$(ci_chaos_suffix "${_entry%% *}")") done + # The NAT lab's host veths (vn{a,b}{token}{0,1}) carry the RUN-wide + # suffix, not any chaos scenario's, so its token appears in no suffix + # above and the reaper's NAT shape would match nothing at all. This is + # the half of that fix that fails silently: widening the regex in + # ci-cleanup.sh without this line looks correct and reaps nothing. + _suffixes+=("$CI_RUN_NAME_SUFFIX") timeout 150 bash "$SCRIPT_DIR/ci-cleanup.sh" \ --label "$CI_LABEL" \ --run-id "$CI_RUN_ID" \ @@ -415,7 +421,7 @@ ci_teardown() { --images "$CI_IMAGE_TEST $CI_IMAGE_APP" \ --veth-suffixes "${_suffixes[*]}" >/dev/null || true - # 3. The static and firewall suites' generated configs are per-run (a + # 3. The static, firewall and nat suites' generated configs are per-run (a # shared directory would let concurrent runs overwrite each other's node # configs), so they are this run's to remove. Only on a green run: after # a failure they are the evidence of what the failing nodes were actually @@ -819,16 +825,138 @@ run_firewall() { fi } +# ── NAT lab: per-run network claim ───────────────────────────────────────── +# +# The lab's two bridges pinned 172.31.254.0/24 (wan) and 172.31.10.0/24 +# (shared-lan), so two concurrent runs asked the daemon for the same address +# space and the second died with `Pool overlaps`. Claim a free /24 for each +# instead and let docker's own create be the atomic arbiter, exactly as +# claim_gateway_lan6 and sidecar's alloc_network do. The run-id-derived offset +# this task considered earlier stays rejected: it makes a collision unlikely +# rather than impossible, and a collision is the failure being removed. +# +# 10.41.0.0/16 is unclaimed in-tree, sits below this host's docker pool +# (10.128.0.0/9, per /etc/docker/daemon.json), and is also outside docker's +# stock 172.17-31 / 192.168 default, so the choice holds either way. +# +# The claim lives here rather than in the suite scripts because +# .github/workflows/ci.yml invokes nat-test.sh, nostr-relay-test.sh and +# stun-faults-test.sh directly and testing/mesh-lab/run-loop.sh invokes +# `nat-test.sh lan`; none of those want a claim, and all tear down with the +# base compose file only. +# +# One claim per SUITE invocation, not per run — matching run_gateway. Both +# labels are stamped so ci-cleanup.sh's label sweep can recover the networks +# when a run is SIGKILLed, which no inline removal can cover. +CI_NAT_NET_BASE="10.41" +CI_NAT_NET_CANDIDATES=256 +CI_NAT_CLAIMED_PREFIX="" + +# Deliberately does NOT discard stderr: only an address-pool conflict is worth +# advancing on. Any other failure is real, and burning through 256 candidates +# would bury the reason. +ci_claim_nat_net() { + local net="$1" i err + CI_NAT_CLAIMED_PREFIX="" + for (( i = 0; i < CI_NAT_NET_CANDIDATES; i++ )); do + if err=$(docker network create \ + --subnet "${CI_NAT_NET_BASE}.${i}.0/24" \ + --label "$CI_LABEL" --label "$CI_LABEL_RUN" \ + "$net" 2>&1); then + CI_NAT_CLAIMED_PREFIX="${CI_NAT_NET_BASE}.${i}" + info "[nat] Claimed $net on ${CI_NAT_CLAIMED_PREFIX}.0/24" + return 0 + fi + case "$err" in + *"Pool overlaps"*|*"pool overlaps"*) continue ;; + *) fail "[nat] docker network create: $err"; return 1 ;; + esac + done + fail "[nat] no free /24 in ${CI_NAT_NET_BASE}.0.0/16 after ${CI_NAT_NET_CANDIDATES} attempts" + return 1 +} + +# Claim both lab networks and export the prefixes every lab address derives +# from. A partial claim is rolled back here, because nothing downstream will +# run to release it. +ci_claim_nat_networks() { + unset NAT_WAN_PREFIX NAT_LAN_PREFIX + export FIPS_NAT_WAN_NET="fips-nat-wan${FIPS_CI_NAME_SUFFIX:-}" + export FIPS_NAT_LAN_NET="fips-nat-shared-lan${FIPS_CI_NAME_SUFFIX:-}" + + ci_claim_nat_net "$FIPS_NAT_WAN_NET" || return 1 + local wan="$CI_NAT_CLAIMED_PREFIX" + if ! ci_claim_nat_net "$FIPS_NAT_LAN_NET"; then + docker network rm "$FIPS_NAT_WAN_NET" >/dev/null 2>&1 || true + return 1 + fi + + export NAT_WAN_PREFIX="$wan" + export NAT_LAN_PREFIX="$CI_NAT_CLAIMED_PREFIX" +} + +# Release both claimed networks. A complete claim left behind would not merely +# leak: the next nat-family suite in this run would hit `network with name ... +# already exists`, which is not a pool overlap, so the allocator correctly +# refuses to advance and fails. One suite failure would cascade into four. +# +# Both halves of run_gateway's shape are needed, and the second alone is a trap. +# `docker network rm` refuses a network that still has endpoints attached, and +# the nat suites deliberately leave their containers UP on a failure path so the +# diagnostics they just dumped can be inspected — which is precisely the path +# this release exists for. Without the `down` the removal silently no-ops +# exactly when it matters and reports success. Nothing is lost by tearing down +# here: the diagnostics are already in the run log, and ci_teardown reaps the +# containers at the end of the run regardless. +ci_release_nat_networks() { + docker compose \ + -f testing/nat/docker-compose.yml \ + -f testing/nat/docker-compose.external-net.yml \ + --profile cone --profile symmetric --profile lan \ + --profile nostr-publish-consume --profile stun-faults \ + down --volumes --remove-orphans >/dev/null 2>&1 || true + [[ -n "${FIPS_NAT_WAN_NET:-}" ]] && \ + { docker network rm "$FIPS_NAT_WAN_NET" >/dev/null 2>&1 || true; } + [[ -n "${FIPS_NAT_LAN_NET:-}" ]] && \ + { docker network rm "$FIPS_NAT_LAN_NET" >/dev/null 2>&1 || true; } + return 0 +} + +# The overlay pointing compose at the claimed networks. APPENDED to any +# caller-supplied chain rather than replacing it: mesh-lab/run-loop.sh sets +# FIPS_NAT_EXTRA_COMPOSE for its trace overlay, and overwriting would silently +# drop that. +ci_nat_extra_compose() { + printf '%s' \ + "${FIPS_NAT_EXTRA_COMPOSE:+${FIPS_NAT_EXTRA_COMPOSE}:}testing/nat/docker-compose.external-net.yml" +} + # Run a NAT scenario (cone, symmetric, lan) run_nat() { local scenario="$1" + local rc=0 export COMPOSE_PROJECT_NAME="$(ci_project nat)" - info "[nat-$scenario] Running NAT lab" - if bash testing/nat/scripts/nat-test.sh "$scenario" 2>&1; then - record "nat-$scenario" 0 - else + + # Early return only BEFORE the claim succeeds, so nothing can leak. Every + # path after it falls through to the single release below — run_gateway's + # shape. Not a trap: bash traps are not function-scoped, so a `trap ... EXIT` + # here would replace the script-level on_exit handler for the rest of the + # run and disable ci_teardown entirely. + info "[nat-$scenario] Claiming lab networks" + if ! ci_claim_nat_networks; then record "nat-$scenario" 1 + return fi + + local _extra + _extra="$(ci_nat_extra_compose)" + local -x FIPS_NAT_EXTRA_COMPOSE="$_extra" + + info "[nat-$scenario] Running NAT lab" + bash testing/nat/scripts/nat-test.sh "$scenario" 2>&1 || rc=1 + + ci_release_nat_networks + record "nat-$scenario" $rc } # Run the Nostr overlay advert publish/consume integration test. @@ -836,13 +964,24 @@ run_nat() { # (A→B publish/consume), Phase 2 (B→A reverse), and Phase 3 (malformed # advert injected directly to the relay; consumer-liveness assertion). run_nostr_publish_consume() { + local rc=0 export COMPOSE_PROJECT_NAME="$(ci_project nat)" - info "[nostr-publish-consume] Running Nostr publish/consume test" - if bash testing/nat/scripts/nostr-relay-test.sh 2>&1; then - record "nostr-publish-consume" 0 - else + + info "[nostr-publish-consume] Claiming lab networks" + if ! ci_claim_nat_networks; then record "nostr-publish-consume" 1 + return fi + + local _extra + _extra="$(ci_nat_extra_compose)" + local -x FIPS_NAT_EXTRA_COMPOSE="$_extra" + + info "[nostr-publish-consume] Running Nostr publish/consume test" + bash testing/nat/scripts/nostr-relay-test.sh 2>&1 || rc=1 + + ci_release_nat_networks + record "nostr-publish-consume" $rc } # Run the STUN fault-injection integration test. @@ -851,13 +990,24 @@ run_nostr_publish_consume() { # kill. Asserts the daemon detects each fault, recovers from delay, and # never panics. run_stun_faults() { + local rc=0 export COMPOSE_PROJECT_NAME="$(ci_project nat)" - info "[stun-faults] Running STUN fault-injection test" - if bash testing/nat/scripts/stun-faults-test.sh 2>&1; then - record "stun-faults" 0 - else + + info "[stun-faults] Claiming lab networks" + if ! ci_claim_nat_networks; then record "stun-faults" 1 + return fi + + local _extra + _extra="$(ci_nat_extra_compose)" + local -x FIPS_NAT_EXTRA_COMPOSE="$_extra" + + info "[stun-faults] Running STUN fault-injection test" + bash testing/nat/scripts/stun-faults-test.sh 2>&1 || rc=1 + + ci_release_nat_networks + record "stun-faults" $rc } # Run dns-resolver harness (multi-distro + e2e scenarios) diff --git a/testing/nat/docker-compose.external-net.yml b/testing/nat/docker-compose.external-net.yml new file mode 100644 index 0000000..10ad975 --- /dev/null +++ b/testing/nat/docker-compose.external-net.yml @@ -0,0 +1,26 @@ +# Override: attach the NAT lab's two bridges to pre-created external networks +# instead of letting compose create them from the base file's fixed pins. +# +# Applied only by ci-local.sh's run_nat / run_nostr_publish_consume / +# run_stun_faults, which claim a free /24 per network per suite invocation +# (ci_claim_nat_networks) and export FIPS_NAT_WAN_NET / FIPS_NAT_LAN_NET and +# NAT_WAN_PREFIX / NAT_LAN_PREFIX before `up`. That is what makes two +# concurrent local NAT runs collision-safe: each claims distinct ranges, so +# neither requests address space the other holds. +# +# The GitHub matrix, testing/mesh-lab/run-loop.sh, the README invocation and +# any standalone `docker compose up` do NOT apply this overlay; they use the +# base file's normal networks with the `:-` defaults, which render today's +# exact addresses. This mirrors static/docker-compose.gateway-external-net.yml +# and sidecar/docker-compose.external-net.yml. +# +# One file carries both networks: they are claimed and released together, and +# splitting them would let a suite attach to one claimed and one compose-made +# bridge. +networks: + wan: + external: true + name: ${FIPS_NAT_WAN_NET:-fips-nat-wan} + shared-lan: + external: true + name: ${FIPS_NAT_LAN_NET:-fips-nat-shared-lan} diff --git a/testing/nat/docker-compose.yml b/testing/nat/docker-compose.yml index 0dd3055..0ff22af 100644 --- a/testing/nat/docker-compose.yml +++ b/testing/nat/docker-compose.yml @@ -5,14 +5,14 @@ networks: - "com.corganlabs.fips-ci=1" ipam: config: - - subnet: 172.31.254.0/24 + - subnet: ${NAT_WAN_PREFIX:-172.31.254}.0/24 shared-lan: driver: bridge labels: - "com.corganlabs.fips-ci=1" ipam: config: - - subnet: 172.31.10.0/24 + - subnet: ${NAT_LAN_PREFIX:-172.31.10}.0/24 volumes: relay-data: @@ -42,9 +42,9 @@ services: - ../docker/resolv.conf:/etc/resolv.conf:ro networks: wan: - ipv4_address: 172.31.254.30 + ipv4_address: ${NAT_WAN_PREFIX:-172.31.254}.30 shared-lan: - ipv4_address: 172.31.10.30 + ipv4_address: ${NAT_LAN_PREFIX:-172.31.10}.30 stun: build: @@ -53,9 +53,9 @@ services: restart: "no" networks: wan: - ipv4_address: 172.31.254.40 + ipv4_address: ${NAT_WAN_PREFIX:-172.31.254}.40 shared-lan: - ipv4_address: 172.31.10.40 + ipv4_address: ${NAT_LAN_PREFIX:-172.31.10}.40 nat-a: build: @@ -74,11 +74,11 @@ services: - WAN_IF=eth0 - LAN_HOST=172.31.1.10 - LAN_SUBNET=172.31.1.0/24 - - WAN_SUBNET=172.31.254.0/24 - - WAN_GATEWAY=172.31.254.1 + - WAN_SUBNET=${NAT_WAN_PREFIX:-172.31.254}.0/24 + - WAN_GATEWAY=${NAT_WAN_PREFIX:-172.31.254}.1 networks: wan: - ipv4_address: 172.31.254.10 + ipv4_address: ${NAT_WAN_PREFIX:-172.31.254}.10 nat-b: build: @@ -97,11 +97,11 @@ services: - WAN_IF=eth0 - LAN_HOST=172.31.2.10 - LAN_SUBNET=172.31.2.0/24 - - WAN_SUBNET=172.31.254.0/24 - - WAN_GATEWAY=172.31.254.1 + - WAN_SUBNET=${NAT_WAN_PREFIX:-172.31.254}.0/24 + - WAN_GATEWAY=${NAT_WAN_PREFIX:-172.31.254}.1 networks: wan: - ipv4_address: 172.31.254.11 + ipv4_address: ${NAT_WAN_PREFIX:-172.31.254}.11 cone-a: <<: *fips-common @@ -117,11 +117,11 @@ services: environment: - RUST_LOG=info,fips::discovery::nostr=debug,fips::node::lifecycle=debug - DATA_IF=eth0 - - ROUTE_SUBNET=172.31.254.0/24 + - ROUTE_SUBNET=${NAT_WAN_PREFIX:-172.31.254}.0/24 - ROUTE_VIA=172.31.1.254 - - RELAY_HOST=172.31.254.30 + - RELAY_HOST=${NAT_WAN_PREFIX:-172.31.254}.30 - RELAY_PORT=7777 - - STUN_HOST=172.31.254.40 + - STUN_HOST=${NAT_WAN_PREFIX:-172.31.254}.40 - STUN_PORT=3478 volumes: - ../docker/resolv.conf:/etc/resolv.conf:ro @@ -143,11 +143,11 @@ services: environment: - RUST_LOG=info,fips::discovery::nostr=debug,fips::node::lifecycle=debug - DATA_IF=eth0 - - ROUTE_SUBNET=172.31.254.0/24 + - ROUTE_SUBNET=${NAT_WAN_PREFIX:-172.31.254}.0/24 - ROUTE_VIA=172.31.2.254 - - RELAY_HOST=172.31.254.30 + - RELAY_HOST=${NAT_WAN_PREFIX:-172.31.254}.30 - RELAY_PORT=7777 - - STUN_HOST=172.31.254.40 + - STUN_HOST=${NAT_WAN_PREFIX:-172.31.254}.40 - STUN_PORT=3478 volumes: - ../docker/resolv.conf:/etc/resolv.conf:ro @@ -169,11 +169,11 @@ services: environment: - RUST_LOG=info,fips::discovery::nostr=debug,fips::node::lifecycle=debug - DATA_IF=eth0 - - ROUTE_SUBNET=172.31.254.0/24 + - ROUTE_SUBNET=${NAT_WAN_PREFIX:-172.31.254}.0/24 - ROUTE_VIA=172.31.1.254 - - RELAY_HOST=172.31.254.30 + - RELAY_HOST=${NAT_WAN_PREFIX:-172.31.254}.30 - RELAY_PORT=7777 - - STUN_HOST=172.31.254.40 + - STUN_HOST=${NAT_WAN_PREFIX:-172.31.254}.40 - STUN_PORT=3478 volumes: - ../docker/resolv.conf:/etc/resolv.conf:ro @@ -195,11 +195,11 @@ services: environment: - RUST_LOG=info,fips::discovery::nostr=debug,fips::node::lifecycle=debug - DATA_IF=eth0 - - ROUTE_SUBNET=172.31.254.0/24 + - ROUTE_SUBNET=${NAT_WAN_PREFIX:-172.31.254}.0/24 - ROUTE_VIA=172.31.2.254 - - RELAY_HOST=172.31.254.30 + - RELAY_HOST=${NAT_WAN_PREFIX:-172.31.254}.30 - RELAY_PORT=7777 - - STUN_HOST=172.31.254.40 + - STUN_HOST=${NAT_WAN_PREFIX:-172.31.254}.40 - STUN_PORT=3478 volumes: - ../docker/resolv.conf:/etc/resolv.conf:ro @@ -220,7 +220,7 @@ services: - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/lan/node-a.yaml:/etc/fips/fips.yaml:ro networks: shared-lan: - ipv4_address: 172.31.10.10 + ipv4_address: ${NAT_LAN_PREFIX:-172.31.10}.10 lan-b: <<: *fips-common @@ -235,7 +235,7 @@ services: - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/lan/node-b.yaml:/etc/fips/fips.yaml:ro networks: shared-lan: - ipv4_address: 172.31.10.11 + ipv4_address: ${NAT_LAN_PREFIX:-172.31.10}.11 # ── Nostr publish/consume profile ────────────────────────────────────── # Two FIPS daemons + the existing strfry relay, exercising the overlay @@ -257,7 +257,7 @@ services: - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/nostr-publish-consume/node-a.yaml:/etc/fips/fips.yaml:ro networks: shared-lan: - ipv4_address: 172.31.10.20 + ipv4_address: ${NAT_LAN_PREFIX:-172.31.10}.20 nostr-pub-b: <<: *fips-common @@ -272,7 +272,7 @@ services: - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/nostr-publish-consume/node-b.yaml:/etc/fips/fips.yaml:ro networks: shared-lan: - ipv4_address: 172.31.10.21 + ipv4_address: ${NAT_LAN_PREFIX:-172.31.10}.21 # ── STUN fault-injection profile ─────────────────────────────────────── # One FIPS daemon + a netns-sharing shim that injects tc/iptables faults @@ -301,7 +301,7 @@ services: - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/stun-faults/stun-fault-node.yaml:/etc/fips/fips.yaml:ro networks: shared-lan: - ipv4_address: 172.31.10.50 + ipv4_address: ${NAT_LAN_PREFIX:-172.31.10}.50 # Fault-free peer that publishes a valid overlay advert, so the # fault-node's NAT-traversal attempt actually reaches @@ -322,7 +322,7 @@ services: - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/stun-faults/stun-fault-peer.yaml:/etc/fips/fips.yaml:ro networks: shared-lan: - ipv4_address: 172.31.10.51 + ipv4_address: ${NAT_LAN_PREFIX:-172.31.10}.51 stun-fault-shim: image: ${FIPS_TEST_IMAGE:-fips-test:latest} diff --git a/testing/nat/scripts/generate-configs.sh b/testing/nat/scripts/generate-configs.sh index ec4c40f..a2d07fb 100755 --- a/testing/nat/scripts/generate-configs.sh +++ b/testing/nat/scripts/generate-configs.sh @@ -33,12 +33,18 @@ npub_a="$(echo "$keys_a" | awk -F= '/^npub=/{print $2}')" nsec_b="$(echo "$keys_b" | awk -F= '/^nsec=/{print $2}')" npub_b="$(echo "$keys_b" | awk -F= '/^npub=/{print $2}')" -relay_addr="ws://172.31.254.30:7777" -stun_addr="stun:172.31.254.40:3478" +# The two lab bridges. ci-local.sh claims a free /24 for each per run and +# exports these; unset renders the addresses the lab has always used, so a +# bare invocation and the GitHub matrix are unaffected. +wan="${NAT_WAN_PREFIX:-172.31.254}" +lan="${NAT_LAN_PREFIX:-172.31.10}" + +relay_addr="ws://${wan}.30:7777" +stun_addr="stun:${wan}.40:3478" if [ "$SCENARIO" = "lan" ] || [ "$SCENARIO" = "nostr-publish-consume" ] \ || [ "$SCENARIO" = "stun-faults" ]; then - relay_addr="ws://172.31.10.30:7777" - stun_addr="stun:172.31.10.40:3478" + relay_addr="ws://${lan}.30:7777" + stun_addr="stun:${lan}.40:3478" fi peer_block_a=$(cat </dev/null || ss -H -uan 2>/dev/null || netstat -anu 2>/dev/null' 2>&1 || true echo "" echo "--- $node: UDP routes to STUN and peer WANs ---" - docker exec "$node" sh -lc 'for ip in 172.31.254.40 172.31.254.10 172.31.254.11; do ip route get "$ip"; done' 2>&1 || true + # Double-quoted so THIS shell expands NAT_WAN; the loop variable is escaped + # so the container's shell still expands that one. Left single-quoted, the + # literal string ${NAT_WAN}.40 would reach `ip route get` and the probe + # would stop probing without anything going red — these are diagnostic + # paths, so the loss would only surface during a failure investigation. + docker exec "$node" sh -lc "for ip in ${NAT_WAN}.40 ${NAT_WAN}.10 ${NAT_WAN}.11; do ip route get \"\$ip\"; done" 2>&1 || true local capture_file capture_file="$(mktemp)" @@ -155,7 +169,7 @@ dump_node_udp_probe() { dump_router_udp_probe() { local router="$1" local source_node="$2" - local stun_host="${3:-172.31.254.40}" + local stun_host="${3:-${NAT_WAN}.40}" local stun_port="${4:-3478}" echo "" @@ -167,7 +181,9 @@ dump_router_udp_probe() { docker exec "$router" sh -lc 'iptables -vnL FORWARD; echo; iptables -t nat -vnL POSTROUTING' 2>&1 || true echo "" echo "--- $router: UDP routes to STUN and peer WANs ---" - docker exec "$router" sh -lc 'for ip in 172.31.254.40 172.31.254.10 172.31.254.11; do ip route get "$ip"; done' 2>&1 || true + # See the note in dump_node_udp_probe: outer shell expands NAT_WAN, inner + # shell expands the loop variable. + docker exec "$router" sh -lc "for ip in ${NAT_WAN}.40 ${NAT_WAN}.10 ${NAT_WAN}.11; do ip route get \"\$ip\"; done" 2>&1 || true local capture_file capture_file="$(mktemp)" @@ -199,7 +215,7 @@ dump_router_udp_probe() { dump_stun_udp_probe() { local source_node="$1" - local stun_host="${2:-172.31.254.40}" + local stun_host="${2:-${NAT_WAN}.40}" local stun_port="${3:-3478}" local helper_image helper_image="$(helper_tcpdump_image)" @@ -229,9 +245,9 @@ dump_stun_udp_probe() { dump_cone_diagnostics() { echo "" echo "=== cone diagnostics ===" - dump_fips_state fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} 172.31.254.30 7777 172.31.254.40 3478 + dump_fips_state fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} ${NAT_WAN}.30 7777 ${NAT_WAN}.40 3478 dump_node_udp_probe fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} - dump_fips_state fips-nat-cone-b${FIPS_CI_NAME_SUFFIX:-} 172.31.254.30 7777 172.31.254.40 3478 + dump_fips_state fips-nat-cone-b${FIPS_CI_NAME_SUFFIX:-} ${NAT_WAN}.30 7777 ${NAT_WAN}.40 3478 dump_node_udp_probe fips-nat-cone-b${FIPS_CI_NAME_SUFFIX:-} dump_container_state fips-nat-router-a${FIPS_CI_NAME_SUFFIX:-} dump_router_udp_probe fips-nat-router-a${FIPS_CI_NAME_SUFFIX:-} fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} @@ -246,8 +262,8 @@ dump_cone_diagnostics() { dump_symmetric_diagnostics() { echo "" echo "=== symmetric diagnostics ===" - dump_fips_state fips-nat-symmetric-a${FIPS_CI_NAME_SUFFIX:-} 172.31.254.30 7777 172.31.254.40 3478 - dump_fips_state fips-nat-symmetric-b${FIPS_CI_NAME_SUFFIX:-} 172.31.254.30 7777 172.31.254.40 3478 + dump_fips_state fips-nat-symmetric-a${FIPS_CI_NAME_SUFFIX:-} ${NAT_WAN}.30 7777 ${NAT_WAN}.40 3478 + dump_fips_state fips-nat-symmetric-b${FIPS_CI_NAME_SUFFIX:-} ${NAT_WAN}.30 7777 ${NAT_WAN}.40 3478 dump_container_state fips-nat-router-a${FIPS_CI_NAME_SUFFIX:-} dump_container_state fips-nat-router-b${FIPS_CI_NAME_SUFFIX:-} dump_container_state fips-nat-relay${FIPS_CI_NAME_SUFFIX:-} @@ -257,8 +273,8 @@ dump_symmetric_diagnostics() { dump_lan_diagnostics() { echo "" echo "=== lan diagnostics ===" - dump_fips_state fips-nat-lan-a${FIPS_CI_NAME_SUFFIX:-} 172.31.10.30 7777 172.31.10.40 3478 - dump_fips_state fips-nat-lan-b${FIPS_CI_NAME_SUFFIX:-} 172.31.10.30 7777 172.31.10.40 3478 + dump_fips_state fips-nat-lan-a${FIPS_CI_NAME_SUFFIX:-} ${NAT_LAN}.30 7777 ${NAT_LAN}.40 3478 + dump_fips_state fips-nat-lan-b${FIPS_CI_NAME_SUFFIX:-} ${NAT_LAN}.30 7777 ${NAT_LAN}.40 3478 dump_container_state fips-nat-relay${FIPS_CI_NAME_SUFFIX:-} dump_container_state fips-nat-stun${FIPS_CI_NAME_SUFFIX:-} } @@ -357,10 +373,10 @@ run_cone() { dump_cone_diagnostics return 1 } - assert_peer_path fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} udp 172.31.254. - assert_peer_path fips-nat-cone-b${FIPS_CI_NAME_SUFFIX:-} udp 172.31.254. - assert_link_path fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} 172.31.254. - assert_link_path fips-nat-cone-b${FIPS_CI_NAME_SUFFIX:-} 172.31.254. + assert_peer_path fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} udp ${NAT_WAN}. + assert_peer_path fips-nat-cone-b${FIPS_CI_NAME_SUFFIX:-} udp ${NAT_WAN}. + assert_link_path fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} ${NAT_WAN}. + assert_link_path fips-nat-cone-b${FIPS_CI_NAME_SUFFIX:-} ${NAT_WAN}. # shellcheck disable=SC1090 source "$CONFIG_DIR/cone/npubs.env" ping_peer fips-nat-cone-a${FIPS_CI_NAME_SUFFIX:-} "$NPUB_B" @@ -382,10 +398,10 @@ run_symmetric() { dump_symmetric_diagnostics return 1 } - assert_peer_path fips-nat-symmetric-a${FIPS_CI_NAME_SUFFIX:-} tcp 172.31.254.11: - assert_peer_path fips-nat-symmetric-b${FIPS_CI_NAME_SUFFIX:-} tcp 172.31.254.10: - assert_link_path fips-nat-symmetric-a${FIPS_CI_NAME_SUFFIX:-} 172.31.254.11: - assert_link_path fips-nat-symmetric-b${FIPS_CI_NAME_SUFFIX:-} 172.31.254.10: + assert_peer_path fips-nat-symmetric-a${FIPS_CI_NAME_SUFFIX:-} tcp ${NAT_WAN}.11: + assert_peer_path fips-nat-symmetric-b${FIPS_CI_NAME_SUFFIX:-} tcp ${NAT_WAN}.10: + assert_link_path fips-nat-symmetric-a${FIPS_CI_NAME_SUFFIX:-} ${NAT_WAN}.11: + assert_link_path fips-nat-symmetric-b${FIPS_CI_NAME_SUFFIX:-} ${NAT_WAN}.10: require_bootstrap_activity fips-nat-symmetric-a${FIPS_CI_NAME_SUFFIX:-} require_bootstrap_activity fips-nat-symmetric-b${FIPS_CI_NAME_SUFFIX:-} # shellcheck disable=SC1090 @@ -408,10 +424,10 @@ run_lan() { dump_lan_diagnostics return 1 } - assert_peer_path fips-nat-lan-a${FIPS_CI_NAME_SUFFIX:-} udp 172.31.10. - assert_peer_path fips-nat-lan-b${FIPS_CI_NAME_SUFFIX:-} udp 172.31.10. - assert_link_path fips-nat-lan-a${FIPS_CI_NAME_SUFFIX:-} 172.31.10. - assert_link_path fips-nat-lan-b${FIPS_CI_NAME_SUFFIX:-} 172.31.10. + assert_peer_path fips-nat-lan-a${FIPS_CI_NAME_SUFFIX:-} udp ${NAT_LAN}. + assert_peer_path fips-nat-lan-b${FIPS_CI_NAME_SUFFIX:-} udp ${NAT_LAN}. + assert_link_path fips-nat-lan-a${FIPS_CI_NAME_SUFFIX:-} ${NAT_LAN}. + assert_link_path fips-nat-lan-b${FIPS_CI_NAME_SUFFIX:-} ${NAT_LAN}. # shellcheck disable=SC1090 source "$CONFIG_DIR/lan/npubs.env" ping_peer fips-nat-lan-a${FIPS_CI_NAME_SUFFIX:-} "$NPUB_B" diff --git a/testing/nat/scripts/nostr-relay-test.sh b/testing/nat/scripts/nostr-relay-test.sh index 26c0b9e..2c32101 100755 --- a/testing/nat/scripts/nostr-relay-test.sh +++ b/testing/nat/scripts/nostr-relay-test.sh @@ -27,9 +27,26 @@ CONFIG_DIR="$NAT_DIR/generated-configs${FIPS_CI_NAME_SUFFIX:-}" PROFILE="nostr-publish-consume" SCENARIO="$PROFILE" COMPOSE=(docker compose -f "$NAT_DIR/docker-compose.yml") + +# Optional extra compose-file overlay chain (colon-separated paths), matching +# nat-test.sh. ci-local.sh appends testing/nat/docker-compose.external-net.yml +# here so this suite attaches to the networks the run already claimed; without +# the hook compose would create its own from the base file's subnet and collide +# with the run's own claim. Paths are relative to ROOT_DIR unless absolute. +if [ -n "${FIPS_NAT_EXTRA_COMPOSE:-}" ]; then + IFS=':' read -ra _NAT_EXTRA <<< "${FIPS_NAT_EXTRA_COMPOSE}" + for _f in "${_NAT_EXTRA[@]}"; do + case "$_f" in + /*) COMPOSE+=(-f "$_f") ;; + *) COMPOSE+=(-f "$ROOT_DIR/$_f") ;; + esac + done +fi + NODE_A="fips-nat-nostr-pub-a${FIPS_CI_NAME_SUFFIX:-}" NODE_B="fips-nat-nostr-pub-b${FIPS_CI_NAME_SUFFIX:-}" -RELAY_HOST="172.31.10.30" +# Claimed per run by ci-local.sh; unset renders the lab's historical address. +RELAY_HOST="${NAT_LAN_PREFIX:-172.31.10}.30" RELAY_PORT=7777 RELAY_CONTAINER="fips-nat-relay${FIPS_CI_NAME_SUFFIX:-}" diff --git a/testing/nat/scripts/setup-topology.sh b/testing/nat/scripts/setup-topology.sh index e5d6130..a7468a6 100755 --- a/testing/nat/scripts/setup-topology.sh +++ b/testing/nat/scripts/setup-topology.sh @@ -4,6 +4,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" NAT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +ROOT_DIR="$(cd "$NAT_DIR/../.." && pwd)" SCENARIO="${1:?usage: setup-topology.sh }" @@ -25,6 +26,29 @@ esac router_a="fips-nat-router-a${FIPS_CI_NAME_SUFFIX:-}" router_b="fips-nat-router-b${FIPS_CI_NAME_SUFFIX:-}" +# Host interface names live in one global namespace and are capped at +# IFNAMSIZ-1 = 15 characters — far short of the run suffix, which is 20-plus at +# the default run-id length. Two concurrent runs both creating `vna0` damage +# each other: setup_pair deletes, adds and moves the interface in three +# separate `docker run`s, so B's re-add inside A's window sends B's interface +# into A's namespace and fails B's own move. +# +# Scope the names with the same four-hex token the chaos simulation uses, and +# derive it by calling sim.naming rather than re-implementing sha1 here, so +# ci-cleanup.sh's reaper (which calls the same module) cannot end up matching a +# different width. Empty suffix yields an empty token and today's exact names. +# +# No fallback on a derivation failure: silently reverting to `vna0` would +# reinstate the collision this exists to remove, so let it fail loudly. +veth_token() { + local suffix="${FIPS_CI_NAME_SUFFIX:-}" + if [ -z "$suffix" ]; then + echo "" + return 0 + fi + PYTHONPATH="$ROOT_DIR/testing/chaos" python3 -m sim.naming "$suffix" +} + helper_image() { if [ -n "${IP_HELPER_IMAGE:-}" ]; then echo "$IP_HELPER_IMAGE" @@ -122,11 +146,14 @@ setup_pair() { main() { cd "$NAT_DIR" - local image + local image token image="$(helper_image)" + token="$(veth_token)" - setup_pair "$image" "$node_a" "$router_a" vna0 vna1 172.31.1.10/24 172.31.1.254/24 - setup_pair "$image" "$node_b" "$router_b" vnb0 vnb1 172.31.2.10/24 172.31.2.254/24 + setup_pair "$image" "$node_a" "$router_a" \ + "vna${token}0" "vna${token}1" 172.31.1.10/24 172.31.1.254/24 + setup_pair "$image" "$node_b" "$router_b" \ + "vnb${token}0" "vnb${token}1" 172.31.2.10/24 172.31.2.254/24 } main "$@" diff --git a/testing/nat/scripts/stun-faults-test.sh b/testing/nat/scripts/stun-faults-test.sh index bc508fe..5a5fafe 100755 --- a/testing/nat/scripts/stun-faults-test.sh +++ b/testing/nat/scripts/stun-faults-test.sh @@ -33,11 +33,28 @@ GENERATE_SCRIPT="$SCRIPT_DIR/generate-configs.sh" PROFILE="stun-faults" SCENARIO="$PROFILE" COMPOSE=(docker compose -f "$NAT_DIR/docker-compose.yml") + +# Optional extra compose-file overlay chain (colon-separated paths), matching +# nat-test.sh. ci-local.sh appends testing/nat/docker-compose.external-net.yml +# here so this suite attaches to the networks the run already claimed; without +# the hook compose would create its own from the base file's subnet and collide +# with the run's own claim. Paths are relative to ROOT_DIR unless absolute. +if [ -n "${FIPS_NAT_EXTRA_COMPOSE:-}" ]; then + IFS=':' read -ra _NAT_EXTRA <<< "${FIPS_NAT_EXTRA_COMPOSE}" + for _f in "${_NAT_EXTRA[@]}"; do + case "$_f" in + /*) COMPOSE+=(-f "$_f") ;; + *) COMPOSE+=(-f "$ROOT_DIR/$_f") ;; + esac + done +fi + NODE="fips-nat-stun-fault-node${FIPS_CI_NAME_SUFFIX:-}" PEER="fips-nat-stun-fault-peer${FIPS_CI_NAME_SUFFIX:-}" SHIM="fips-nat-stun-fault-shim${FIPS_CI_NAME_SUFFIX:-}" STUN_CONTAINER="fips-nat-stun${FIPS_CI_NAME_SUFFIX:-}" -STUN_HOST="172.31.10.40" +# Claimed per run by ci-local.sh; unset renders the lab's historical address. +STUN_HOST="${NAT_LAN_PREFIX:-172.31.10}.40" STUN_PORT=3478 DEV="eth0"