diff --git a/testing/ci-local.sh b/testing/ci-local.sh index f879ff4..3fe7796 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -390,15 +390,20 @@ ci_teardown() { --images "$CI_IMAGE_TEST $CI_IMAGE_APP" \ --veth-suffixes "${_suffixes[*]}" >/dev/null || true - # 3. The static suite's generated configs are per-run (a shared directory - # would let concurrent runs overwrite each other's node configs), so - # they are this run's to remove. Only on a green run: after a failure - # they are the evidence of what the failing nodes were actually + # 3. The static and firewall suites' generated configs are per-run (a + # shared directory would let concurrent runs overwrite each other's node + # configs), so they are this run's to remove. Only on a green run: after + # a failure they are the evidence of what the failing nodes were actually # configured with. Guarded on a non-empty suffix too, since without one # the path is the unscoped working directory a developer uses by hand, # which is not ours to delete. + # + # Every suite whose configs become per-run owes a line here. acl-allowlist + # also generates per-run now but is not in this runner's suite list, so + # this teardown never creates its directory and must not remove one. if [[ $run_status -eq 0 && -n "${CI_RUN_NAME_SUFFIX:-}" ]]; then rm -rf "$SCRIPT_DIR/static/generated-configs${CI_RUN_NAME_SUFFIX}" + rm -rf "$SCRIPT_DIR/firewall/generated-configs${CI_RUN_NAME_SUFFIX}" fi } diff --git a/testing/firewall/.gitignore b/testing/firewall/.gitignore index 31315f6..b06c7df 100644 --- a/testing/firewall/.gitignore +++ b/testing/firewall/.gitignore @@ -1 +1 @@ -generated-configs +generated-configs* diff --git a/testing/firewall/README.md b/testing/firewall/README.md index cecd186..3a6bd99 100644 --- a/testing/firewall/README.md +++ b/testing/firewall/README.md @@ -24,10 +24,18 @@ actively DROP'd by the fips chain (not silently unrouted). Two FIPS nodes peered over UDP on a Docker bridge network: -| Container | Hostname | docker IPv4 | Firewall | -|-------------------------|----------|---------------|----------| -| `fips-fw-container-a` | `host-a` | 172.32.0.10 | none (probe) | -| `fips-fw-container-b` | `host-b` | 172.32.0.11 | `fips.nft` + drop-in | +| Container | Hostname | Firewall | +|-------------------------|----------|----------| +| `fips-fw-container-a` | `host-a` | none (probe) | +| `fips-fw-container-b` | `host-b` | `fips.nft` + drop-in | + +The bridge network requests no subnet, so docker assigns one from its own +address pool and two concurrent runs never contend for a fixed range. No +node's IPv4 address is therefore known before startup, and the generated peer +stanzas address each other by docker hostname, resolved through the +container's dnsmasq to docker's embedded DNS. The firewall assertions +themselves are unaffected: they run over the fips0 overlay, whose addresses +are derived from the node npubs. `node-b` mounts the production `packaging/common/fips.nft` read-only at `/etc/fips/fips.nft`, plus a drop-in at `/etc/fips/fips.d/services.nft` @@ -108,4 +116,6 @@ docker compose -f testing/firewall/docker-compose.yml down ## Generated fixture location -`testing/firewall/generated-configs/` (gitignored). +`testing/firewall/generated-configs/` (gitignored), or +`generated-configs/` when `FIPS_CI_NAME_SUFFIX` is set, which is how +concurrent runs keep their fixtures apart. diff --git a/testing/firewall/docker-compose.yml b/testing/firewall/docker-compose.yml index 60b16ea..88110dd 100644 --- a/testing/firewall/docker-compose.yml +++ b/testing/firewall/docker-compose.yml @@ -1,11 +1,21 @@ networks: + # No subnet is requested: docker assigns one from the daemon's address pool. + # A fixed request is honoured verbatim, so two runs asking for the same range + # collide on "Pool overlaps" — which is why the generated peer addresses are + # docker hostnames (host-a, host-b) rather than literal IPs. Docker's embedded + # DNS is per-network, so the same hostname in two concurrent runs resolves + # inside each run's own subnet. + # + # That removes one of the two obstacles to running this suite twice at once, + # not both. The compose project name is still fixed, so two runs that do not + # set COMPOSE_PROJECT_NAME share a project, and the second `up` recreates the + # first's containers while either `down` removes both. The local CI runner + # scopes it externally (run_firewall in ci-local.sh); a bare hand run does + # not. Do not read the floating subnet as making concurrent bare runs safe. fw-net: driver: bridge labels: - "com.corganlabs.fips-ci=1" - ipam: - config: - - subnet: 172.32.0.0/24 x-fips-common: &fips-common build: @@ -31,12 +41,11 @@ services: hostname: host-a volumes: - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs/node-a/hosts:/etc/fips/hosts:ro - - ./generated-configs/node-a/fips.yaml:/etc/fips/fips.yaml:ro - - ./generated-configs/node-a/fips.key:/etc/fips/fips.key:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/hosts:/etc/fips/hosts:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/fips.yaml:/etc/fips/fips.yaml:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/fips.key:/etc/fips/fips.key:ro networks: - fw-net: - ipv4_address: 172.32.0.10 + - fw-net service-b: <<: *fips-common @@ -44,11 +53,10 @@ services: hostname: host-b volumes: - ../docker/resolv.conf:/etc/resolv.conf:ro - - ./generated-configs/node-b/hosts:/etc/fips/hosts:ro - - ./generated-configs/node-b/fips.yaml:/etc/fips/fips.yaml:ro - - ./generated-configs/node-b/fips.key:/etc/fips/fips.key:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/hosts:/etc/fips/hosts:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.yaml:/etc/fips/fips.yaml:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.key:/etc/fips/fips.key:ro - ../../packaging/common/fips.nft:/etc/fips/fips.nft:ro - - ./generated-configs/node-b/fips.d:/etc/fips/fips.d:ro + - ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.d:/etc/fips/fips.d:ro networks: - fw-net: - ipv4_address: 172.32.0.11 + - fw-net diff --git a/testing/firewall/generate-configs.sh b/testing/firewall/generate-configs.sh index 1fca1de..8ecd328 100755 --- a/testing/firewall/generate-configs.sh +++ b/testing/firewall/generate-configs.sh @@ -9,7 +9,12 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -GENERATED_DIR="$SCRIPT_DIR/generated-configs" + +# Scoped by the per-run suffix because this directory is wiped and rewritten +# below: two runs sharing one output directory would delete each other's +# fixtures out from under running containers. Unset (a bare hand run, or the +# GitHub-hosted path) it collapses to the historical "generated-configs". +GENERATED_DIR="$SCRIPT_DIR/generated-configs${FIPS_CI_NAME_SUFFIX:-}" # Deterministic test identities (mirrors the acl-allowlist style). NPUB_A="npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m" @@ -32,6 +37,10 @@ node-b $NPUB_B EOF } +# Peers are addressed by the docker hostname the compose file assigns (host-a, +# host-b), not by IP. The network requests no subnet so that two concurrent +# runs cannot collide on one address range, which means neither node's address +# is knowable before `docker compose up`. echo "Generating firewall fixtures..." rm -rf "$GENERATED_DIR" @@ -58,7 +67,7 @@ peers: alias: "node-b" addresses: - transport: udp - addr: "172.32.0.11:2121" + addr: "host-b:2121" connect_policy: auto_connect EOF @@ -89,7 +98,7 @@ peers: alias: "node-a" addresses: - transport: udp - addr: "172.32.0.10:2121" + addr: "host-a:2121" connect_policy: auto_connect EOF diff --git a/testing/firewall/test.sh b/testing/firewall/test.sh index 4d942e7..f4327ca 100755 --- a/testing/firewall/test.sh +++ b/testing/firewall/test.sh @@ -230,14 +230,20 @@ log "Case (b): node-b initiates outbound TCP, expects reply via conntrack" # node-b → node-a:8000 on the fips overlay. node-a has http.server on # [::]:8000 and is NOT firewalled, so this is purely a test of node-b's # outbound + ct state established,related path on the way back. +# +# mktemp rather than a fixed /tmp name: two concurrent runs of this suite +# would otherwise share one host file, and either one's `rm` between the +# other's write and read leaves an empty read that fails the http_code check +# for a reason that has nothing to do with the firewall. +CURL_OUT="$(mktemp)" set +e docker exec "$CONTAINER_B" curl -6 --silent --max-time 5 \ --output /dev/null --write-out '%{http_code}' \ - "http://[${ADDR_A}]:${OUTBOUND_TARGET_PORT}/" >/tmp/fw_b_rc 2>/dev/null + "http://[${ADDR_A}]:${OUTBOUND_TARGET_PORT}/" >"$CURL_OUT" 2>/dev/null RC=$? set -e -HTTP_CODE="$(cat /tmp/fw_b_rc 2>/dev/null || true)" -rm -f /tmp/fw_b_rc +HTTP_CODE="$(cat "$CURL_OUT" 2>/dev/null || true)" +rm -f "$CURL_OUT" if [ "$RC" -ne 0 ]; then fail "(b) outbound from node-b failed (curl rc=$RC, http=$HTTP_CODE) — conntrack reply path broken" fi