Files
fips/testing/static/docker-compose.yml
Johnathan Corgan 965de26239 testing: make ci-local.sh preemption-safe with per-run docker isolation
A CI worker may preempt an in-flight ci-local.sh run (SIGTERM, then SIGKILL
after a grace period) to restart on a newer commit. For that kill to be safe,
the script must clean up after itself and never let a dying run collide with
its restart. It previously had no signal handling, shared the default compose
project name across runs, and tore down each suite only at the suite end.

- Derive a per-run id (honoring FIPS_CI_RUN_ID, else short-sha+random) and
  namespace every docker resource to it: a fipsci_<run>_<suite> compose project
  per suite and per parallel chaos child, and per-run image tags
  (fips-test:<run>, fips-test-app:<run>) retagged to :latest only after both
  builds succeed so :latest never points at a half-built image.
- Install a bounded, idempotent teardown trap on SIGTERM/SIGINT (+ EXIT): reap
  parallel chaos children, then force-remove this run's docker resources via
  the new ci-cleanup.sh, wrapped in timeout so a stuck down cannot wedge it.
- Exit 143 (SIGTERM) / 130 (SIGINT), distinct from 0 (pass) / 1 (failed), so a
  preempting worker tells a cancelled run from a real failure.
- Add ci-cleanup.sh (also ci-local.sh --reap): force-removes leftover CI
  resources by the com.corganlabs.fips-ci=1 label and the fipsci_ project
  prefix, robust to however a prior run died.
- Label every per-suite docker resource so the label sweep reaps it after a
  SIGKILL regardless of network name: direct docker run/network resources, the
  sidecar compose services, and every per-suite compose network (acl-allowlist,
  boringtun, firewall, nat, static, both tor suites, and the chaos generator
  template). Parametrize the static/sidecar compose image refs so the per-run
  tags are honored.
- Give each parallel chaos child a unique /24 from 10.30.x (a new --subnet
  override on the sim CLI, assigned per-child in ci-local.sh) so parallel
  children never collide on a shared docker subnet, and a chaos net can never
  span a fixed-subnet suite (sidecar/static in 172.20.x). 10.30.x sits outside
  docker's default-address-pool range, so an auto-assigned net cannot land on
  it either; node IPs derive from the subnet, so no scenario config changes.
2026-06-29 14:23:46 +00:00

580 lines
17 KiB
YAML

networks:
fips-net:
driver: bridge
labels:
- "com.corganlabs.fips-ci=1"
ipam:
config:
- subnet: 172.20.0.0/24
gateway-lan:
driver: bridge
labels:
- "com.corganlabs.fips-ci=1"
enable_ipv6: true
ipam:
config:
- subnet: 172.20.1.0/24
- subnet: fd02::/64
x-fips-common: &fips-common
image: ${FIPS_TEST_IMAGE:-fips-test:latest}
cap_add:
- NET_ADMIN
devices:
- /dev/net/tun:/dev/net/tun
sysctls:
- net.ipv6.conf.all.disable_ipv6=0
restart: "no"
env_file:
- ./generated-configs/npubs.env
environment:
- RUST_LOG=info
# Passthrough for A/B benchmarking — set on the host shell before
# `docker compose up` to override the worker-pool defaults.
- FIPS_ENCRYPT_WORKERS=${FIPS_ENCRYPT_WORKERS:-}
- FIPS_DECRYPT_WORKERS=${FIPS_DECRYPT_WORKERS:-}
- FIPS_PERF=${FIPS_PERF:-0}
- FIPS_PERF_INTERVAL_SECS=${FIPS_PERF_INTERVAL_SECS:-5}
services:
# ── Mesh topology ──────────────────────────────────────────────
node-a:
<<: *fips-common
profiles: ["mesh"]
container_name: fips-node-a
hostname: node-a
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh/node-a.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.10
node-b:
<<: *fips-common
profiles: ["mesh"]
container_name: fips-node-b
hostname: node-b
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh/node-b.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.11
node-c:
<<: *fips-common
profiles: ["mesh"]
container_name: fips-node-c
hostname: node-c
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh/node-c.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.12
node-d:
<<: *fips-common
profiles: ["mesh"]
container_name: fips-node-d
hostname: node-d
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh/node-d.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.13
node-e:
<<: *fips-common
profiles: ["mesh"]
container_name: fips-node-e
hostname: node-e
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh/node-e.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.14
# ── Mesh-public topology (mesh + external public node) ────────
pub-a:
<<: *fips-common
profiles: ["mesh-public"]
container_name: fips-node-a
hostname: node-a
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh-public/node-a.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.10
pub-b:
<<: *fips-common
profiles: ["mesh-public"]
container_name: fips-node-b
hostname: node-b
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh-public/node-b.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.11
pub-c:
<<: *fips-common
profiles: ["mesh-public"]
container_name: fips-node-c
hostname: node-c
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh-public/node-c.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.12
pub-d:
<<: *fips-common
profiles: ["mesh-public"]
container_name: fips-node-d
hostname: node-d
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh-public/node-d.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.13
pub-e:
<<: *fips-common
profiles: ["mesh-public"]
container_name: fips-node-e
hostname: node-e
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/mesh-public/node-e.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.14
# ── Chain topology (A-B-C-D-E) ────────────────────────────────
chain-a:
<<: *fips-common
profiles: ["chain"]
container_name: fips-node-a
hostname: node-a
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/chain/node-a.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.10
chain-b:
<<: *fips-common
profiles: ["chain"]
container_name: fips-node-b
hostname: node-b
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/chain/node-b.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.11
chain-c:
<<: *fips-common
profiles: ["chain"]
container_name: fips-node-c
hostname: node-c
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/chain/node-c.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.12
chain-d:
<<: *fips-common
profiles: ["chain"]
container_name: fips-node-d
hostname: node-d
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/chain/node-d.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.13
chain-e:
<<: *fips-common
profiles: ["chain"]
container_name: fips-node-e
hostname: node-e
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/chain/node-e.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.14
# ── Rekey integration test (mesh + aggressive rekey timers) ──
rekey-a:
<<: *fips-common
profiles: ["rekey"]
container_name: fips-node-a
hostname: node-a
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey/node-a.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.10
rekey-b:
<<: *fips-common
profiles: ["rekey"]
container_name: fips-node-b
hostname: node-b
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey/node-b.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.11
rekey-c:
<<: *fips-common
profiles: ["rekey"]
container_name: fips-node-c
hostname: node-c
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey/node-c.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.12
rekey-d:
<<: *fips-common
profiles: ["rekey"]
container_name: fips-node-d
hostname: node-d
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey/node-d.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.13
rekey-e:
<<: *fips-common
profiles: ["rekey"]
container_name: fips-node-e
hostname: node-e
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey/node-e.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.14
# ── Rekey + accept_connections=false on node b ──────────────────
# Exercises the auto_connect-initiator-with-accept-off regression
# class. Same 5-node mesh, but node b's generated config has
# `transports.udp.accept_connections: false` (injected by
# rekey-test.sh's inject-config when REKEY_ACCEPT_OFF_NODES=b).
rekey-accept-off-a:
<<: *fips-common
profiles: ["rekey-accept-off"]
container_name: fips-node-a
hostname: node-a
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-accept-off/node-a.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.10
rekey-accept-off-b:
<<: *fips-common
profiles: ["rekey-accept-off"]
container_name: fips-node-b
hostname: node-b
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-accept-off/node-b.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.11
rekey-accept-off-c:
<<: *fips-common
profiles: ["rekey-accept-off"]
container_name: fips-node-c
hostname: node-c
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-accept-off/node-c.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.12
rekey-accept-off-d:
<<: *fips-common
profiles: ["rekey-accept-off"]
container_name: fips-node-d
hostname: node-d
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-accept-off/node-d.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.13
rekey-accept-off-e:
<<: *fips-common
profiles: ["rekey-accept-off"]
container_name: fips-node-e
hostname: node-e
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-accept-off/node-e.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.14
# ── Rekey topology with udp.outbound_only=true on node b ────
# Exercises the udp.outbound_only rekey-msg1 admission regression
# observed in production 2026-04-30. Same 5-node mesh as
# rekey-accept-off, but node-b's generated config has
# `transports.udp.outbound_only: true` (forces ephemeral bind +
# accept_connections false) AND node-b's peer-c address is
# rewritten to the docker hostname (`node-c:2121`) so the
# `addr_to_link` lookup misses on the inbound numeric source addr.
# Injected by rekey-test.sh's inject-config when
# REKEY_OUTBOUND_ONLY_NODES=b.
rekey-outbound-only-a:
<<: *fips-common
profiles: ["rekey-outbound-only"]
container_name: fips-node-a
hostname: node-a
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-outbound-only/node-a.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.10
rekey-outbound-only-b:
<<: *fips-common
profiles: ["rekey-outbound-only"]
container_name: fips-node-b
hostname: node-b
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-outbound-only/node-b.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.11
rekey-outbound-only-c:
<<: *fips-common
profiles: ["rekey-outbound-only"]
container_name: fips-node-c
hostname: node-c
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-outbound-only/node-c.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.12
rekey-outbound-only-d:
<<: *fips-common
profiles: ["rekey-outbound-only"]
container_name: fips-node-d
hostname: node-d
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-outbound-only/node-d.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.13
rekey-outbound-only-e:
<<: *fips-common
profiles: ["rekey-outbound-only"]
container_name: fips-node-e
hostname: node-e
environment:
- RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/rekey-outbound-only/node-e.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.14
# ── TCP chain topology (A-B-C) ───────────────────────────────
tcp-a:
<<: *fips-common
profiles: ["tcp-chain"]
container_name: fips-node-a
hostname: node-a
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/tcp-chain/node-a.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.10
tcp-b:
<<: *fips-common
profiles: ["tcp-chain"]
container_name: fips-node-b
hostname: node-b
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/tcp-chain/node-b.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.11
tcp-c:
<<: *fips-common
profiles: ["tcp-chain"]
container_name: fips-node-c
hostname: node-c
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/tcp-chain/node-c.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.12
# ── Gateway integration test (gateway + server + non-FIPS client) ─
gw-gateway:
<<: *fips-common
profiles: ["gateway"]
container_name: fips-gw-gateway
hostname: gw-gateway
# Privileged required: gateway must enable IPv6 on eth1 (second network,
# attached after container start) and manage nftables NAT rules.
privileged: true
environment:
- RUST_LOG=info
- FIPS_TEST_MODE=gateway
sysctls:
- net.ipv6.conf.all.disable_ipv6=0
- net.ipv6.conf.default.disable_ipv6=0
- net.ipv6.conf.all.forwarding=1
- net.ipv6.conf.all.proxy_ndp=1
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/gateway/node-a.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.10
gateway-lan:
ipv4_address: 172.20.1.10
ipv6_address: fd02::10
gw-server:
<<: *fips-common
profiles: ["gateway"]
container_name: fips-gw-server
hostname: gw-server
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/gateway/node-b.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.11
# Second mesh destination — gives gw-client-2 a distinct npub to target
# so the gateway allocates a separate virtual-IP mapping per LAN client.
# Mirrors gw-server; not on gateway-lan.
gw-server-2:
<<: *fips-common
profiles: ["gateway"]
container_name: fips-gw-server-2
hostname: gw-server-2
volumes:
- ../docker/resolv.conf:/etc/resolv.conf:ro
- ./generated-configs/gateway/node-c.yaml:/etc/fips/fips.yaml:ro
networks:
fips-net:
ipv4_address: 172.20.0.12
gw-client:
image: ${FIPS_TEST_APP_IMAGE:-fips-test-app:latest}
profiles: ["gateway"]
container_name: fips-gw-client
hostname: gw-client
cap_add:
- NET_ADMIN
sysctls:
- net.ipv6.conf.all.disable_ipv6=0
volumes:
- ./configs/gateway-resolv.conf:/etc/resolv.conf:ro
networks:
gateway-lan:
ipv4_address: 172.20.1.20
ipv6_address: fd02::20
restart: "no"
env_file:
- ./generated-configs/npubs.env
# Second LAN client — exercises concurrent multi-client mappings.
# Same image and gateway-lan attachment as
# gw-client; the gateway must allocate a distinct virtual IP for it.
gw-client-2:
image: ${FIPS_TEST_APP_IMAGE:-fips-test-app:latest}
profiles: ["gateway"]
container_name: fips-gw-client-2
hostname: gw-client-2
cap_add:
- NET_ADMIN
sysctls:
- net.ipv6.conf.all.disable_ipv6=0
volumes:
- ./configs/gateway-resolv.conf:/etc/resolv.conf:ro
networks:
gateway-lan:
ipv4_address: 172.20.1.21
ipv6_address: fd02::21
restart: "no"
env_file:
- ./generated-configs/npubs.env