mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Make the gateway integration suite safe for concurrent CI runs
gateway-lan pinned 172.20.1.0/24 and fd02::/64, so two concurrent local CI runs collided on "Pool overlaps" at network creation. The IPv4 subnet has no consumer -- the whole gateway LAN path is IPv6 -- so drop it and let docker auto-assign. The IPv6 side cannot float, because the LAN clients' resolv.conf pins the gateway's address as their nameserver and that must be a literal known before they start, so claim a free /64 per run and thread the prefix through the compose address pins, a generated resolv.conf, and the test via a single exported variable. The claim and the external network ship as a harness-only compose overlay applied by run_gateway; the base compose keeps a normal gateway-lan network, so the GitHub matrix and any standalone bring-up are unaffected. With no claim every address renders exactly as before.
This commit is contained in:
Johnathan Corgan
+63
-15
@@ -617,29 +617,77 @@ run_chaos() {
|
||||
return $rc
|
||||
}
|
||||
|
||||
# Claim a free /64 for this run's gateway-lan network (Mechanism B).
|
||||
#
|
||||
# gateway-lan cannot float like fips-net: the LAN clients' resolv.conf pins the
|
||||
# gateway's LAN address as their nameserver, which must be a literal known
|
||||
# before they start. So instead of a fixed fd02::/64 that two concurrent runs
|
||||
# both request (the second failing on "Pool overlaps"), claim-and-advance a
|
||||
# candidate /64 and let docker's create be the atomic arbiter. The claimed
|
||||
# prefix is threaded — via the exported FIPS_GW_LAN6_PREFIX — into the compose
|
||||
# ipv6_address pins, the generated resolv.conf, and gateway-test.sh, so every
|
||||
# LAN address moves with the claim. i=0 renders today's fd02::/64.
|
||||
#
|
||||
# Mirrors sidecar/scripts/test-sidecar.sh:alloc_network. Deliberately does NOT
|
||||
# discard stderr: only an address-pool conflict is worth advancing on; any other
|
||||
# failure is real and burning through 256 candidates would bury the reason.
|
||||
claim_gateway_lan6() {
|
||||
local net="$1" i err
|
||||
for (( i = 0; i < 256; i++ )); do
|
||||
if err=$(docker network create --ipv6 \
|
||||
--subnet "fd02:0:0:${i}::/64" \
|
||||
--label "$CI_LABEL" --label "$CI_LABEL_RUN" \
|
||||
"$net" 2>&1); then
|
||||
export FIPS_GW_LAN6_PREFIX="fd02:0:0:${i}"
|
||||
info "[gateway] Claimed $net on fd02:0:0:${i}::/64"
|
||||
return 0
|
||||
fi
|
||||
case "$err" in
|
||||
*"Pool overlaps"*|*"pool overlaps"*) continue ;;
|
||||
*) fail "[gateway] docker network create: $err"; return 1 ;;
|
||||
esac
|
||||
done
|
||||
fail "[gateway] no free /64 in fd02:0:0:0-255::/64 after 256 attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Run gateway integration test
|
||||
run_gateway() {
|
||||
local compose="testing/static/docker-compose.yml"
|
||||
local ext="testing/static/docker-compose.gateway-external-net.yml"
|
||||
local rc=0
|
||||
export COMPOSE_PROJECT_NAME="$(ci_project static)"
|
||||
export FIPS_GW_LAN_NET="fips-gateway-lan${FIPS_CI_NAME_SUFFIX:-}"
|
||||
|
||||
info "[gateway] Generating configs"
|
||||
bash testing/static/scripts/generate-configs.sh gateway gateway-test || { record "gateway" 1; return; }
|
||||
bash testing/static/scripts/gateway-test.sh inject-config || { record "gateway" 1; return; }
|
||||
|
||||
info "[gateway] Starting containers"
|
||||
docker compose -f "$compose" --profile gateway up -d || { record "gateway" 1; return; }
|
||||
|
||||
info "[gateway] Running gateway test"
|
||||
if bash testing/static/scripts/gateway-test.sh; then
|
||||
rc=0
|
||||
else
|
||||
rc=1
|
||||
info "[gateway] Collecting failure logs"
|
||||
docker compose -f "$compose" --profile gateway logs --no-color 2>&1 | tail -100
|
||||
# Claim the LAN /64 first: generate-configs writes resolv.conf from the
|
||||
# claimed prefix, and inject-config renders the port-forward targets from
|
||||
# it, so both must see FIPS_GW_LAN6_PREFIX before they run.
|
||||
info "[gateway] Claiming LAN network"
|
||||
if ! claim_gateway_lan6 "$FIPS_GW_LAN_NET"; then
|
||||
record "gateway" 1
|
||||
return
|
||||
fi
|
||||
|
||||
docker compose -f "$compose" --profile gateway down --volumes --remove-orphans 2>/dev/null
|
||||
info "[gateway] Generating configs"
|
||||
if bash testing/static/scripts/generate-configs.sh gateway gateway-test \
|
||||
&& bash testing/static/scripts/gateway-test.sh inject-config \
|
||||
&& { info "[gateway] Starting containers"; \
|
||||
docker compose -f "$compose" -f "$ext" --profile gateway up -d; }; then
|
||||
info "[gateway] Running gateway test"
|
||||
if bash testing/static/scripts/gateway-test.sh; then
|
||||
rc=0
|
||||
else
|
||||
rc=1
|
||||
info "[gateway] Collecting failure logs"
|
||||
docker compose -f "$compose" -f "$ext" --profile gateway logs --no-color 2>&1 | tail -100
|
||||
fi
|
||||
else
|
||||
rc=1
|
||||
fi
|
||||
|
||||
# compose down does not remove an external network, so drop it explicitly.
|
||||
docker compose -f "$compose" -f "$ext" --profile gateway down --volumes --remove-orphans 2>/dev/null
|
||||
docker network rm "$FIPS_GW_LAN_NET" >/dev/null 2>&1 || true
|
||||
record "gateway" $rc
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# Override: attach gateway-lan to a pre-created external network instead of
|
||||
# letting compose create it from the base file's fd02::/64 pin.
|
||||
#
|
||||
# Applied only by ci-local.sh's run_gateway, which claims a free /64 per run
|
||||
# (claim_gateway_lan6) and exports FIPS_GW_LAN_NET / FIPS_GW_LAN6_PREFIX before
|
||||
# `up`. This is what makes two concurrent local gateway runs collision-safe:
|
||||
# each claims a distinct /64, so neither requests a fixed range the other holds.
|
||||
#
|
||||
# The GitHub matrix and any standalone `docker compose up` do NOT apply this
|
||||
# overlay; they use the base file's normal gateway-lan network unchanged. This
|
||||
# mirrors sidecar/docker-compose.external-net.yml.
|
||||
networks:
|
||||
gateway-lan:
|
||||
external: true
|
||||
name: ${FIPS_GW_LAN_NET:-fips-gateway-lan}
|
||||
@@ -8,6 +8,10 @@ networks:
|
||||
driver: bridge
|
||||
labels:
|
||||
- "com.corganlabs.fips-ci=1"
|
||||
# IPv4 is dropped so docker auto-assigns it (nothing reads a LAN IPv4 — the
|
||||
# whole gateway LAN path is IPv6). The fd02::/64 pin stays for the standalone /
|
||||
# GitHub path; concurrent local runs replace this network with a per-run
|
||||
# claimed /64 via docker-compose.gateway-external-net.yml (see run_gateway).
|
||||
gateway-lan:
|
||||
driver: bridge
|
||||
labels:
|
||||
@@ -15,7 +19,6 @@ networks:
|
||||
enable_ipv6: true
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.20.1.0/24
|
||||
- subnet: fd02::/64
|
||||
|
||||
x-fips-common: &fips-common
|
||||
@@ -475,8 +478,7 @@ services:
|
||||
networks:
|
||||
fips-net:
|
||||
gateway-lan:
|
||||
ipv4_address: 172.20.1.10
|
||||
ipv6_address: fd02::10
|
||||
ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::10
|
||||
|
||||
gw-server:
|
||||
<<: *fips-common
|
||||
@@ -513,11 +515,10 @@ services:
|
||||
sysctls:
|
||||
- net.ipv6.conf.all.disable_ipv6=0
|
||||
volumes:
|
||||
- ./configs/gateway-resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/gateway/resolv.conf:/etc/resolv.conf:ro
|
||||
networks:
|
||||
gateway-lan:
|
||||
ipv4_address: 172.20.1.20
|
||||
ipv6_address: fd02::20
|
||||
ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::20
|
||||
restart: "no"
|
||||
env_file:
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env
|
||||
@@ -535,11 +536,10 @@ services:
|
||||
sysctls:
|
||||
- net.ipv6.conf.all.disable_ipv6=0
|
||||
volumes:
|
||||
- ./configs/gateway-resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/gateway/resolv.conf:/etc/resolv.conf:ro
|
||||
networks:
|
||||
gateway-lan:
|
||||
ipv4_address: 172.20.1.21
|
||||
ipv6_address: fd02::21
|
||||
ipv6_address: ${FIPS_GW_LAN6_PREFIX:-fd02}::21
|
||||
restart: "no"
|
||||
env_file:
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env
|
||||
|
||||
@@ -26,6 +26,16 @@ SERVER2="fips-gw-server-2${FIPS_CI_NAME_SUFFIX:-}"
|
||||
CLIENT="fips-gw-client${FIPS_CI_NAME_SUFFIX:-}"
|
||||
CLIENT2="fips-gw-client-2${FIPS_CI_NAME_SUFFIX:-}"
|
||||
|
||||
# LAN-side IPv6 addressing. run_gateway claims a per-run /64 and exports
|
||||
# FIPS_GW_LAN6_PREFIX; unset (standalone / GitHub) these render the base
|
||||
# compose's fd02:: addresses, byte-identical to before. GW_DNS is the gateway's
|
||||
# LAN address (nameserver + route next-hop); GW_CLIENT_LAN is gw-client's LAN
|
||||
# address (inbound port-forward target). fd01::/112 (the virtual pool) is NOT
|
||||
# claimed and stays literal below.
|
||||
GW_LAN6_PREFIX="${FIPS_GW_LAN6_PREFIX:-fd02}"
|
||||
GW_DNS="${GW_LAN6_PREFIX}::10"
|
||||
GW_CLIENT_LAN="${GW_LAN6_PREFIX}::20"
|
||||
|
||||
# ── inject-config subcommand ─────────────────────────────────────────────
|
||||
|
||||
inject_gateway_config() {
|
||||
@@ -58,21 +68,21 @@ cfg['gateway'] = {
|
||||
{
|
||||
'listen_port': 18080,
|
||||
'proto': 'tcp',
|
||||
'target': '[fd02::20]:8080',
|
||||
'target': '[${GW_CLIENT_LAN}]:8080',
|
||||
},
|
||||
# 6B: second TCP forward — exercises multiple simultaneous TCP
|
||||
# rules sharing the same LAN backend on a different listen port.
|
||||
{
|
||||
'listen_port': 18082,
|
||||
'proto': 'tcp',
|
||||
'target': '[fd02::20]:8081',
|
||||
'target': '[${GW_CLIENT_LAN}]:8081',
|
||||
},
|
||||
# 6A: UDP forward — exercises the runtime UDP DNAT path (rule
|
||||
# shape + conntrack handling) end-to-end.
|
||||
{
|
||||
'listen_port': 18081,
|
||||
'proto': 'udp',
|
||||
'target': '[fd02::20]:8081',
|
||||
'target': '[${GW_CLIENT_LAN}]:8081',
|
||||
},
|
||||
],
|
||||
}
|
||||
@@ -130,7 +140,7 @@ for i in $(seq 1 30); do
|
||||
# Try resolving the server's npub via the gateway DNS from the client.
|
||||
# Match fd01:: specifically (the pool prefix) to avoid false-positive
|
||||
# matches on error messages containing fd02::10.
|
||||
local_result=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @fd02::10 2>/dev/null || true)
|
||||
local_result=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @${GW_DNS} 2>/dev/null || true)
|
||||
if echo "$local_result" | grep -q "^fd01::"; then
|
||||
echo " Gateway DNS responding after ${i}s"
|
||||
DNS_READY=true
|
||||
@@ -146,23 +156,23 @@ fi
|
||||
# Phase 3: Client network setup — route virtual IP pool via gateway
|
||||
echo ""
|
||||
echo "Phase 3: Client network setup"
|
||||
docker exec "$CLIENT" ip -6 route add fd01::/112 via fd02::10 2>/dev/null || true
|
||||
echo " Added route fd01::/112 via fd02::10 on $CLIENT"
|
||||
docker exec "$CLIENT2" ip -6 route add fd01::/112 via fd02::10 2>/dev/null || true
|
||||
echo " Added route fd01::/112 via fd02::10 on $CLIENT2"
|
||||
docker exec "$CLIENT" ip -6 route add fd01::/112 via ${GW_DNS} 2>/dev/null || true
|
||||
echo " Added route fd01::/112 via ${GW_DNS} on $CLIENT"
|
||||
docker exec "$CLIENT2" ip -6 route add fd01::/112 via ${GW_DNS} 2>/dev/null || true
|
||||
echo " Added route fd01::/112 via ${GW_DNS} on $CLIENT2"
|
||||
|
||||
# Phase 4: DNS resolution test — resolve server npub from both clients,
|
||||
# exercising concurrent multi-client mappings.
|
||||
echo ""
|
||||
echo "Phase 4: DNS resolution"
|
||||
VIRTUAL_IP=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @fd02::10 2>/dev/null | head -1)
|
||||
VIRTUAL_IP=$(docker exec "$CLIENT" dig +short AAAA "${NPUB_B}.fips" @${GW_DNS} 2>/dev/null | head -1)
|
||||
if [ -n "$VIRTUAL_IP" ] && echo "$VIRTUAL_IP" | grep -q "fd01"; then
|
||||
check "Resolve ${NPUB_B:0:20}...fips on $CLIENT → $VIRTUAL_IP" 0
|
||||
else
|
||||
check "Resolve ${NPUB_B:0:20}...fips on $CLIENT (got: '$VIRTUAL_IP')" 1
|
||||
fi
|
||||
|
||||
VIRTUAL_IP_2=$(docker exec "$CLIENT2" dig +short AAAA "${NPUB_C}.fips" @fd02::10 2>/dev/null | head -1)
|
||||
VIRTUAL_IP_2=$(docker exec "$CLIENT2" dig +short AAAA "${NPUB_C}.fips" @${GW_DNS} 2>/dev/null | head -1)
|
||||
if [ -n "$VIRTUAL_IP_2" ] && echo "$VIRTUAL_IP_2" | grep -q "fd01"; then
|
||||
check "Resolve ${NPUB_C:0:20}...fips on $CLIENT2 → $VIRTUAL_IP_2" 0
|
||||
else
|
||||
@@ -357,7 +367,7 @@ else
|
||||
# 8080 backend serves "inbound-forward-ok" (no -2 suffix) — distinct
|
||||
# from the 8081 backend so a misrouted response would be detectable.
|
||||
if echo "$FWD_RESPONSE" | grep -qE '^inbound-forward-ok$'; then
|
||||
check "Inbound HTTP via TCP forward 18080 → [fd02::20]:8080" 0
|
||||
check "Inbound HTTP via TCP forward 18080 → [${GW_CLIENT_LAN}]:8080" 0
|
||||
else
|
||||
check "Inbound HTTP via TCP forward 18080 (response: '${FWD_RESPONSE:0:80}')" 1
|
||||
fi
|
||||
@@ -365,7 +375,7 @@ else
|
||||
FWD_RESPONSE_2=$(docker exec "$SERVER" curl -6 -s --max-time 10 \
|
||||
"http://[${GW_MESH_IP}]:18082/" 2>&1) || true
|
||||
if echo "$FWD_RESPONSE_2" | grep -q "inbound-forward-ok-2"; then
|
||||
check "Inbound HTTP via TCP forward 18082 → [fd02::20]:8081 (6B)" 0
|
||||
check "Inbound HTTP via TCP forward 18082 → [${GW_CLIENT_LAN}]:8081 (6B)" 0
|
||||
else
|
||||
check "Inbound HTTP via TCP forward 18082 (response: '${FWD_RESPONSE_2:0:80}')" 1
|
||||
fi
|
||||
@@ -384,7 +394,7 @@ except Exception as e:
|
||||
sys.stdout.write('ERR: ' + str(e))
|
||||
" 2>&1) || true
|
||||
if echo "$UDP_RESPONSE" | grep -q "udp-forward-ok:ping-via-udp-fwd"; then
|
||||
check "Inbound UDP via forward 18081 → [fd02::20]:8081 (6A)" 0
|
||||
check "Inbound UDP via forward 18081 → [${GW_CLIENT_LAN}]:8081 (6A)" 0
|
||||
else
|
||||
check "Inbound UDP via forward 18081 (response: '${UDP_RESPONSE:0:80}')" 1
|
||||
fi
|
||||
@@ -444,8 +454,8 @@ docker exec "$GATEWAY" pkill -f "^fips --config" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Gateway upstream timeout is 5s, so dig must wait longer than that.
|
||||
SERVFAIL_RESULT=$(docker exec "$CLIENT" dig +short +tries=1 +time=8 AAAA "test-servfail.fips" @fd02::10 2>&1 || true)
|
||||
SERVFAIL_STATUS=$(docker exec "$CLIENT" dig +tries=1 +time=8 AAAA "test-servfail.fips" @fd02::10 2>&1 | grep -c "SERVFAIL" || true)
|
||||
SERVFAIL_RESULT=$(docker exec "$CLIENT" dig +short +tries=1 +time=8 AAAA "test-servfail.fips" @${GW_DNS} 2>&1 || true)
|
||||
SERVFAIL_STATUS=$(docker exec "$CLIENT" dig +tries=1 +time=8 AAAA "test-servfail.fips" @${GW_DNS} 2>&1 | grep -c "SERVFAIL" || true)
|
||||
if [ "$SERVFAIL_STATUS" -ge 1 ]; then
|
||||
check "SERVFAIL when daemon DNS is down" 0
|
||||
else
|
||||
|
||||
@@ -274,6 +274,16 @@ generate_topology() {
|
||||
echo "${var_name}=$(get_key RESOLVED_NPUB "$node_id")" >> "$env_file"
|
||||
done
|
||||
echo " ✓ Generated $env_file"
|
||||
|
||||
# Phase 4 (gateway only): write the LAN-client resolv.conf. Its nameserver
|
||||
# is the gateway's LAN address, which must be a literal known before the
|
||||
# client starts. run_gateway claims a per-run /64 and exports
|
||||
# FIPS_GW_LAN6_PREFIX before calling this; unset (standalone / GitHub) it
|
||||
# renders the base compose's fd02::10.
|
||||
if [ "$topology_name" = "gateway" ]; then
|
||||
echo "nameserver ${FIPS_GW_LAN6_PREFIX:-fd02}::10" > "$output_dir/resolv.conf"
|
||||
echo " ✓ Generated $output_dir/resolv.conf"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
|
||||
Reference in New Issue
Block a user