mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Claim a free address range for the sidecar network instead of hardcoding one
Making the sidecar network name unique per run exposed a latent problem rather than causing a new one. With a fixed name, compose found any existing network and reused it, which quietly masked the fact that the address range was hardcoded. With a unique name there is nothing to reuse, so the range is requested every time and the run fails outright if anything already holds it. Claim the range instead of assuming it. The suite now tries to create its network on a candidate range and, if the daemon reports an overlap, moves to the next candidate. The daemon's own address pool becomes the arbiter, so two concurrent runs cannot land on the same range at all - as opposed to a range derived from the run identifier, which only makes a collision unlikely, and a collision is the precise failure being avoided. The node addresses are derived from whichever range is claimed rather than written alongside it, including the gateway address the isolation check probes. Moved to a range that nothing in the tree claims and that sits outside the daemon's default pool. Only an overlap is worth advancing on. Any other creation failure is real and is reported with the daemon's own message, rather than being retried sixty-four times and buried - a lesson from a failure elsewhere in the harness this week that was undiagnosable because its error output was discarded. All three nodes now attach to the pre-created network as external, so teardown removes it explicitly. Verified: three concurrent claims take three distinct ranges; a non-overlap error fails fast rather than looping; the suite passes 18/18; and the claimed gateway address is confirmed pingable when not firewalled, so the isolation assertions still mean something.
This commit is contained in:
@@ -33,10 +33,20 @@ NETWORK_NAME="fips-sidecar-test${FIPS_CI_NAME_SUFFIX:-}"
|
||||
PROJ_A="sidecar-a${FIPS_CI_NAME_SUFFIX:-}"
|
||||
PROJ_B="sidecar-b${FIPS_CI_NAME_SUFFIX:-}"
|
||||
PROJ_C="sidecar-c${FIPS_CI_NAME_SUFFIX:-}"
|
||||
SUBNET="172.20.2.0/24"
|
||||
NODE_A_IP="172.20.2.10"
|
||||
NODE_B_IP="172.20.2.11"
|
||||
NODE_C_IP="172.20.2.12"
|
||||
# Network address range — claimed at startup by alloc_network below rather than
|
||||
# hardcoded, so two concurrent runs cannot request the same range.
|
||||
#
|
||||
# 10.40.0.0/16 is unclaimed in-tree: the suites use 172.20-172.32 and 172.99,
|
||||
# the chaos children 10.30.x, boringtun's inner network 10.99.x, and docker's
|
||||
# own default pool is 172.17-31 plus 192.168.
|
||||
NET_BASE="10.40"
|
||||
NET_CANDIDATES=64
|
||||
|
||||
SUBNET=""
|
||||
GATEWAY_IP=""
|
||||
NODE_A_IP=""
|
||||
NODE_B_IP=""
|
||||
NODE_C_IP=""
|
||||
|
||||
CONVERGE_TIMEOUT=30
|
||||
PASSED=0
|
||||
@@ -52,13 +62,50 @@ log() { echo "=== $*"; }
|
||||
pass() { echo " PASS: $*"; PASSED=$((PASSED + 1)); }
|
||||
fail() { echo " FAIL: $*"; FAILED=$((FAILED + 1)); }
|
||||
|
||||
# Claim a free /24 for this run's network.
|
||||
#
|
||||
# Claim-and-advance rather than deriving an offset from the run id: we attempt
|
||||
# creation on a candidate range and, if docker reports the range is taken, move
|
||||
# to the next. Docker's own address pool is then the arbiter, which makes an
|
||||
# overlap between two concurrent runs *impossible*. A hashed offset would only
|
||||
# make it unlikely, and an overlap is precisely the failure being avoided here.
|
||||
#
|
||||
# Deliberately does NOT discard stderr: only an address-pool conflict is worth
|
||||
# advancing on. Any other failure (name already taken, daemon error) is real,
|
||||
# and retrying it 64 times would bury the reason.
|
||||
alloc_network() {
|
||||
local i err
|
||||
for (( i = 0; i < NET_CANDIDATES; i++ )); do
|
||||
SUBNET="${NET_BASE}.${i}.0/24"
|
||||
if err=$(docker network create \
|
||||
--subnet "$SUBNET" \
|
||||
--label com.corganlabs.fips-ci=1 \
|
||||
--label "com.corganlabs.fips-ci.run=${FIPS_CI_RUN_ID:-manual}" \
|
||||
"$NETWORK_NAME" 2>&1); then
|
||||
GATEWAY_IP="${NET_BASE}.${i}.1"
|
||||
NODE_A_IP="${NET_BASE}.${i}.10"
|
||||
NODE_B_IP="${NET_BASE}.${i}.11"
|
||||
NODE_C_IP="${NET_BASE}.${i}.12"
|
||||
log "Claimed network $NETWORK_NAME on $SUBNET"
|
||||
return 0
|
||||
fi
|
||||
case "$err" in
|
||||
*"Pool overlaps"*|*"pool overlaps"*) continue ;;
|
||||
*) echo " FAIL: docker network create: $err" >&2; return 1 ;;
|
||||
esac
|
||||
done
|
||||
echo " FAIL: no free /24 in ${NET_BASE}.0.0/16 after ${NET_CANDIDATES} attempts" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
log "Cleaning up..."
|
||||
# Tear down B and C first (they reference A's network as external)
|
||||
# Tear down C and B first, then A. All three attach to the network as
|
||||
# external, so none of them removes it — we do that last, by name.
|
||||
docker compose $COMPOSE_EXT -p "$PROJ_C" down --volumes --remove-orphans 2>/dev/null || true
|
||||
docker compose $COMPOSE_EXT -p "$PROJ_B" down --volumes --remove-orphans 2>/dev/null || true
|
||||
# Tear down A last (it owns the network)
|
||||
docker compose $COMPOSE_BASE -p "$PROJ_A" down --volumes --remove-orphans 2>/dev/null || true
|
||||
docker compose $COMPOSE_EXT -p "$PROJ_A" down --volumes --remove-orphans 2>/dev/null || true
|
||||
docker network rm "$NETWORK_NAME" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# Always clean up on exit
|
||||
@@ -80,7 +127,9 @@ fi
|
||||
# node-b: peers with A (middle node, transit router)
|
||||
# node-c: peers with B (end node)
|
||||
|
||||
log "Starting node-a (no peers, creates network)..."
|
||||
alloc_network || exit 1
|
||||
|
||||
log "Starting node-a (no peers, joins the claimed network)..."
|
||||
# node-a is the root: explicitly clear FIPS_PEER_* so it does not inherit the
|
||||
# external peer default from .env (which points at a real public mesh node).
|
||||
# Without this, node-a auto-connects to the live mesh and the chain attaches
|
||||
@@ -91,7 +140,7 @@ FIPS_PEER_ADDR="" \
|
||||
FIPS_NETWORK="$NETWORK_NAME" \
|
||||
FIPS_SUBNET="$SUBNET" \
|
||||
FIPS_IPV4="$NODE_A_IP" \
|
||||
docker compose $COMPOSE_BASE -p "$PROJ_A" up -d
|
||||
docker compose $COMPOSE_EXT -p "$PROJ_A" up -d
|
||||
|
||||
log "Starting node-b (peers with node-a, joins external network)..."
|
||||
FIPS_NSEC="$NODE_B_NSEC" \
|
||||
@@ -217,7 +266,7 @@ for node in a b c; do
|
||||
log " Checking $container..."
|
||||
|
||||
# IPv4 gateway should be unreachable (iptables DROP on eth0)
|
||||
if docker exec "$container" ping -c1 -W2 172.20.2.1 >/dev/null 2>&1; then
|
||||
if docker exec "$container" ping -c1 -W2 "$GATEWAY_IP" >/dev/null 2>&1; then
|
||||
fail "$container can reach IPv4 gateway (isolation broken!)"
|
||||
else
|
||||
pass "$container cannot reach IPv4 gateway (IPv4 blocked)"
|
||||
|
||||
Reference in New Issue
Block a user