mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Johnathan Corgan
26a579b1c9
Making the sidecar network name unique per run exposed a latent problem rather than causing a new one. With a fixed name, compose found any existing network and reused it, which quietly masked the fact that the address range was hardcoded. With a unique name there is nothing to reuse, so the range is requested every time and the run fails outright if anything already holds it. Claim the range instead of assuming it. The suite now tries to create its network on a candidate range and, if the daemon reports an overlap, moves to the next candidate. The daemon's own address pool becomes the arbiter, so two concurrent runs cannot land on the same range at all - as opposed to a range derived from the run identifier, which only makes a collision unlikely, and a collision is the precise failure being avoided. The node addresses are derived from whichever range is claimed rather than written alongside it, including the gateway address the isolation check probes. Moved to a range that nothing in the tree claims and that sits outside the daemon's default pool. Only an overlap is worth advancing on. Any other creation failure is real and is reported with the daemon's own message, rather than being retried sixty-four times and buried - a lesson from a failure elsewhere in the harness this week that was undiagnosable because its error output was discarded. All three nodes now attach to the pre-created network as external, so teardown removes it explicitly. Verified: three concurrent claims take three distinct ranges; a non-overlap error fails fast rather than looping; the suite passes 18/18; and the claimed gateway address is confirmed pingable when not firewalled, so the isolation assertions still mean something.
306 lines
12 KiB
Bash
Executable File
306 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# Integration test for the FIPS sidecar deployment.
|
|
#
|
|
# Starts a 3-node chain (A—B—C) using standalone sidecar instances,
|
|
# verifies link establishment, multi-hop connectivity, and network
|
|
# isolation on each app container.
|
|
#
|
|
# Usage: ./test-sidecar.sh [--skip-build]
|
|
#
|
|
# Exit codes:
|
|
# 0 — all tests passed
|
|
# 1 — test failure
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SIDECAR_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
# shellcheck source=../../lib/wait-converge.sh
|
|
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
|
|
|
|
# Deterministic keys derived from: derive-keys.py sidecar-test node-{a,b,c}
|
|
NODE_A_NSEC="9e688d0879fa9cd025fea0487ac23495080e3de626070fdb9b78dc1f619dd453"
|
|
NODE_A_NPUB="npub1jvren5hnege54lu3p2nzqacctmulqvkgp68yvfuj5jme5dtgnhxsdh6788"
|
|
NODE_B_NSEC="3e4e10614c0490575fa5e994524ff3f4deaac2f20db189cc9c9a79da0d90f17a"
|
|
NODE_B_NPUB="npub15h7z0ljzudqe9pgwx99cjsz2c0ennuyvkcc8zvtk3lg97xwzex9ska6g4y"
|
|
NODE_C_NSEC="15148ed0131f7da43fd13e369dfedede14fb64698f3756636b569c3a3e87438f"
|
|
NODE_C_NPUB="npub1zhezcykd0e34z4fxtranl45jaasgnlxv0kjqwlq2v56ggssn0w4qelcrvr"
|
|
|
|
NETWORK_NAME="fips-sidecar-test${FIPS_CI_NAME_SUFFIX:-}"
|
|
|
|
# Compose project names. These are explicit (they override COMPOSE_PROJECT_NAME),
|
|
# so they carry the run suffix themselves — otherwise concurrent runs would share
|
|
# one project and tear down each other's containers.
|
|
PROJ_A="sidecar-a${FIPS_CI_NAME_SUFFIX:-}"
|
|
PROJ_B="sidecar-b${FIPS_CI_NAME_SUFFIX:-}"
|
|
PROJ_C="sidecar-c${FIPS_CI_NAME_SUFFIX:-}"
|
|
# Network address range — claimed at startup by alloc_network below rather than
|
|
# hardcoded, so two concurrent runs cannot request the same range.
|
|
#
|
|
# 10.40.0.0/16 is unclaimed in-tree: the suites use 172.20-172.32 and 172.99,
|
|
# the chaos children 10.30.x, boringtun's inner network 10.99.x, and docker's
|
|
# own default pool is 172.17-31 plus 192.168.
|
|
NET_BASE="10.40"
|
|
NET_CANDIDATES=64
|
|
|
|
SUBNET=""
|
|
GATEWAY_IP=""
|
|
NODE_A_IP=""
|
|
NODE_B_IP=""
|
|
NODE_C_IP=""
|
|
|
|
CONVERGE_TIMEOUT=30
|
|
PASSED=0
|
|
FAILED=0
|
|
|
|
# Compose file paths
|
|
COMPOSE_BASE="-f $SIDECAR_DIR/docker-compose.yml"
|
|
COMPOSE_EXT="$COMPOSE_BASE -f $SIDECAR_DIR/docker-compose.external-net.yml"
|
|
|
|
# ── Helpers ────────────────────────────────────────────────────────────────
|
|
|
|
log() { echo "=== $*"; }
|
|
pass() { echo " PASS: $*"; PASSED=$((PASSED + 1)); }
|
|
fail() { echo " FAIL: $*"; FAILED=$((FAILED + 1)); }
|
|
|
|
# Claim a free /24 for this run's network.
|
|
#
|
|
# Claim-and-advance rather than deriving an offset from the run id: we attempt
|
|
# creation on a candidate range and, if docker reports the range is taken, move
|
|
# to the next. Docker's own address pool is then the arbiter, which makes an
|
|
# overlap between two concurrent runs *impossible*. A hashed offset would only
|
|
# make it unlikely, and an overlap is precisely the failure being avoided here.
|
|
#
|
|
# Deliberately does NOT discard stderr: only an address-pool conflict is worth
|
|
# advancing on. Any other failure (name already taken, daemon error) is real,
|
|
# and retrying it 64 times would bury the reason.
|
|
alloc_network() {
|
|
local i err
|
|
for (( i = 0; i < NET_CANDIDATES; i++ )); do
|
|
SUBNET="${NET_BASE}.${i}.0/24"
|
|
if err=$(docker network create \
|
|
--subnet "$SUBNET" \
|
|
--label com.corganlabs.fips-ci=1 \
|
|
--label "com.corganlabs.fips-ci.run=${FIPS_CI_RUN_ID:-manual}" \
|
|
"$NETWORK_NAME" 2>&1); then
|
|
GATEWAY_IP="${NET_BASE}.${i}.1"
|
|
NODE_A_IP="${NET_BASE}.${i}.10"
|
|
NODE_B_IP="${NET_BASE}.${i}.11"
|
|
NODE_C_IP="${NET_BASE}.${i}.12"
|
|
log "Claimed network $NETWORK_NAME on $SUBNET"
|
|
return 0
|
|
fi
|
|
case "$err" in
|
|
*"Pool overlaps"*|*"pool overlaps"*) continue ;;
|
|
*) echo " FAIL: docker network create: $err" >&2; return 1 ;;
|
|
esac
|
|
done
|
|
echo " FAIL: no free /24 in ${NET_BASE}.0.0/16 after ${NET_CANDIDATES} attempts" >&2
|
|
return 1
|
|
}
|
|
|
|
cleanup() {
|
|
log "Cleaning up..."
|
|
# Tear down C and B first, then A. All three attach to the network as
|
|
# external, so none of them removes it — we do that last, by name.
|
|
docker compose $COMPOSE_EXT -p "$PROJ_C" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker compose $COMPOSE_EXT -p "$PROJ_B" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker compose $COMPOSE_EXT -p "$PROJ_A" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker network rm "$NETWORK_NAME" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Always clean up on exit
|
|
trap cleanup EXIT
|
|
|
|
# ── Build ──────────────────────────────────────────────────────────────────
|
|
|
|
if [[ "${1:-}" != "--skip-build" ]]; then
|
|
log "Building test images..."
|
|
DOCKER_DIR="$(cd "$SIDECAR_DIR/../docker" && pwd)"
|
|
docker build -t fips-test:latest "$DOCKER_DIR"
|
|
docker build -t fips-test-app:latest -f "$DOCKER_DIR/Dockerfile.app" "$DOCKER_DIR"
|
|
fi
|
|
|
|
# ── Start nodes ────────────────────────────────────────────────────────────
|
|
#
|
|
# Chain topology: A — B — C
|
|
# node-a: no outbound peers (accepts inbound from B)
|
|
# node-b: peers with A (middle node, transit router)
|
|
# node-c: peers with B (end node)
|
|
|
|
alloc_network || exit 1
|
|
|
|
log "Starting node-a (no peers, joins the claimed network)..."
|
|
# node-a is the root: explicitly clear FIPS_PEER_* so it does not inherit the
|
|
# external peer default from .env (which points at a real public mesh node).
|
|
# Without this, node-a auto-connects to the live mesh and the chain attaches
|
|
# under an external root, inflating tree depth and breaking isolation.
|
|
FIPS_NSEC="$NODE_A_NSEC" \
|
|
FIPS_PEER_NPUB="" \
|
|
FIPS_PEER_ADDR="" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_A_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_A" up -d
|
|
|
|
log "Starting node-b (peers with node-a, joins external network)..."
|
|
FIPS_NSEC="$NODE_B_NSEC" \
|
|
FIPS_PEER_NPUB="$NODE_A_NPUB" \
|
|
FIPS_PEER_ADDR="${NODE_A_IP}:2121" \
|
|
FIPS_PEER_ALIAS="node-a" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_B_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_B" up -d
|
|
|
|
log "Starting node-c (peers with node-b, joins external network)..."
|
|
FIPS_NSEC="$NODE_C_NSEC" \
|
|
FIPS_PEER_NPUB="$NODE_B_NPUB" \
|
|
FIPS_PEER_ADDR="${NODE_B_IP}:2121" \
|
|
FIPS_PEER_ALIAS="node-b" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_C_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_C" up -d
|
|
|
|
# ── Wait for convergence ──────────────────────────────────────────────────
|
|
|
|
log "Waiting for link establishment (up to ${CONVERGE_TIMEOUT}s)..."
|
|
|
|
converged=false
|
|
for i in $(seq 1 "$CONVERGE_TIMEOUT"); do
|
|
# node-b should have 2 links (A and C)
|
|
link_count=$(docker exec "${PROJ_B}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
if [ "$link_count" -ge 2 ]; then
|
|
converged=true
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
if [ "$converged" = true ]; then
|
|
log "Links established after ${i}s"
|
|
else
|
|
log "TIMEOUT: links did not converge in ${CONVERGE_TIMEOUT}s"
|
|
log "node-a links:"
|
|
docker exec "${PROJ_A}-fips-1" fipsctl show links 2>&1 || true
|
|
log "node-b links:"
|
|
docker exec "${PROJ_B}-fips-1" fipsctl show links 2>&1 || true
|
|
log "node-c links:"
|
|
docker exec "${PROJ_C}-fips-1" fipsctl show links 2>&1 || true
|
|
exit 1
|
|
fi
|
|
|
|
# Wait for end-to-end multi-hop connectivity (the same app-to-app pings
|
|
# the test asserts on) with a progress-aware deadline, instead of a
|
|
# blind fixed sleep that can fire before coordinates propagate across the
|
|
# chain. The directed pings below remain the actual assertions.
|
|
_sidecar_converged() {
|
|
PASSED=0; FAILED=0
|
|
if docker exec "${PROJ_B}-app-1" ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
if docker exec "${PROJ_C}-app-1" ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
if docker exec "${PROJ_A}-app-1" ping6 -c1 -W2 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
}
|
|
wait_until_connected _sidecar_converged "$CONVERGE_TIMEOUT" 10 || true
|
|
|
|
# ── Link verification ─────────────────────────────────────────────────────
|
|
|
|
log "Verifying link counts..."
|
|
|
|
a_links=$(docker exec "${PROJ_A}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
b_links=$(docker exec "${PROJ_B}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
c_links=$(docker exec "${PROJ_C}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
|
|
[ "$a_links" -ge 1 ] && pass "node-a has $a_links link(s)" || fail "node-a has $a_links links (expected >= 1)"
|
|
[ "$b_links" -ge 2 ] && pass "node-b has $b_links link(s)" || fail "node-b has $b_links links (expected >= 2)"
|
|
[ "$c_links" -ge 1 ] && pass "node-c has $c_links link(s)" || fail "node-c has $c_links links (expected >= 1)"
|
|
|
|
# ── Direct connectivity (adjacent nodes) ──────────────────────────────────
|
|
|
|
log "Testing direct connectivity (B app → A via fips0)..."
|
|
|
|
if docker exec "${PROJ_B}-app-1" ping6 -c2 -W5 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-b app can ping node-a via fips0"
|
|
else
|
|
fail "node-b app cannot ping node-a via fips0"
|
|
fi
|
|
|
|
# ── Multi-hop connectivity (C → A through B) ─────────────────────────────
|
|
|
|
log "Testing multi-hop connectivity (C app → A via fips0, through B)..."
|
|
|
|
if docker exec "${PROJ_C}-app-1" ping6 -c2 -W10 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-c app can ping node-a via fips0 (multi-hop through B)"
|
|
else
|
|
fail "node-c app cannot ping node-a via fips0 (multi-hop through B)"
|
|
fi
|
|
|
|
# ── Reverse direction (A → C through B) ──────────────────────────────────
|
|
|
|
log "Testing reverse multi-hop (A app → C via fips0, through B)..."
|
|
|
|
if docker exec "${PROJ_A}-app-1" ping6 -c2 -W10 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-a app can ping node-c via fips0 (multi-hop through B)"
|
|
else
|
|
fail "node-a app cannot ping node-c via fips0 (multi-hop through B)"
|
|
fi
|
|
|
|
# ── Network isolation verification ────────────────────────────────────────
|
|
#
|
|
# This is the critical security assertion: app containers must NOT be able
|
|
# to reach anything outside the FIPS mesh.
|
|
|
|
log "Verifying network isolation on app containers..."
|
|
|
|
for node in a b c; do
|
|
container="sidecar-${node}-app-1"
|
|
# Pick a peer IP that isn't this node's own address
|
|
case $node in
|
|
a) peer_ip="$NODE_B_IP" ;;
|
|
b) peer_ip="$NODE_C_IP" ;;
|
|
c) peer_ip="$NODE_A_IP" ;;
|
|
esac
|
|
log " Checking $container..."
|
|
|
|
# IPv4 gateway should be unreachable (iptables DROP on eth0)
|
|
if docker exec "$container" ping -c1 -W2 "$GATEWAY_IP" >/dev/null 2>&1; then
|
|
fail "$container can reach IPv4 gateway (isolation broken!)"
|
|
else
|
|
pass "$container cannot reach IPv4 gateway (IPv4 blocked)"
|
|
fi
|
|
|
|
# IPv4 peer should be unreachable (iptables DROP on eth0)
|
|
if docker exec "$container" ping -c1 -W2 "$peer_ip" >/dev/null 2>&1; then
|
|
fail "$container can reach peer IPv4 (isolation broken!)"
|
|
else
|
|
pass "$container cannot reach peer IPv4 (IPv4 blocked)"
|
|
fi
|
|
|
|
# Loopback should work
|
|
if docker exec "$container" ping -c1 -W2 127.0.0.1 >/dev/null 2>&1; then
|
|
pass "$container can reach loopback (expected)"
|
|
else
|
|
fail "$container cannot reach loopback"
|
|
fi
|
|
done
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────
|
|
|
|
echo ""
|
|
log "Results: $PASSED passed, $FAILED failed"
|
|
|
|
if [ "$FAILED" -gt 0 ]; then
|
|
log "Dumping logs for failed run..."
|
|
for node in a b c; do
|
|
echo "--- sidecar-${node} logs ---"
|
|
docker logs "sidecar-${node}-fips-1" 2>&1 | tail -30
|
|
echo ""
|
|
done
|
|
exit 1
|
|
fi
|
|
|
|
log "All tests passed."
|