mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 11:36:15 +00:00
Johnathan Corgan
49163befd5
Scoping the image tag was never sufficient on its own. Every build read one unscoped directory in the working tree, into which each run copies the binaries it just built, so two concurrent runs raced on the contents of the context as well as on the name of the result — and a run could produce a correctly-per-run-tagged image built from the other run's binaries. The run now copies the context's tracked files into its own directory, installs its binaries there, and builds from it, exporting the path so every other consumer follows. Deliberately not carried over: a previous run's binaries, since inheriting them is the failure this prevents. The path is absolute because compose resolves a relative build context against the compose file's own directory rather than the working directory, which was measured rather than assumed. The chaos entry script gated the whole simulation on a binary in the shared directory by literal path, so it moves in this same commit: left behind, it would have failed on a clean checkout and, worse on a host with leftovers, passed while reading a binary that was not the one under test. Teardown removes the directory on red runs as well as green, since it holds only reproducible content and is never the evidence of a failure. The worker's SIGKILL runs no trap, so the cleanup script also sweeps contexts left by a preempted run, and the ignore rule keeps a concurrent run from showing up as untracked working-tree noise.
313 lines
13 KiB
Bash
Executable File
313 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
# Integration test for the FIPS sidecar deployment.
|
|
#
|
|
# Starts a 3-node chain (A—B—C) using standalone sidecar instances,
|
|
# verifies link establishment, multi-hop connectivity, and network
|
|
# isolation on each app container.
|
|
#
|
|
# Usage: ./test-sidecar.sh [--skip-build]
|
|
#
|
|
# Exit codes:
|
|
# 0 — all tests passed
|
|
# 1 — test failure
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SIDECAR_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
# shellcheck source=../../lib/wait-converge.sh
|
|
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
|
|
|
|
# Deterministic keys derived from: derive-keys.py sidecar-test node-{a,b,c}
|
|
NODE_A_NSEC="9e688d0879fa9cd025fea0487ac23495080e3de626070fdb9b78dc1f619dd453"
|
|
NODE_A_NPUB="npub1jvren5hnege54lu3p2nzqacctmulqvkgp68yvfuj5jme5dtgnhxsdh6788"
|
|
NODE_B_NSEC="3e4e10614c0490575fa5e994524ff3f4deaac2f20db189cc9c9a79da0d90f17a"
|
|
NODE_B_NPUB="npub15h7z0ljzudqe9pgwx99cjsz2c0ennuyvkcc8zvtk3lg97xwzex9ska6g4y"
|
|
NODE_C_NSEC="15148ed0131f7da43fd13e369dfedede14fb64698f3756636b569c3a3e87438f"
|
|
NODE_C_NPUB="npub1zhezcykd0e34z4fxtranl45jaasgnlxv0kjqwlq2v56ggssn0w4qelcrvr"
|
|
|
|
NETWORK_NAME="fips-sidecar-test${FIPS_CI_NAME_SUFFIX:-}"
|
|
|
|
# Compose project names. These are explicit (they override COMPOSE_PROJECT_NAME),
|
|
# so they carry the run suffix themselves — otherwise concurrent runs would share
|
|
# one project and tear down each other's containers.
|
|
PROJ_A="sidecar-a${FIPS_CI_NAME_SUFFIX:-}"
|
|
PROJ_B="sidecar-b${FIPS_CI_NAME_SUFFIX:-}"
|
|
PROJ_C="sidecar-c${FIPS_CI_NAME_SUFFIX:-}"
|
|
# Network address range — claimed at startup by alloc_network below rather than
|
|
# hardcoded, so two concurrent runs cannot request the same range.
|
|
#
|
|
# 10.40.0.0/16 is unclaimed in-tree: the suites use 172.20-172.32 and 172.99,
|
|
# the chaos children 10.30.x, boringtun's inner network 10.99.x, and docker's
|
|
# own default pool is 172.17-31 plus 192.168.
|
|
NET_BASE="10.40"
|
|
NET_CANDIDATES=64
|
|
|
|
SUBNET=""
|
|
GATEWAY_IP=""
|
|
NODE_A_IP=""
|
|
NODE_B_IP=""
|
|
NODE_C_IP=""
|
|
|
|
CONVERGE_TIMEOUT=30
|
|
PASSED=0
|
|
FAILED=0
|
|
|
|
# Compose file paths
|
|
COMPOSE_BASE="-f $SIDECAR_DIR/docker-compose.yml"
|
|
COMPOSE_EXT="$COMPOSE_BASE -f $SIDECAR_DIR/docker-compose.external-net.yml"
|
|
|
|
# ── Helpers ────────────────────────────────────────────────────────────────
|
|
|
|
log() { echo "=== $*"; }
|
|
pass() { echo " PASS: $*"; PASSED=$((PASSED + 1)); }
|
|
fail() { echo " FAIL: $*"; FAILED=$((FAILED + 1)); }
|
|
|
|
# Claim a free /24 for this run's network.
|
|
#
|
|
# Claim-and-advance rather than deriving an offset from the run id: we attempt
|
|
# creation on a candidate range and, if docker reports the range is taken, move
|
|
# to the next. Docker's own address pool is then the arbiter, which makes an
|
|
# overlap between two concurrent runs *impossible*. A hashed offset would only
|
|
# make it unlikely, and an overlap is precisely the failure being avoided here.
|
|
#
|
|
# Deliberately does NOT discard stderr: only an address-pool conflict is worth
|
|
# advancing on. Any other failure (name already taken, daemon error) is real,
|
|
# and retrying it 64 times would bury the reason.
|
|
alloc_network() {
|
|
local i err
|
|
for (( i = 0; i < NET_CANDIDATES; i++ )); do
|
|
SUBNET="${NET_BASE}.${i}.0/24"
|
|
if err=$(docker network create \
|
|
--subnet "$SUBNET" \
|
|
--label com.corganlabs.fips-ci=1 \
|
|
--label "com.corganlabs.fips-ci.run=${FIPS_CI_RUN_ID:-manual}" \
|
|
"$NETWORK_NAME" 2>&1); then
|
|
GATEWAY_IP="${NET_BASE}.${i}.1"
|
|
NODE_A_IP="${NET_BASE}.${i}.10"
|
|
NODE_B_IP="${NET_BASE}.${i}.11"
|
|
NODE_C_IP="${NET_BASE}.${i}.12"
|
|
log "Claimed network $NETWORK_NAME on $SUBNET"
|
|
return 0
|
|
fi
|
|
case "$err" in
|
|
*"Pool overlaps"*|*"pool overlaps"*) continue ;;
|
|
*) echo " FAIL: docker network create: $err" >&2; return 1 ;;
|
|
esac
|
|
done
|
|
echo " FAIL: no free /24 in ${NET_BASE}.0.0/16 after ${NET_CANDIDATES} attempts" >&2
|
|
return 1
|
|
}
|
|
|
|
cleanup() {
|
|
log "Cleaning up..."
|
|
# Tear down C and B first, then A. All three attach to the network as
|
|
# external, so none of them removes it — we do that last, by name.
|
|
docker compose $COMPOSE_EXT -p "$PROJ_C" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker compose $COMPOSE_EXT -p "$PROJ_B" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker compose $COMPOSE_EXT -p "$PROJ_A" down --volumes --remove-orphans 2>/dev/null || true
|
|
docker network rm "$NETWORK_NAME" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Always clean up on exit
|
|
trap cleanup EXIT
|
|
|
|
# ── Build ──────────────────────────────────────────────────────────────────
|
|
|
|
if [[ "${1:-}" != "--skip-build" ]]; then
|
|
log "Building test images..."
|
|
DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$(cd "$SIDECAR_DIR/../docker" && pwd)}"
|
|
docker build -t fips-test:latest "$DOCKER_DIR"
|
|
docker build -t fips-test-app:latest -f "$DOCKER_DIR/Dockerfile.app" "$DOCKER_DIR"
|
|
fi
|
|
|
|
# ── Start nodes ────────────────────────────────────────────────────────────
|
|
#
|
|
# Chain topology: A — B — C
|
|
# node-a: no outbound peers (accepts inbound from B)
|
|
# node-b: peers with A (middle node, transit router)
|
|
# node-c: peers with B (end node)
|
|
|
|
alloc_network || exit 1
|
|
|
|
log "Starting node-a (no peers, joins the claimed network)..."
|
|
# node-a is the root: explicitly clear FIPS_PEER_* so it does not inherit the
|
|
# external peer default from .env (which points at a real public mesh node).
|
|
# Without this, node-a auto-connects to the live mesh and the chain attaches
|
|
# under an external root, inflating tree depth and breaking isolation.
|
|
FIPS_NSEC="$NODE_A_NSEC" \
|
|
FIPS_PEER_NPUB="" \
|
|
FIPS_PEER_ADDR="" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_A_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_A" up -d
|
|
|
|
log "Starting node-b (peers with node-a, joins external network)..."
|
|
FIPS_NSEC="$NODE_B_NSEC" \
|
|
FIPS_PEER_NPUB="$NODE_A_NPUB" \
|
|
FIPS_PEER_ADDR="${NODE_A_IP}:2121" \
|
|
FIPS_PEER_ALIAS="node-a" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_B_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_B" up -d
|
|
|
|
log "Starting node-c (peers with node-b, joins external network)..."
|
|
FIPS_NSEC="$NODE_C_NSEC" \
|
|
FIPS_PEER_NPUB="$NODE_B_NPUB" \
|
|
FIPS_PEER_ADDR="${NODE_B_IP}:2121" \
|
|
FIPS_PEER_ALIAS="node-b" \
|
|
FIPS_NETWORK="$NETWORK_NAME" \
|
|
FIPS_SUBNET="$SUBNET" \
|
|
FIPS_IPV4="$NODE_C_IP" \
|
|
docker compose $COMPOSE_EXT -p "$PROJ_C" up -d
|
|
|
|
# ── Wait for convergence ──────────────────────────────────────────────────
|
|
|
|
log "Waiting for link establishment (up to ${CONVERGE_TIMEOUT}s)..."
|
|
|
|
converged=false
|
|
for i in $(seq 1 "$CONVERGE_TIMEOUT"); do
|
|
# node-b should have 2 links (A and C)
|
|
link_count=$(docker exec "${PROJ_B}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
if [ "$link_count" -ge 2 ]; then
|
|
converged=true
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
if [ "$converged" = true ]; then
|
|
log "Links established after ${i}s"
|
|
else
|
|
log "TIMEOUT: links did not converge in ${CONVERGE_TIMEOUT}s"
|
|
log "node-a links:"
|
|
docker exec "${PROJ_A}-fips-1" fipsctl show links 2>&1 || true
|
|
log "node-b links:"
|
|
docker exec "${PROJ_B}-fips-1" fipsctl show links 2>&1 || true
|
|
log "node-c links:"
|
|
docker exec "${PROJ_C}-fips-1" fipsctl show links 2>&1 || true
|
|
exit 1
|
|
fi
|
|
|
|
# Wait for end-to-end multi-hop connectivity (the same app-to-app pings
|
|
# the test asserts on) with a progress-aware deadline, instead of a
|
|
# blind fixed sleep that can fire before coordinates propagate across the
|
|
# chain. The directed pings below remain the actual assertions.
|
|
_sidecar_converged() {
|
|
PASSED=0; FAILED=0
|
|
if docker exec "${PROJ_B}-app-1" ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
if docker exec "${PROJ_C}-app-1" ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
if docker exec "${PROJ_A}-app-1" ping6 -c1 -W2 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
|
}
|
|
wait_until_connected _sidecar_converged "$CONVERGE_TIMEOUT" 10 || true
|
|
|
|
# ── Link verification ─────────────────────────────────────────────────────
|
|
|
|
log "Verifying link counts..."
|
|
|
|
a_links=$(docker exec "${PROJ_A}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
b_links=$(docker exec "${PROJ_B}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
c_links=$(docker exec "${PROJ_C}-fips-1" fipsctl show links 2>/dev/null \
|
|
| grep -c '"state": "connected"' || true)
|
|
|
|
[ "$a_links" -ge 1 ] && pass "node-a has $a_links link(s)" || fail "node-a has $a_links links (expected >= 1)"
|
|
[ "$b_links" -ge 2 ] && pass "node-b has $b_links link(s)" || fail "node-b has $b_links links (expected >= 2)"
|
|
[ "$c_links" -ge 1 ] && pass "node-c has $c_links link(s)" || fail "node-c has $c_links links (expected >= 1)"
|
|
|
|
# ── Direct connectivity (adjacent nodes) ──────────────────────────────────
|
|
|
|
log "Testing direct connectivity (B app → A via fips0)..."
|
|
|
|
if docker exec "${PROJ_B}-app-1" ping6 -c2 -W5 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-b app can ping node-a via fips0"
|
|
else
|
|
fail "node-b app cannot ping node-a via fips0"
|
|
fi
|
|
|
|
# ── Multi-hop connectivity (C → A through B) ─────────────────────────────
|
|
|
|
log "Testing multi-hop connectivity (C app → A via fips0, through B)..."
|
|
|
|
if docker exec "${PROJ_C}-app-1" ping6 -c2 -W10 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-c app can ping node-a via fips0 (multi-hop through B)"
|
|
else
|
|
fail "node-c app cannot ping node-a via fips0 (multi-hop through B)"
|
|
fi
|
|
|
|
# ── Reverse direction (A → C through B) ──────────────────────────────────
|
|
|
|
log "Testing reverse multi-hop (A app → C via fips0, through B)..."
|
|
|
|
if docker exec "${PROJ_A}-app-1" ping6 -c2 -W10 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then
|
|
pass "node-a app can ping node-c via fips0 (multi-hop through B)"
|
|
else
|
|
fail "node-a app cannot ping node-c via fips0 (multi-hop through B)"
|
|
fi
|
|
|
|
# ── Network isolation verification ────────────────────────────────────────
|
|
#
|
|
# This is the critical security assertion: app containers must NOT be able
|
|
# to reach anything outside the FIPS mesh.
|
|
|
|
log "Verifying network isolation on app containers..."
|
|
|
|
for node in a b c; do
|
|
container="sidecar-${node}${FIPS_CI_NAME_SUFFIX:-}-app-1"
|
|
# Fail loudly if the container is missing. Without this the two isolation
|
|
# assertions below pass vacuously: they assert a ping FAILS, and a ping into
|
|
# a non-existent container fails for the wrong reason. Only the loopback
|
|
# check, which expects success, would notice — so a naming slip here would
|
|
# silently turn the security assertions into no-ops.
|
|
docker inspect "$container" >/dev/null 2>&1 \
|
|
|| { fail "$container does not exist — isolation assertions cannot be trusted"; continue; }
|
|
# Pick a peer IP that isn't this node's own address
|
|
case $node in
|
|
a) peer_ip="$NODE_B_IP" ;;
|
|
b) peer_ip="$NODE_C_IP" ;;
|
|
c) peer_ip="$NODE_A_IP" ;;
|
|
esac
|
|
log " Checking $container..."
|
|
|
|
# IPv4 gateway should be unreachable (iptables DROP on eth0)
|
|
if docker exec "$container" ping -c1 -W2 "$GATEWAY_IP" >/dev/null 2>&1; then
|
|
fail "$container can reach IPv4 gateway (isolation broken!)"
|
|
else
|
|
pass "$container cannot reach IPv4 gateway (IPv4 blocked)"
|
|
fi
|
|
|
|
# IPv4 peer should be unreachable (iptables DROP on eth0)
|
|
if docker exec "$container" ping -c1 -W2 "$peer_ip" >/dev/null 2>&1; then
|
|
fail "$container can reach peer IPv4 (isolation broken!)"
|
|
else
|
|
pass "$container cannot reach peer IPv4 (IPv4 blocked)"
|
|
fi
|
|
|
|
# Loopback should work
|
|
if docker exec "$container" ping -c1 -W2 127.0.0.1 >/dev/null 2>&1; then
|
|
pass "$container can reach loopback (expected)"
|
|
else
|
|
fail "$container cannot reach loopback"
|
|
fi
|
|
done
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────
|
|
|
|
echo ""
|
|
log "Results: $PASSED passed, $FAILED failed"
|
|
|
|
if [ "$FAILED" -gt 0 ]; then
|
|
log "Dumping logs for failed run..."
|
|
for node in a b c; do
|
|
echo "--- sidecar-${node} logs ---"
|
|
docker logs "sidecar-${node}${FIPS_CI_NAME_SUFFIX:-}-fips-1" 2>&1 | tail -30
|
|
echo ""
|
|
done
|
|
exit 1
|
|
fi
|
|
|
|
log "All tests passed."
|