laantungir/fips
1
0
Fork 0
mirror of https://github.com/jmcorgan/fips.git synced 2026-07-30 19:46:15 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
e4a854f6b0cf51011976b393e884663df7e8c747
fips/testing/sidecar/scripts/test-sidecar.sh
T
Johnathan Corgan e4a854f6b0 Stop concurrent local CI runs from clobbering each other's containers 
Two runs on one host destroyed each other's containers, producing
mid-test "No such container" failures that look like real defects. The
automated builder runs a full local CI on the same box every few minutes,
so the machine is contended almost always and this has red-ed both a hand
run and an automated gate.

Two independent causes are fixed here. Container names were hardcoded, and
docker names are global rather than scoped by compose project, so two runs
collided on the same name; every name now takes an optional suffix that
the harness sets from the run id. And the cleanup sweep matched a label
shared by every run, so one run's teardown force-removed another's
containers; resources now also carry a per-run label and the sweep can be
narrowed to it.

A third hazard turned up that was not in the original report: the sidecar
suite passes explicit compose project names, which override the run-scoped
project and put it outside the shared prefix entirely. Its project names,
network, and derived container references are now scoped too.

Both are default-off. With the suffix unset, names render exactly as they
do today and a bare compose invocation is unchanged, which is what keeps
the hosted CI and the documentation correct. A cleanup run with no run id
still reaps everything, which is what a manual "clear the box" wants.

The literal-name sweep was not sufficient: six scripts build container
names dynamically from node labels, and two suites create their own
containers outside compose. Those are handled at their construction sites.

Verified: syntax check on all modified scripts; compose validation on
every modified file with the suffix both set and unset; and a synthetic
two-run reproduction that shows the old cleanup destroying a bystander run
and the new one leaving it alone.

Known gap: subnets are still hardcoded, so two concurrent full runs will
still collide on address-pool overlap. That fix reaches into topology
configs, chaos scenarios, diagrams and production source, so it is left
for its own change rather than half-done here.
2026-07-19 06:41:25 +00:00

257 lines
10 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# Integration test for the FIPS sidecar deployment.
#
# Starts a 3-node chain (A—B—C) using standalone sidecar instances,
# verifies link establishment, multi-hop connectivity, and network
# isolation on each app container.
#
# Usage: ./test-sidecar.sh [--skip-build]
#
# Exit codes:
# 0 — all tests passed
# 1 — test failure
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SIDECAR_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
# shellcheck source=../../lib/wait-converge.sh
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
# Deterministic keys derived from: derive-keys.py sidecar-test node-{a,b,c}
NODE_A_NSEC="9e688d0879fa9cd025fea0487ac23495080e3de626070fdb9b78dc1f619dd453"
NODE_A_NPUB="npub1jvren5hnege54lu3p2nzqacctmulqvkgp68yvfuj5jme5dtgnhxsdh6788"
NODE_B_NSEC="3e4e10614c0490575fa5e994524ff3f4deaac2f20db189cc9c9a79da0d90f17a"
NODE_B_NPUB="npub15h7z0ljzudqe9pgwx99cjsz2c0ennuyvkcc8zvtk3lg97xwzex9ska6g4y"
NODE_C_NSEC="15148ed0131f7da43fd13e369dfedede14fb64698f3756636b569c3a3e87438f"
NODE_C_NPUB="npub1zhezcykd0e34z4fxtranl45jaasgnlxv0kjqwlq2v56ggssn0w4qelcrvr"
NETWORK_NAME="fips-sidecar-test${FIPS_CI_NAME_SUFFIX:-}"
# Compose project names. These are explicit (they override COMPOSE_PROJECT_NAME),
# so they carry the run suffix themselves — otherwise concurrent runs would share
# one project and tear down each other's containers.
PROJ_A="sidecar-a${FIPS_CI_NAME_SUFFIX:-}"
PROJ_B="sidecar-b${FIPS_CI_NAME_SUFFIX:-}"
PROJ_C="sidecar-c${FIPS_CI_NAME_SUFFIX:-}"
SUBNET="172.20.2.0/24"
NODE_A_IP="172.20.2.10"
NODE_B_IP="172.20.2.11"
NODE_C_IP="172.20.2.12"
CONVERGE_TIMEOUT=30
PASSED=0
FAILED=0
# Compose file paths
COMPOSE_BASE="-f $SIDECAR_DIR/docker-compose.yml"
COMPOSE_EXT="$COMPOSE_BASE -f $SIDECAR_DIR/docker-compose.external-net.yml"
# ── Helpers ────────────────────────────────────────────────────────────────
log() { echo "=== $*"; }
pass() { echo " PASS: $*"; PASSED=$((PASSED + 1)); }
fail() { echo " FAIL: $*"; FAILED=$((FAILED + 1)); }
cleanup() {
log "Cleaning up..."
# Tear down B and C first (they reference A's network as external)
docker compose $COMPOSE_EXT -p "$PROJ_C" down --volumes --remove-orphans 2>/dev/null || true
docker compose $COMPOSE_EXT -p "$PROJ_B" down --volumes --remove-orphans 2>/dev/null || true
# Tear down A last (it owns the network)
docker compose $COMPOSE_BASE -p "$PROJ_A" down --volumes --remove-orphans 2>/dev/null || true
}
# Always clean up on exit
trap cleanup EXIT
# ── Build ──────────────────────────────────────────────────────────────────
if [[ "${1:-}" != "--skip-build" ]]; then
log "Building test images..."
DOCKER_DIR="$(cd "$SIDECAR_DIR/../docker" && pwd)"
docker build -t fips-test:latest "$DOCKER_DIR"
docker build -t fips-test-app:latest -f "$DOCKER_DIR/Dockerfile.app" "$DOCKER_DIR"
fi
# ── Start nodes ────────────────────────────────────────────────────────────
#
# Chain topology: A — B — C
# node-a: no outbound peers (accepts inbound from B)
# node-b: peers with A (middle node, transit router)
# node-c: peers with B (end node)
log "Starting node-a (no peers, creates network)..."
# node-a is the root: explicitly clear FIPS_PEER_* so it does not inherit the
# external peer default from .env (which points at a real public mesh node).
# Without this, node-a auto-connects to the live mesh and the chain attaches
# under an external root, inflating tree depth and breaking isolation.
FIPS_NSEC="$NODE_A_NSEC" \
FIPS_PEER_NPUB="" \
FIPS_PEER_ADDR="" \
FIPS_NETWORK="$NETWORK_NAME" \
FIPS_SUBNET="$SUBNET" \
FIPS_IPV4="$NODE_A_IP" \
docker compose $COMPOSE_BASE -p "$PROJ_A" up -d
log "Starting node-b (peers with node-a, joins external network)..."
FIPS_NSEC="$NODE_B_NSEC" \
FIPS_PEER_NPUB="$NODE_A_NPUB" \
FIPS_PEER_ADDR="${NODE_A_IP}:2121" \
FIPS_PEER_ALIAS="node-a" \
FIPS_NETWORK="$NETWORK_NAME" \
FIPS_SUBNET="$SUBNET" \
FIPS_IPV4="$NODE_B_IP" \
docker compose $COMPOSE_EXT -p "$PROJ_B" up -d
log "Starting node-c (peers with node-b, joins external network)..."
FIPS_NSEC="$NODE_C_NSEC" \
FIPS_PEER_NPUB="$NODE_B_NPUB" \
FIPS_PEER_ADDR="${NODE_B_IP}:2121" \
FIPS_PEER_ALIAS="node-b" \
FIPS_NETWORK="$NETWORK_NAME" \
FIPS_SUBNET="$SUBNET" \
FIPS_IPV4="$NODE_C_IP" \
docker compose $COMPOSE_EXT -p "$PROJ_C" up -d
# ── Wait for convergence ──────────────────────────────────────────────────
log "Waiting for link establishment (up to ${CONVERGE_TIMEOUT}s)..."
converged=false
for i in $(seq 1 "$CONVERGE_TIMEOUT"); do
# node-b should have 2 links (A and C)
link_count=$(docker exec "${PROJ_B}-fips-1" fipsctl show links 2>/dev/null \
| grep -c '"state": "connected"' || true)
if [ "$link_count" -ge 2 ]; then
converged=true
break
fi
sleep 1
done
if [ "$converged" = true ]; then
log "Links established after ${i}s"
else
log "TIMEOUT: links did not converge in ${CONVERGE_TIMEOUT}s"
log "node-a links:"
docker exec "${PROJ_A}-fips-1" fipsctl show links 2>&1 || true
log "node-b links:"
docker exec "${PROJ_B}-fips-1" fipsctl show links 2>&1 || true
log "node-c links:"
docker exec "${PROJ_C}-fips-1" fipsctl show links 2>&1 || true
exit 1
fi
# Wait for end-to-end multi-hop connectivity (the same app-to-app pings
# the test asserts on) with a progress-aware deadline, instead of a
# blind fixed sleep that can fire before coordinates propagate across the
# chain. The directed pings below remain the actual assertions.
_sidecar_converged() {
PASSED=0; FAILED=0
if docker exec "${PROJ_B}-app-1" ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
if docker exec "${PROJ_C}-app-1" ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
if docker exec "${PROJ_A}-app-1" ping6 -c1 -W2 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
}
wait_until_connected _sidecar_converged "$CONVERGE_TIMEOUT" 10 || true
# ── Link verification ─────────────────────────────────────────────────────
log "Verifying link counts..."
a_links=$(docker exec "${PROJ_A}-fips-1" fipsctl show links 2>/dev/null \
| grep -c '"state": "connected"' || true)
b_links=$(docker exec "${PROJ_B}-fips-1" fipsctl show links 2>/dev/null \
| grep -c '"state": "connected"' || true)
c_links=$(docker exec "${PROJ_C}-fips-1" fipsctl show links 2>/dev/null \
| grep -c '"state": "connected"' || true)
[ "$a_links" -ge 1 ] && pass "node-a has $a_links link(s)" || fail "node-a has $a_links links (expected >= 1)"
[ "$b_links" -ge 2 ] && pass "node-b has $b_links link(s)" || fail "node-b has $b_links links (expected >= 2)"
[ "$c_links" -ge 1 ] && pass "node-c has $c_links link(s)" || fail "node-c has $c_links links (expected >= 1)"
# ── Direct connectivity (adjacent nodes) ──────────────────────────────────
log "Testing direct connectivity (B app → A via fips0)..."
if docker exec "${PROJ_B}-app-1" ping6 -c2 -W5 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then
pass "node-b app can ping node-a via fips0"
else
fail "node-b app cannot ping node-a via fips0"
fi
# ── Multi-hop connectivity (C → A through B) ─────────────────────────────
log "Testing multi-hop connectivity (C app → A via fips0, through B)..."
if docker exec "${PROJ_C}-app-1" ping6 -c2 -W10 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then
pass "node-c app can ping node-a via fips0 (multi-hop through B)"
else
fail "node-c app cannot ping node-a via fips0 (multi-hop through B)"
fi
# ── Reverse direction (A → C through B) ──────────────────────────────────
log "Testing reverse multi-hop (A app → C via fips0, through B)..."
if docker exec "${PROJ_A}-app-1" ping6 -c2 -W10 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then
pass "node-a app can ping node-c via fips0 (multi-hop through B)"
else
fail "node-a app cannot ping node-c via fips0 (multi-hop through B)"
fi
# ── Network isolation verification ────────────────────────────────────────
#
# This is the critical security assertion: app containers must NOT be able
# to reach anything outside the FIPS mesh.
log "Verifying network isolation on app containers..."
for node in a b c; do
container="sidecar-${node}-app-1"
# Pick a peer IP that isn't this node's own address
case $node in
a) peer_ip="$NODE_B_IP" ;;
b) peer_ip="$NODE_C_IP" ;;
c) peer_ip="$NODE_A_IP" ;;
esac
log " Checking $container..."
# IPv4 gateway should be unreachable (iptables DROP on eth0)
if docker exec "$container" ping -c1 -W2 172.20.2.1 >/dev/null 2>&1; then
fail "$container can reach IPv4 gateway (isolation broken!)"
else
pass "$container cannot reach IPv4 gateway (IPv4 blocked)"
fi
# IPv4 peer should be unreachable (iptables DROP on eth0)
if docker exec "$container" ping -c1 -W2 "$peer_ip" >/dev/null 2>&1; then
fail "$container can reach peer IPv4 (isolation broken!)"
else
pass "$container cannot reach peer IPv4 (IPv4 blocked)"
fi
# Loopback should work
if docker exec "$container" ping -c1 -W2 127.0.0.1 >/dev/null 2>&1; then
pass "$container can reach loopback (expected)"
else
fail "$container cannot reach loopback"
fi
done
# ── Summary ───────────────────────────────────────────────────────────────
echo ""
log "Results: $PASSED passed, $FAILED failed"
if [ "$FAILED" -gt 0 ]; then
log "Dumping logs for failed run..."
for node in a b c; do
echo "--- sidecar-${node} logs ---"
docker logs "sidecar-${node}-fips-1" 2>&1 | tail -30
echo ""
done
exit 1
fi
log "All tests passed."
Reference in New Issue View Git Blame Copy Permalink