Files
Johnathan Corgan e4a854f6b0 Stop concurrent local CI runs from clobbering each other's containers
Two runs on one host destroyed each other's containers, producing
mid-test "No such container" failures that look like real defects. The
automated builder runs a full local CI on the same box every few minutes,
so the machine is contended almost always and this has red-ed both a hand
run and an automated gate.

Two independent causes are fixed here. Container names were hardcoded, and
docker names are global rather than scoped by compose project, so two runs
collided on the same name; every name now takes an optional suffix that
the harness sets from the run id. And the cleanup sweep matched a label
shared by every run, so one run's teardown force-removed another's
containers; resources now also carry a per-run label and the sweep can be
narrowed to it.

A third hazard turned up that was not in the original report: the sidecar
suite passes explicit compose project names, which override the run-scoped
project and put it outside the shared prefix entirely. Its project names,
network, and derived container references are now scoped too.

Both are default-off. With the suffix unset, names render exactly as they
do today and a bare compose invocation is unchanged, which is what keeps
the hosted CI and the documentation correct. A cleanup run with no run id
still reaps everything, which is what a manual "clear the box" wants.

The literal-name sweep was not sufficient: six scripts build container
names dynamically from node labels, and two suites create their own
containers outside compose. Those are handled at their construction sites.

Verified: syntax check on all modified scripts; compose validation on
every modified file with the suffix both set and unset; and a synthetic
two-run reproduction that shows the old cleanup destroying a bystander run
and the new one leaving it alone.

Known gap: subnets are still hardcoded, so two concurrent full runs will
still collide on address-pool overlap. That fix reaches into topology
configs, chaos scenarios, diagrams and production source, so it is left
for its own change rather than half-done here.
2026-07-19 06:41:25 +00:00

261 lines
8.7 KiB
Bash
Executable File

#!/bin/bash
# Tor directory-mode integration test.
#
# Validates end-to-end connectivity through a Tor onion service managed
# by HiddenServiceDir (directory mode) with Sandbox 1:
# fips-a creates onion service via Tor-managed HiddenServiceDir
# fips-b connects outbound to fips-a's .onion address via SOCKS5
#
# Both containers run Tor + FIPS co-located. This is the recommended
# production deployment mode.
#
# Requires internet — the Tor daemon must bootstrap to the network
# and publish the onion service descriptor for .onion routing.
#
# Usage: ./directory-test.sh
set -e
trap 'echo ""; echo "Test interrupted — cleaning up..."; docker compose down 2>/dev/null; exit 130' INT
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TEST_DIR="$SCRIPT_DIR/.."
DERIVE_KEYS="$SCRIPT_DIR/../../../lib/derive_keys.py"
cd "$TEST_DIR"
PASSED=0
FAILED=0
TIMEOUT_PING=15
MAX_WAIT_ONION=120
MAX_WAIT_PEER=180
# Count connected peers for a node using fipsctl show peers JSON output
count_connected_peers() {
local container="$1"
docker exec "$container" fipsctl show peers 2>/dev/null \
| python3 -c "
import json, sys
try:
data = json.load(sys.stdin)
print(sum(1 for p in data.get('peers', []) if p.get('connectivity') == 'connected'))
except:
print(0)
" 2>/dev/null || echo 0
}
echo "=== FIPS Tor Directory-Mode Integration Test ==="
echo ""
# ── Phase 0: Setup ───────────────────────────────────────────────
echo "Phase 0: Setup..."
# Copy binaries from common (built externally)
if [ ! -f fips ] || [ ! -f fipsctl ]; then
if [ -f ../common/fips ] && [ -f ../common/fipsctl ]; then
cp ../common/fips ../common/fipsctl .
echo " Copied binaries from ../common/"
else
echo " ERROR: fips and fipsctl binaries not found."
echo " Build them first: cargo build --release"
echo " Then copy to testing/tor/common/ or testing/tor/directory-mode/"
exit 1
fi
fi
# Generate ephemeral identities
MESH_NAME="dir-test-$(date +%s)-$$"
echo " Mesh name: $MESH_NAME"
KEYS_A=$(python3 "$DERIVE_KEYS" "$MESH_NAME" "a")
NSEC_A=$(echo "$KEYS_A" | grep "^nsec=" | cut -d= -f2)
NPUB_A=$(echo "$KEYS_A" | grep "^npub=" | cut -d= -f2)
KEYS_B=$(python3 "$DERIVE_KEYS" "$MESH_NAME" "b")
NSEC_B=$(echo "$KEYS_B" | grep "^nsec=" | cut -d= -f2)
NPUB_B=$(echo "$KEYS_B" | grep "^npub=" | cut -d= -f2)
echo " Node A: $NPUB_A"
echo " Node B: $NPUB_B"
# Generate node-a config
sed "s/{{NSEC_A}}/$NSEC_A/" configs/node-a.yaml.tmpl > configs/node-a.yaml
echo " Node A config generated"
echo ""
# ── Phase 1: Start node A (Tor + FIPS co-located) ────────────────
echo "Phase 1: Starting node A (Tor+FIPS, directory-mode onion service)..."
docker compose down 2>/dev/null || true
docker compose up -d fips-a
echo ""
# ── Phase 2: Wait for onion service creation ─────────────────────
echo "Phase 2: Waiting for node A's onion service (up to ${MAX_WAIT_ONION}s)..."
echo " (Tor bootstrap + HiddenServiceDir publication)"
ONION_ADDR=""
elapsed=0
while [ "$elapsed" -lt "$MAX_WAIT_ONION" ]; do
# Extract .onion address from structured log: onion_address=<addr>.onion
# Strip ANSI color codes before matching (tracing emits them by default)
ONION_ADDR=$(docker logs fips-dir-a${FIPS_CI_NAME_SUFFIX:-} 2>&1 \
| sed 's/\x1b\[[0-9;]*m//g' \
| grep -oE 'onion_address=[a-z2-7]{56}\.onion' \
| head -1 \
| cut -d= -f2)
if [ -n "$ONION_ADDR" ]; then
echo " Onion service active after ${elapsed}s"
echo " Address: $ONION_ADDR"
break
fi
sleep 5
elapsed=$((elapsed + 5))
echo " ${elapsed}s..."
done
if [ -z "$ONION_ADDR" ]; then
echo " FAIL: Onion service not created within ${MAX_WAIT_ONION}s"
echo ""
echo "Node A logs (last 30 lines):"
docker logs fips-dir-a${FIPS_CI_NAME_SUFFIX:-} 2>&1 | tail -30
docker compose down
exit 1
fi
echo ""
# ── Phase 3: Start node B with .onion address ────────────────────
echo "Phase 3: Starting node B (Tor+FIPS, socks5-only)..."
# Generate node-b config with discovered .onion address + virtual port
ONION_CONNECT="${ONION_ADDR}:8443"
sed -e "s/{{NSEC_B}}/$NSEC_B/" \
-e "s/{{NPUB_A}}/$NPUB_A/" \
-e "s/{{ONION_ADDR_A}}/$ONION_CONNECT/" \
configs/node-b.yaml.tmpl > configs/node-b.yaml
echo " Node B config generated (target: $ONION_CONNECT)"
docker compose up -d fips-b
echo ""
# ── Phase 4: Wait for peer connection ────────────────────────────
echo "Phase 4: Waiting for peer connection (up to ${MAX_WAIT_PEER}s)..."
echo " (SOCKS5 circuit setup + .onion routing may take a while)"
peers_a=0
peers_b=0
elapsed=0
while [ "$elapsed" -lt "$MAX_WAIT_PEER" ]; do
peers_a=$(count_connected_peers fips-dir-a${FIPS_CI_NAME_SUFFIX:-})
peers_b=$(count_connected_peers fips-dir-b${FIPS_CI_NAME_SUFFIX:-})
if [ "$peers_a" -ge 1 ] && [ "$peers_b" -ge 1 ]; then
echo " Both nodes connected after ${elapsed}s (A: ${peers_a}, B: ${peers_b})"
break
fi
sleep 10
elapsed=$((elapsed + 10))
echo " ${elapsed}s... (A peers: ${peers_a}, B peers: ${peers_b})"
done
if [ "$peers_a" -lt 1 ] || [ "$peers_b" -lt 1 ]; then
echo " FAIL: Peers not established within ${MAX_WAIT_PEER}s"
echo ""
echo "Node A logs (last 30 lines):"
docker logs fips-dir-a${FIPS_CI_NAME_SUFFIX:-} 2>&1 | tail -30
echo ""
echo "Node B logs (last 30 lines):"
docker logs fips-dir-b${FIPS_CI_NAME_SUFFIX:-} 2>&1 | tail -30
docker compose down
exit 1
fi
# Extra convergence time for routing
echo " Waiting 10s for routing convergence..."
sleep 10
echo ""
# ── Phase 5: Connectivity tests ──────────────────────────────────
echo "Phase 5: Connectivity tests"
PING_COUNT=11
ping_series() {
local from="$1"
local to_npub="$2"
local label="$3"
echo " $label ($PING_COUNT pings, dropping first):"
local rtts=()
local fails=0
for i in $(seq 1 "$PING_COUNT"); do
local output
if output=$(docker exec "$from" ping6 -c 1 -W "$TIMEOUT_PING" "${to_npub}.fips" 2>&1); then
local rtt
rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2)
if [ -n "$rtt" ]; then
printf " %2d: %s ms\n" "$i" "$rtt"
rtts+=("$rtt")
else
printf " %2d: OK (no rtt)\n" "$i"
fi
else
printf " %2d: FAIL\n" "$i"
fails=$((fails + 1))
fi
done
if [ "$fails" -gt 0 ]; then
FAILED=$((FAILED + fails))
fi
# Drop first ping, compute average of remaining
if [ "${#rtts[@]}" -ge 2 ]; then
local avg
local csv
csv=$(IFS=,; echo "${rtts[*]}")
avg=$(python3 -c "
rtts = [$csv]
trimmed = rtts[1:]
print(f'{sum(trimmed)/len(trimmed):.1f}')
")
echo " Avg (excluding first): ${avg} ms"
PASSED=$((PASSED + ${#rtts[@]}))
elif [ "${#rtts[@]}" -eq 1 ]; then
echo " Only 1 successful ping, no average"
PASSED=$((PASSED + 1))
else
echo " No successful pings"
fi
echo ""
}
echo ""
echo " Ping via Tor onion service (directory mode, Sandbox 1):"
ping_series fips-dir-a${FIPS_CI_NAME_SUFFIX:-} "$NPUB_B" "A → B"
ping_series fips-dir-b${FIPS_CI_NAME_SUFFIX:-} "$NPUB_A" "B → A"
echo ""
# ── Phase 6: Log analysis ────────────────────────────────────────
echo "Phase 6: Log analysis"
for node in fips-dir-a${FIPS_CI_NAME_SUFFIX:-} fips-dir-b${FIPS_CI_NAME_SUFFIX:-}; do
panics=$(docker logs "$node" 2>&1 | grep -ci "panic" || true)
errors=$(docker logs "$node" 2>&1 | grep -ci "error" || true)
onion=$(docker logs "$node" 2>&1 | grep -ci "onion" || true)
directory=$(docker logs "$node" 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep -ci "directory.mode" || true)
echo " $node: panics=$panics errors=$errors onion_mentions=$onion directory_mentions=$directory"
if [ "$panics" -gt 0 ]; then
echo " WARNING: panics detected in $node logs"
fi
done
echo ""
# ── Cleanup ──────────────────────────────────────────────────────
echo "Cleaning up..."
docker compose down
rm -f configs/node-a.yaml configs/node-b.yaml
echo ""
echo "=== Results: $PASSED passed, $FAILED failed ==="
[ "$FAILED" -eq 0 ] && exit 0 || exit 1