Merge branch 'maint'

The static compose's mixed-profile services are next-only, so the floating
subnet arrived without them: four services kept ipv4_address pins on a
network that no longer declares a subnet, which docker rejects outright at
`up`. They now float with the rest, take the per-run config directory, and
carry the run name suffix their container_name was missing.

Phase 3 of the admission-cap test conflicted for a real reason rather than a
textual one. This line asserts size-agnostically — sustained inbound from
each denied peer plus "was never promoted" — because the XX handshake
message sizes differ from IK and may change again. That assertion is kept;
what comes across from maint is the address handling underneath it, which
matters here for the same reason: with the subnet floating, a restarted
container can change address, and matching on a single one silently
under-counts the very evidence the assertion rests on.
This commit is contained in:
Johnathan Corgan
2026-07-23 02:08:52 +00:00
15 changed files with 266 additions and 152 deletions
+83 -11
View File
@@ -39,7 +39,7 @@ TOPO_FILE="$SCRIPT_DIR/../configs/topologies/$TOPOLOGY.yaml"
# before building Docker images.
if [ "${1:-}" = "inject-config" ]; then
echo "Injecting node.limits.max_peers: $MAX_PEERS into node-$CAP_NODE ($TOPOLOGY topology)..."
cfg="$SCRIPT_DIR/../generated-configs/$TOPOLOGY/node-$CAP_NODE.yaml"
cfg="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/$TOPOLOGY/node-$CAP_NODE.yaml"
if [ ! -f "$cfg" ]; then
echo " Error: $cfg not found (run generate-configs.sh $TOPOLOGY first)" >&2
exit 1
@@ -67,11 +67,22 @@ info() { echo "[$(stamp)] $*"; }
fail() { echo "[$(stamp)] FAIL: $*"; exit 1; }
pass() { echo "[$(stamp)] PASS: $*"; }
# Extract docker_ip for a node from the topology file
# A node's docker address, read from the running container.
#
# NOT from the topology file's docker_ip: fips-net requests no subnet, so that
# two concurrent CI runs cannot collide on one fixed range, and docker assigns
# the addresses at `up`. A topology literal would no longer match anything on
# the wire, and the phase-3 tcpdump assertions are built from these addresses
# — a stale one turns "no Msg2 leaked" into a check that cannot fail.
# Takes the first attachment only: these nodes have one, and concatenating two
# would yield a string that is not an address at all. The `|| true` keeps a
# missing container from killing the script under `set -e` before the caller
# can say which container it was.
node_ip() {
grep -A 5 "^ $1:" "$TOPO_FILE" \
| grep -m1 'docker_ip:' \
| sed 's/.*: *"*\([^"]*\)".*/\1/'
docker inspect \
-f '{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}' \
"fips-node-${1}${FIPS_CI_NAME_SUFFIX:-}" 2>/dev/null \
| awk '{print $1}' || true
}
# Extract npub for a node from the topology file
@@ -91,7 +102,7 @@ node_peers() {
}
CAP_IP=$(node_ip "$CAP_NODE")
[ -n "$CAP_IP" ] || fail "could not resolve docker_ip for node-$CAP_NODE in $TOPO_FILE"
[ -n "$CAP_IP" ] || fail "could not read the docker address of container fips-node-${CAP_NODE}${FIPS_CI_NAME_SUFFIX:-}"
info "cap'd node: node-$CAP_NODE (ip $CAP_IP, max_peers=$MAX_PEERS)"
# Read the cap'd node's peer_count, or the empty string if it did not answer.
@@ -138,6 +149,32 @@ info "denied (sustained-retry): ${DENIED:-<none>}"
[ -n "$DENIED" ] \
|| fail "no denied peers — test setup wrong (cap=$MAX_PEERS too high vs configured peers)"
# Every address a denied peer holds during the capture, one "node ip" line per
# observation.
#
# It is not one address per node. `fips-net` requests no subnet so that
# concurrent CI runs cannot collide on one, and the load driver below restarts
# these containers repeatedly — docker frees the address on stop and may hand
# back a different one, which was impossible while the compose pinned
# ipv4_address. Observed live: node-d went 10.128.2.4 → 10.128.2.6 mid-window.
# Phase 3 matches against the union, because an address that held for only part
# of the window under-counts Msg1 and, worse, satisfies the expect-zero Msg2
# assertion for the wrong reason.
ADDR_FILE=$(mktemp /tmp/admission-cap-addrs.XXXXXX)
record_denied_addrs() {
local n n_ip
for n in $DENIED; do
n_ip=$(node_ip "$n")
[ -n "$n_ip" ] || continue
grep -qxF "$n $n_ip" "$ADDR_FILE" 2>/dev/null || echo "$n $n_ip" >> "$ADDR_FILE"
done
}
record_denied_addrs
for n in $DENIED; do
grep -q "^$n " "$ADDR_FILE" \
|| fail "could not read the docker address of denied peer node-$n"
done
# ── Phase 2: capture wire traffic for CAPTURE_SECS seconds ───────────
# Drives sustained load by restarting denied peer containers on a cadence
# during the capture window. Each restart resets the auto-reconnect
@@ -156,10 +193,18 @@ HELPER_IMAGE=$(docker inspect -f '{{.Config.Image}}' "fips-node-${CAP_NODE}${FIP
while [ $elapsed -lt $((CAPTURE_SECS - 5)) ]; do
sleep 15
elapsed=$((elapsed + 15))
# ONE AT A TIME, deliberately. Restarting them together frees both
# addresses at once and docker reallocates in completion order, so the
# two peers SWAP — observed live, and it destroys per-peer attribution
# because both then match the same address set. Restarted singly, a
# container frees its address and immediately reclaims it as the
# lowest free one, so each keeps its own.
for n in $DENIED; do
docker restart "fips-node-${n}${FIPS_CI_NAME_SUFFIX:-}" >/dev/null 2>&1 &
docker restart "fips-node-${n}${FIPS_CI_NAME_SUFFIX:-}" >/dev/null 2>&1 || true
done
wait
# Belt and braces: a restart may still move an address, so re-read
# rather than assuming the pre-capture snapshot still holds.
record_denied_addrs
info " [load-driver] restarted denied peers ($DENIED) at t+${elapsed}s"
done
) &
@@ -185,17 +230,44 @@ info "captured $captured tcpdump lines → $CAP_FILE"
info "phase 3: per-denied-peer assertion (sustained inbound retries > 0, not promoted)"
OVERALL=0
TOTAL_IN=0
# One last observation: the final restart round may have moved an address after
# the driver's own record.
record_denied_addrs
# The cap'd node is never restarted, so its address must not have moved. If it
# did, every pattern below covers only part of the window and the counts mean
# nothing — that is a harness failure, not a cap regression.
cap_ip_now=$(node_ip "$CAP_NODE")
[ "$cap_ip_now" = "$CAP_IP" ] \
|| fail "cap'd node address moved during the capture ($CAP_IP${cap_ip_now:-<unreadable>}) though it was never restarted"
cap_re=$(printf '%s' "$CAP_IP" | sed 's/\./\\./g')
# Two denied peers must never have held the same address, or the per-peer
# counts below are not per-peer: each would match the other's traffic and the
# "this peer is sustained-retrying" assertion could be satisfied entirely by
# its neighbour. Serialized restarts above are what prevent it; this is the
# check that says so out loud if they ever stop working.
dup=$(awk '{ if (seen[$2] != "" && seen[$2] != $1) print $2; seen[$2] = $1 }' "$ADDR_FILE" | sort -u)
[ -z "$dup" ] \
|| fail "denied peers shared an address during the capture ($(echo "$dup" | paste -sd, -)); per-peer attribution is not possible"
FINAL_PEERS=$(docker exec "fips-node-${CAP_NODE}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show peers 2>/dev/null \
| grep -oE 'npub1[a-z0-9]+' | sort -u || true)
for n in $DENIED; do
n_ip=$(node_ip "$n")
# Match every address this peer held during the window, not just its last:
# a restart can move it, and grepping for one of several under-counts the
# inbound evidence this assertion rests on.
n_re=$(awk -v n="$n" '$1 == n { gsub(/\./, "\\.", $2); printf "%s%s", (c++ ? "|" : ""), $2 }' "$ADDR_FILE")
[ -n "$n_re" ] \
|| fail "no docker address was ever recorded for denied peer node-$n"
n_seen=$(awk -v n="$n" '$1 == n {print $2}' "$ADDR_FILE" | paste -sd, -)
n_npub=$(node_npub "$n")
# Sustained retry: any inbound UDP from the denied peer to the cap'd
# node, regardless of handshake message size.
in_count=$(grep -cE "IP $n_ip\.[0-9]+ > $CAP_IP\.2121:" "$CAP_FILE" || true)
in_count=$(grep -cE "IP ($n_re)\.[0-9]+ > $cap_re\.2121:" "$CAP_FILE" || true)
promoted="no"
if echo "$FINAL_PEERS" | grep -q "$n_npub"; then promoted="yes"; fi
info " node-$n ($n_ip): inbound packets = $in_count, promoted = $promoted"
info " node-$n ($n_seen): inbound packets = $in_count, promoted = $promoted"
TOTAL_IN=$((TOTAL_IN + in_count))
if [ "$in_count" -eq 0 ]; then
info " FAIL: no inbound from denied peer (peer not sustained-retrying?)"
+1 -1
View File
@@ -31,7 +31,7 @@ if ! [[ "$RUNS" =~ ^[1-9][0-9]*$ ]] || [ "$RUNS" -lt 1 ]; then
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="$SCRIPT_DIR/../generated-configs/npubs.env"
ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env"
if [ ! -f "$ENV_FILE" ]; then
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
exit 1
+1 -1
View File
@@ -17,7 +17,7 @@ trap 'echo ""; echo "Test interrupted"; exit 130' INT
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
GENERATED_DIR="$SCRIPT_DIR/../generated-configs"
GENERATED_DIR="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}"
ENV_FILE="$GENERATED_DIR/npubs.env"
GATEWAY="fips-gw-gateway${FIPS_CI_NAME_SUFFIX:-}"
+50 -7
View File
@@ -10,25 +10,47 @@ set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_DIR="$SCRIPT_DIR/../configs"
GENERATED_DIR="$SCRIPT_DIR/../generated-configs"
# Scoped by the CI run suffix so two concurrent runs cannot overwrite each
# other's generated configs or npubs.env. Unset (a bare invocation) renders
# the historical unscoped path.
GENERATED_DIR="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}"
TEMPLATE_FILE="$CONFIG_DIR/node.template.yaml"
DERIVE_KEYS="$SCRIPT_DIR/../../lib/derive_keys.py"
# Every line belonging to one node, from its key to the next node key.
#
# Bounded by the block rather than by a fixed number of lines: a node that
# omits an attribute would otherwise read the NEXT node's value for it, which
# is silent and wrong in both directions — an external node followed by an
# internal one would be classified as internal, and a node without docker_host
# would dial the following node's container.
node_block() {
local topology_file="$1"
local node_id="$2"
awk -v id="$node_id" '
$0 ~ "^ " id ":" { inblock = 1; next }
inblock && /^ [a-zA-Z]/ { exit }
inblock { print }
' "$topology_file"
}
# Parse topology YAML to extract node attributes
# Usage: get_node_attr <topology_file> <node_id> <attr_name>
get_node_attr() {
local topology_file="$1"
local node_id="$2"
local attr="$3"
local block
block=$(node_block "$topology_file" "$node_id")
# Handle both docker_ip and external_ip as "address"
if [ "$attr" = "address" ]; then
local ip=$(grep -A 10 "^ $node_id:" "$topology_file" | grep "docker_ip:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/')
local ip=$(echo "$block" | grep "docker_ip:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/')
if [ -z "$ip" ]; then
ip=$(grep -A 10 "^ $node_id:" "$topology_file" | grep "external_ip:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/')
ip=$(echo "$block" | grep "external_ip:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/')
fi
echo "$ip"
else
grep -A 10 "^ $node_id:" "$topology_file" | grep "${attr}:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/'
echo "$block" | grep "${attr}:" | head -1 | sed 's/.*: *"*\([^"]*\)".*/\1/'
fi
}
@@ -36,10 +58,24 @@ get_node_attr() {
is_external_node() {
local topology_file="$1"
local node_id="$2"
local docker_ip=$(grep -A 10 "^ $node_id:" "$topology_file" | grep "docker_ip:" | head -1)
local docker_ip
docker_ip=$(node_block "$topology_file" "$node_id" | grep "docker_ip:" | head -1)
[ -z "$docker_ip" ]
}
# Docker hostname of an internal node. Peers address each other by name so the
# compose network can be auto-assigned, which is what makes two concurrent runs
# safe: with no fixed subnet requested there is nothing for them to contend
# for. Defaults to node-<id>, the compose `hostname:` every static profile
# uses; a topology whose services are named otherwise declares docker_host.
docker_host_name() {
local topology_file="$1"
local node_id="$2"
local host
host=$(get_node_attr "$topology_file" "$node_id" "docker_host")
echo "${host:-node-$node_id}"
}
# Get peers list from topology
get_peers() {
local topology_file="$1"
@@ -98,7 +134,14 @@ generate_peer_block() {
local peer_id="$2"
local peer_npub="$(get_key RESOLVED_NPUB "$peer_id")"
local peer_ip=$(get_node_attr "$topology_file" "$peer_id" "address")
local peer_addr
if is_external_node "$topology_file" "$peer_id"; then
# An external peer is not ours to name — use the address the
# topology gives it.
peer_addr=$(get_node_attr "$topology_file" "$peer_id" "address")
else
peer_addr=$(docker_host_name "$topology_file" "$peer_id")
fi
local transport=$(get_default_transport "$topology_file")
local port=$(transport_port "$transport")
@@ -107,7 +150,7 @@ generate_peer_block() {
alias: "node-$peer_id"
addresses:
- transport: $transport
addr: "$peer_ip:$port"
addr: "$peer_addr:$port"
connect_policy: auto_connect
EOF
}
+1 -1
View File
@@ -27,7 +27,7 @@ FAILED=0
# Node identities (from generated env file)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="$SCRIPT_DIR/../generated-configs/npubs.env"
ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env"
if [ ! -f "$ENV_FILE" ]; then
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
exit 1
+7 -7
View File
@@ -27,7 +27,7 @@ if [ "${1:-}" = "inject-config" ]; then
echo "Injecting mixed-profile config overrides..."
# Node C: non-routing
cfg="$SCRIPT_DIR/../generated-configs/$TOPOLOGY/node-c.yaml"
cfg="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/$TOPOLOGY/node-c.yaml"
if [ ! -f "$cfg" ]; then
echo " Error: $cfg not found" >&2
exit 1
@@ -43,7 +43,7 @@ with open('$cfg', 'w') as f:
echo " ✓ node-c (disable_routing: true)"
# Node D: leaf
cfg="$SCRIPT_DIR/../generated-configs/$TOPOLOGY/node-d.yaml"
cfg="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/$TOPOLOGY/node-d.yaml"
if [ ! -f "$cfg" ]; then
echo " Error: $cfg not found" >&2
exit 1
@@ -64,7 +64,7 @@ fi
# ── Full test ─────────────────────────────────────────────────────────
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
ENV_FILE="$SCRIPT_DIR/../generated-configs/npubs.env"
ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env"
if [ ! -f "$ENV_FILE" ]; then
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
exit 1
@@ -102,10 +102,10 @@ echo "Phase 1: Link convergence"
# B: peers with A, C → 2 links
# C (NonRouting): peers with A, B → 2 links
# D (Leaf): peers with A → 1 link
wait_for_peers fips-node-a 3 30 || true
wait_for_peers fips-node-b 2 30 || true
wait_for_peers fips-node-c 2 30 || true
wait_for_peers fips-node-d 1 30 || true
wait_for_peers fips-node-a${FIPS_CI_NAME_SUFFIX:-} 3 30 || true
wait_for_peers fips-node-b${FIPS_CI_NAME_SUFFIX:-} 2 30 || true
wait_for_peers fips-node-c${FIPS_CI_NAME_SUFFIX:-} 2 30 || true
wait_for_peers fips-node-d${FIPS_CI_NAME_SUFFIX:-} 1 30 || true
# Phase 2: Verify link counts (already validated by wait_for_peers above)
echo ""
+1 -1
View File
@@ -19,7 +19,7 @@ FAILED=0
# Node identities (from generated env file)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
ENV_FILE="$SCRIPT_DIR/../generated-configs/npubs.env"
ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env"
if [ ! -f "$ENV_FILE" ]; then
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
exit 1
+29 -28
View File
@@ -33,13 +33,16 @@ NODES="a b c d e"
REKEY_ACCEPT_OFF_NODES="${REKEY_ACCEPT_OFF_NODES:-}"
# Comma-separated list of node IDs to set udp.outbound_only=true on
# during inject-config. For each such node, peer addresses are also
# rewritten from numeric docker IPs to docker hostnames (e.g.
# 172.20.0.12:2121 → node-c:2121). This reproduces the production
# scenario where peer configs carry hostnames so the `addr_to_link`
# key is hostname-form while inbound packet source addrs are numeric,
# making the should_admit_msg1 carve-out's `addr_to_link.contains_key`
# check miss.
# during inject-config.
#
# The other half of this scenario — peer addresses in hostname form
# (`node-c:2121`) rather than numeric — is no longer injected here. The
# generator now emits a docker hostname for every internal peer in every
# static topology, so the condition holds for all nodes unconditionally and a
# rewrite step here would match nothing. What it reproduces is unchanged: the
# `addr_to_link` key is hostname-form while inbound packet source addrs are
# numeric, so the should_admit_msg1 carve-out's `addr_to_link.contains_key`
# check misses.
REKEY_OUTBOUND_ONLY_NODES="${REKEY_OUTBOUND_ONLY_NODES:-}"
# Rekey timing configuration
@@ -54,10 +57,10 @@ if [ "${1:-}" = "inject-config" ]; then
echo " Setting udp.accept_connections=false on nodes: $REKEY_ACCEPT_OFF_NODES"
fi
if [ -n "$REKEY_OUTBOUND_ONLY_NODES" ]; then
echo " Setting udp.outbound_only=true + rewriting peer addrs to docker hostnames on nodes: $REKEY_OUTBOUND_ONLY_NODES"
echo " Setting udp.outbound_only=true (peer addrs already hostname-form) on nodes: $REKEY_OUTBOUND_ONLY_NODES"
fi
for node in $NODES; do
cfg="$SCRIPT_DIR/../generated-configs/$TOPOLOGY/node-$node.yaml"
cfg="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/$TOPOLOGY/node-$node.yaml"
if [ ! -f "$cfg" ]; then
echo " Error: $cfg not found" >&2
exit 1
@@ -79,6 +82,7 @@ if [ "${1:-}" = "inject-config" ]; then
done
fi
python3 -c "
import re
import yaml
with open('$cfg') as f:
cfg = yaml.safe_load(f)
@@ -103,29 +107,26 @@ if '$outbound_only' == 'true':
transports['udp'] = udp
if isinstance(udp, dict):
udp['outbound_only'] = True
# Rewrite peer addrs to docker hostnames so the addr_to_link key
# is hostname-form (mirroring production peer configs that carry
# hostnames). Without this, peer addrs are numeric and the
# carve-out's addr_to_link lookup matches inbound numeric source
# addrs, masking the bug.
ip_to_host = {
'172.20.0.10': 'node-a',
'172.20.0.11': 'node-b',
'172.20.0.12': 'node-c',
'172.20.0.13': 'node-d',
'172.20.0.14': 'node-e',
}
# Assert, rather than create, the hostname-form peer addrs this
# scenario depends on: the addr_to_link key must be a name so the
# carve-out's lookup misses the numeric inbound source addr. The
# generator emits hostnames for every internal peer, so a numeric addr
# here means the generator regressed and the suite would otherwise go
# on passing while testing nothing.
for peer in cfg.get('peers', []) or []:
for addr in peer.get('addresses', []) or []:
t = addr.get('transport')
if t is not None and t != 'udp':
continue
a = addr.get('addr', '')
for ip, host in ip_to_host.items():
if a.startswith(ip + ':'):
port = a.split(':', 1)[1]
addr['addr'] = f'{host}:{port}'
break
host = a.rsplit(':', 1)[0]
# A bracketed IPv6 literal is numeric too, and rsplit leaves the
# brackets on, so it would not match the v4 pattern.
if host.startswith('[') or re.fullmatch(r'[0-9.]+', host):
raise SystemExit(
'outbound_only premise broken on node-$node: peer addr '
+ a + ' is numeric, expected a docker hostname'
)
with open('$cfg', 'w') as f:
yaml.dump(cfg, f, default_flow_style=False, sort_keys=False)
"
@@ -134,7 +135,7 @@ with open('$cfg', 'w') as f:
suffix=" (accept_connections=false)"
fi
if [ "$outbound_only" = "true" ]; then
suffix=" (outbound_only=true, hostname peer addrs)"
suffix=" (outbound_only=true, hostname peer addrs verified)"
fi
echo " ✓ node-$node$suffix"
done
@@ -197,7 +198,7 @@ TOTAL_PASSED=0
TOTAL_FAILED=0
# Node identities
ENV_FILE="$SCRIPT_DIR/../generated-configs/npubs.env"
ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env"
if [ ! -f "$ENV_FILE" ]; then
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
exit 1