mirror of
https://github.com/jmcorgan/fips.git
synced 2026-10-05 19:18:25 +00:00
Johnathan Corgan
957f042282
count_log_pattern prints unreadable:<container> and fails when a node's logs cannot be read, but only two of its callers checked that. The rest compared the text as a number: the first-rekey check reported "no FMP rekey cutover observed", the second-cycle wait aborted the run on an unbound variable before the log analysis, and the log analysis read the rekey totals as zero. Each now checks the status and reports the count as not established.
1227 lines
52 KiB
Bash
Executable File
1227 lines
52 KiB
Bash
Executable File
#!/bin/bash
|
|
# Mixed-version interop test driver.
|
|
#
|
|
# Brings up an N-node full mesh from a NODE-SPEC (a multiset of image
|
|
# slots), verifies every pair interoperates — FMP link, FSP session,
|
|
# connectivity, rekey survival — and runs a per-node, per-pair log
|
|
# analysis tuned to surface INTEROP problems: handshake failures,
|
|
# FSP/FMP decrypt failures, `unknown FMP version`, replay storms,
|
|
# link/session teardowns.
|
|
#
|
|
# The harness's job is NOT to test a single version. It is to find any
|
|
# place where two DIFFERENT versions fail to interoperate in a way a
|
|
# same-version pair would not. Every failure is attributed to a specific
|
|
# (version-X <-> version-Y) pair, classified same-version vs MIXED.
|
|
#
|
|
# A node-spec like `a a b c` produces a same-version pair (a1<->a2) — the
|
|
# CONTROL ARM — alongside the mixed pairs. The control pair is what makes
|
|
# a netem run interpretable: a failure on a mixed pair that the control
|
|
# pair does not share is an interop regression; a failure both share is
|
|
# loss noise, not version-specific.
|
|
#
|
|
# Prerequisites:
|
|
# 1. bash build-images.sh <ref-a> <ref-b> <ref-c> (builds fips-interop:a/b/c)
|
|
# 2. configs are (re)generated automatically below for the given spec.
|
|
#
|
|
# Usage:
|
|
# ./interop-test.sh [--topology <name>] [node-spec...]
|
|
#
|
|
# --topology built-in multi-hop topology selecting a node-spec + edge
|
|
# set + default data-plane streams. Known: multihop-3v-cycle
|
|
# (6 nodes, 2 of each version, one cycle, two leaves —
|
|
# exercises cross-version forwarding + mesh-size). Without
|
|
# it the mesh is a full mesh from the positional node-spec.
|
|
# node-spec space-separated slot letters (a/b/c), default `a b c`.
|
|
# e.g. `a a b c` (4-node, one same-version control pair),
|
|
# `a a a` (3-node same-version flake rig).
|
|
#
|
|
# Beyond FMP/FSP rekey survival, a --topology run also verifies multi-hop
|
|
# FORWARDING (all-pairs ping over non-adjacent pairs), DATA-PLANE
|
|
# CONTINUITY across rekey (control-differential ping-loss stream,
|
|
# Phase 5b), and MESH-SIZE estimate convergence across versions
|
|
# (Phase 7).
|
|
#
|
|
# Environment:
|
|
# FIPS_INTEROP_NETEM tc-netem arg string applied to every container's
|
|
# eth0, e.g. "delay 10ms 5ms 25% loss 1%". Unset =
|
|
# no impairment.
|
|
# FIPS_INTEROP_EDGES explicit undirected edge list (overrides the
|
|
# full mesh) — see generate-configs.sh. Usually
|
|
# set for you by --topology.
|
|
# FIPS_INTEROP_STREAMS data-plane stream pairs (`nid-nid` tokens, both
|
|
# directions streamed). Enables Phase 1b/5b. Set
|
|
# by --topology; empty = streams off.
|
|
# STREAM_LOSS_MARGIN_PCT rekey-vs-control loss margin (default 5).
|
|
# CONTROL_STREAM_SECS quiet control-window length (default 12).
|
|
# CONTROL_MAX_ATTEMPTS control windows measured before Phase 5b
|
|
# abstains because every one saw a rekey
|
|
# cutover or could not be checked (default 3).
|
|
# MESH_SIZE_WARMUP Phase 7 bloom warmup, from mesh start, before
|
|
# any estimate counts (default 300).
|
|
# MESH_SIZE_SETTLE Phase 7 unbroken in-band window a node must
|
|
# hold to be credited (default 60).
|
|
# MESH_SIZE_TIMEOUT Phase 7 poll budget beyond warmup+settle
|
|
# (default 180).
|
|
# FIPS_INTEROP_KEEP_UP 1 = leave containers running after the test (debug).
|
|
# REKEY_AFTER_SECS rekey interval to generate configs with (default
|
|
# 35; multihop-3v-cycle defaults it to 50).
|
|
# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/,
|
|
# .stress-runs/, generated-configs/). When
|
|
# unset, falls back to in-tree paths under
|
|
# testing/interop/ and prints a warning to
|
|
# stderr; set it to a path outside the source
|
|
# tree to keep generated artefacts out of the
|
|
# checkout.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
source "$SCRIPT_DIR/../lib/wait-converge.sh"
|
|
|
|
# ── Scratch-dir root ─────────────────────────────────────────────────
|
|
#
|
|
# FIPS_INTEROP_RUNS_DIR controls where the harness writes its scratch
|
|
# directories (.build/, .stress-runs/, generated-configs/). When unset
|
|
# we fall back to in-tree paths under testing/interop/ and warn the
|
|
# operator, so the warning fires exactly once per invocation. When a
|
|
# parent script has already warned it exports _FIPS_INTEROP_WARNED=1
|
|
# to suppress duplicate warnings in child scripts.
|
|
if [[ -n "${FIPS_INTEROP_RUNS_DIR:-}" ]]; then
|
|
RUNS_BASE="$FIPS_INTEROP_RUNS_DIR"
|
|
mkdir -p "$RUNS_BASE"
|
|
else
|
|
RUNS_BASE="$SCRIPT_DIR"
|
|
if [[ -z "${_FIPS_INTEROP_WARNED:-}" ]]; then
|
|
echo >&2 "WARNING: FIPS_INTEROP_RUNS_DIR not set; harness output will be written under the source tree at $RUNS_BASE. Set FIPS_INTEROP_RUNS_DIR to a path outside the source tree to avoid this."
|
|
export _FIPS_INTEROP_WARNED=1
|
|
fi
|
|
fi
|
|
|
|
GEN_DIR="$RUNS_BASE/generated-configs"
|
|
COMPOSE_FILE="$GEN_DIR/docker-compose.generated.yml"
|
|
NODES_ENV="$GEN_DIR/nodes.env"
|
|
REFS_ENV="$RUNS_BASE/.build/refs.env"
|
|
|
|
# ── Built-in topology selection ──────────────────────────────────────
|
|
#
|
|
# --topology <name> selects a node-spec + an explicit edge set (a
|
|
# multi-hop, mixed-version, leaf-bearing graph) + a default data-plane
|
|
# stream set, exercising forwarding / routing / mesh-size convergence
|
|
# across versions. Without it, positional args are the node-spec and the
|
|
# mesh is a full mesh (historical behavior).
|
|
TOPOLOGY_NAME=""
|
|
_ARGS=()
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--topology) TOPOLOGY_NAME="${2:-}"; shift 2 ;;
|
|
--topology=*) TOPOLOGY_NAME="${1#--topology=}"; shift ;;
|
|
*) _ARGS+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
set -- ${_ARGS[@]+"${_ARGS[@]}"}
|
|
|
|
if [ -n "$TOPOLOGY_NAME" ]; then
|
|
case "$TOPOLOGY_NAME" in
|
|
multihop-3v-cycle)
|
|
# 6 nodes (2 of each version), one cycle, two leaves.
|
|
# a1
|
|
# / \ leaves: a2, c2
|
|
# b1 c1 cycle: b1-a1-c1-b2-b1
|
|
# / \ \
|
|
# a2 b2------c2-edge(b2-c1)
|
|
set -- a a b b c c
|
|
export FIPS_INTEROP_EDGES="a1-b1 a1-c1 b1-a2 b1-b2 c1-c2 b2-c1"
|
|
: "${FIPS_INTEROP_STREAMS:=a2-c2 a1-b1}"
|
|
export FIPS_INTEROP_STREAMS
|
|
# Multi-hop convergence + a clean pre-rekey control window
|
|
# want more headroom than the 35s full-mesh default.
|
|
: "${REKEY_AFTER_SECS:=50}"
|
|
;;
|
|
*)
|
|
echo "ERROR: unknown --topology '$TOPOLOGY_NAME' (known: multihop-3v-cycle)" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
REKEY_AFTER_SECS="${REKEY_AFTER_SECS:-35}"
|
|
|
|
# ── Node-spec ────────────────────────────────────────────────────────
|
|
|
|
SPEC=("$@")
|
|
if [ "${#SPEC[@]}" -eq 0 ]; then
|
|
SPEC=(a b c)
|
|
fi
|
|
SPEC_STR="${SPEC[*]}"
|
|
|
|
# ── Timing ───────────────────────────────────────────────────────────
|
|
|
|
CONVERGENCE_TIMEOUT="${CONVERGENCE_TIMEOUT:-90}" # detector settle budget (env-overridable; larger meshes under loss converge slower)
|
|
PING_TIMEOUT=5
|
|
MAX_PING_ATTEMPTS=4 # strict-ping retry (mirrors rekey-test.sh)
|
|
PING_RETRY_DELAY=1
|
|
# First rekey should follow shortly after the interval once converged.
|
|
FIRST_REKEY_TIMEOUT=$((REKEY_AFTER_SECS + 20))
|
|
SECOND_REKEY_WAIT=$((REKEY_AFTER_SECS + 5))
|
|
REKEY_SETTLE=12 # FSP-cutover settle budget (Phase 6)
|
|
# Post-rekey reconvergence is polled, not fixed-slept: the mesh is given
|
|
# up to this long to restore full connectivity after a rekey before the
|
|
# strict assertion sweep runs. A genuinely stuck pair still fails — the
|
|
# poll times out and the recording sweep captures it.
|
|
POST_REKEY_TIMEOUT=45
|
|
# Progress-aware stall budget for the convergence detector
|
|
# (wait_for_full_baseline → wait_until_connected). If no additional pair
|
|
# becomes reachable for this long while more than the near-converged
|
|
# slack of pairs is still down, the detector gives up early instead of
|
|
# burning the whole convergence/post-rekey window; any progress resets
|
|
# the clock, so a slow-but-converging mesh under netem keeps polling.
|
|
RECONVERGE_STALL=15
|
|
LOG_POLL_INTERVAL=2
|
|
|
|
# Data-plane continuity stream (control-differential). Streams run a
|
|
# sustained ping6 over the overlay across the rekey window vs a quiet
|
|
# control window; loss is compared to prove rekey is hitless.
|
|
# The control window cannot be lengthened much: it has to fit inside one
|
|
# rekey interval (REKEY_AFTER_SECS, 35s by default) to stand a chance of
|
|
# being cutover-free, and it already gets contaminated sometimes at 12s.
|
|
# So the sample is raised by rate instead. At 20 Hz the control window is
|
|
# 240 packets and the rekey window ~3100, where at 5 Hz they were 60 and
|
|
# ~775 and a 1% margin came to 0.6 of a control packet — below the
|
|
# quantisation of its own measurement, so a one-packet difference decided
|
|
# the verdict and the netem arm failed on same-version pairs as readily
|
|
# as on mixed ones.
|
|
STREAM_RATE_HZ=20
|
|
CONTROL_STREAM_SECS="${CONTROL_STREAM_SECS:-12}" # quiet pre-rekey window
|
|
# A control window that saw a rekey cutover, or whose cutover count could
|
|
# not be read, is no baseline. Phase 1b re-measures it, after waiting for
|
|
# the cutover count to hold still for CONTROL_QUIET_SECS (at most
|
|
# CONTROL_QUIET_MAX) so the retry does not land in the same cluster of
|
|
# cutovers, and Phase 5b abstains when no attempt comes out clean.
|
|
CONTROL_MAX_ATTEMPTS="${CONTROL_MAX_ATTEMPTS:-3}"
|
|
CONTROL_QUIET_SECS=5
|
|
CONTROL_QUIET_MAX=30
|
|
# 5% is 12 packets of the 240-packet control window and ~155 of the
|
|
# ~3100-packet rekey window. On a path losing up to 6% (the range the
|
|
# v0.4.2 netem runs baselined at, under `loss 2%` over several hops) the
|
|
# difference of the two windows has a standard deviation below 1.6%, so
|
|
# 5% is past three sigma and sampling spread can no longer produce a red.
|
|
# It still catches what this check exists for: a rekey that is not
|
|
# hitless blackholes until the session reconverges, which the harness
|
|
# budgets 45s for, against the ~8s of traffic 5% of the rekey window is.
|
|
STREAM_LOSS_MARGIN_PCT="${STREAM_LOSS_MARGIN_PCT:-5}"
|
|
# The rekey-window stream must span Phases 2-5 (both cutovers + reconverge).
|
|
REKEY_STREAM_SECS=$(( FIRST_REKEY_TIMEOUT + SECOND_REKEY_WAIT + POST_REKEY_TIMEOUT + 15 ))
|
|
|
|
# Mesh-size estimate convergence (strict ±25% of true N). The archived
|
|
# design note puts bloom-filter warmup at ~5 minutes from node start and
|
|
# calls the estimate unreliable before that, so nothing sampled inside
|
|
# MESH_SIZE_WARMUP is evidence of convergence, and a node is credited
|
|
# only after holding the band unbroken for MESH_SIZE_SETTLE afterwards.
|
|
# MESH_SIZE_TIMEOUT is the extra budget for reaching that state, on top
|
|
# of the warmup and settle windows rather than covering them.
|
|
MESH_SIZE_WARMUP="${MESH_SIZE_WARMUP:-300}"
|
|
MESH_SIZE_SETTLE="${MESH_SIZE_SETTLE:-60}"
|
|
MESH_SIZE_TIMEOUT="${MESH_SIZE_TIMEOUT:-180}"
|
|
MESH_SIZE_POLL=5
|
|
|
|
# ── Counters ─────────────────────────────────────────────────────────
|
|
|
|
PASSED=0
|
|
FAILED=0
|
|
TOTAL_PASSED=0
|
|
TOTAL_FAILED=0
|
|
# INTEROP_FAILURES collects human-readable, pair-attributed failure lines.
|
|
INTEROP_FAILURES=()
|
|
|
|
# ── Preflight: docker + images ───────────────────────────────────────
|
|
|
|
if ! docker info >/dev/null 2>&1; then
|
|
echo "ERROR: Docker daemon is not reachable" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# A node-spec only ever references the three image slots, regardless of
|
|
# how many nodes it has. Check the slots actually present in the spec.
|
|
for slot in $(printf '%s\n' "${SPEC[@]}" | sort -u); do
|
|
case "$slot" in
|
|
a|b|c) ;;
|
|
*) echo "ERROR: invalid slot '$slot' in node-spec" >&2; exit 2 ;;
|
|
esac
|
|
if ! docker image inspect "fips-interop:$slot" >/dev/null 2>&1; then
|
|
echo "ERROR: image fips-interop:$slot not present." >&2
|
|
echo "Build the three images first:" >&2
|
|
echo " bash $SCRIPT_DIR/build-images.sh <ref-a> <ref-b> <ref-c>" >&2
|
|
exit 2
|
|
fi
|
|
done
|
|
|
|
# ── (Re)generate configs for this node-spec ──────────────────────────
|
|
#
|
|
# Always regenerate when the spec on disk does not match the requested
|
|
# spec (or no manifest exists). Generation is deterministic and cheap.
|
|
|
|
need_regen=1
|
|
if [ -f "$NODES_ENV" ]; then
|
|
existing_spec="$(sed -n 's/^INTEROP_SPEC="\(.*\)"$/\1/p' "$NODES_ENV")"
|
|
existing_edges="$(sed -n 's/^INTEROP_TOPOLOGY_EDGES="\(.*\)"$/\1/p' "$NODES_ENV")"
|
|
if [ "$existing_spec" = "$SPEC_STR" ] \
|
|
&& [ "$existing_edges" = "${FIPS_INTEROP_EDGES:-}" ]; then
|
|
need_regen=0
|
|
fi
|
|
fi
|
|
if [ "$need_regen" -eq 1 ]; then
|
|
echo "Generating mesh configs for spec '$SPEC_STR' (rekey.after_secs=$REKEY_AFTER_SECS)..."
|
|
REKEY_AFTER_SECS="$REKEY_AFTER_SECS" bash "$SCRIPT_DIR/generate-configs.sh" \
|
|
"${SPEC[@]}"
|
|
echo ""
|
|
fi
|
|
|
|
# ── Load the manifest ────────────────────────────────────────────────
|
|
#
|
|
# nodes.env gives the ordered node list and the per-node slot/container/
|
|
# ip/npub maps. npubs.env gives NPUB_<NODEID> vars used by ping_one.
|
|
|
|
# shellcheck disable=SC1090
|
|
source "$NODES_ENV"
|
|
# shellcheck disable=SC1091
|
|
source "$GEN_DIR/npubs.env"
|
|
|
|
read -r -a NODES <<< "$INTEROP_NODE_IDS"
|
|
|
|
declare -A SLOT_OF CONTAINER NODE_IP_OF NPUB_OF
|
|
parse_map() {
|
|
# parse_map <assoc-array-name> <space-separated nodeid:value tokens>
|
|
local -n _dst="$1"
|
|
local tok nid val
|
|
for tok in $2; do
|
|
nid="${tok%%:*}"
|
|
val="${tok#*:}"
|
|
_dst["$nid"]="$val"
|
|
done
|
|
}
|
|
parse_map SLOT_OF "$INTEROP_NODE_SLOTS"
|
|
parse_map CONTAINER "$INTEROP_NODE_CONTAINERS"
|
|
parse_map NODE_IP_OF "$INTEROP_NODE_IPS"
|
|
parse_map NPUB_OF "$INTEROP_NODE_NPUBS"
|
|
|
|
# Topology metadata: per-node expected direct-peer degree, and the
|
|
# undirected direct-edge set (for direct-vs-routed pair labeling).
|
|
TOPOLOGY_KIND="${INTEROP_TOPOLOGY:-full-mesh}"
|
|
declare -A DEGREE_OF
|
|
parse_map DEGREE_OF "${INTEROP_NODE_DEGREE:-}"
|
|
|
|
declare -A IS_DIRECT_EDGE
|
|
if [ -n "${INTEROP_TOPOLOGY_EDGES:-}" ]; then
|
|
for _e in $INTEROP_TOPOLOGY_EDGES; do
|
|
_x="${_e%%-*}"; _y="${_e#*-}"
|
|
IS_DIRECT_EDGE["$_x|$_y"]=1
|
|
IS_DIRECT_EDGE["$_y|$_x"]=1
|
|
done
|
|
else
|
|
# Full mesh: every ordered pair is a direct edge.
|
|
for _a in "${NODES[@]}"; do
|
|
for _b in "${NODES[@]}"; do
|
|
[ "$_a" = "$_b" ] && continue
|
|
IS_DIRECT_EDGE["$_a|$_b"]=1
|
|
done
|
|
done
|
|
fi
|
|
|
|
pair_is_direct() { [ -n "${IS_DIRECT_EDGE[$1|$2]:-}" ]; }
|
|
hop_label() { if pair_is_direct "$1" "$2"; then echo "direct"; else echo "routed"; fi; }
|
|
|
|
# Data-plane stream directed pairs (both directions of each token).
|
|
STREAM_PAIRS=()
|
|
if [ -n "${FIPS_INTEROP_STREAMS:-}" ]; then
|
|
for _tok in ${FIPS_INTEROP_STREAMS//,/ }; do
|
|
_x="${_tok%%-*}"; _y="${_tok#*-}"
|
|
STREAM_PAIRS+=("$_x $_y" "$_y $_x")
|
|
done
|
|
fi
|
|
|
|
# Unordered pairs of the mesh.
|
|
PAIRS=()
|
|
for ((i = 0; i < ${#NODES[@]}; i++)); do
|
|
for ((j = i + 1; j < ${#NODES[@]}; j++)); do
|
|
PAIRS+=("${NODES[$i]} ${NODES[$j]}")
|
|
done
|
|
done
|
|
|
|
NUM_NODES="${#NODES[@]}"
|
|
NUM_PAIRS="${#PAIRS[@]}"
|
|
NUM_DIRECTED=$((NUM_PAIRS * 2))
|
|
PEERS_EXPECTED=$((NUM_NODES - 1))
|
|
|
|
# ── Ref / version metadata ───────────────────────────────────────────
|
|
#
|
|
# refs.env (written by build-images.sh) records what each SLOT was built
|
|
# from. If absent, fall back to the image labels, then to "unknown".
|
|
# These maps are keyed by SLOT (a/b/c), not by node id.
|
|
|
|
declare -A SLOT_REF SLOT_SHA
|
|
|
|
load_slot_metadata() {
|
|
local slot upper
|
|
for slot in a b c; do
|
|
SLOT_REF[$slot]="unknown"
|
|
SLOT_SHA[$slot]="unknown"
|
|
done
|
|
if [ -f "$REFS_ENV" ]; then
|
|
# shellcheck disable=SC1090
|
|
source "$REFS_ENV"
|
|
fi
|
|
for slot in a b c; do
|
|
upper="$(echo "$slot" | tr '[:lower:]' '[:upper:]')"
|
|
local ref_var="INTEROP_REF_${upper}"
|
|
local sha_var="INTEROP_SHA_${upper}"
|
|
if [ -n "${!ref_var:-}" ]; then
|
|
SLOT_REF[$slot]="${!ref_var}"
|
|
SLOT_SHA[$slot]="${!sha_var:-unknown}"
|
|
continue
|
|
fi
|
|
local lbl_ref lbl_sha
|
|
lbl_ref="$(docker image inspect "fips-interop:$slot" \
|
|
--format '{{ index .Config.Labels "fips.interop.ref" }}' 2>/dev/null || true)"
|
|
lbl_sha="$(docker image inspect "fips-interop:$slot" \
|
|
--format '{{ index .Config.Labels "fips.interop.sha" }}' 2>/dev/null || true)"
|
|
[ -n "$lbl_ref" ] && SLOT_REF[$slot]="$lbl_ref"
|
|
[ -n "$lbl_sha" ] && SLOT_SHA[$slot]="$lbl_sha"
|
|
done
|
|
}
|
|
|
|
# A pair is "mixed-version" iff the two nodes' image slots resolve to
|
|
# different built SHAs. SHA is the precise discriminator; ref names can
|
|
# differ yet point at the same commit.
|
|
pair_is_mixed() {
|
|
local n1="$1" n2="$2"
|
|
local s1="${SLOT_OF[$n1]}" s2="${SLOT_OF[$n2]}"
|
|
if [ "${SLOT_SHA[$s1]}" = "${SLOT_SHA[$s2]}" ] \
|
|
&& [ "${SLOT_SHA[$s1]}" != "unknown" ]; then
|
|
return 1 # same version
|
|
fi
|
|
return 0 # mixed (or unknown — treat as mixed for scrutiny)
|
|
}
|
|
|
|
pair_kind() {
|
|
if pair_is_mixed "$1" "$2"; then
|
|
echo "MIXED"
|
|
else
|
|
echo "same"
|
|
fi
|
|
}
|
|
|
|
pair_label() {
|
|
# e.g. "a1[A fix/...@3045212] <-> c1[C v0.3.0@b11b639]"
|
|
local n1="$1" n2="$2"
|
|
local s1="${SLOT_OF[$n1]}" s2="${SLOT_OF[$n2]}"
|
|
local u1 u2
|
|
u1="$(echo "$s1" | tr '[:lower:]' '[:upper:]')"
|
|
u2="$(echo "$s2" | tr '[:lower:]' '[:upper:]')"
|
|
echo "${n1}[$u1 ${SLOT_REF[$s1]}@${SLOT_SHA[$s1]}] <-> ${n2}[$u2 ${SLOT_REF[$s2]}@${SLOT_SHA[$s2]}]"
|
|
}
|
|
|
|
# ── Helpers ──────────────────────────────────────────────────────────
|
|
|
|
# ping6 from one container to another node's mesh address (npub.fips).
|
|
ping_one() {
|
|
local from_node="$1" to_node="$2" quiet="${3:-}"
|
|
local max_attempts="${4:-1}"
|
|
local from_ctr="${CONTAINER[$from_node]}"
|
|
local to_npub="${NPUB_OF[$to_node]}"
|
|
local label="$from_node -> $to_node"
|
|
|
|
local attempt=1 output rtt
|
|
while (( attempt <= max_attempts )); do
|
|
(( attempt > 1 )) && sleep "$PING_RETRY_DELAY"
|
|
if output=$(docker exec "$from_ctr" ping6 -c 1 -W "$PING_TIMEOUT" \
|
|
"${to_npub}.fips" 2>&1); then
|
|
rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2)
|
|
if [ -z "$quiet" ]; then
|
|
echo " $label ... OK (${rtt:-?}ms${attempt:+, attempt $attempt})"
|
|
fi
|
|
PASSED=$((PASSED + 1))
|
|
return 0
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
if [ -z "$quiet" ]; then
|
|
echo " $label ... FAIL (after $max_attempts attempt(s))"
|
|
fi
|
|
FAILED=$((FAILED + 1))
|
|
return 1
|
|
}
|
|
|
|
# All directed pairs of the mesh. Records pair-attributed failures into
|
|
# INTEROP_FAILURES, with the mixed/same classification.
|
|
ping_all_pairs() {
|
|
local quiet="${1:-}" max_attempts="${2:-1}" context="${3:-connectivity}"
|
|
PASSED=0
|
|
FAILED=0
|
|
local i j
|
|
for i in "${NODES[@]}"; do
|
|
for j in "${NODES[@]}"; do
|
|
[ "$i" = "$j" ] && continue
|
|
if ! ping_one "$i" "$j" "$quiet" "$max_attempts"; then
|
|
# The `convergence` context is the wait_for_full_baseline
|
|
# detector poll, NOT an assertion: a miss there only means
|
|
# "not converged yet, keep polling". Recording detector
|
|
# misses as interop failures is wrong — it false-fails
|
|
# every rep that takes a moment to converge. Only the
|
|
# strict assertion sweeps (baseline / post-rekey-*) record.
|
|
if [ "$context" != "convergence" ]; then
|
|
local kind hop
|
|
kind="$(pair_kind "$i" "$j")"
|
|
hop="$(hop_label "$i" "$j")"
|
|
# NOTE: keep "$kind pair" contiguous — interop-stress.sh
|
|
# greps "MIXED pair"/"same pair". The [hop] tag is additive.
|
|
INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j") [$hop]: ping $i->$j FAILED")
|
|
fi
|
|
fi
|
|
done
|
|
done
|
|
}
|
|
|
|
# Convergence detector probe: one full all-pairs ping sweep in the
|
|
# "convergence" context (which ping_all_pairs deliberately does NOT
|
|
# record as a failure), setting PASSED/FAILED for wait_until_connected.
|
|
_baseline_probe() {
|
|
ping_all_pairs quiet 1 "convergence"
|
|
}
|
|
|
|
# Poll until every directed pair pings clean, or until timeout. This is a
|
|
# convergence DETECTOR — used at establishment (Phase 1) and after each
|
|
# rekey (Phases 3/5). It pings with the "convergence" context, which
|
|
# ping_all_pairs deliberately does not record as a failure; the caller
|
|
# runs a separate strict assertion sweep afterwards.
|
|
#
|
|
# Delegates to the shared progress-aware wait_until_connected so the
|
|
# deadline extends while more pairs are still coming up and gives up fast
|
|
# on a genuine stall, instead of the prior fixed-deadline poll that could
|
|
# false-time-out under heavy CI contention even while still converging.
|
|
# Returns 0 once every pair is reachable, 1 on stall/timeout.
|
|
wait_for_full_baseline() {
|
|
local timeout="$1"
|
|
wait_until_connected _baseline_probe "$timeout" "$RECONVERGE_STALL"
|
|
}
|
|
|
|
phase_result() {
|
|
local phase="$1"
|
|
TOTAL_PASSED=$((TOTAL_PASSED + PASSED))
|
|
TOTAL_FAILED=$((TOTAL_FAILED + FAILED))
|
|
if [ "$FAILED" -eq 0 ]; then
|
|
echo " PASS $phase: $PASSED/$((PASSED + FAILED))"
|
|
else
|
|
echo " FAIL $phase: $PASSED passed, $FAILED FAILED"
|
|
fi
|
|
}
|
|
|
|
# Count a pattern across all node logs.
|
|
#
|
|
# A node whose logs cannot be read makes the whole count unusable rather than
|
|
# contributing 0. The previous form ended each read `| grep -cE … || true`, so a
|
|
# failed `docker logs` yielded 0 for that node and the eight expect-zero
|
|
# assertions in the GLOBAL_PATTERNS loop read a clean result from a node that was
|
|
# never consulted. Same defect and same fix as rekey-test.sh.
|
|
count_log_pattern() {
|
|
local pattern="$1" total=0 n logs count
|
|
for n in "${NODES[@]}"; do
|
|
if ! logs=$(docker logs "${CONTAINER[$n]}" 2>&1); then
|
|
echo "unreadable:${CONTAINER[$n]}"
|
|
return 1
|
|
fi
|
|
count=$(grep -cE "$pattern" <<<"$logs" || true)
|
|
total=$((total + count))
|
|
done
|
|
echo "$total"
|
|
return 0
|
|
}
|
|
|
|
# Count completed FMP initiator rekey cutovers across all node logs.
|
|
#
|
|
# The pattern is a literal argument so the log-string guard can check it
|
|
# against the daemon source. Prints the count, or `unreadable:<ctr>` with
|
|
# status 1 when a node's logs cannot be read.
|
|
fmp_cutover_count() {
|
|
count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped'
|
|
return $?
|
|
}
|
|
|
|
# Wait until the FMP cutover count has held unchanged for
|
|
# CONTROL_QUIET_SECS, giving up after CONTROL_QUIET_MAX. An unreadable
|
|
# count is treated as a change, so it never counts as quiet.
|
|
wait_cutover_quiet() {
|
|
local start=$SECONDS still=$SECONDS last cur
|
|
last="$(fmp_cutover_count)" || last="unreadable"
|
|
while (( SECONDS - still < CONTROL_QUIET_SECS )); do
|
|
if (( SECONDS - start >= CONTROL_QUIET_MAX )); then
|
|
echo " cutover count still moving after ${CONTROL_QUIET_MAX}s; measuring anyway"
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
cur="$(fmp_cutover_count)" || cur="unreadable"
|
|
if [ "$cur" != "$last" ] || [ "$cur" = "unreadable" ]; then
|
|
last="$cur"
|
|
still=$SECONDS
|
|
fi
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# Per-node count of a pattern.
|
|
count_node_pattern() {
|
|
local node="$1" pattern="$2"
|
|
docker logs "${CONTAINER[$node]}" 2>&1 | grep -cE "$pattern" || true
|
|
}
|
|
|
|
# Wait until the count of a pattern across all node logs reaches
|
|
# min_count. Returns 0 when it does, 1 on timeout, and 2 at once when a
|
|
# node's logs cannot be read, since no count is established then; the
|
|
# caller reports that when it takes its own count.
|
|
wait_for_log_pattern_count() {
|
|
local pattern="$1" min_count="$2" timeout="$3"
|
|
local start=$SECONDS count
|
|
while :; do
|
|
count="$(count_log_pattern "$pattern")" || return 2
|
|
[ "$count" -ge "$min_count" ] && return 0
|
|
(( SECONDS - start < timeout )) || return 1
|
|
sleep "$LOG_POLL_INTERVAL"
|
|
done
|
|
}
|
|
|
|
# One node's bloom-union mesh-size estimate, or "null" when the daemon
|
|
# has no estimate yet or cannot be reached.
|
|
mesh_estimate() {
|
|
docker exec "${CONTAINER[$1]}" fipsctl show status 2>/dev/null \
|
|
| python3 -c "import sys,json; v=json.load(sys.stdin).get('estimated_mesh_size'); print(v if v is not None else 'null')" 2>/dev/null \
|
|
|| echo null
|
|
}
|
|
|
|
# ── Data-plane continuity streams ────────────────────────────────────
|
|
#
|
|
# A sustained ping6 stream over the overlay (<npub>.fips) is data-plane
|
|
# traffic: it traverses TUN → FSP session → FMP link → forwarding, the
|
|
# same path application packets take. We run streams across the rekey
|
|
# window and across a quiet control window, then compare loss — a rekey
|
|
# that drops data shows up as rekey-window loss above the control.
|
|
|
|
declare -A STREAM_TX STREAM_RX
|
|
_STREAM_PIDS=()
|
|
_STREAM_FILES=() # "label:from->to:resultfile"
|
|
_STREAM_TMP=""
|
|
|
|
# One directed ping6 stream of <dur>s at STREAM_RATE_HZ; writes "tx rx".
|
|
_stream_one() {
|
|
local from="$1" to="$2" dur="$3" outfile="$4"
|
|
local from_ctr="${CONTAINER[$from]}" to_npub="${NPUB_OF[$to]}"
|
|
local count=$(( dur * STREAM_RATE_HZ ))
|
|
local out tx rx interval
|
|
# Interval and count both derive from STREAM_RATE_HZ, so the packet
|
|
# count the margin is reasoned about cannot drift from the rate sent.
|
|
interval=$(awk -v hz="$STREAM_RATE_HZ" 'BEGIN{ printf "%.3f", 1/hz }')
|
|
out=$(docker exec "$from_ctr" ping6 -i "$interval" -c "$count" -W "$PING_TIMEOUT" \
|
|
"${to_npub}.fips" 2>&1)
|
|
tx=$(echo "$out" | grep -oE '[0-9]+ packets transmitted' | grep -oE '^[0-9]+')
|
|
rx=$(echo "$out" | grep -oE '[0-9]+ received' | grep -oE '^[0-9]+')
|
|
echo "${tx:-0} ${rx:-0}" > "$outfile"
|
|
}
|
|
|
|
# Launch all stream pairs in the background for <label> over <dur>s.
|
|
launch_streams() {
|
|
local label="$1" dur="$2"
|
|
_STREAM_TMP="$(mktemp -d)"
|
|
_STREAM_PIDS=(); _STREAM_FILES=()
|
|
local sp from to
|
|
for sp in "${STREAM_PAIRS[@]}"; do
|
|
read -r from to <<< "$sp"
|
|
_stream_one "$from" "$to" "$dur" "$_STREAM_TMP/$label-$from-$to" &
|
|
_STREAM_PIDS+=("$!")
|
|
_STREAM_FILES+=("$label:$from->$to:$_STREAM_TMP/$label-$from-$to")
|
|
done
|
|
}
|
|
|
|
# Wait for the launched streams and read tx/rx into STREAM_TX/STREAM_RX.
|
|
collect_streams() {
|
|
[ "${#_STREAM_PIDS[@]}" -gt 0 ] && wait "${_STREAM_PIDS[@]}" 2>/dev/null || true
|
|
local entry label rest key f tx rx
|
|
for entry in "${_STREAM_FILES[@]}"; do
|
|
label="${entry%%:*}"; rest="${entry#*:}"; key="${rest%%:*}"; f="${rest#*:}"
|
|
if read -r tx rx < "$f" 2>/dev/null; then :; else tx=0; rx=0; fi
|
|
STREAM_TX["$label:$key"]="${tx:-0}"
|
|
STREAM_RX["$label:$key"]="${rx:-0}"
|
|
done
|
|
[ -n "$_STREAM_TMP" ] && rm -rf "$_STREAM_TMP"
|
|
_STREAM_TMP=""
|
|
}
|
|
|
|
# loss% from tx/rx (one decimal), or "NA" when tx==0.
|
|
_loss_pct() {
|
|
awk -v t="${1:-0}" -v r="${2:-0}" \
|
|
'BEGIN{ if (t>0) printf "%.1f", (t-r)*100/t; else print "NA" }'
|
|
}
|
|
|
|
dump_diagnostics() {
|
|
echo ""
|
|
echo "=== Peer / link snapshot ==="
|
|
local n s
|
|
for n in "${NODES[@]}"; do
|
|
s="${SLOT_OF[$n]}"
|
|
echo "--- $n (slot $s, ${SLOT_REF[$s]}@${SLOT_SHA[$s]}) ---"
|
|
docker exec "${CONTAINER[$n]}" fipsctl show peers 2>/dev/null || true
|
|
docker exec "${CONTAINER[$n]}" fipsctl show links 2>/dev/null || true
|
|
echo ""
|
|
done
|
|
echo "=== Recent node logs (interop-relevant lines) ==="
|
|
for n in "${NODES[@]}"; do
|
|
echo "--- $n ---"
|
|
docker logs "${CONTAINER[$n]}" 2>&1 \
|
|
| grep -E "(ERROR|PANIC|panicked|FMP version|handshake|Handshake|decrypt|Decrypt|teardown|rekey|Rekey|K-bit|established|Established)" \
|
|
| tail -40
|
|
echo ""
|
|
done
|
|
}
|
|
|
|
compose() {
|
|
docker compose -f "$COMPOSE_FILE" "$@"
|
|
}
|
|
|
|
cleanup() {
|
|
# Reap any in-flight stream tmpdir (e.g. on interrupt mid-window).
|
|
[ -n "${_STREAM_TMP:-}" ] && rm -rf "$_STREAM_TMP" 2>/dev/null
|
|
if [ "${FIPS_INTEROP_KEEP_UP:-}" = "1" ]; then
|
|
echo ""
|
|
echo "FIPS_INTEROP_KEEP_UP=1 — leaving mesh running. Tear down with:"
|
|
echo " docker compose -f $COMPOSE_FILE down --volumes --remove-orphans"
|
|
return
|
|
fi
|
|
compose down --volumes --remove-orphans >/dev/null 2>&1 || true
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'echo ""; echo "Interrupted"; exit 130' INT
|
|
|
|
load_slot_metadata
|
|
|
|
# ── Banner ───────────────────────────────────────────────────────────
|
|
|
|
echo "=============================================================="
|
|
echo " FIPS Mixed-Version Interop Test"
|
|
echo "=============================================================="
|
|
echo ""
|
|
echo "Node-spec: $SPEC_STR ($NUM_NODES nodes, $NUM_PAIRS pairs, $NUM_DIRECTED directed)"
|
|
echo "Topology : $TOPOLOGY_KIND${TOPOLOGY_NAME:+ ($TOPOLOGY_NAME)}"
|
|
if [ "$TOPOLOGY_KIND" != "full-mesh" ]; then
|
|
echo " edges: ${INTEROP_TOPOLOGY_EDGES:-}"
|
|
fi
|
|
echo ""
|
|
echo "Mesh nodes:"
|
|
for n in "${NODES[@]}"; do
|
|
s="${SLOT_OF[$n]}"
|
|
u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
|
|
echo " $n slot $u ${SLOT_REF[$s]} @ ${SLOT_SHA[$s]} deg=${DEGREE_OF[$n]:-?} (${NODE_IP_OF[$n]})"
|
|
done
|
|
echo ""
|
|
echo "Mesh pairs:"
|
|
for p in "${PAIRS[@]}"; do
|
|
read -r n1 n2 <<< "$p"
|
|
echo " $(pair_kind "$n1" "$n2") $(pair_label "$n1" "$n2") [$(hop_label "$n1" "$n2")]"
|
|
done
|
|
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
|
|
echo ""
|
|
echo "Data-plane streams (continuity across rekey):"
|
|
for sp in "${STREAM_PAIRS[@]}"; do
|
|
read -r sf st <<< "$sp"
|
|
echo " $sf -> $st [$(hop_label "$sf" "$st")]"
|
|
done
|
|
fi
|
|
echo ""
|
|
if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then
|
|
echo "Netem impairment: $FIPS_INTEROP_NETEM"
|
|
echo ""
|
|
fi
|
|
echo "rekey.after_secs=$REKEY_AFTER_SECS"
|
|
echo ""
|
|
|
|
# ── Phase 0: bring up the mesh ───────────────────────────────────────
|
|
|
|
echo "Phase 0: Starting mesh"
|
|
compose down --volumes --remove-orphans >/dev/null 2>&1 || true
|
|
if ! compose up -d; then
|
|
echo " FAIL compose up failed"
|
|
exit 1
|
|
fi
|
|
# Phase 7 measures bloom-filter warmup from node start, not from its own
|
|
# start, so the clock has to be taken here.
|
|
MESH_UP_AT=$SECONDS
|
|
|
|
# Optional netem: applied via `docker exec ... tc qdisc` on each
|
|
# container's eth0 — host bridge qdisc does NOT shape inter-container
|
|
# port-to-port traffic, so the impairment must live inside each
|
|
# container. Same mechanism as testing/flake-lab/run-loop.sh.
|
|
if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then
|
|
echo " Applying netem to each container eth0: $FIPS_INTEROP_NETEM"
|
|
for n in "${NODES[@]}"; do
|
|
if docker exec "${CONTAINER[$n]}" tc qdisc add dev eth0 root \
|
|
netem ${FIPS_INTEROP_NETEM} >/dev/null 2>&1; then
|
|
echo " $n: netem applied"
|
|
else
|
|
echo " $n: WARN netem apply failed"
|
|
fi
|
|
done
|
|
fi
|
|
echo ""
|
|
|
|
# ── Phase 1: FMP link + FSP session establishment ────────────────────
|
|
|
|
echo "Phase 1: Link/session establishment + connectivity baseline"
|
|
PASSED=0; FAILED=0
|
|
|
|
# Every node must reach its DIRECT-peer degree of authenticated peers (a
|
|
# peer in `show peers` is an authenticated FMP-link peer). In a full mesh
|
|
# that degree is N-1; in a multi-hop topology it is the node's adjacency.
|
|
for n in "${NODES[@]}"; do
|
|
exp="${DEGREE_OF[$n]:-$PEERS_EXPECTED}"
|
|
if ! wait_for_peers "${CONTAINER[$n]}" "$exp" "$CONVERGENCE_TIMEOUT"; then
|
|
echo " $n did not reach $exp authenticated direct peer(s)"
|
|
fi
|
|
done
|
|
|
|
# The all-pairs ping over fips0 is the definitive reachability check.
|
|
# Direct-neighbor pairs prove their FSP session; NON-adjacent pairs in a
|
|
# multi-hop topology can only succeed via forwarding, so all-pairs ping
|
|
# is also the cross-version FORWARDING test (we keep pinging every pair).
|
|
# The convergence detector is an ADVISORY settle-wait, not the assertion.
|
|
# It needs one fully-clean, no-retry sweep of every directed pair; under
|
|
# packet loss that is statistically unlikely on a large mesh even when the
|
|
# mesh is perfectly healthy (e.g. 30 pairs at 2% loss => ~55% of sweeps
|
|
# are clean), so its timeout must NOT be fatal. The strict, retrying ping
|
|
# below is the real baseline assertion — it decides pass/fail.
|
|
if ! wait_for_full_baseline "$CONVERGENCE_TIMEOUT"; then
|
|
echo " detector saw no fully-clean sweep within ${CONVERGENCE_TIMEOUT}s (best $PASSED/$NUM_DIRECTED); strict re-ping decides"
|
|
fi
|
|
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
|
|
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
|
|
if [ "$FAILED" -ne 0 ]; then
|
|
dump_diagnostics
|
|
echo ""
|
|
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
|
echo "FAIL: mesh did not converge — see pair attribution above."
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
|
|
# ── Phase 1b: data-plane control window (quiet, pre-rekey) ───────────
|
|
#
|
|
# Measure stream loss over a window with NO rekey cutover, as the control
|
|
# baseline for the differential. A window counts only when the FMP cutover
|
|
# count was read before and after it and did not advance. A window that
|
|
# saw a cutover, or could not be checked, is re-measured up to
|
|
# CONTROL_MAX_ATTEMPTS times; if none is clean, Phase 5b abstains rather
|
|
# than computing a verdict from a contaminated baseline. Then launch the
|
|
# rekey-window streams, which run in the background across Phases 2-5 and
|
|
# are collected/asserted in Phase 5b.
|
|
control_abstain=0
|
|
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
|
|
echo "Phase 1b: Data-plane control stream (${CONTROL_STREAM_SECS}s quiet window, up to ${CONTROL_MAX_ATTEMPTS} attempts)"
|
|
control_ok=0
|
|
for ((attempt = 1; attempt <= CONTROL_MAX_ATTEMPTS; attempt++)); do
|
|
pre_rc=0
|
|
post_rc=0
|
|
pre_cut="$(fmp_cutover_count)" || pre_rc=$?
|
|
launch_streams CONTROL "$CONTROL_STREAM_SECS"
|
|
collect_streams
|
|
post_cut="$(fmp_cutover_count)" || post_rc=$?
|
|
if [ "$pre_rc" -ne 0 ] || ! [[ "$pre_cut" =~ ^[0-9]+$ ]]; then
|
|
why="cutover count unreadable ($pre_cut)"
|
|
elif [ "$post_rc" -ne 0 ] || ! [[ "$post_cut" =~ ^[0-9]+$ ]]; then
|
|
why="cutover count unreadable ($post_cut)"
|
|
elif [ "$post_cut" -ne "$pre_cut" ]; then
|
|
why="a rekey cutover occurred (+$((post_cut - pre_cut)))"
|
|
else
|
|
control_ok=1
|
|
echo " control window accepted on attempt $attempt/$CONTROL_MAX_ATTEMPTS"
|
|
break
|
|
fi
|
|
if [ "$attempt" -lt "$CONTROL_MAX_ATTEMPTS" ]; then
|
|
echo " control window attempt $attempt/$CONTROL_MAX_ATTEMPTS: $why; re-measuring"
|
|
wait_cutover_quiet || true
|
|
else
|
|
echo " control window attempt $attempt/$CONTROL_MAX_ATTEMPTS: $why"
|
|
fi
|
|
done
|
|
if [ "$control_ok" -eq 0 ]; then
|
|
control_abstain=1
|
|
echo " ABSTAIN control window contaminated on all $CONTROL_MAX_ATTEMPTS attempts; Phase 5b will not return a verdict"
|
|
fi
|
|
ctl_note=""
|
|
[ "$control_abstain" -eq 1 ] && ctl_note=" (last attempt, contaminated)"
|
|
for sp in "${STREAM_PAIRS[@]}"; do
|
|
read -r sf st <<< "$sp"
|
|
key="$sf->$st"
|
|
echo " control $key: tx=${STREAM_TX[CONTROL:$key]:-0} rx=${STREAM_RX[CONTROL:$key]:-0} loss=$(_loss_pct "${STREAM_TX[CONTROL:$key]:-0}" "${STREAM_RX[CONTROL:$key]:-0}")%$ctl_note"
|
|
done
|
|
echo " Launching rekey-window streams (${REKEY_STREAM_SECS}s, spanning Phases 2-5)"
|
|
launch_streams REKEY "$REKEY_STREAM_SECS"
|
|
echo ""
|
|
fi
|
|
|
|
# ── Phase 2: first rekey cycle ───────────────────────────────────────
|
|
|
|
echo "Phase 2: First rekey cycle (waiting up to ${FIRST_REKEY_TIMEOUT}s)"
|
|
PASSED=0; FAILED=0
|
|
wait_for_log_pattern_count \
|
|
"Rekey cutover complete \(initiator\), K-bit flipped" 1 \
|
|
"$FIRST_REKEY_TIMEOUT" || true
|
|
if ! fmp_cutovers="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"; then
|
|
echo " FAIL FMP rekey cutovers: node logs unreadable ($fmp_cutovers), not established"
|
|
FAILED=$((FAILED + 1))
|
|
INTEROP_FAILURES+=("[log] FMP rekey cutovers: node logs unreadable ($fmp_cutovers)")
|
|
elif [ "$fmp_cutovers" -ge 1 ]; then
|
|
echo " PASS FMP rekey initiator cutovers: $fmp_cutovers"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
echo " FAIL no FMP rekey cutover observed within ${FIRST_REKEY_TIMEOUT}s"
|
|
FAILED=$((FAILED + 1))
|
|
INTEROP_FAILURES+=("[rekey] no FMP rekey cutover completed across the mesh")
|
|
fi
|
|
phase_result "First rekey cycle"
|
|
echo ""
|
|
|
|
# ── Phase 3: post-first-rekey connectivity ───────────────────────────
|
|
|
|
echo "Phase 3: Post-first-rekey connectivity (reconverge within ${POST_REKEY_TIMEOUT}s)"
|
|
PASSED=0; FAILED=0
|
|
# Poll until the mesh has reconverged after the rekey, then assert once.
|
|
# The detector poll records nothing (see ping_all_pairs); only the strict
|
|
# sweep below records, so a brief rekey-induced disruption is not a fail.
|
|
wait_for_full_baseline "$POST_REKEY_TIMEOUT" || true
|
|
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "post-rekey-1"
|
|
phase_result "Post-first-rekey (all $NUM_DIRECTED directed pairs)"
|
|
echo ""
|
|
|
|
# ── Phase 4: second rekey cycle ──────────────────────────────────────
|
|
|
|
echo "Phase 4: Second rekey cycle (waiting up to ${SECOND_REKEY_WAIT}s for the next cutover)"
|
|
# Poll for the next FMP cutover beyond what Phases 2/3 already saw, using
|
|
# the same pre/post cutover-count delta convention as the control window
|
|
# (Phase 1b), instead of a blind sleep. Bounded by SECOND_REKEY_WAIT so a
|
|
# stalled rekey falls through to the strict Phase 5/6 assertions.
|
|
# An unreadable baseline gives nothing to wait beyond, so do not wait.
|
|
# Phase 4 keeps no PASSED/FAILED of its own; the INTEROP_FAILURES entry
|
|
# is what turns the run red.
|
|
if fmp_cutovers_before="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"; then
|
|
wait_for_log_pattern_count \
|
|
"Rekey cutover complete \(initiator\), K-bit flipped" \
|
|
"$((fmp_cutovers_before + 1))" "$SECOND_REKEY_WAIT" || true
|
|
else
|
|
echo " FAIL second rekey cycle: node logs unreadable ($fmp_cutovers_before), baseline not established"
|
|
INTEROP_FAILURES+=("[log] second rekey cycle: node logs unreadable ($fmp_cutovers_before)")
|
|
fi
|
|
echo ""
|
|
|
|
echo "Phase 5: Post-second-rekey connectivity (reconverge within ${POST_REKEY_TIMEOUT}s)"
|
|
PASSED=0; FAILED=0
|
|
wait_for_full_baseline "$POST_REKEY_TIMEOUT" || true
|
|
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "post-rekey-2"
|
|
phase_result "Post-second-rekey (all $NUM_DIRECTED directed pairs)"
|
|
echo ""
|
|
|
|
# ── Phase 5b: data-plane continuity across rekey (control-differential)
|
|
#
|
|
# Collect the rekey-window streams launched in Phase 1b and compare their
|
|
# loss to the control window. A hitless rekey keeps rekey-window loss at
|
|
# or below the control (plus a small margin); excess loss is data the
|
|
# rekey dropped — the failure the cutover fixes (4af3730/6e5cb89) and the
|
|
# pipelined wire layout are supposed to prevent.
|
|
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
|
|
echo "Phase 5b: Data-plane continuity across rekey (control-differential, margin ${STREAM_LOSS_MARGIN_PCT}%)"
|
|
PASSED=0; FAILED=0
|
|
collect_streams
|
|
printf ' %-16s %11s %11s %8s %s\n' "stream" "ctrl-loss%" "rekey-loss%" "delta" "verdict"
|
|
for sp in "${STREAM_PAIRS[@]}"; do
|
|
read -r sf st <<< "$sp"
|
|
key="$sf->$st"
|
|
cpct="$(_loss_pct "${STREAM_TX[CONTROL:$key]:-0}" "${STREAM_RX[CONTROL:$key]:-0}")"
|
|
rpct="$(_loss_pct "${STREAM_TX[REKEY:$key]:-0}" "${STREAM_RX[REKEY:$key]:-0}")"
|
|
verdict="$(awk -v c="$cpct" -v k="$rpct" -v m="$STREAM_LOSS_MARGIN_PCT" 'BEGIN{
|
|
if (c=="NA" || k=="NA") { print "NODATA"; exit }
|
|
if (k <= c + m) print "PASS"; else print "FAIL"
|
|
}')"
|
|
delta="$(awk -v c="$cpct" -v k="$rpct" 'BEGIN{ if(c=="NA"||k=="NA") print "NA"; else printf "%+.1f", k-c }')"
|
|
# No clean control window: report the losses for reading, but
|
|
# compute no verdict from a contaminated baseline.
|
|
if [ "$control_abstain" -eq 1 ]; then
|
|
printf ' %-16s %11s %11s %8s %s\n' "$key" "$cpct" "$rpct" "$delta" "ABSTAIN"
|
|
continue
|
|
fi
|
|
printf ' %-16s %11s %11s %8s %s\n' "$key" "$cpct" "$rpct" "$delta" "$verdict"
|
|
if [ "$verdict" = "PASS" ]; then
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
FAILED=$((FAILED + 1))
|
|
INTEROP_FAILURES+=("[stream] $key ($(hop_label "$sf" "$st")): rekey-window loss ${rpct}% vs control ${cpct}% (+${STREAM_LOSS_MARGIN_PCT}% margin) -> $verdict")
|
|
fi
|
|
done
|
|
if [ "$control_abstain" -eq 1 ]; then
|
|
echo " ABSTAIN Data-plane continuity across rekey: no uncontaminated control window in $CONTROL_MAX_ATTEMPTS attempts"
|
|
else
|
|
phase_result "Data-plane continuity across rekey"
|
|
fi
|
|
echo ""
|
|
fi
|
|
|
|
# ── Phase 6: per-node, per-pair interop log analysis ─────────────────
|
|
#
|
|
# This is the interop-specific analysis. For each unordered pair it
|
|
# reports same vs mixed-version, then scans BOTH endpoints' logs for
|
|
# interop failure signatures. A signature firing on a mixed pair that
|
|
# does not fire on a same-version pair is the harness's primary signal.
|
|
|
|
echo "Phase 6: Interop log analysis"
|
|
PASSED=0; FAILED=0
|
|
|
|
# Give FSP rekey (which trails FMP rekey) a bounded chance to complete
|
|
# at least one cutover before the final assertions.
|
|
wait_for_log_pattern_count "FSP rekey cutover complete" 1 "$REKEY_SETTLE" || true
|
|
|
|
# Global negative checks — these must be zero on EVERY node regardless
|
|
# of version pairing. A non-zero count is attributed to the node and,
|
|
# where the count is asymmetric across versions, flagged as interop.
|
|
echo ""
|
|
echo " -- Global health (all $NUM_NODES nodes) --"
|
|
|
|
declare -A GLOBAL_PATTERNS=(
|
|
["PANIC|panicked"]="panics"
|
|
["ERROR"]="error-level log lines"
|
|
["unknown FMP version|Unknown FMP version"]="unknown-FMP-version drops"
|
|
["MMP link teardown"]="MMP link teardowns"
|
|
["Excessive decryption failures"]="excessive-decryption-failure removals"
|
|
["Session AEAD decryption failed"]="FSP AEAD decrypt failures"
|
|
["Rekey msg2 processing failed"]="rekey msg2 failures"
|
|
["Handshake failed|handshake failed"]="handshake failures"
|
|
)
|
|
|
|
for pat in "${!GLOBAL_PATTERNS[@]}"; do
|
|
desc="${GLOBAL_PATTERNS[$pat]}"
|
|
if ! total="$(count_log_pattern "$pat")"; then
|
|
echo " FAIL $desc: node logs unreadable ($total), zero not established"
|
|
FAILED=$((FAILED + 1))
|
|
INTEROP_FAILURES+=("[log] $desc: node logs unreadable ($total)")
|
|
continue
|
|
fi
|
|
if [ "$total" -eq 0 ]; then
|
|
echo " PASS $desc: 0"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
echo " FAIL $desc: $total (expected 0)"
|
|
FAILED=$((FAILED + 1))
|
|
# Per-node breakdown so the count can be attributed to a build.
|
|
for n in "${NODES[@]}"; do
|
|
c="$(count_node_pattern "$n" "$pat")"
|
|
if [ "$c" -gt 0 ]; then
|
|
s="${SLOT_OF[$n]}"
|
|
u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
|
|
echo " $n [$u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}]: $c"
|
|
INTEROP_FAILURES+=("[log] node $n ($u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}): $c x '$desc'")
|
|
fi
|
|
done
|
|
fi
|
|
done
|
|
|
|
# Positive checks — the rekey machinery actually exercised both layers.
|
|
echo ""
|
|
echo " -- Rekey machinery exercised --"
|
|
# An unreadable log means the harness observed nothing, so it is a FAIL
|
|
# on either layer; the FSP WARN below is for a readable zero only.
|
|
if ! fmp_total="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"; then
|
|
echo " FAIL FMP rekey cutovers: node logs unreadable ($fmp_total), not established"
|
|
FAILED=$((FAILED + 1))
|
|
elif [ "$fmp_total" -ge 1 ]; then
|
|
echo " PASS FMP rekey cutovers across mesh: $fmp_total"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
echo " FAIL FMP rekey cutovers: $fmp_total (expected >= 1)"
|
|
FAILED=$((FAILED + 1))
|
|
fi
|
|
if ! fsp_total="$(count_log_pattern 'FSP rekey cutover complete')"; then
|
|
echo " FAIL FSP rekey cutovers: node logs unreadable ($fsp_total), not established"
|
|
FAILED=$((FAILED + 1))
|
|
elif [ "$fsp_total" -ge 1 ]; then
|
|
echo " PASS FSP rekey cutovers across mesh: $fsp_total"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
# FSP rekey not completing is a soft signal: report it but only
|
|
# hard-fail if a pair also lost connectivity. The connectivity
|
|
# phases already cover the user-visible impact.
|
|
echo " WARN FSP rekey cutovers: $fsp_total (expected >= 1)"
|
|
fi
|
|
|
|
# Per-pair summary: classify each unordered pair and report whether it
|
|
# stayed healthy through the run. "Healthy" = no connectivity failure
|
|
# recorded for either direction of the pair.
|
|
echo ""
|
|
echo " -- Per-pair interop summary --"
|
|
for p in "${PAIRS[@]}"; do
|
|
read -r n1 n2 <<< "$p"
|
|
kind="$(pair_kind "$n1" "$n2")"
|
|
label="$(pair_label "$n1" "$n2") [$(hop_label "$n1" "$n2")]"
|
|
pair_failed=0
|
|
if [ "${#INTEROP_FAILURES[@]}" -gt 0 ]; then
|
|
for f in "${INTEROP_FAILURES[@]}"; do
|
|
case "$f" in
|
|
*"ping $n1->$n2 FAILED"*|*"ping $n2->$n1 FAILED"*)
|
|
pair_failed=1 ;;
|
|
esac
|
|
done
|
|
fi
|
|
if [ "$pair_failed" -eq 0 ]; then
|
|
echo " PASS $kind pair $label: stayed healthy"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
echo " FAIL $kind pair $label: connectivity failed during the run"
|
|
FAILED=$((FAILED + 1))
|
|
fi
|
|
done
|
|
|
|
phase_result "Interop log analysis"
|
|
echo ""
|
|
|
|
# ── Phase 7: mesh-size estimate convergence (strict ±25%) ────────────
|
|
#
|
|
# Each node's bloom-union mesh-size estimate (fipsctl show status
|
|
# .estimated_mesh_size) should converge to the true node count across
|
|
# versions. A mixed-version bloom/tree-encoding divergence shows up as a
|
|
# node that never produces an in-band estimate (or returns null). Strict
|
|
# band = [0.75N, 1.25N].
|
|
#
|
|
# The verdict is taken on a settled estimate rather than on a node's
|
|
# first tolerable reading. Two windows enforce that. Nothing sampled
|
|
# before MESH_SIZE_WARMUP has elapsed since the mesh came up counts at
|
|
# all, because the estimate is documented as unreliable during warmup;
|
|
# after it, a node is credited only once it has held the band unbroken
|
|
# for MESH_SIZE_SETTLE. Every node is re-polled every round and any
|
|
# out-of-band reading restarts that node's settle clock, so a node
|
|
# cannot be credited for a sample it has since contradicted.
|
|
echo "Phase 7: Mesh-size estimate convergence (strict ±25% of true N=$NUM_NODES)"
|
|
PASSED=0; FAILED=0
|
|
ms_lo="$(awk -v n="$NUM_NODES" 'BEGIN{printf "%.2f", 0.75*n}')"
|
|
ms_hi="$(awk -v n="$NUM_NODES" 'BEGIN{printf "%.2f", 1.25*n}')"
|
|
declare -A MS_EST MS_OK MS_INBAND_SINCE
|
|
ms_accept_after=$(( MESH_UP_AT + MESH_SIZE_WARMUP ))
|
|
echo " band [$ms_lo, $ms_hi], warmup ends $(( ms_accept_after - SECONDS ))s from now, then ${MESH_SIZE_SETTLE}s settled, poll up to ${MESH_SIZE_TIMEOUT}s beyond that"
|
|
|
|
# Poll through the warmup as well. Nothing here is asserted on — it is
|
|
# the trajectory the phase has never recorded, and it is what an
|
|
# undercount that never recovers would show up in.
|
|
while [ "$SECONDS" -lt "$ms_accept_after" ]; do
|
|
ms_line=""
|
|
for n in "${NODES[@]}"; do
|
|
MS_EST[$n]="$(mesh_estimate "$n")"
|
|
ms_line+=" $n=${MS_EST[$n]}"
|
|
done
|
|
echo " warmup, $(( ms_accept_after - SECONDS ))s to go:$ms_line"
|
|
sleep 30
|
|
done
|
|
|
|
ms_deadline=$(( SECONDS + MESH_SIZE_SETTLE + MESH_SIZE_TIMEOUT ))
|
|
while :; do
|
|
all_ok=1
|
|
for n in "${NODES[@]}"; do
|
|
est="$(mesh_estimate "$n")"
|
|
MS_EST[$n]="$est"
|
|
if [ "$est" != "null" ] && awk "BEGIN{exit !($est>=$ms_lo && $est<=$ms_hi)}"; then
|
|
[ -z "${MS_INBAND_SINCE[$n]:-}" ] && MS_INBAND_SINCE[$n]="$SECONDS"
|
|
else
|
|
unset "MS_INBAND_SINCE[$n]"
|
|
fi
|
|
if [ -n "${MS_INBAND_SINCE[$n]:-}" ] \
|
|
&& [ $(( SECONDS - ${MS_INBAND_SINCE[$n]} )) -ge "$MESH_SIZE_SETTLE" ]; then
|
|
MS_OK[$n]=1
|
|
else
|
|
MS_OK[$n]=0
|
|
all_ok=0
|
|
fi
|
|
done
|
|
[ "$all_ok" = "1" ] && break
|
|
[ "$SECONDS" -ge "$ms_deadline" ] && break
|
|
sleep "$MESH_SIZE_POLL"
|
|
done
|
|
for n in "${NODES[@]}"; do
|
|
s="${SLOT_OF[$n]}"; u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
|
|
if [ "${MS_OK[$n]:-0}" = "1" ]; then
|
|
echo " PASS $n [$u]: estimated_mesh_size=${MS_EST[$n]} (held band ${MESH_SIZE_SETTLE}s+ after warmup)"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
if [ -n "${MS_INBAND_SINCE[$n]:-}" ]; then
|
|
ms_why="held band only $(( SECONDS - ${MS_INBAND_SINCE[$n]} ))s of the ${MESH_SIZE_SETTLE}s settle window"
|
|
else
|
|
ms_why="outside [$ms_lo,$ms_hi]"
|
|
fi
|
|
echo " FAIL $n [$u]: estimated_mesh_size=${MS_EST[$n]:-null} ($ms_why)"
|
|
FAILED=$((FAILED + 1))
|
|
INTEROP_FAILURES+=("[mesh-size] node $n ($u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}): estimate=${MS_EST[$n]:-null} $ms_why")
|
|
fi
|
|
done
|
|
# The band is per-node, so a green says every node was plausible, not
|
|
# that the nodes agreed. Print the spread so nobody has to infer it.
|
|
ms_spread="$(printf '%s\n' "${MS_EST[@]}" | sort -n | awk '/^[0-9]+$/{ if (lo=="") lo=$1; hi=$1 } END{ if (lo=="") print "no numeric estimates"; else if (lo==hi) print "all nodes agree on " lo; else print "nodes disagree: " lo " to " hi }')"
|
|
echo " NOTE: final estimates — $ms_spread (agreement is reported, not asserted)"
|
|
phase_result "Mesh-size estimate convergence"
|
|
echo ""
|
|
|
|
# ── Summary ──────────────────────────────────────────────────────────
|
|
|
|
echo "=============================================================="
|
|
echo " Results: $TOTAL_PASSED checks passed, $TOTAL_FAILED failed"
|
|
echo "=============================================================="
|
|
|
|
# `${#arr[@]}` cannot be combined with `:-`; count into a plain var.
|
|
INTEROP_FAILURE_COUNT="${#INTEROP_FAILURES[@]}"
|
|
|
|
# An abstaining Phase 5b adds no check, so say so: otherwise a green run
|
|
# reads as having covered data-plane continuity across rekey.
|
|
if [ "${control_abstain:-0}" -eq 1 ]; then
|
|
echo ""
|
|
echo "NOTE: Phase 5b abstained (control window contaminated); its rekey-loss check did not run."
|
|
fi
|
|
|
|
if [ "$TOTAL_FAILED" -eq 0 ] && [ "$INTEROP_FAILURE_COUNT" -eq 0 ]; then
|
|
echo ""
|
|
echo "PASS: all versions interoperate cleanly across the mesh (spec '$SPEC_STR')."
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "INTEROP FAILURES (attributed to specific version pairs / builds):"
|
|
if [ "$INTEROP_FAILURE_COUNT" -gt 0 ]; then
|
|
printf '%s\n' "${INTEROP_FAILURES[@]}" | sort -u | sed 's/^/ - /'
|
|
else
|
|
echo " (no per-pair failure recorded; see phase output above)"
|
|
fi
|
|
echo ""
|
|
|
|
# Highlight whether failures concentrate on mixed-version pairs — the
|
|
# defining interop signal.
|
|
mixed_hits=0
|
|
same_hits=0
|
|
if [ "$INTEROP_FAILURE_COUNT" -gt 0 ]; then
|
|
for f in "${INTEROP_FAILURES[@]}"; do
|
|
case "$f" in
|
|
*"MIXED pair"*) mixed_hits=$((mixed_hits + 1)) ;;
|
|
*"same pair"*) same_hits=$((same_hits + 1)) ;;
|
|
esac
|
|
done
|
|
fi
|
|
echo "Connectivity-failure attribution: mixed-version=$mixed_hits same-version=$same_hits"
|
|
if [ "$mixed_hits" -gt 0 ] && [ "$same_hits" -eq 0 ]; then
|
|
echo "=> Failures are MIXED-VERSION ONLY: a genuine interop regression."
|
|
elif [ "$mixed_hits" -gt 0 ] && [ "$same_hits" -gt 0 ]; then
|
|
echo "=> Failures hit both mixed and same-version pairs: likely a general"
|
|
echo " instability, not version-specific — investigate the common cause."
|
|
elif [ "$same_hits" -gt 0 ]; then
|
|
echo "=> Failures are SAME-VERSION ONLY: not an interop issue per se;"
|
|
echo " a build is unstable even against itself."
|
|
fi
|
|
|
|
dump_diagnostics
|
|
exit 1
|