mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 08:14:42 +00:00
701 lines
27 KiB
Bash
Executable File
701 lines
27 KiB
Bash
Executable File
#!/bin/bash
|
|
# Mixed-version interop test driver.
|
|
#
|
|
# Brings up an N-node full mesh from a NODE-SPEC (a multiset of image
|
|
# slots), verifies every pair interoperates — FMP link, FSP session,
|
|
# connectivity, rekey survival — and runs a per-node, per-pair log
|
|
# analysis tuned to surface INTEROP problems: handshake failures,
|
|
# FSP/FMP decrypt failures, `unknown FMP version`, replay storms,
|
|
# link/session teardowns.
|
|
#
|
|
# The harness's job is NOT to test a single version. It is to find any
|
|
# place where two DIFFERENT versions fail to interoperate in a way a
|
|
# same-version pair would not. Every failure is attributed to a specific
|
|
# (version-X <-> version-Y) pair, classified same-version vs MIXED.
|
|
#
|
|
# A node-spec like `a a b c` produces a same-version pair (a1<->a2) — the
|
|
# CONTROL ARM — alongside the mixed pairs. The control pair is what makes
|
|
# a netem run interpretable: a failure on a mixed pair that the control
|
|
# pair does not share is an interop regression; a failure both share is
|
|
# loss noise, not version-specific.
|
|
#
|
|
# Prerequisites:
|
|
# 1. bash build-images.sh <ref-a> <ref-b> <ref-c> (builds fips-interop:a/b/c)
|
|
# 2. configs are (re)generated automatically below for the given spec.
|
|
#
|
|
# Usage:
|
|
# ./interop-test.sh [node-spec...]
|
|
#
|
|
# node-spec space-separated slot letters (a/b/c), default `a b c`.
|
|
# e.g. `a a b c` (4-node, one same-version control pair),
|
|
# `a a a` (3-node same-version flake rig).
|
|
#
|
|
# Environment:
|
|
# FIPS_INTEROP_NETEM tc-netem arg string applied to every container's
|
|
# eth0, e.g. "delay 10ms 5ms 25% loss 1%". Unset =
|
|
# no impairment.
|
|
# FIPS_INTEROP_KEEP_UP 1 = leave containers running after the test (debug).
|
|
# REKEY_AFTER_SECS rekey interval to generate configs with (default 35).
|
|
# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/,
|
|
# .stress-runs/, generated-configs/). When
|
|
# unset, falls back to in-tree paths under
|
|
# testing/interop/ and prints a warning to
|
|
# stderr; set it to a path outside the source
|
|
# tree to keep generated artefacts out of the
|
|
# checkout.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
source "$SCRIPT_DIR/../lib/wait-converge.sh"
|
|
|
|
# ── Scratch-dir root ─────────────────────────────────────────────────
|
|
#
|
|
# FIPS_INTEROP_RUNS_DIR controls where the harness writes its scratch
|
|
# directories (.build/, .stress-runs/, generated-configs/). When unset
|
|
# we fall back to in-tree paths under testing/interop/ and warn the
|
|
# operator, so the warning fires exactly once per invocation. When a
|
|
# parent script has already warned it exports _FIPS_INTEROP_WARNED=1
|
|
# to suppress duplicate warnings in child scripts.
|
|
if [[ -n "${FIPS_INTEROP_RUNS_DIR:-}" ]]; then
|
|
RUNS_BASE="$FIPS_INTEROP_RUNS_DIR"
|
|
mkdir -p "$RUNS_BASE"
|
|
else
|
|
RUNS_BASE="$SCRIPT_DIR"
|
|
if [[ -z "${_FIPS_INTEROP_WARNED:-}" ]]; then
|
|
echo >&2 "WARNING: FIPS_INTEROP_RUNS_DIR not set; harness output will be written under the source tree at $RUNS_BASE. Set FIPS_INTEROP_RUNS_DIR to a path outside the source tree to avoid this."
|
|
export _FIPS_INTEROP_WARNED=1
|
|
fi
|
|
fi
|
|
|
|
GEN_DIR="$RUNS_BASE/generated-configs"
|
|
COMPOSE_FILE="$GEN_DIR/docker-compose.generated.yml"
|
|
NODES_ENV="$GEN_DIR/nodes.env"
|
|
REFS_ENV="$RUNS_BASE/.build/refs.env"
|
|
|
|
REKEY_AFTER_SECS="${REKEY_AFTER_SECS:-35}"
|
|
|
|
# ── Node-spec ────────────────────────────────────────────────────────
|
|
|
|
SPEC=("$@")
|
|
if [ "${#SPEC[@]}" -eq 0 ]; then
|
|
SPEC=(a b c)
|
|
fi
|
|
SPEC_STR="${SPEC[*]}"
|
|
|
|
# ── Timing ───────────────────────────────────────────────────────────
|
|
|
|
CONVERGENCE_TIMEOUT=60 # wait for full mesh + clean ping baseline
|
|
PING_TIMEOUT=5
|
|
MAX_PING_ATTEMPTS=4 # strict-ping retry (mirrors rekey-test.sh)
|
|
PING_RETRY_DELAY=1
|
|
# First rekey should follow shortly after the interval once converged.
|
|
FIRST_REKEY_TIMEOUT=$((REKEY_AFTER_SECS + 20))
|
|
SECOND_REKEY_WAIT=$((REKEY_AFTER_SECS + 5))
|
|
REKEY_SETTLE=12 # FSP-cutover settle budget (Phase 6)
|
|
# Post-rekey reconvergence is polled, not fixed-slept: the mesh is given
|
|
# up to this long to restore full connectivity after a rekey before the
|
|
# strict assertion sweep runs. A genuinely stuck pair still fails — the
|
|
# poll times out and the recording sweep captures it.
|
|
POST_REKEY_TIMEOUT=45
|
|
LOG_POLL_INTERVAL=2
|
|
|
|
# ── Counters ─────────────────────────────────────────────────────────
|
|
|
|
PASSED=0
|
|
FAILED=0
|
|
TOTAL_PASSED=0
|
|
TOTAL_FAILED=0
|
|
# INTEROP_FAILURES collects human-readable, pair-attributed failure lines.
|
|
INTEROP_FAILURES=()
|
|
|
|
# ── Preflight: docker + images ───────────────────────────────────────
|
|
|
|
if ! docker info >/dev/null 2>&1; then
|
|
echo "ERROR: Docker daemon is not reachable" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# A node-spec only ever references the three image slots, regardless of
|
|
# how many nodes it has. Check the slots actually present in the spec.
|
|
for slot in $(printf '%s\n' "${SPEC[@]}" | sort -u); do
|
|
case "$slot" in
|
|
a|b|c) ;;
|
|
*) echo "ERROR: invalid slot '$slot' in node-spec" >&2; exit 2 ;;
|
|
esac
|
|
if ! docker image inspect "fips-interop:$slot" >/dev/null 2>&1; then
|
|
echo "ERROR: image fips-interop:$slot not present." >&2
|
|
echo "Build the three images first:" >&2
|
|
echo " bash $SCRIPT_DIR/build-images.sh <ref-a> <ref-b> <ref-c>" >&2
|
|
exit 2
|
|
fi
|
|
done
|
|
|
|
# ── (Re)generate configs for this node-spec ──────────────────────────
|
|
#
|
|
# Always regenerate when the spec on disk does not match the requested
|
|
# spec (or no manifest exists). Generation is deterministic and cheap.
|
|
|
|
need_regen=1
|
|
if [ -f "$NODES_ENV" ]; then
|
|
existing_spec="$(sed -n 's/^INTEROP_SPEC="\(.*\)"$/\1/p' "$NODES_ENV")"
|
|
if [ "$existing_spec" = "$SPEC_STR" ]; then
|
|
need_regen=0
|
|
fi
|
|
fi
|
|
if [ "$need_regen" -eq 1 ]; then
|
|
echo "Generating mesh configs for spec '$SPEC_STR' (rekey.after_secs=$REKEY_AFTER_SECS)..."
|
|
REKEY_AFTER_SECS="$REKEY_AFTER_SECS" bash "$SCRIPT_DIR/generate-configs.sh" \
|
|
"${SPEC[@]}"
|
|
echo ""
|
|
fi
|
|
|
|
# ── Load the manifest ────────────────────────────────────────────────
|
|
#
|
|
# nodes.env gives the ordered node list and the per-node slot/container/
|
|
# ip/npub maps. npubs.env gives NPUB_<NODEID> vars used by ping_one.
|
|
|
|
# shellcheck disable=SC1090
|
|
source "$NODES_ENV"
|
|
# shellcheck disable=SC1091
|
|
source "$GEN_DIR/npubs.env"
|
|
|
|
read -r -a NODES <<< "$INTEROP_NODE_IDS"
|
|
|
|
declare -A SLOT_OF CONTAINER NODE_IP_OF NPUB_OF
|
|
parse_map() {
|
|
# parse_map <assoc-array-name> <space-separated nodeid:value tokens>
|
|
local -n _dst="$1"
|
|
local tok nid val
|
|
for tok in $2; do
|
|
nid="${tok%%:*}"
|
|
val="${tok#*:}"
|
|
_dst["$nid"]="$val"
|
|
done
|
|
}
|
|
parse_map SLOT_OF "$INTEROP_NODE_SLOTS"
|
|
parse_map CONTAINER "$INTEROP_NODE_CONTAINERS"
|
|
parse_map NODE_IP_OF "$INTEROP_NODE_IPS"
|
|
parse_map NPUB_OF "$INTEROP_NODE_NPUBS"
|
|
|
|
# Unordered pairs of the mesh.
|
|
PAIRS=()
|
|
for ((i = 0; i < ${#NODES[@]}; i++)); do
|
|
for ((j = i + 1; j < ${#NODES[@]}; j++)); do
|
|
PAIRS+=("${NODES[$i]} ${NODES[$j]}")
|
|
done
|
|
done
|
|
|
|
NUM_NODES="${#NODES[@]}"
|
|
NUM_PAIRS="${#PAIRS[@]}"
|
|
NUM_DIRECTED=$((NUM_PAIRS * 2))
|
|
PEERS_EXPECTED=$((NUM_NODES - 1))
|
|
|
|
# ── Ref / version metadata ───────────────────────────────────────────
|
|
#
|
|
# refs.env (written by build-images.sh) records what each SLOT was built
|
|
# from. If absent, fall back to the image labels, then to "unknown".
|
|
# These maps are keyed by SLOT (a/b/c), not by node id.
|
|
|
|
declare -A SLOT_REF SLOT_SHA
|
|
|
|
load_slot_metadata() {
|
|
local slot upper
|
|
for slot in a b c; do
|
|
SLOT_REF[$slot]="unknown"
|
|
SLOT_SHA[$slot]="unknown"
|
|
done
|
|
if [ -f "$REFS_ENV" ]; then
|
|
# shellcheck disable=SC1090
|
|
source "$REFS_ENV"
|
|
fi
|
|
for slot in a b c; do
|
|
upper="$(echo "$slot" | tr '[:lower:]' '[:upper:]')"
|
|
local ref_var="INTEROP_REF_${upper}"
|
|
local sha_var="INTEROP_SHA_${upper}"
|
|
if [ -n "${!ref_var:-}" ]; then
|
|
SLOT_REF[$slot]="${!ref_var}"
|
|
SLOT_SHA[$slot]="${!sha_var:-unknown}"
|
|
continue
|
|
fi
|
|
local lbl_ref lbl_sha
|
|
lbl_ref="$(docker image inspect "fips-interop:$slot" \
|
|
--format '{{ index .Config.Labels "fips.interop.ref" }}' 2>/dev/null || true)"
|
|
lbl_sha="$(docker image inspect "fips-interop:$slot" \
|
|
--format '{{ index .Config.Labels "fips.interop.sha" }}' 2>/dev/null || true)"
|
|
[ -n "$lbl_ref" ] && SLOT_REF[$slot]="$lbl_ref"
|
|
[ -n "$lbl_sha" ] && SLOT_SHA[$slot]="$lbl_sha"
|
|
done
|
|
}
|
|
|
|
# A pair is "mixed-version" iff the two nodes' image slots resolve to
|
|
# different built SHAs. SHA is the precise discriminator; ref names can
|
|
# differ yet point at the same commit.
|
|
pair_is_mixed() {
|
|
local n1="$1" n2="$2"
|
|
local s1="${SLOT_OF[$n1]}" s2="${SLOT_OF[$n2]}"
|
|
if [ "${SLOT_SHA[$s1]}" = "${SLOT_SHA[$s2]}" ] \
|
|
&& [ "${SLOT_SHA[$s1]}" != "unknown" ]; then
|
|
return 1 # same version
|
|
fi
|
|
return 0 # mixed (or unknown — treat as mixed for scrutiny)
|
|
}
|
|
|
|
pair_kind() {
|
|
if pair_is_mixed "$1" "$2"; then
|
|
echo "MIXED"
|
|
else
|
|
echo "same"
|
|
fi
|
|
}
|
|
|
|
pair_label() {
|
|
# e.g. "a1[A fix/...@3045212] <-> c1[C v0.3.0@b11b639]"
|
|
local n1="$1" n2="$2"
|
|
local s1="${SLOT_OF[$n1]}" s2="${SLOT_OF[$n2]}"
|
|
local u1 u2
|
|
u1="$(echo "$s1" | tr '[:lower:]' '[:upper:]')"
|
|
u2="$(echo "$s2" | tr '[:lower:]' '[:upper:]')"
|
|
echo "${n1}[$u1 ${SLOT_REF[$s1]}@${SLOT_SHA[$s1]}] <-> ${n2}[$u2 ${SLOT_REF[$s2]}@${SLOT_SHA[$s2]}]"
|
|
}
|
|
|
|
# ── Helpers ──────────────────────────────────────────────────────────
|
|
|
|
# ping6 from one container to another node's mesh address (npub.fips).
|
|
ping_one() {
|
|
local from_node="$1" to_node="$2" quiet="${3:-}"
|
|
local max_attempts="${4:-1}"
|
|
local from_ctr="${CONTAINER[$from_node]}"
|
|
local to_npub="${NPUB_OF[$to_node]}"
|
|
local label="$from_node -> $to_node"
|
|
|
|
local attempt=1 output rtt
|
|
while (( attempt <= max_attempts )); do
|
|
(( attempt > 1 )) && sleep "$PING_RETRY_DELAY"
|
|
if output=$(docker exec "$from_ctr" ping6 -c 1 -W "$PING_TIMEOUT" \
|
|
"${to_npub}.fips" 2>&1); then
|
|
rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2)
|
|
if [ -z "$quiet" ]; then
|
|
echo " $label ... OK (${rtt:-?}ms${attempt:+, attempt $attempt})"
|
|
fi
|
|
PASSED=$((PASSED + 1))
|
|
return 0
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
if [ -z "$quiet" ]; then
|
|
echo " $label ... FAIL (after $max_attempts attempt(s))"
|
|
fi
|
|
FAILED=$((FAILED + 1))
|
|
return 1
|
|
}
|
|
|
|
# All directed pairs of the mesh. Records pair-attributed failures into
|
|
# INTEROP_FAILURES, with the mixed/same classification.
|
|
ping_all_pairs() {
|
|
local quiet="${1:-}" max_attempts="${2:-1}" context="${3:-connectivity}"
|
|
PASSED=0
|
|
FAILED=0
|
|
local i j
|
|
for i in "${NODES[@]}"; do
|
|
for j in "${NODES[@]}"; do
|
|
[ "$i" = "$j" ] && continue
|
|
if ! ping_one "$i" "$j" "$quiet" "$max_attempts"; then
|
|
# The `convergence` context is the wait_for_full_baseline
|
|
# detector poll, NOT an assertion: a miss there only means
|
|
# "not converged yet, keep polling". Recording detector
|
|
# misses as interop failures is wrong — it false-fails
|
|
# every rep that takes a moment to converge. Only the
|
|
# strict assertion sweeps (baseline / post-rekey-*) record.
|
|
if [ "$context" != "convergence" ]; then
|
|
local kind
|
|
kind="$(pair_kind "$i" "$j")"
|
|
INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j"): ping $i->$j FAILED")
|
|
fi
|
|
fi
|
|
done
|
|
done
|
|
}
|
|
|
|
# Poll until every directed pair pings clean, or until timeout. This is a
|
|
# convergence DETECTOR — used at establishment (Phase 1) and after each
|
|
# rekey (Phases 3/5). It pings with the "convergence" context, which
|
|
# ping_all_pairs deliberately does not record as a failure; the caller
|
|
# runs a separate strict assertion sweep afterwards.
|
|
wait_for_full_baseline() {
|
|
local timeout="$1"
|
|
local start=$SECONDS
|
|
local best_passed=0 best_failed="$NUM_DIRECTED"
|
|
while (( SECONDS - start < timeout )); do
|
|
ping_all_pairs quiet 1 "convergence"
|
|
if [ "$PASSED" -gt "$best_passed" ]; then
|
|
best_passed="$PASSED"; best_failed="$FAILED"
|
|
fi
|
|
[ "$FAILED" -eq 0 ] && return 0
|
|
sleep 1
|
|
done
|
|
PASSED="$best_passed"; FAILED="$best_failed"
|
|
return 1
|
|
}
|
|
|
|
phase_result() {
|
|
local phase="$1"
|
|
TOTAL_PASSED=$((TOTAL_PASSED + PASSED))
|
|
TOTAL_FAILED=$((TOTAL_FAILED + FAILED))
|
|
if [ "$FAILED" -eq 0 ]; then
|
|
echo " PASS $phase: $PASSED/$((PASSED + FAILED))"
|
|
else
|
|
echo " FAIL $phase: $PASSED passed, $FAILED FAILED"
|
|
fi
|
|
}
|
|
|
|
# Count a pattern across all node logs.
|
|
count_log_pattern() {
|
|
local pattern="$1" total=0 n count
|
|
for n in "${NODES[@]}"; do
|
|
count=$(docker logs "${CONTAINER[$n]}" 2>&1 | grep -cE "$pattern" || true)
|
|
total=$((total + count))
|
|
done
|
|
echo "$total"
|
|
}
|
|
|
|
# Per-node count of a pattern.
|
|
count_node_pattern() {
|
|
local node="$1" pattern="$2"
|
|
docker logs "${CONTAINER[$node]}" 2>&1 | grep -cE "$pattern" || true
|
|
}
|
|
|
|
wait_for_log_pattern_count() {
|
|
local pattern="$1" min_count="$2" timeout="$3"
|
|
local start=$SECONDS
|
|
while (( SECONDS - start < timeout )); do
|
|
[ "$(count_log_pattern "$pattern")" -ge "$min_count" ] && return 0
|
|
sleep "$LOG_POLL_INTERVAL"
|
|
done
|
|
[ "$(count_log_pattern "$pattern")" -ge "$min_count" ]
|
|
}
|
|
|
|
dump_diagnostics() {
|
|
echo ""
|
|
echo "=== Peer / link snapshot ==="
|
|
local n s
|
|
for n in "${NODES[@]}"; do
|
|
s="${SLOT_OF[$n]}"
|
|
echo "--- $n (slot $s, ${SLOT_REF[$s]}@${SLOT_SHA[$s]}) ---"
|
|
docker exec "${CONTAINER[$n]}" fipsctl show peers 2>/dev/null || true
|
|
docker exec "${CONTAINER[$n]}" fipsctl show links 2>/dev/null || true
|
|
echo ""
|
|
done
|
|
echo "=== Recent node logs (interop-relevant lines) ==="
|
|
for n in "${NODES[@]}"; do
|
|
echo "--- $n ---"
|
|
docker logs "${CONTAINER[$n]}" 2>&1 \
|
|
| grep -E "(ERROR|PANIC|panicked|FMP version|handshake|Handshake|decrypt|Decrypt|teardown|rekey|Rekey|K-bit|established|Established)" \
|
|
| tail -40
|
|
echo ""
|
|
done
|
|
}
|
|
|
|
compose() {
|
|
docker compose -f "$COMPOSE_FILE" "$@"
|
|
}
|
|
|
|
cleanup() {
|
|
if [ "${FIPS_INTEROP_KEEP_UP:-}" = "1" ]; then
|
|
echo ""
|
|
echo "FIPS_INTEROP_KEEP_UP=1 — leaving mesh running. Tear down with:"
|
|
echo " docker compose -f $COMPOSE_FILE down --volumes --remove-orphans"
|
|
return
|
|
fi
|
|
compose down --volumes --remove-orphans >/dev/null 2>&1 || true
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'echo ""; echo "Interrupted"; exit 130' INT
|
|
|
|
load_slot_metadata
|
|
|
|
# ── Banner ───────────────────────────────────────────────────────────
|
|
|
|
echo "=============================================================="
|
|
echo " FIPS Mixed-Version Interop Test"
|
|
echo "=============================================================="
|
|
echo ""
|
|
echo "Node-spec: $SPEC_STR ($NUM_NODES nodes, $NUM_PAIRS pairs, $NUM_DIRECTED directed)"
|
|
echo ""
|
|
echo "Mesh nodes:"
|
|
for n in "${NODES[@]}"; do
|
|
s="${SLOT_OF[$n]}"
|
|
u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
|
|
echo " $n slot $u ${SLOT_REF[$s]} @ ${SLOT_SHA[$s]} (${NODE_IP_OF[$n]})"
|
|
done
|
|
echo ""
|
|
echo "Mesh pairs:"
|
|
for p in "${PAIRS[@]}"; do
|
|
read -r n1 n2 <<< "$p"
|
|
echo " $(pair_kind "$n1" "$n2") $(pair_label "$n1" "$n2")"
|
|
done
|
|
echo ""
|
|
if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then
|
|
echo "Netem impairment: $FIPS_INTEROP_NETEM"
|
|
echo ""
|
|
fi
|
|
echo "rekey.after_secs=$REKEY_AFTER_SECS"
|
|
echo ""
|
|
|
|
# ── Phase 0: bring up the mesh ───────────────────────────────────────
|
|
|
|
echo "Phase 0: Starting mesh"
|
|
compose down --volumes --remove-orphans >/dev/null 2>&1 || true
|
|
if ! compose up -d; then
|
|
echo " FAIL compose up failed"
|
|
exit 1
|
|
fi
|
|
|
|
# Optional netem: applied via `docker exec ... tc qdisc` on each
|
|
# container's eth0 — host bridge qdisc does NOT shape inter-container
|
|
# port-to-port traffic, so the impairment must live inside each
|
|
# container. Same mechanism as testing/flake-lab/run-loop.sh.
|
|
if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then
|
|
echo " Applying netem to each container eth0: $FIPS_INTEROP_NETEM"
|
|
for n in "${NODES[@]}"; do
|
|
if docker exec "${CONTAINER[$n]}" tc qdisc add dev eth0 root \
|
|
netem ${FIPS_INTEROP_NETEM} >/dev/null 2>&1; then
|
|
echo " $n: netem applied"
|
|
else
|
|
echo " $n: WARN netem apply failed"
|
|
fi
|
|
done
|
|
fi
|
|
echo ""
|
|
|
|
# ── Phase 1: FMP link + FSP session establishment ────────────────────
|
|
|
|
echo "Phase 1: Link/session establishment + connectivity baseline"
|
|
PASSED=0; FAILED=0
|
|
|
|
# Every node must reach N-1 authenticated peers. A peer in `show peers`
|
|
# is an authenticated FMP-link peer.
|
|
for n in "${NODES[@]}"; do
|
|
if ! wait_for_peers "${CONTAINER[$n]}" "$PEERS_EXPECTED" "$CONVERGENCE_TIMEOUT"; then
|
|
echo " $n did not reach $PEERS_EXPECTED authenticated peers"
|
|
fi
|
|
done
|
|
|
|
# The all-pairs ping over fips0 is the definitive FSP-session check: in
|
|
# a full mesh every pair is a direct neighbor, so a successful ping
|
|
# proves that pair's FSP session carries traffic in that direction.
|
|
if wait_for_full_baseline "$CONVERGENCE_TIMEOUT"; then
|
|
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
|
|
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
|
|
else
|
|
echo " Best baseline before timeout: $PASSED/$((PASSED + FAILED))"
|
|
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
|
|
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
|
|
dump_diagnostics
|
|
echo ""
|
|
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
|
echo "FAIL: mesh did not converge — see pair attribution above."
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
|
|
# ── Phase 2: first rekey cycle ───────────────────────────────────────
|
|
|
|
echo "Phase 2: First rekey cycle (waiting up to ${FIRST_REKEY_TIMEOUT}s)"
|
|
PASSED=0; FAILED=0
|
|
wait_for_log_pattern_count \
|
|
"Rekey cutover complete \(initiator\), K-bit flipped" 1 \
|
|
"$FIRST_REKEY_TIMEOUT" || true
|
|
fmp_cutovers="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"
|
|
if [ "$fmp_cutovers" -ge 1 ]; then
|
|
echo " PASS FMP rekey initiator cutovers: $fmp_cutovers"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
echo " FAIL no FMP rekey cutover observed within ${FIRST_REKEY_TIMEOUT}s"
|
|
FAILED=$((FAILED + 1))
|
|
INTEROP_FAILURES+=("[rekey] no FMP rekey cutover completed across the mesh")
|
|
fi
|
|
phase_result "First rekey cycle"
|
|
echo ""
|
|
|
|
# ── Phase 3: post-first-rekey connectivity ───────────────────────────
|
|
|
|
echo "Phase 3: Post-first-rekey connectivity (reconverge within ${POST_REKEY_TIMEOUT}s)"
|
|
PASSED=0; FAILED=0
|
|
# Poll until the mesh has reconverged after the rekey, then assert once.
|
|
# The detector poll records nothing (see ping_all_pairs); only the strict
|
|
# sweep below records, so a brief rekey-induced disruption is not a fail.
|
|
wait_for_full_baseline "$POST_REKEY_TIMEOUT" || true
|
|
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "post-rekey-1"
|
|
phase_result "Post-first-rekey (all $NUM_DIRECTED directed pairs)"
|
|
echo ""
|
|
|
|
# ── Phase 4: second rekey cycle ──────────────────────────────────────
|
|
|
|
echo "Phase 4: Second rekey cycle (waiting ${SECOND_REKEY_WAIT}s)"
|
|
sleep "$SECOND_REKEY_WAIT"
|
|
echo ""
|
|
|
|
echo "Phase 5: Post-second-rekey connectivity (reconverge within ${POST_REKEY_TIMEOUT}s)"
|
|
PASSED=0; FAILED=0
|
|
wait_for_full_baseline "$POST_REKEY_TIMEOUT" || true
|
|
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "post-rekey-2"
|
|
phase_result "Post-second-rekey (all $NUM_DIRECTED directed pairs)"
|
|
echo ""
|
|
|
|
# ── Phase 6: per-node, per-pair interop log analysis ─────────────────
|
|
#
|
|
# This is the interop-specific analysis. For each unordered pair it
|
|
# reports same vs mixed-version, then scans BOTH endpoints' logs for
|
|
# interop failure signatures. A signature firing on a mixed pair that
|
|
# does not fire on a same-version pair is the harness's primary signal.
|
|
|
|
echo "Phase 6: Interop log analysis"
|
|
PASSED=0; FAILED=0
|
|
|
|
# Give FSP rekey (which trails FMP rekey) a bounded chance to complete
|
|
# at least one cutover before the final assertions.
|
|
wait_for_log_pattern_count "FSP rekey cutover complete" 1 "$REKEY_SETTLE" || true
|
|
|
|
# Global negative checks — these must be zero on EVERY node regardless
|
|
# of version pairing. A non-zero count is attributed to the node and,
|
|
# where the count is asymmetric across versions, flagged as interop.
|
|
echo ""
|
|
echo " -- Global health (all $NUM_NODES nodes) --"
|
|
|
|
declare -A GLOBAL_PATTERNS=(
|
|
["PANIC|panicked"]="panics"
|
|
["ERROR"]="error-level log lines"
|
|
["unknown FMP version|Unknown FMP version"]="unknown-FMP-version drops"
|
|
["MMP link teardown"]="MMP link teardowns"
|
|
["Excessive decrypt failures"]="excessive-decrypt-failure removals"
|
|
["Session AEAD decryption failed"]="FSP AEAD decrypt failures"
|
|
["Rekey msg2 processing failed"]="rekey msg2 failures"
|
|
["Handshake failed|handshake failed|Handshake error"]="handshake failures"
|
|
)
|
|
|
|
for pat in "${!GLOBAL_PATTERNS[@]}"; do
|
|
desc="${GLOBAL_PATTERNS[$pat]}"
|
|
total="$(count_log_pattern "$pat")"
|
|
if [ "$total" -eq 0 ]; then
|
|
echo " PASS $desc: 0"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
echo " FAIL $desc: $total (expected 0)"
|
|
FAILED=$((FAILED + 1))
|
|
# Per-node breakdown so the count can be attributed to a build.
|
|
for n in "${NODES[@]}"; do
|
|
c="$(count_node_pattern "$n" "$pat")"
|
|
if [ "$c" -gt 0 ]; then
|
|
s="${SLOT_OF[$n]}"
|
|
u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
|
|
echo " $n [$u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}]: $c"
|
|
INTEROP_FAILURES+=("[log] node $n ($u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}): $c x '$desc'")
|
|
fi
|
|
done
|
|
fi
|
|
done
|
|
|
|
# Positive checks — the rekey machinery actually exercised both layers.
|
|
echo ""
|
|
echo " -- Rekey machinery exercised --"
|
|
fmp_total="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"
|
|
fsp_total="$(count_log_pattern 'FSP rekey cutover complete')"
|
|
if [ "$fmp_total" -ge 1 ]; then
|
|
echo " PASS FMP rekey cutovers across mesh: $fmp_total"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
echo " FAIL FMP rekey cutovers: $fmp_total (expected >= 1)"
|
|
FAILED=$((FAILED + 1))
|
|
fi
|
|
if [ "$fsp_total" -ge 1 ]; then
|
|
echo " PASS FSP rekey cutovers across mesh: $fsp_total"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
# FSP rekey not completing is a soft signal: report it but only
|
|
# hard-fail if a pair also lost connectivity. The connectivity
|
|
# phases already cover the user-visible impact.
|
|
echo " WARN FSP rekey cutovers: $fsp_total (expected >= 1)"
|
|
fi
|
|
|
|
# Per-pair summary: classify each unordered pair and report whether it
|
|
# stayed healthy through the run. "Healthy" = no connectivity failure
|
|
# recorded for either direction of the pair.
|
|
echo ""
|
|
echo " -- Per-pair interop summary --"
|
|
for p in "${PAIRS[@]}"; do
|
|
read -r n1 n2 <<< "$p"
|
|
kind="$(pair_kind "$n1" "$n2")"
|
|
label="$(pair_label "$n1" "$n2")"
|
|
pair_failed=0
|
|
if [ "${#INTEROP_FAILURES[@]}" -gt 0 ]; then
|
|
for f in "${INTEROP_FAILURES[@]}"; do
|
|
case "$f" in
|
|
*"ping $n1->$n2 FAILED"*|*"ping $n2->$n1 FAILED"*)
|
|
pair_failed=1 ;;
|
|
esac
|
|
done
|
|
fi
|
|
if [ "$pair_failed" -eq 0 ]; then
|
|
echo " PASS $kind pair $label: stayed healthy"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
echo " FAIL $kind pair $label: connectivity failed during the run"
|
|
FAILED=$((FAILED + 1))
|
|
fi
|
|
done
|
|
|
|
phase_result "Interop log analysis"
|
|
echo ""
|
|
|
|
# ── Summary ──────────────────────────────────────────────────────────
|
|
|
|
echo "=============================================================="
|
|
echo " Results: $TOTAL_PASSED checks passed, $TOTAL_FAILED failed"
|
|
echo "=============================================================="
|
|
|
|
# `${#arr[@]}` cannot be combined with `:-`; count into a plain var.
|
|
INTEROP_FAILURE_COUNT="${#INTEROP_FAILURES[@]}"
|
|
|
|
if [ "$TOTAL_FAILED" -eq 0 ] && [ "$INTEROP_FAILURE_COUNT" -eq 0 ]; then
|
|
echo ""
|
|
echo "PASS: all versions interoperate cleanly across the mesh (spec '$SPEC_STR')."
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "INTEROP FAILURES (attributed to specific version pairs / builds):"
|
|
if [ "$INTEROP_FAILURE_COUNT" -gt 0 ]; then
|
|
printf '%s\n' "${INTEROP_FAILURES[@]}" | sort -u | sed 's/^/ - /'
|
|
else
|
|
echo " (no per-pair failure recorded; see phase output above)"
|
|
fi
|
|
echo ""
|
|
|
|
# Highlight whether failures concentrate on mixed-version pairs — the
|
|
# defining interop signal.
|
|
mixed_hits=0
|
|
same_hits=0
|
|
if [ "$INTEROP_FAILURE_COUNT" -gt 0 ]; then
|
|
for f in "${INTEROP_FAILURES[@]}"; do
|
|
case "$f" in
|
|
*"MIXED pair"*) mixed_hits=$((mixed_hits + 1)) ;;
|
|
*"same pair"*) same_hits=$((same_hits + 1)) ;;
|
|
esac
|
|
done
|
|
fi
|
|
echo "Connectivity-failure attribution: mixed-version=$mixed_hits same-version=$same_hits"
|
|
if [ "$mixed_hits" -gt 0 ] && [ "$same_hits" -eq 0 ]; then
|
|
echo "=> Failures are MIXED-VERSION ONLY: a genuine interop regression."
|
|
elif [ "$mixed_hits" -gt 0 ] && [ "$same_hits" -gt 0 ]; then
|
|
echo "=> Failures hit both mixed and same-version pairs: likely a general"
|
|
echo " instability, not version-specific — investigate the common cause."
|
|
elif [ "$same_hits" -gt 0 ]; then
|
|
echo "=> Failures are SAME-VERSION ONLY: not an interop issue per se;"
|
|
echo " a build is unstable even against itself."
|
|
fi
|
|
|
|
dump_diagnostics
|
|
exit 1
|