mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
testing/interop: add multi-hop mixed-version topology with forwarding, continuity, and mesh-size checks
The interop harness only tested a full mesh, where every pair is a direct one-hop FMP link. That left multi-hop forwarding, routing, coordinate handling, and mesh-size estimation untested across versions. Add an opt-in multi-hop topology and three new checks: - generate-configs.sh: FIPS_INTEROP_EDGES selects an explicit undirected edge list instead of the full mesh, with symmetric peering, connectivity validation (no isolated node, graph connected), and per-node degree plus edge metadata in nodes.env. Unset means full mesh, unchanged. - interop-test.sh: a --topology flag with a built-in multihop-3v-cycle (six nodes, two of each version, one cycle, two leaves). The per-node peer check now expects each node's adjacency degree rather than N-1, and the all-pairs ping now also exercises cross-version forwarding over non-adjacent pairs. Adds a control-differential data-plane continuity stream across the rekey window (Phase 5b) and a strict plus/minus 25 percent mesh-size convergence check (Phase 7). - interop-stress.sh: forward --topology through to the per-rep driver (mutually exclusive with a positional node-spec). Phase 1 makes the strict retrying ping authoritative. The convergence detector (one fully-clean, no-retry sweep of every directed pair) is now an advisory settle-wait whose timeout is non-fatal; the strict retrying ping decides pass/fail, matching how the post-rekey phases already work. Under packet loss a clean no-retry sweep of a large mesh is statistically unlikely even when the mesh is healthy (30 pairs at 2 percent loss leaves only about 55 percent of sweeps clean), so the detector otherwise falsely failed runs whose strict ping reported full reachability. CONVERGENCE_TIMEOUT is now env-overridable and defaults to 90s for larger meshes under loss. Full-mesh runs are unaffected. Validated end-to-end with the multihop-3v-cycle topology across three versions: all 30 directed pairs reachable including multi-hop routed pairs, zero data-plane loss through both rekey cutovers, and mesh-size converging to the true node count on every node.
This commit is contained in:
@@ -18,7 +18,9 @@
|
|||||||
# - IPv4 = 172.30.0.1<index>, index = 0-based position in the spec
|
# - IPv4 = 172.30.0.1<index>, index = 0-based position in the spec
|
||||||
# (10, 11, 12, 13, ...).
|
# (10, 11, 12, 13, ...).
|
||||||
#
|
#
|
||||||
# Topology is a FULL MESH: every node auto-connects to every other node.
|
# Topology is a FULL MESH by default: every node auto-connects to every
|
||||||
|
# other node. Set FIPS_INTEROP_EDGES (see Environment) to an explicit
|
||||||
|
# undirected edge list for a multi-hop / leaf topology instead.
|
||||||
# Node identities are derived deterministically via the shared
|
# Node identities are derived deterministically via the shared
|
||||||
# derive_keys.py helper, keyed by a unique per-node string so that two
|
# derive_keys.py helper, keyed by a unique per-node string so that two
|
||||||
# nodes of the same slot (a1, a2) still get DISTINCT identities.
|
# nodes of the same slot (a1, a2) still get DISTINCT identities.
|
||||||
@@ -41,6 +43,13 @@
|
|||||||
# Environment:
|
# Environment:
|
||||||
# REKEY_AFTER_SECS FSP/FMP rekey interval (default 35, matching
|
# REKEY_AFTER_SECS FSP/FMP rekey interval (default 35, matching
|
||||||
# the static rekey suite).
|
# the static rekey suite).
|
||||||
|
# FIPS_INTEROP_EDGES Optional undirected edge list (space/comma
|
||||||
|
# separated `nid-nid` tokens, e.g.
|
||||||
|
# "a1-b1 a1-c1 b1-a2"). Unset ⇒ full mesh. When
|
||||||
|
# set, each node peers only with its listed
|
||||||
|
# neighbors; the graph must be connected with no
|
||||||
|
# isolated node. Enables multi-hop forwarding /
|
||||||
|
# leaf-node topologies.
|
||||||
# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/,
|
# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/,
|
||||||
# .stress-runs/, generated-configs/). When
|
# .stress-runs/, generated-configs/). When
|
||||||
# unset, falls back to in-tree paths under
|
# unset, falls back to in-tree paths under
|
||||||
@@ -124,6 +133,79 @@ for slot in "${SPEC[@]}"; do
|
|||||||
index=$(( index + 1 ))
|
index=$(( index + 1 ))
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# ── Topology: adjacency (full mesh by default, or explicit edges) ─────
|
||||||
|
#
|
||||||
|
# FIPS_INTEROP_EDGES, when set, is a space/comma list of `nid-nid` tokens
|
||||||
|
# describing UNDIRECTED edges (direct FMP peer links), e.g.
|
||||||
|
# "a1-b1 a1-c1 b1-a2". Empty ⇒ FULL MESH (every node peers with every
|
||||||
|
# other, the historical behavior). Edges are symmetric: `x-y` makes x and
|
||||||
|
# y each other's auto_connect peer.
|
||||||
|
|
||||||
|
declare -A ADJ # ADJ[nid] = space-separated neighbor ids
|
||||||
|
declare -A _ADJ_SEEN # "x|y" dedup guard
|
||||||
|
TOPOLOGY="full-mesh"
|
||||||
|
EDGES_LIST=()
|
||||||
|
|
||||||
|
add_edge() {
|
||||||
|
local x="$1" y="$2"
|
||||||
|
if [ "$x" = "$y" ]; then
|
||||||
|
echo "ERROR: self-edge '$x-$x' not allowed" >&2; exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "${NODE_SLOT[$x]:-}" ]; then
|
||||||
|
echo "ERROR: edge references unknown node '$x'" >&2; exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "${NODE_SLOT[$y]:-}" ]; then
|
||||||
|
echo "ERROR: edge references unknown node '$y'" >&2; exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "${_ADJ_SEEN[$x|$y]:-}" ]; then
|
||||||
|
ADJ[$x]="${ADJ[$x]:+${ADJ[$x]} }$y"; _ADJ_SEEN[$x|$y]=1
|
||||||
|
fi
|
||||||
|
if [ -z "${_ADJ_SEEN[$y|$x]:-}" ]; then
|
||||||
|
ADJ[$y]="${ADJ[$y]:+${ADJ[$y]} }$x"; _ADJ_SEEN[$y|$x]=1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ -n "${FIPS_INTEROP_EDGES:-}" ]; then
|
||||||
|
TOPOLOGY="custom"
|
||||||
|
for tok in ${FIPS_INTEROP_EDGES//,/ }; do
|
||||||
|
x="${tok%%-*}"; y="${tok#*-}"
|
||||||
|
if [ "$x" = "$tok" ] || [ -z "$x" ] || [ -z "$y" ] || [[ "$y" == *-* ]]; then
|
||||||
|
echo "ERROR: malformed edge token '$tok' (want nid-nid)" >&2; exit 1
|
||||||
|
fi
|
||||||
|
add_edge "$x" "$y"
|
||||||
|
EDGES_LIST+=("$x-$y")
|
||||||
|
done
|
||||||
|
# Validate: no isolated node, graph connected (BFS from node 0).
|
||||||
|
for nid in "${NODE_IDS[@]}"; do
|
||||||
|
if [ -z "${ADJ[$nid]:-}" ]; then
|
||||||
|
echo "ERROR: node '$nid' has no edges (isolated)" >&2; exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
declare -A _BFS_SEEN=()
|
||||||
|
bfs_queue=("${NODE_IDS[0]}"); _BFS_SEEN[${NODE_IDS[0]}]=1; bfs_reached=1
|
||||||
|
while [ "${#bfs_queue[@]}" -gt 0 ]; do
|
||||||
|
cur="${bfs_queue[0]}"; bfs_queue=("${bfs_queue[@]:1}")
|
||||||
|
for nb in ${ADJ[$cur]}; do
|
||||||
|
if [ -z "${_BFS_SEEN[$nb]:-}" ]; then
|
||||||
|
_BFS_SEEN[$nb]=1; bfs_reached=$(( bfs_reached + 1 ))
|
||||||
|
bfs_queue+=("$nb")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
|
if [ "$bfs_reached" -ne "${#NODE_IDS[@]}" ]; then
|
||||||
|
echo "ERROR: topology is disconnected ($bfs_reached/${#NODE_IDS[@]} reachable from ${NODE_IDS[0]})" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Full mesh: every node adjacent to every other, in spec order.
|
||||||
|
for a in "${NODE_IDS[@]}"; do
|
||||||
|
for b in "${NODE_IDS[@]}"; do
|
||||||
|
[ "$a" = "$b" ] && continue
|
||||||
|
ADJ[$a]="${ADJ[$a]:+${ADJ[$a]} }$b"
|
||||||
|
done
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
mkdir -p "$OUT_DIR"
|
mkdir -p "$OUT_DIR"
|
||||||
|
|
||||||
# Clear stale per-node configs from a previous (differently-shaped) spec
|
# Clear stale per-node configs from a previous (differently-shaped) spec
|
||||||
@@ -186,8 +268,7 @@ for nid in "${NODE_IDS[@]}"; do
|
|||||||
echo " mtu: 1472"
|
echo " mtu: 1472"
|
||||||
echo ""
|
echo ""
|
||||||
echo "peers:"
|
echo "peers:"
|
||||||
for pid in "${NODE_IDS[@]}"; do
|
for pid in ${ADJ[$nid]}; do
|
||||||
[ "$pid" = "$nid" ] && continue
|
|
||||||
emit_peer_block "$pid"
|
emit_peer_block "$pid"
|
||||||
done
|
done
|
||||||
} > "$cfg"
|
} > "$cfg"
|
||||||
@@ -259,6 +340,19 @@ MANIFEST="$OUT_DIR/nodes.env"
|
|||||||
echo "# Node-spec: ${SPEC[*]}"
|
echo "# Node-spec: ${SPEC[*]}"
|
||||||
echo "INTEROP_SPEC=\"${SPEC[*]}\""
|
echo "INTEROP_SPEC=\"${SPEC[*]}\""
|
||||||
echo "INTEROP_NODE_IDS=\"${NODE_IDS[*]}\""
|
echo "INTEROP_NODE_IDS=\"${NODE_IDS[*]}\""
|
||||||
|
echo "INTEROP_TOPOLOGY=\"$TOPOLOGY\""
|
||||||
|
echo "INTEROP_TOPOLOGY_EDGES=\"${EDGES_LIST[*]:-}\""
|
||||||
|
# Per-node expected DIRECT-peer count (adjacency degree). The driver
|
||||||
|
# uses this for the per-node peer-convergence check instead of N-1.
|
||||||
|
{
|
||||||
|
printf 'INTEROP_NODE_DEGREE="'
|
||||||
|
for nid in "${NODE_IDS[@]}"; do
|
||||||
|
d=0
|
||||||
|
for _nb in ${ADJ[$nid]}; do d=$(( d + 1 )); done
|
||||||
|
printf '%s:%s ' "$nid" "$d"
|
||||||
|
done
|
||||||
|
printf '"\n'
|
||||||
|
}
|
||||||
# Per-node maps, one var each, space-separated 'nodeid:value' tokens.
|
# Per-node maps, one var each, space-separated 'nodeid:value' tokens.
|
||||||
{
|
{
|
||||||
printf 'INTEROP_NODE_SLOTS="'
|
printf 'INTEROP_NODE_SLOTS="'
|
||||||
@@ -297,7 +391,11 @@ ENV_FILE="$OUT_DIR/npubs.env"
|
|||||||
echo " generated $ENV_FILE"
|
echo " generated $ENV_FILE"
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "Mesh configs ready (spec='${SPEC[*]}', rekey.after_secs=$REKEY_AFTER_SECS):"
|
echo "Mesh configs ready (spec='${SPEC[*]}', topology=$TOPOLOGY, rekey.after_secs=$REKEY_AFTER_SECS):"
|
||||||
for nid in "${NODE_IDS[@]}"; do
|
for nid in "${NODE_IDS[@]}"; do
|
||||||
echo " $nid slot ${NODE_SLOT[$nid]} ${NODE_IP[$nid]} ${NPUB[$nid]}"
|
d=0; for _nb in ${ADJ[$nid]}; do d=$(( d + 1 )); done
|
||||||
|
echo " $nid slot ${NODE_SLOT[$nid]} ${NODE_IP[$nid]} deg=$d ${NPUB[$nid]}"
|
||||||
done
|
done
|
||||||
|
if [ "$TOPOLOGY" = "custom" ]; then
|
||||||
|
echo " edges: ${EDGES_LIST[*]}"
|
||||||
|
fi
|
||||||
|
|||||||
@@ -26,9 +26,13 @@
|
|||||||
# fixed Docker network, so two reps must never overlap.
|
# fixed Docker network, so two reps must never overlap.
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
# ./interop-stress.sh [--reps N] [node-spec...]
|
# ./interop-stress.sh [--reps N] [--topology <name> | node-spec...]
|
||||||
#
|
#
|
||||||
# --reps N repetitions (default 10).
|
# --reps N repetitions (default 10).
|
||||||
|
# --topology built-in multi-hop topology forwarded to interop-test.sh
|
||||||
|
# (e.g. multihop-3v-cycle). Mutually exclusive with a
|
||||||
|
# positional node-spec. Adds multi-hop forwarding,
|
||||||
|
# data-plane continuity, and mesh-size checks to each rep.
|
||||||
# node-spec slot letters (a/b/c), default `a a b c` (the control
|
# node-spec slot letters (a/b/c), default `a a b c` (the control
|
||||||
# topology — one same-version pair + five mixed pairs).
|
# topology — one same-version pair + five mixed pairs).
|
||||||
#
|
#
|
||||||
@@ -81,8 +85,17 @@ RUNS_BASE="$INTEROP_RUNS_BASE/.stress-runs"
|
|||||||
|
|
||||||
REPS=10
|
REPS=10
|
||||||
SPEC=()
|
SPEC=()
|
||||||
|
TOPOLOGY_ARG=""
|
||||||
while [ "$#" -gt 0 ]; do
|
while [ "$#" -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
|
--topology)
|
||||||
|
TOPOLOGY_ARG="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--topology=*)
|
||||||
|
TOPOLOGY_ARG="${1#--topology=}"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
--reps)
|
--reps)
|
||||||
REPS="${2:-}"
|
REPS="${2:-}"
|
||||||
if ! [[ "$REPS" =~ ^[0-9]+$ ]] || [ "$REPS" -lt 1 ]; then
|
if ! [[ "$REPS" =~ ^[0-9]+$ ]] || [ "$REPS" -lt 1 ]; then
|
||||||
@@ -119,10 +132,23 @@ while [ "$#" -gt 0 ]; do
|
|||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
if [ "${#SPEC[@]}" -eq 0 ]; then
|
# A --topology selects spec + edges inside the driver; otherwise use the
|
||||||
SPEC=(a a b c)
|
# positional node-spec (default the control-arm topology `a a b c`).
|
||||||
|
DRIVER_ARGS=()
|
||||||
|
if [ -n "$TOPOLOGY_ARG" ]; then
|
||||||
|
if [ "${#SPEC[@]}" -gt 0 ]; then
|
||||||
|
echo "ERROR: pass either --topology or a node-spec, not both" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
DRIVER_ARGS=(--topology "$TOPOLOGY_ARG")
|
||||||
|
SPEC_STR="(topology: $TOPOLOGY_ARG)"
|
||||||
|
else
|
||||||
|
if [ "${#SPEC[@]}" -eq 0 ]; then
|
||||||
|
SPEC=(a a b c)
|
||||||
|
fi
|
||||||
|
DRIVER_ARGS=("${SPEC[@]}")
|
||||||
|
SPEC_STR="${SPEC[*]}"
|
||||||
fi
|
fi
|
||||||
SPEC_STR="${SPEC[*]}"
|
|
||||||
|
|
||||||
# ── Preflight ────────────────────────────────────────────────────────
|
# ── Preflight ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -175,7 +201,7 @@ for ((rep = 1; rep <= REPS; rep++)); do
|
|||||||
# Run the driver, capturing full output and exit code. Netem is
|
# Run the driver, capturing full output and exit code. Netem is
|
||||||
# passed through the environment; interop-test.sh applies it.
|
# passed through the environment; interop-test.sh applies it.
|
||||||
FIPS_INTEROP_NETEM="${FIPS_INTEROP_NETEM:-}" \
|
FIPS_INTEROP_NETEM="${FIPS_INTEROP_NETEM:-}" \
|
||||||
bash "$DRIVER" "${SPEC[@]}" >"$driver_log" 2>&1
|
bash "$DRIVER" "${DRIVER_ARGS[@]}" >"$driver_log" 2>&1
|
||||||
rc=$?
|
rc=$?
|
||||||
|
|
||||||
if [ "$rc" -eq 0 ]; then
|
if [ "$rc" -eq 0 ]; then
|
||||||
|
|||||||
+330
-23
@@ -24,18 +24,39 @@
|
|||||||
# 2. configs are (re)generated automatically below for the given spec.
|
# 2. configs are (re)generated automatically below for the given spec.
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
# ./interop-test.sh [node-spec...]
|
# ./interop-test.sh [--topology <name>] [node-spec...]
|
||||||
#
|
#
|
||||||
|
# --topology built-in multi-hop topology selecting a node-spec + edge
|
||||||
|
# set + default data-plane streams. Known: multihop-3v-cycle
|
||||||
|
# (6 nodes, 2 of each version, one cycle, two leaves —
|
||||||
|
# exercises cross-version forwarding + mesh-size). Without
|
||||||
|
# it the mesh is a full mesh from the positional node-spec.
|
||||||
# node-spec space-separated slot letters (a/b/c), default `a b c`.
|
# node-spec space-separated slot letters (a/b/c), default `a b c`.
|
||||||
# e.g. `a a b c` (4-node, one same-version control pair),
|
# e.g. `a a b c` (4-node, one same-version control pair),
|
||||||
# `a a a` (3-node same-version flake rig).
|
# `a a a` (3-node same-version flake rig).
|
||||||
#
|
#
|
||||||
|
# Beyond FMP/FSP rekey survival, a --topology run also verifies multi-hop
|
||||||
|
# FORWARDING (all-pairs ping over non-adjacent pairs), DATA-PLANE
|
||||||
|
# CONTINUITY across rekey (control-differential ping-loss stream,
|
||||||
|
# Phase 5b), and MESH-SIZE estimate convergence across versions
|
||||||
|
# (Phase 7).
|
||||||
|
#
|
||||||
# Environment:
|
# Environment:
|
||||||
# FIPS_INTEROP_NETEM tc-netem arg string applied to every container's
|
# FIPS_INTEROP_NETEM tc-netem arg string applied to every container's
|
||||||
# eth0, e.g. "delay 10ms 5ms 25% loss 1%". Unset =
|
# eth0, e.g. "delay 10ms 5ms 25% loss 1%". Unset =
|
||||||
# no impairment.
|
# no impairment.
|
||||||
|
# FIPS_INTEROP_EDGES explicit undirected edge list (overrides the
|
||||||
|
# full mesh) — see generate-configs.sh. Usually
|
||||||
|
# set for you by --topology.
|
||||||
|
# FIPS_INTEROP_STREAMS data-plane stream pairs (`nid-nid` tokens, both
|
||||||
|
# directions streamed). Enables Phase 1b/5b. Set
|
||||||
|
# by --topology; empty = streams off.
|
||||||
|
# STREAM_LOSS_MARGIN_PCT rekey-vs-control loss margin (default 1).
|
||||||
|
# CONTROL_STREAM_SECS quiet control-window length (default 12).
|
||||||
|
# MESH_SIZE_TIMEOUT Phase 7 convergence poll budget (default 180).
|
||||||
# FIPS_INTEROP_KEEP_UP 1 = leave containers running after the test (debug).
|
# FIPS_INTEROP_KEEP_UP 1 = leave containers running after the test (debug).
|
||||||
# REKEY_AFTER_SECS rekey interval to generate configs with (default 35).
|
# REKEY_AFTER_SECS rekey interval to generate configs with (default
|
||||||
|
# 35; multihop-3v-cycle defaults it to 50).
|
||||||
# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/,
|
# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/,
|
||||||
# .stress-runs/, generated-configs/). When
|
# .stress-runs/, generated-configs/). When
|
||||||
# unset, falls back to in-tree paths under
|
# unset, falls back to in-tree paths under
|
||||||
@@ -74,6 +95,48 @@ COMPOSE_FILE="$GEN_DIR/docker-compose.generated.yml"
|
|||||||
NODES_ENV="$GEN_DIR/nodes.env"
|
NODES_ENV="$GEN_DIR/nodes.env"
|
||||||
REFS_ENV="$RUNS_BASE/.build/refs.env"
|
REFS_ENV="$RUNS_BASE/.build/refs.env"
|
||||||
|
|
||||||
|
# ── Built-in topology selection ──────────────────────────────────────
|
||||||
|
#
|
||||||
|
# --topology <name> selects a node-spec + an explicit edge set (a
|
||||||
|
# multi-hop, mixed-version, leaf-bearing graph) + a default data-plane
|
||||||
|
# stream set, exercising forwarding / routing / mesh-size convergence
|
||||||
|
# across versions. Without it, positional args are the node-spec and the
|
||||||
|
# mesh is a full mesh (historical behavior).
|
||||||
|
TOPOLOGY_NAME=""
|
||||||
|
_ARGS=()
|
||||||
|
while [ "$#" -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--topology) TOPOLOGY_NAME="${2:-}"; shift 2 ;;
|
||||||
|
--topology=*) TOPOLOGY_NAME="${1#--topology=}"; shift ;;
|
||||||
|
*) _ARGS+=("$1"); shift ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
set -- ${_ARGS[@]+"${_ARGS[@]}"}
|
||||||
|
|
||||||
|
if [ -n "$TOPOLOGY_NAME" ]; then
|
||||||
|
case "$TOPOLOGY_NAME" in
|
||||||
|
multihop-3v-cycle)
|
||||||
|
# 6 nodes (2 of each version), one cycle, two leaves.
|
||||||
|
# a1
|
||||||
|
# / \ leaves: a2, c2
|
||||||
|
# b1 c1 cycle: b1-a1-c1-b2-b1
|
||||||
|
# / \ \
|
||||||
|
# a2 b2------c2-edge(b2-c1)
|
||||||
|
set -- a a b b c c
|
||||||
|
export FIPS_INTEROP_EDGES="a1-b1 a1-c1 b1-a2 b1-b2 c1-c2 b2-c1"
|
||||||
|
: "${FIPS_INTEROP_STREAMS:=a2-c2 a1-b1}"
|
||||||
|
export FIPS_INTEROP_STREAMS
|
||||||
|
# Multi-hop convergence + a clean pre-rekey control window
|
||||||
|
# want more headroom than the 35s full-mesh default.
|
||||||
|
: "${REKEY_AFTER_SECS:=50}"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: unknown --topology '$TOPOLOGY_NAME' (known: multihop-3v-cycle)" >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
REKEY_AFTER_SECS="${REKEY_AFTER_SECS:-35}"
|
REKEY_AFTER_SECS="${REKEY_AFTER_SECS:-35}"
|
||||||
|
|
||||||
# ── Node-spec ────────────────────────────────────────────────────────
|
# ── Node-spec ────────────────────────────────────────────────────────
|
||||||
@@ -86,7 +149,7 @@ SPEC_STR="${SPEC[*]}"
|
|||||||
|
|
||||||
# ── Timing ───────────────────────────────────────────────────────────
|
# ── Timing ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
CONVERGENCE_TIMEOUT=60 # wait for full mesh + clean ping baseline
|
CONVERGENCE_TIMEOUT="${CONVERGENCE_TIMEOUT:-90}" # detector settle budget (env-overridable; larger meshes under loss converge slower)
|
||||||
PING_TIMEOUT=5
|
PING_TIMEOUT=5
|
||||||
MAX_PING_ATTEMPTS=4 # strict-ping retry (mirrors rekey-test.sh)
|
MAX_PING_ATTEMPTS=4 # strict-ping retry (mirrors rekey-test.sh)
|
||||||
PING_RETRY_DELAY=1
|
PING_RETRY_DELAY=1
|
||||||
@@ -101,6 +164,19 @@ REKEY_SETTLE=12 # FSP-cutover settle budget (Phase 6)
|
|||||||
POST_REKEY_TIMEOUT=45
|
POST_REKEY_TIMEOUT=45
|
||||||
LOG_POLL_INTERVAL=2
|
LOG_POLL_INTERVAL=2
|
||||||
|
|
||||||
|
# Data-plane continuity stream (control-differential). Streams run a
|
||||||
|
# sustained ping6 over the overlay across the rekey window vs a quiet
|
||||||
|
# control window; loss is compared to prove rekey is hitless.
|
||||||
|
STREAM_RATE_HZ=5 # ping6 -i 0.2
|
||||||
|
CONTROL_STREAM_SECS="${CONTROL_STREAM_SECS:-12}" # quiet pre-rekey window
|
||||||
|
STREAM_LOSS_MARGIN_PCT="${STREAM_LOSS_MARGIN_PCT:-1}"
|
||||||
|
# The rekey-window stream must span Phases 2-5 (both cutovers + reconverge).
|
||||||
|
REKEY_STREAM_SECS=$(( FIRST_REKEY_TIMEOUT + SECOND_REKEY_WAIT + POST_REKEY_TIMEOUT + 15 ))
|
||||||
|
|
||||||
|
# Mesh-size estimate convergence (strict ±25% of true N). Generous poll
|
||||||
|
# budget — the bloom-union estimate converges over minutes.
|
||||||
|
MESH_SIZE_TIMEOUT="${MESH_SIZE_TIMEOUT:-180}"
|
||||||
|
|
||||||
# ── Counters ─────────────────────────────────────────────────────────
|
# ── Counters ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
PASSED=0
|
PASSED=0
|
||||||
@@ -140,7 +216,9 @@ done
|
|||||||
need_regen=1
|
need_regen=1
|
||||||
if [ -f "$NODES_ENV" ]; then
|
if [ -f "$NODES_ENV" ]; then
|
||||||
existing_spec="$(sed -n 's/^INTEROP_SPEC="\(.*\)"$/\1/p' "$NODES_ENV")"
|
existing_spec="$(sed -n 's/^INTEROP_SPEC="\(.*\)"$/\1/p' "$NODES_ENV")"
|
||||||
if [ "$existing_spec" = "$SPEC_STR" ]; then
|
existing_edges="$(sed -n 's/^INTEROP_TOPOLOGY_EDGES="\(.*\)"$/\1/p' "$NODES_ENV")"
|
||||||
|
if [ "$existing_spec" = "$SPEC_STR" ] \
|
||||||
|
&& [ "$existing_edges" = "${FIPS_INTEROP_EDGES:-}" ]; then
|
||||||
need_regen=0
|
need_regen=0
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@@ -179,6 +257,41 @@ parse_map CONTAINER "$INTEROP_NODE_CONTAINERS"
|
|||||||
parse_map NODE_IP_OF "$INTEROP_NODE_IPS"
|
parse_map NODE_IP_OF "$INTEROP_NODE_IPS"
|
||||||
parse_map NPUB_OF "$INTEROP_NODE_NPUBS"
|
parse_map NPUB_OF "$INTEROP_NODE_NPUBS"
|
||||||
|
|
||||||
|
# Topology metadata: per-node expected direct-peer degree, and the
|
||||||
|
# undirected direct-edge set (for direct-vs-routed pair labeling).
|
||||||
|
TOPOLOGY_KIND="${INTEROP_TOPOLOGY:-full-mesh}"
|
||||||
|
declare -A DEGREE_OF
|
||||||
|
parse_map DEGREE_OF "${INTEROP_NODE_DEGREE:-}"
|
||||||
|
|
||||||
|
declare -A IS_DIRECT_EDGE
|
||||||
|
if [ -n "${INTEROP_TOPOLOGY_EDGES:-}" ]; then
|
||||||
|
for _e in $INTEROP_TOPOLOGY_EDGES; do
|
||||||
|
_x="${_e%%-*}"; _y="${_e#*-}"
|
||||||
|
IS_DIRECT_EDGE["$_x|$_y"]=1
|
||||||
|
IS_DIRECT_EDGE["$_y|$_x"]=1
|
||||||
|
done
|
||||||
|
else
|
||||||
|
# Full mesh: every ordered pair is a direct edge.
|
||||||
|
for _a in "${NODES[@]}"; do
|
||||||
|
for _b in "${NODES[@]}"; do
|
||||||
|
[ "$_a" = "$_b" ] && continue
|
||||||
|
IS_DIRECT_EDGE["$_a|$_b"]=1
|
||||||
|
done
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
pair_is_direct() { [ -n "${IS_DIRECT_EDGE[$1|$2]:-}" ]; }
|
||||||
|
hop_label() { if pair_is_direct "$1" "$2"; then echo "direct"; else echo "routed"; fi; }
|
||||||
|
|
||||||
|
# Data-plane stream directed pairs (both directions of each token).
|
||||||
|
STREAM_PAIRS=()
|
||||||
|
if [ -n "${FIPS_INTEROP_STREAMS:-}" ]; then
|
||||||
|
for _tok in ${FIPS_INTEROP_STREAMS//,/ }; do
|
||||||
|
_x="${_tok%%-*}"; _y="${_tok#*-}"
|
||||||
|
STREAM_PAIRS+=("$_x $_y" "$_y $_x")
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
# Unordered pairs of the mesh.
|
# Unordered pairs of the mesh.
|
||||||
PAIRS=()
|
PAIRS=()
|
||||||
for ((i = 0; i < ${#NODES[@]}; i++)); do
|
for ((i = 0; i < ${#NODES[@]}; i++)); do
|
||||||
@@ -309,9 +422,12 @@ ping_all_pairs() {
|
|||||||
# every rep that takes a moment to converge. Only the
|
# every rep that takes a moment to converge. Only the
|
||||||
# strict assertion sweeps (baseline / post-rekey-*) record.
|
# strict assertion sweeps (baseline / post-rekey-*) record.
|
||||||
if [ "$context" != "convergence" ]; then
|
if [ "$context" != "convergence" ]; then
|
||||||
local kind
|
local kind hop
|
||||||
kind="$(pair_kind "$i" "$j")"
|
kind="$(pair_kind "$i" "$j")"
|
||||||
INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j"): ping $i->$j FAILED")
|
hop="$(hop_label "$i" "$j")"
|
||||||
|
# NOTE: keep "$kind pair" contiguous — interop-stress.sh
|
||||||
|
# greps "MIXED pair"/"same pair". The [hop] tag is additive.
|
||||||
|
INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j") [$hop]: ping $i->$j FAILED")
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
@@ -376,6 +492,66 @@ wait_for_log_pattern_count() {
|
|||||||
[ "$(count_log_pattern "$pattern")" -ge "$min_count" ]
|
[ "$(count_log_pattern "$pattern")" -ge "$min_count" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Data-plane continuity streams ────────────────────────────────────
|
||||||
|
#
|
||||||
|
# A sustained ping6 stream over the overlay (<npub>.fips) is data-plane
|
||||||
|
# traffic: it traverses TUN → FSP session → FMP link → forwarding, the
|
||||||
|
# same path application packets take. We run streams across the rekey
|
||||||
|
# window and across a quiet control window, then compare loss — a rekey
|
||||||
|
# that drops data shows up as rekey-window loss above the control.
|
||||||
|
|
||||||
|
declare -A STREAM_TX STREAM_RX
|
||||||
|
_STREAM_PIDS=()
|
||||||
|
_STREAM_FILES=() # "label:from->to:resultfile"
|
||||||
|
_STREAM_TMP=""
|
||||||
|
|
||||||
|
# One directed ping6 stream of <dur>s at STREAM_RATE_HZ; writes "tx rx".
|
||||||
|
_stream_one() {
|
||||||
|
local from="$1" to="$2" dur="$3" outfile="$4"
|
||||||
|
local from_ctr="${CONTAINER[$from]}" to_npub="${NPUB_OF[$to]}"
|
||||||
|
local count=$(( dur * STREAM_RATE_HZ ))
|
||||||
|
local out tx rx
|
||||||
|
out=$(docker exec "$from_ctr" ping6 -i 0.2 -c "$count" -W "$PING_TIMEOUT" \
|
||||||
|
"${to_npub}.fips" 2>&1)
|
||||||
|
tx=$(echo "$out" | grep -oE '[0-9]+ packets transmitted' | grep -oE '^[0-9]+')
|
||||||
|
rx=$(echo "$out" | grep -oE '[0-9]+ received' | grep -oE '^[0-9]+')
|
||||||
|
echo "${tx:-0} ${rx:-0}" > "$outfile"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Launch all stream pairs in the background for <label> over <dur>s.
|
||||||
|
launch_streams() {
|
||||||
|
local label="$1" dur="$2"
|
||||||
|
_STREAM_TMP="$(mktemp -d)"
|
||||||
|
_STREAM_PIDS=(); _STREAM_FILES=()
|
||||||
|
local sp from to
|
||||||
|
for sp in "${STREAM_PAIRS[@]}"; do
|
||||||
|
read -r from to <<< "$sp"
|
||||||
|
_stream_one "$from" "$to" "$dur" "$_STREAM_TMP/$label-$from-$to" &
|
||||||
|
_STREAM_PIDS+=("$!")
|
||||||
|
_STREAM_FILES+=("$label:$from->$to:$_STREAM_TMP/$label-$from-$to")
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Wait for the launched streams and read tx/rx into STREAM_TX/STREAM_RX.
|
||||||
|
collect_streams() {
|
||||||
|
[ "${#_STREAM_PIDS[@]}" -gt 0 ] && wait "${_STREAM_PIDS[@]}" 2>/dev/null || true
|
||||||
|
local entry label rest key f tx rx
|
||||||
|
for entry in "${_STREAM_FILES[@]}"; do
|
||||||
|
label="${entry%%:*}"; rest="${entry#*:}"; key="${rest%%:*}"; f="${rest#*:}"
|
||||||
|
if read -r tx rx < "$f" 2>/dev/null; then :; else tx=0; rx=0; fi
|
||||||
|
STREAM_TX["$label:$key"]="${tx:-0}"
|
||||||
|
STREAM_RX["$label:$key"]="${rx:-0}"
|
||||||
|
done
|
||||||
|
[ -n "$_STREAM_TMP" ] && rm -rf "$_STREAM_TMP"
|
||||||
|
_STREAM_TMP=""
|
||||||
|
}
|
||||||
|
|
||||||
|
# loss% from tx/rx (one decimal), or "NA" when tx==0.
|
||||||
|
_loss_pct() {
|
||||||
|
awk -v t="${1:-0}" -v r="${2:-0}" \
|
||||||
|
'BEGIN{ if (t>0) printf "%.1f", (t-r)*100/t; else print "NA" }'
|
||||||
|
}
|
||||||
|
|
||||||
dump_diagnostics() {
|
dump_diagnostics() {
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Peer / link snapshot ==="
|
echo "=== Peer / link snapshot ==="
|
||||||
@@ -402,6 +578,8 @@ compose() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
|
# Reap any in-flight stream tmpdir (e.g. on interrupt mid-window).
|
||||||
|
[ -n "${_STREAM_TMP:-}" ] && rm -rf "$_STREAM_TMP" 2>/dev/null
|
||||||
if [ "${FIPS_INTEROP_KEEP_UP:-}" = "1" ]; then
|
if [ "${FIPS_INTEROP_KEEP_UP:-}" = "1" ]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "FIPS_INTEROP_KEEP_UP=1 — leaving mesh running. Tear down with:"
|
echo "FIPS_INTEROP_KEEP_UP=1 — leaving mesh running. Tear down with:"
|
||||||
@@ -422,19 +600,31 @@ echo " FIPS Mixed-Version Interop Test"
|
|||||||
echo "=============================================================="
|
echo "=============================================================="
|
||||||
echo ""
|
echo ""
|
||||||
echo "Node-spec: $SPEC_STR ($NUM_NODES nodes, $NUM_PAIRS pairs, $NUM_DIRECTED directed)"
|
echo "Node-spec: $SPEC_STR ($NUM_NODES nodes, $NUM_PAIRS pairs, $NUM_DIRECTED directed)"
|
||||||
|
echo "Topology : $TOPOLOGY_KIND${TOPOLOGY_NAME:+ ($TOPOLOGY_NAME)}"
|
||||||
|
if [ "$TOPOLOGY_KIND" != "full-mesh" ]; then
|
||||||
|
echo " edges: ${INTEROP_TOPOLOGY_EDGES:-}"
|
||||||
|
fi
|
||||||
echo ""
|
echo ""
|
||||||
echo "Mesh nodes:"
|
echo "Mesh nodes:"
|
||||||
for n in "${NODES[@]}"; do
|
for n in "${NODES[@]}"; do
|
||||||
s="${SLOT_OF[$n]}"
|
s="${SLOT_OF[$n]}"
|
||||||
u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
|
u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
|
||||||
echo " $n slot $u ${SLOT_REF[$s]} @ ${SLOT_SHA[$s]} (${NODE_IP_OF[$n]})"
|
echo " $n slot $u ${SLOT_REF[$s]} @ ${SLOT_SHA[$s]} deg=${DEGREE_OF[$n]:-?} (${NODE_IP_OF[$n]})"
|
||||||
done
|
done
|
||||||
echo ""
|
echo ""
|
||||||
echo "Mesh pairs:"
|
echo "Mesh pairs:"
|
||||||
for p in "${PAIRS[@]}"; do
|
for p in "${PAIRS[@]}"; do
|
||||||
read -r n1 n2 <<< "$p"
|
read -r n1 n2 <<< "$p"
|
||||||
echo " $(pair_kind "$n1" "$n2") $(pair_label "$n1" "$n2")"
|
echo " $(pair_kind "$n1" "$n2") $(pair_label "$n1" "$n2") [$(hop_label "$n1" "$n2")]"
|
||||||
done
|
done
|
||||||
|
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
|
||||||
|
echo ""
|
||||||
|
echo "Data-plane streams (continuity across rekey):"
|
||||||
|
for sp in "${STREAM_PAIRS[@]}"; do
|
||||||
|
read -r sf st <<< "$sp"
|
||||||
|
echo " $sf -> $st [$(hop_label "$sf" "$st")]"
|
||||||
|
done
|
||||||
|
fi
|
||||||
echo ""
|
echo ""
|
||||||
if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then
|
if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then
|
||||||
echo "Netem impairment: $FIPS_INTEROP_NETEM"
|
echo "Netem impairment: $FIPS_INTEROP_NETEM"
|
||||||
@@ -474,24 +664,32 @@ echo ""
|
|||||||
echo "Phase 1: Link/session establishment + connectivity baseline"
|
echo "Phase 1: Link/session establishment + connectivity baseline"
|
||||||
PASSED=0; FAILED=0
|
PASSED=0; FAILED=0
|
||||||
|
|
||||||
# Every node must reach N-1 authenticated peers. A peer in `show peers`
|
# Every node must reach its DIRECT-peer degree of authenticated peers (a
|
||||||
# is an authenticated FMP-link peer.
|
# peer in `show peers` is an authenticated FMP-link peer). In a full mesh
|
||||||
|
# that degree is N-1; in a multi-hop topology it is the node's adjacency.
|
||||||
for n in "${NODES[@]}"; do
|
for n in "${NODES[@]}"; do
|
||||||
if ! wait_for_peers "${CONTAINER[$n]}" "$PEERS_EXPECTED" "$CONVERGENCE_TIMEOUT"; then
|
exp="${DEGREE_OF[$n]:-$PEERS_EXPECTED}"
|
||||||
echo " $n did not reach $PEERS_EXPECTED authenticated peers"
|
if ! wait_for_peers "${CONTAINER[$n]}" "$exp" "$CONVERGENCE_TIMEOUT"; then
|
||||||
|
echo " $n did not reach $exp authenticated direct peer(s)"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# The all-pairs ping over fips0 is the definitive FSP-session check: in
|
# The all-pairs ping over fips0 is the definitive reachability check.
|
||||||
# a full mesh every pair is a direct neighbor, so a successful ping
|
# Direct-neighbor pairs prove their FSP session; NON-adjacent pairs in a
|
||||||
# proves that pair's FSP session carries traffic in that direction.
|
# multi-hop topology can only succeed via forwarding, so all-pairs ping
|
||||||
if wait_for_full_baseline "$CONVERGENCE_TIMEOUT"; then
|
# is also the cross-version FORWARDING test (we keep pinging every pair).
|
||||||
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
|
# The convergence detector is an ADVISORY settle-wait, not the assertion.
|
||||||
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
|
# It needs one fully-clean, no-retry sweep of every directed pair; under
|
||||||
else
|
# packet loss that is statistically unlikely on a large mesh even when the
|
||||||
echo " Best baseline before timeout: $PASSED/$((PASSED + FAILED))"
|
# mesh is perfectly healthy (e.g. 30 pairs at 2% loss => ~55% of sweeps
|
||||||
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
|
# are clean), so its timeout must NOT be fatal. The strict, retrying ping
|
||||||
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
|
# below is the real baseline assertion — it decides pass/fail.
|
||||||
|
if ! wait_for_full_baseline "$CONVERGENCE_TIMEOUT"; then
|
||||||
|
echo " detector saw no fully-clean sweep within ${CONVERGENCE_TIMEOUT}s (best $PASSED/$NUM_DIRECTED); strict re-ping decides"
|
||||||
|
fi
|
||||||
|
ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline"
|
||||||
|
phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)"
|
||||||
|
if [ "$FAILED" -ne 0 ]; then
|
||||||
dump_diagnostics
|
dump_diagnostics
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
||||||
@@ -500,6 +698,34 @@ else
|
|||||||
fi
|
fi
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
# ── Phase 1b: data-plane control window (quiet, pre-rekey) ───────────
|
||||||
|
#
|
||||||
|
# Measure stream loss over a window with NO rekey cutover, as the control
|
||||||
|
# baseline for the differential. Validated cutover-free by confirming the
|
||||||
|
# FMP cutover count did not advance during the window. Then launch the
|
||||||
|
# rekey-window streams, which run in the background across Phases 2-5 and
|
||||||
|
# are collected/asserted in Phase 5b.
|
||||||
|
control_contaminated=0
|
||||||
|
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
|
||||||
|
echo "Phase 1b: Data-plane control stream (${CONTROL_STREAM_SECS}s quiet window)"
|
||||||
|
pre_cut="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"
|
||||||
|
launch_streams CONTROL "$CONTROL_STREAM_SECS"
|
||||||
|
collect_streams
|
||||||
|
post_cut="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"
|
||||||
|
if [ "$post_cut" -ne "$pre_cut" ]; then
|
||||||
|
control_contaminated=1
|
||||||
|
echo " WARN a rekey cutover occurred during the control window — control loss may be contaminated"
|
||||||
|
fi
|
||||||
|
for sp in "${STREAM_PAIRS[@]}"; do
|
||||||
|
read -r sf st <<< "$sp"
|
||||||
|
key="$sf->$st"
|
||||||
|
echo " control $key: tx=${STREAM_TX[CONTROL:$key]:-0} rx=${STREAM_RX[CONTROL:$key]:-0} loss=$(_loss_pct "${STREAM_TX[CONTROL:$key]:-0}" "${STREAM_RX[CONTROL:$key]:-0}")%"
|
||||||
|
done
|
||||||
|
echo " Launching rekey-window streams (${REKEY_STREAM_SECS}s, spanning Phases 2-5)"
|
||||||
|
launch_streams REKEY "$REKEY_STREAM_SECS"
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Phase 2: first rekey cycle ───────────────────────────────────────
|
# ── Phase 2: first rekey cycle ───────────────────────────────────────
|
||||||
|
|
||||||
echo "Phase 2: First rekey cycle (waiting up to ${FIRST_REKEY_TIMEOUT}s)"
|
echo "Phase 2: First rekey cycle (waiting up to ${FIRST_REKEY_TIMEOUT}s)"
|
||||||
@@ -544,6 +770,41 @@ ping_all_pairs "" "$MAX_PING_ATTEMPTS" "post-rekey-2"
|
|||||||
phase_result "Post-second-rekey (all $NUM_DIRECTED directed pairs)"
|
phase_result "Post-second-rekey (all $NUM_DIRECTED directed pairs)"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
# ── Phase 5b: data-plane continuity across rekey (control-differential)
|
||||||
|
#
|
||||||
|
# Collect the rekey-window streams launched in Phase 1b and compare their
|
||||||
|
# loss to the control window. A hitless rekey keeps rekey-window loss at
|
||||||
|
# or below the control (plus a small margin); excess loss is data the
|
||||||
|
# rekey dropped — the failure the cutover fixes (4af3730/6e5cb89) and the
|
||||||
|
# pipelined wire layout are supposed to prevent.
|
||||||
|
if [ "${#STREAM_PAIRS[@]}" -gt 0 ]; then
|
||||||
|
echo "Phase 5b: Data-plane continuity across rekey (control-differential, margin ${STREAM_LOSS_MARGIN_PCT}%)"
|
||||||
|
PASSED=0; FAILED=0
|
||||||
|
collect_streams
|
||||||
|
printf ' %-16s %11s %11s %8s %s\n' "stream" "ctrl-loss%" "rekey-loss%" "delta" "verdict"
|
||||||
|
for sp in "${STREAM_PAIRS[@]}"; do
|
||||||
|
read -r sf st <<< "$sp"
|
||||||
|
key="$sf->$st"
|
||||||
|
cpct="$(_loss_pct "${STREAM_TX[CONTROL:$key]:-0}" "${STREAM_RX[CONTROL:$key]:-0}")"
|
||||||
|
rpct="$(_loss_pct "${STREAM_TX[REKEY:$key]:-0}" "${STREAM_RX[REKEY:$key]:-0}")"
|
||||||
|
verdict="$(awk -v c="$cpct" -v k="$rpct" -v m="$STREAM_LOSS_MARGIN_PCT" 'BEGIN{
|
||||||
|
if (c=="NA" || k=="NA") { print "NODATA"; exit }
|
||||||
|
if (k <= c + m) print "PASS"; else print "FAIL"
|
||||||
|
}')"
|
||||||
|
delta="$(awk -v c="$cpct" -v k="$rpct" 'BEGIN{ if(c=="NA"||k=="NA") print "NA"; else printf "%+.1f", k-c }')"
|
||||||
|
printf ' %-16s %11s %11s %8s %s\n' "$key" "$cpct" "$rpct" "$delta" "$verdict"
|
||||||
|
if [ "$verdict" = "PASS" ]; then
|
||||||
|
PASSED=$((PASSED + 1))
|
||||||
|
else
|
||||||
|
FAILED=$((FAILED + 1))
|
||||||
|
INTEROP_FAILURES+=("[stream] $key ($(hop_label "$sf" "$st")): rekey-window loss ${rpct}% vs control ${cpct}% (+${STREAM_LOSS_MARGIN_PCT}% margin) -> $verdict")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
[ "${control_contaminated:-0}" -eq 1 ] && echo " NOTE: control window saw a cutover; differential may understate rekey loss."
|
||||||
|
phase_result "Data-plane continuity across rekey"
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Phase 6: per-node, per-pair interop log analysis ─────────────────
|
# ── Phase 6: per-node, per-pair interop log analysis ─────────────────
|
||||||
#
|
#
|
||||||
# This is the interop-specific analysis. For each unordered pair it
|
# This is the interop-specific analysis. For each unordered pair it
|
||||||
@@ -627,7 +888,7 @@ echo " -- Per-pair interop summary --"
|
|||||||
for p in "${PAIRS[@]}"; do
|
for p in "${PAIRS[@]}"; do
|
||||||
read -r n1 n2 <<< "$p"
|
read -r n1 n2 <<< "$p"
|
||||||
kind="$(pair_kind "$n1" "$n2")"
|
kind="$(pair_kind "$n1" "$n2")"
|
||||||
label="$(pair_label "$n1" "$n2")"
|
label="$(pair_label "$n1" "$n2") [$(hop_label "$n1" "$n2")]"
|
||||||
pair_failed=0
|
pair_failed=0
|
||||||
if [ "${#INTEROP_FAILURES[@]}" -gt 0 ]; then
|
if [ "${#INTEROP_FAILURES[@]}" -gt 0 ]; then
|
||||||
for f in "${INTEROP_FAILURES[@]}"; do
|
for f in "${INTEROP_FAILURES[@]}"; do
|
||||||
@@ -649,6 +910,52 @@ done
|
|||||||
phase_result "Interop log analysis"
|
phase_result "Interop log analysis"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
# ── Phase 7: mesh-size estimate convergence (strict ±25%) ────────────
|
||||||
|
#
|
||||||
|
# Each node's bloom-union mesh-size estimate (fipsctl show status
|
||||||
|
# .estimated_mesh_size) should converge to the true node count across
|
||||||
|
# versions. A mixed-version bloom/tree-encoding divergence shows up as a
|
||||||
|
# node that never produces an in-band estimate (or returns null). Strict
|
||||||
|
# band = [0.75N, 1.25N]; polled up to MESH_SIZE_TIMEOUT (the estimate
|
||||||
|
# converges over minutes — see ISSUE-2026-0046 on its transient jitter).
|
||||||
|
echo "Phase 7: Mesh-size estimate convergence (strict ±25% of true N=$NUM_NODES)"
|
||||||
|
PASSED=0; FAILED=0
|
||||||
|
ms_lo="$(awk -v n="$NUM_NODES" 'BEGIN{printf "%.2f", 0.75*n}')"
|
||||||
|
ms_hi="$(awk -v n="$NUM_NODES" 'BEGIN{printf "%.2f", 1.25*n}')"
|
||||||
|
echo " band [$ms_lo, $ms_hi], poll up to ${MESH_SIZE_TIMEOUT}s"
|
||||||
|
declare -A MS_EST MS_OK
|
||||||
|
ms_deadline=$(( SECONDS + MESH_SIZE_TIMEOUT ))
|
||||||
|
while :; do
|
||||||
|
all_ok=1
|
||||||
|
for n in "${NODES[@]}"; do
|
||||||
|
[ "${MS_OK[$n]:-0}" = "1" ] && continue
|
||||||
|
est="$(docker exec "${CONTAINER[$n]}" fipsctl show status 2>/dev/null \
|
||||||
|
| python3 -c "import sys,json; v=json.load(sys.stdin).get('estimated_mesh_size'); print(v if v is not None else 'null')" 2>/dev/null || echo null)"
|
||||||
|
MS_EST[$n]="$est"
|
||||||
|
if [ "$est" != "null" ] && awk "BEGIN{exit !($est>=$ms_lo && $est<=$ms_hi)}"; then
|
||||||
|
MS_OK[$n]=1
|
||||||
|
else
|
||||||
|
all_ok=0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
[ "$all_ok" = "1" ] && break
|
||||||
|
[ "$SECONDS" -ge "$ms_deadline" ] && break
|
||||||
|
sleep 3
|
||||||
|
done
|
||||||
|
for n in "${NODES[@]}"; do
|
||||||
|
s="${SLOT_OF[$n]}"; u="$(echo "$s" | tr '[:lower:]' '[:upper:]')"
|
||||||
|
if [ "${MS_OK[$n]:-0}" = "1" ]; then
|
||||||
|
echo " PASS $n [$u]: estimated_mesh_size=${MS_EST[$n]}"
|
||||||
|
PASSED=$((PASSED + 1))
|
||||||
|
else
|
||||||
|
echo " FAIL $n [$u]: estimated_mesh_size=${MS_EST[$n]:-null} (outside [$ms_lo,$ms_hi] after ${MESH_SIZE_TIMEOUT}s)"
|
||||||
|
FAILED=$((FAILED + 1))
|
||||||
|
INTEROP_FAILURES+=("[mesh-size] node $n ($u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}): estimate=${MS_EST[$n]:-null} outside [$ms_lo,$ms_hi]")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
phase_result "Mesh-size estimate convergence"
|
||||||
|
echo ""
|
||||||
|
|
||||||
# ── Summary ──────────────────────────────────────────────────────────
|
# ── Summary ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
echo "=============================================================="
|
echo "=============================================================="
|
||||||
|
|||||||
Reference in New Issue
Block a user