diff --git a/testing/interop/generate-configs.sh b/testing/interop/generate-configs.sh index dd32dd9..d21266a 100755 --- a/testing/interop/generate-configs.sh +++ b/testing/interop/generate-configs.sh @@ -18,7 +18,9 @@ # - IPv4 = 172.30.0.1, index = 0-based position in the spec # (10, 11, 12, 13, ...). # -# Topology is a FULL MESH: every node auto-connects to every other node. +# Topology is a FULL MESH by default: every node auto-connects to every +# other node. Set FIPS_INTEROP_EDGES (see Environment) to an explicit +# undirected edge list for a multi-hop / leaf topology instead. # Node identities are derived deterministically via the shared # derive_keys.py helper, keyed by a unique per-node string so that two # nodes of the same slot (a1, a2) still get DISTINCT identities. @@ -41,6 +43,13 @@ # Environment: # REKEY_AFTER_SECS FSP/FMP rekey interval (default 35, matching # the static rekey suite). +# FIPS_INTEROP_EDGES Optional undirected edge list (space/comma +# separated `nid-nid` tokens, e.g. +# "a1-b1 a1-c1 b1-a2"). Unset ⇒ full mesh. When +# set, each node peers only with its listed +# neighbors; the graph must be connected with no +# isolated node. Enables multi-hop forwarding / +# leaf-node topologies. # FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/, # .stress-runs/, generated-configs/). When # unset, falls back to in-tree paths under @@ -124,6 +133,79 @@ for slot in "${SPEC[@]}"; do index=$(( index + 1 )) done +# ── Topology: adjacency (full mesh by default, or explicit edges) ───── +# +# FIPS_INTEROP_EDGES, when set, is a space/comma list of `nid-nid` tokens +# describing UNDIRECTED edges (direct FMP peer links), e.g. +# "a1-b1 a1-c1 b1-a2". Empty ⇒ FULL MESH (every node peers with every +# other, the historical behavior). Edges are symmetric: `x-y` makes x and +# y each other's auto_connect peer. + +declare -A ADJ # ADJ[nid] = space-separated neighbor ids +declare -A _ADJ_SEEN # "x|y" dedup guard +TOPOLOGY="full-mesh" +EDGES_LIST=() + +add_edge() { + local x="$1" y="$2" + if [ "$x" = "$y" ]; then + echo "ERROR: self-edge '$x-$x' not allowed" >&2; exit 1 + fi + if [ -z "${NODE_SLOT[$x]:-}" ]; then + echo "ERROR: edge references unknown node '$x'" >&2; exit 1 + fi + if [ -z "${NODE_SLOT[$y]:-}" ]; then + echo "ERROR: edge references unknown node '$y'" >&2; exit 1 + fi + if [ -z "${_ADJ_SEEN[$x|$y]:-}" ]; then + ADJ[$x]="${ADJ[$x]:+${ADJ[$x]} }$y"; _ADJ_SEEN[$x|$y]=1 + fi + if [ -z "${_ADJ_SEEN[$y|$x]:-}" ]; then + ADJ[$y]="${ADJ[$y]:+${ADJ[$y]} }$x"; _ADJ_SEEN[$y|$x]=1 + fi +} + +if [ -n "${FIPS_INTEROP_EDGES:-}" ]; then + TOPOLOGY="custom" + for tok in ${FIPS_INTEROP_EDGES//,/ }; do + x="${tok%%-*}"; y="${tok#*-}" + if [ "$x" = "$tok" ] || [ -z "$x" ] || [ -z "$y" ] || [[ "$y" == *-* ]]; then + echo "ERROR: malformed edge token '$tok' (want nid-nid)" >&2; exit 1 + fi + add_edge "$x" "$y" + EDGES_LIST+=("$x-$y") + done + # Validate: no isolated node, graph connected (BFS from node 0). + for nid in "${NODE_IDS[@]}"; do + if [ -z "${ADJ[$nid]:-}" ]; then + echo "ERROR: node '$nid' has no edges (isolated)" >&2; exit 1 + fi + done + declare -A _BFS_SEEN=() + bfs_queue=("${NODE_IDS[0]}"); _BFS_SEEN[${NODE_IDS[0]}]=1; bfs_reached=1 + while [ "${#bfs_queue[@]}" -gt 0 ]; do + cur="${bfs_queue[0]}"; bfs_queue=("${bfs_queue[@]:1}") + for nb in ${ADJ[$cur]}; do + if [ -z "${_BFS_SEEN[$nb]:-}" ]; then + _BFS_SEEN[$nb]=1; bfs_reached=$(( bfs_reached + 1 )) + bfs_queue+=("$nb") + fi + done + done + if [ "$bfs_reached" -ne "${#NODE_IDS[@]}" ]; then + echo "ERROR: topology is disconnected ($bfs_reached/${#NODE_IDS[@]} reachable from ${NODE_IDS[0]})" >&2 + exit 1 + fi +else + # Full mesh: every node adjacent to every other, in spec order. + for a in "${NODE_IDS[@]}"; do + for b in "${NODE_IDS[@]}"; do + [ "$a" = "$b" ] && continue + ADJ[$a]="${ADJ[$a]:+${ADJ[$a]} }$b" + done + done +fi + mkdir -p "$OUT_DIR" # Clear stale per-node configs from a previous (differently-shaped) spec @@ -186,8 +268,7 @@ for nid in "${NODE_IDS[@]}"; do echo " mtu: 1472" echo "" echo "peers:" - for pid in "${NODE_IDS[@]}"; do - [ "$pid" = "$nid" ] && continue + for pid in ${ADJ[$nid]}; do emit_peer_block "$pid" done } > "$cfg" @@ -259,6 +340,19 @@ MANIFEST="$OUT_DIR/nodes.env" echo "# Node-spec: ${SPEC[*]}" echo "INTEROP_SPEC=\"${SPEC[*]}\"" echo "INTEROP_NODE_IDS=\"${NODE_IDS[*]}\"" + echo "INTEROP_TOPOLOGY=\"$TOPOLOGY\"" + echo "INTEROP_TOPOLOGY_EDGES=\"${EDGES_LIST[*]:-}\"" + # Per-node expected DIRECT-peer count (adjacency degree). The driver + # uses this for the per-node peer-convergence check instead of N-1. + { + printf 'INTEROP_NODE_DEGREE="' + for nid in "${NODE_IDS[@]}"; do + d=0 + for _nb in ${ADJ[$nid]}; do d=$(( d + 1 )); done + printf '%s:%s ' "$nid" "$d" + done + printf '"\n' + } # Per-node maps, one var each, space-separated 'nodeid:value' tokens. { printf 'INTEROP_NODE_SLOTS="' @@ -297,7 +391,11 @@ ENV_FILE="$OUT_DIR/npubs.env" echo " generated $ENV_FILE" echo "" -echo "Mesh configs ready (spec='${SPEC[*]}', rekey.after_secs=$REKEY_AFTER_SECS):" +echo "Mesh configs ready (spec='${SPEC[*]}', topology=$TOPOLOGY, rekey.after_secs=$REKEY_AFTER_SECS):" for nid in "${NODE_IDS[@]}"; do - echo " $nid slot ${NODE_SLOT[$nid]} ${NODE_IP[$nid]} ${NPUB[$nid]}" + d=0; for _nb in ${ADJ[$nid]}; do d=$(( d + 1 )); done + echo " $nid slot ${NODE_SLOT[$nid]} ${NODE_IP[$nid]} deg=$d ${NPUB[$nid]}" done +if [ "$TOPOLOGY" = "custom" ]; then + echo " edges: ${EDGES_LIST[*]}" +fi diff --git a/testing/interop/interop-stress.sh b/testing/interop/interop-stress.sh index b3554a3..6cf9895 100755 --- a/testing/interop/interop-stress.sh +++ b/testing/interop/interop-stress.sh @@ -26,9 +26,13 @@ # fixed Docker network, so two reps must never overlap. # # Usage: -# ./interop-stress.sh [--reps N] [node-spec...] +# ./interop-stress.sh [--reps N] [--topology | node-spec...] # # --reps N repetitions (default 10). +# --topology built-in multi-hop topology forwarded to interop-test.sh +# (e.g. multihop-3v-cycle). Mutually exclusive with a +# positional node-spec. Adds multi-hop forwarding, +# data-plane continuity, and mesh-size checks to each rep. # node-spec slot letters (a/b/c), default `a a b c` (the control # topology — one same-version pair + five mixed pairs). # @@ -81,8 +85,17 @@ RUNS_BASE="$INTEROP_RUNS_BASE/.stress-runs" REPS=10 SPEC=() +TOPOLOGY_ARG="" while [ "$#" -gt 0 ]; do case "$1" in + --topology) + TOPOLOGY_ARG="${2:-}" + shift 2 + ;; + --topology=*) + TOPOLOGY_ARG="${1#--topology=}" + shift + ;; --reps) REPS="${2:-}" if ! [[ "$REPS" =~ ^[0-9]+$ ]] || [ "$REPS" -lt 1 ]; then @@ -119,10 +132,23 @@ while [ "$#" -gt 0 ]; do esac done -if [ "${#SPEC[@]}" -eq 0 ]; then - SPEC=(a a b c) +# A --topology selects spec + edges inside the driver; otherwise use the +# positional node-spec (default the control-arm topology `a a b c`). +DRIVER_ARGS=() +if [ -n "$TOPOLOGY_ARG" ]; then + if [ "${#SPEC[@]}" -gt 0 ]; then + echo "ERROR: pass either --topology or a node-spec, not both" >&2 + exit 1 + fi + DRIVER_ARGS=(--topology "$TOPOLOGY_ARG") + SPEC_STR="(topology: $TOPOLOGY_ARG)" +else + if [ "${#SPEC[@]}" -eq 0 ]; then + SPEC=(a a b c) + fi + DRIVER_ARGS=("${SPEC[@]}") + SPEC_STR="${SPEC[*]}" fi -SPEC_STR="${SPEC[*]}" # ── Preflight ──────────────────────────────────────────────────────── @@ -175,7 +201,7 @@ for ((rep = 1; rep <= REPS; rep++)); do # Run the driver, capturing full output and exit code. Netem is # passed through the environment; interop-test.sh applies it. FIPS_INTEROP_NETEM="${FIPS_INTEROP_NETEM:-}" \ - bash "$DRIVER" "${SPEC[@]}" >"$driver_log" 2>&1 + bash "$DRIVER" "${DRIVER_ARGS[@]}" >"$driver_log" 2>&1 rc=$? if [ "$rc" -eq 0 ]; then diff --git a/testing/interop/interop-test.sh b/testing/interop/interop-test.sh index ecad4f6..177532c 100755 --- a/testing/interop/interop-test.sh +++ b/testing/interop/interop-test.sh @@ -24,18 +24,39 @@ # 2. configs are (re)generated automatically below for the given spec. # # Usage: -# ./interop-test.sh [node-spec...] +# ./interop-test.sh [--topology ] [node-spec...] # +# --topology built-in multi-hop topology selecting a node-spec + edge +# set + default data-plane streams. Known: multihop-3v-cycle +# (6 nodes, 2 of each version, one cycle, two leaves — +# exercises cross-version forwarding + mesh-size). Without +# it the mesh is a full mesh from the positional node-spec. # node-spec space-separated slot letters (a/b/c), default `a b c`. # e.g. `a a b c` (4-node, one same-version control pair), # `a a a` (3-node same-version flake rig). # +# Beyond FMP/FSP rekey survival, a --topology run also verifies multi-hop +# FORWARDING (all-pairs ping over non-adjacent pairs), DATA-PLANE +# CONTINUITY across rekey (control-differential ping-loss stream, +# Phase 5b), and MESH-SIZE estimate convergence across versions +# (Phase 7). +# # Environment: # FIPS_INTEROP_NETEM tc-netem arg string applied to every container's # eth0, e.g. "delay 10ms 5ms 25% loss 1%". Unset = # no impairment. +# FIPS_INTEROP_EDGES explicit undirected edge list (overrides the +# full mesh) — see generate-configs.sh. Usually +# set for you by --topology. +# FIPS_INTEROP_STREAMS data-plane stream pairs (`nid-nid` tokens, both +# directions streamed). Enables Phase 1b/5b. Set +# by --topology; empty = streams off. +# STREAM_LOSS_MARGIN_PCT rekey-vs-control loss margin (default 1). +# CONTROL_STREAM_SECS quiet control-window length (default 12). +# MESH_SIZE_TIMEOUT Phase 7 convergence poll budget (default 180). # FIPS_INTEROP_KEEP_UP 1 = leave containers running after the test (debug). -# REKEY_AFTER_SECS rekey interval to generate configs with (default 35). +# REKEY_AFTER_SECS rekey interval to generate configs with (default +# 35; multihop-3v-cycle defaults it to 50). # FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/, # .stress-runs/, generated-configs/). When # unset, falls back to in-tree paths under @@ -74,6 +95,48 @@ COMPOSE_FILE="$GEN_DIR/docker-compose.generated.yml" NODES_ENV="$GEN_DIR/nodes.env" REFS_ENV="$RUNS_BASE/.build/refs.env" +# ── Built-in topology selection ────────────────────────────────────── +# +# --topology selects a node-spec + an explicit edge set (a +# multi-hop, mixed-version, leaf-bearing graph) + a default data-plane +# stream set, exercising forwarding / routing / mesh-size convergence +# across versions. Without it, positional args are the node-spec and the +# mesh is a full mesh (historical behavior). +TOPOLOGY_NAME="" +_ARGS=() +while [ "$#" -gt 0 ]; do + case "$1" in + --topology) TOPOLOGY_NAME="${2:-}"; shift 2 ;; + --topology=*) TOPOLOGY_NAME="${1#--topology=}"; shift ;; + *) _ARGS+=("$1"); shift ;; + esac +done +set -- ${_ARGS[@]+"${_ARGS[@]}"} + +if [ -n "$TOPOLOGY_NAME" ]; then + case "$TOPOLOGY_NAME" in + multihop-3v-cycle) + # 6 nodes (2 of each version), one cycle, two leaves. + # a1 + # / \ leaves: a2, c2 + # b1 c1 cycle: b1-a1-c1-b2-b1 + # / \ \ + # a2 b2------c2-edge(b2-c1) + set -- a a b b c c + export FIPS_INTEROP_EDGES="a1-b1 a1-c1 b1-a2 b1-b2 c1-c2 b2-c1" + : "${FIPS_INTEROP_STREAMS:=a2-c2 a1-b1}" + export FIPS_INTEROP_STREAMS + # Multi-hop convergence + a clean pre-rekey control window + # want more headroom than the 35s full-mesh default. + : "${REKEY_AFTER_SECS:=50}" + ;; + *) + echo "ERROR: unknown --topology '$TOPOLOGY_NAME' (known: multihop-3v-cycle)" >&2 + exit 2 + ;; + esac +fi + REKEY_AFTER_SECS="${REKEY_AFTER_SECS:-35}" # ── Node-spec ──────────────────────────────────────────────────────── @@ -86,7 +149,7 @@ SPEC_STR="${SPEC[*]}" # ── Timing ─────────────────────────────────────────────────────────── -CONVERGENCE_TIMEOUT=60 # wait for full mesh + clean ping baseline +CONVERGENCE_TIMEOUT="${CONVERGENCE_TIMEOUT:-90}" # detector settle budget (env-overridable; larger meshes under loss converge slower) PING_TIMEOUT=5 MAX_PING_ATTEMPTS=4 # strict-ping retry (mirrors rekey-test.sh) PING_RETRY_DELAY=1 @@ -101,6 +164,19 @@ REKEY_SETTLE=12 # FSP-cutover settle budget (Phase 6) POST_REKEY_TIMEOUT=45 LOG_POLL_INTERVAL=2 +# Data-plane continuity stream (control-differential). Streams run a +# sustained ping6 over the overlay across the rekey window vs a quiet +# control window; loss is compared to prove rekey is hitless. +STREAM_RATE_HZ=5 # ping6 -i 0.2 +CONTROL_STREAM_SECS="${CONTROL_STREAM_SECS:-12}" # quiet pre-rekey window +STREAM_LOSS_MARGIN_PCT="${STREAM_LOSS_MARGIN_PCT:-1}" +# The rekey-window stream must span Phases 2-5 (both cutovers + reconverge). +REKEY_STREAM_SECS=$(( FIRST_REKEY_TIMEOUT + SECOND_REKEY_WAIT + POST_REKEY_TIMEOUT + 15 )) + +# Mesh-size estimate convergence (strict ±25% of true N). Generous poll +# budget — the bloom-union estimate converges over minutes. +MESH_SIZE_TIMEOUT="${MESH_SIZE_TIMEOUT:-180}" + # ── Counters ───────────────────────────────────────────────────────── PASSED=0 @@ -140,7 +216,9 @@ done need_regen=1 if [ -f "$NODES_ENV" ]; then existing_spec="$(sed -n 's/^INTEROP_SPEC="\(.*\)"$/\1/p' "$NODES_ENV")" - if [ "$existing_spec" = "$SPEC_STR" ]; then + existing_edges="$(sed -n 's/^INTEROP_TOPOLOGY_EDGES="\(.*\)"$/\1/p' "$NODES_ENV")" + if [ "$existing_spec" = "$SPEC_STR" ] \ + && [ "$existing_edges" = "${FIPS_INTEROP_EDGES:-}" ]; then need_regen=0 fi fi @@ -179,6 +257,41 @@ parse_map CONTAINER "$INTEROP_NODE_CONTAINERS" parse_map NODE_IP_OF "$INTEROP_NODE_IPS" parse_map NPUB_OF "$INTEROP_NODE_NPUBS" +# Topology metadata: per-node expected direct-peer degree, and the +# undirected direct-edge set (for direct-vs-routed pair labeling). +TOPOLOGY_KIND="${INTEROP_TOPOLOGY:-full-mesh}" +declare -A DEGREE_OF +parse_map DEGREE_OF "${INTEROP_NODE_DEGREE:-}" + +declare -A IS_DIRECT_EDGE +if [ -n "${INTEROP_TOPOLOGY_EDGES:-}" ]; then + for _e in $INTEROP_TOPOLOGY_EDGES; do + _x="${_e%%-*}"; _y="${_e#*-}" + IS_DIRECT_EDGE["$_x|$_y"]=1 + IS_DIRECT_EDGE["$_y|$_x"]=1 + done +else + # Full mesh: every ordered pair is a direct edge. + for _a in "${NODES[@]}"; do + for _b in "${NODES[@]}"; do + [ "$_a" = "$_b" ] && continue + IS_DIRECT_EDGE["$_a|$_b"]=1 + done + done +fi + +pair_is_direct() { [ -n "${IS_DIRECT_EDGE[$1|$2]:-}" ]; } +hop_label() { if pair_is_direct "$1" "$2"; then echo "direct"; else echo "routed"; fi; } + +# Data-plane stream directed pairs (both directions of each token). +STREAM_PAIRS=() +if [ -n "${FIPS_INTEROP_STREAMS:-}" ]; then + for _tok in ${FIPS_INTEROP_STREAMS//,/ }; do + _x="${_tok%%-*}"; _y="${_tok#*-}" + STREAM_PAIRS+=("$_x $_y" "$_y $_x") + done +fi + # Unordered pairs of the mesh. PAIRS=() for ((i = 0; i < ${#NODES[@]}; i++)); do @@ -309,9 +422,12 @@ ping_all_pairs() { # every rep that takes a moment to converge. Only the # strict assertion sweeps (baseline / post-rekey-*) record. if [ "$context" != "convergence" ]; then - local kind + local kind hop kind="$(pair_kind "$i" "$j")" - INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j"): ping $i->$j FAILED") + hop="$(hop_label "$i" "$j")" + # NOTE: keep "$kind pair" contiguous — interop-stress.sh + # greps "MIXED pair"/"same pair". The [hop] tag is additive. + INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j") [$hop]: ping $i->$j FAILED") fi fi done @@ -376,6 +492,66 @@ wait_for_log_pattern_count() { [ "$(count_log_pattern "$pattern")" -ge "$min_count" ] } +# ── Data-plane continuity streams ──────────────────────────────────── +# +# A sustained ping6 stream over the overlay (.fips) is data-plane +# traffic: it traverses TUN → FSP session → FMP link → forwarding, the +# same path application packets take. We run streams across the rekey +# window and across a quiet control window, then compare loss — a rekey +# that drops data shows up as rekey-window loss above the control. + +declare -A STREAM_TX STREAM_RX +_STREAM_PIDS=() +_STREAM_FILES=() # "label:from->to:resultfile" +_STREAM_TMP="" + +# One directed ping6 stream of s at STREAM_RATE_HZ; writes "tx rx". +_stream_one() { + local from="$1" to="$2" dur="$3" outfile="$4" + local from_ctr="${CONTAINER[$from]}" to_npub="${NPUB_OF[$to]}" + local count=$(( dur * STREAM_RATE_HZ )) + local out tx rx + out=$(docker exec "$from_ctr" ping6 -i 0.2 -c "$count" -W "$PING_TIMEOUT" \ + "${to_npub}.fips" 2>&1) + tx=$(echo "$out" | grep -oE '[0-9]+ packets transmitted' | grep -oE '^[0-9]+') + rx=$(echo "$out" | grep -oE '[0-9]+ received' | grep -oE '^[0-9]+') + echo "${tx:-0} ${rx:-0}" > "$outfile" +} + +# Launch all stream pairs in the background for