From 38f003bc6f0fcde71d16892252ff1e4093dfe7a8 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Tue, 25 Aug 2026 20:25:43 +0100 Subject: [PATCH] Make the interop gate's Phase 7 assert a settled mesh-size estimate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 7 was labelled "mesh-size estimate convergence" but recorded each node's first in-band reading: once a node landed inside [0.75N, 1.25N] it was marked OK and never polled again, and the loop exited as soon as every node had been in band once. The sample was therefore taken inside the documented bloom-filter warmup — the design notes put that at roughly five minutes from node start and call estimates unreliable before it — so the phase could report a converged mesh from a measurement that cannot show convergence. The v0.4.2 multihop run printed 6/6 with two of six nodes reporting 5. The phase now waits out the warmup before any reading counts (MESH_SIZE_WARMUP, default 300s, measured from `compose up` rather than from the phase's own start), then requires each node to hold the band unbroken for MESH_SIZE_SETTLE (default 60s). Every node is re-polled every round and an out-of-band reading restarts that node's settle clock, so a node can no longer be credited for a sample it has since contradicted. MESH_SIZE_TIMEOUT is now the extra budget on top of those two windows. The warmup is polled and printed rather than slept through, which records the estimate trajectory no run has ever captured. The band is unchanged: whether the undercounting nodes would reach N given time is not established, and tightening it without that evidence would only add an unexplained red. A closing NOTE reports whether the nodes agree on a value, since a green asserts per-node plausibility and not agreement — which is how a prior release's evidence came to be read. (cherry picked from commit ee978c1a57b2bfc0f7e33a0475806d19fcac3935) --- testing/interop/interop-test.sh | 91 +++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 15 deletions(-) diff --git a/testing/interop/interop-test.sh b/testing/interop/interop-test.sh index 83c1ac9f..8c4731a9 100755 --- a/testing/interop/interop-test.sh +++ b/testing/interop/interop-test.sh @@ -53,7 +53,12 @@ # by --topology; empty = streams off. # STREAM_LOSS_MARGIN_PCT rekey-vs-control loss margin (default 1). # CONTROL_STREAM_SECS quiet control-window length (default 12). -# MESH_SIZE_TIMEOUT Phase 7 convergence poll budget (default 180). +# MESH_SIZE_WARMUP Phase 7 bloom warmup, from mesh start, before +# any estimate counts (default 300). +# MESH_SIZE_SETTLE Phase 7 unbroken in-band window a node must +# hold to be credited (default 60). +# MESH_SIZE_TIMEOUT Phase 7 poll budget beyond warmup+settle +# (default 180). # FIPS_INTEROP_KEEP_UP 1 = leave containers running after the test (debug). # REKEY_AFTER_SECS rekey interval to generate configs with (default # 35; multihop-3v-cycle defaults it to 50). @@ -180,9 +185,17 @@ STREAM_LOSS_MARGIN_PCT="${STREAM_LOSS_MARGIN_PCT:-1}" # The rekey-window stream must span Phases 2-5 (both cutovers + reconverge). REKEY_STREAM_SECS=$(( FIRST_REKEY_TIMEOUT + SECOND_REKEY_WAIT + POST_REKEY_TIMEOUT + 15 )) -# Mesh-size estimate convergence (strict ±25% of true N). Generous poll -# budget — the bloom-union estimate converges over minutes. +# Mesh-size estimate convergence (strict ±25% of true N). The archived +# design note puts bloom-filter warmup at ~5 minutes from node start and +# calls the estimate unreliable before that, so nothing sampled inside +# MESH_SIZE_WARMUP is evidence of convergence, and a node is credited +# only after holding the band unbroken for MESH_SIZE_SETTLE afterwards. +# MESH_SIZE_TIMEOUT is the extra budget for reaching that state, on top +# of the warmup and settle windows rather than covering them. +MESH_SIZE_WARMUP="${MESH_SIZE_WARMUP:-300}" +MESH_SIZE_SETTLE="${MESH_SIZE_SETTLE:-60}" MESH_SIZE_TIMEOUT="${MESH_SIZE_TIMEOUT:-180}" +MESH_SIZE_POLL=5 # ── Counters ───────────────────────────────────────────────────────── @@ -512,6 +525,14 @@ wait_for_log_pattern_count() { [ "$(count_log_pattern "$pattern")" -ge "$min_count" ] } +# One node's bloom-union mesh-size estimate, or "null" when the daemon +# has no estimate yet or cannot be reached. +mesh_estimate() { + docker exec "${CONTAINER[$1]}" fipsctl show status 2>/dev/null \ + | python3 -c "import sys,json; v=json.load(sys.stdin).get('estimated_mesh_size'); print(v if v is not None else 'null')" 2>/dev/null \ + || echo null +} + # ── Data-plane continuity streams ──────────────────────────────────── # # A sustained ping6 stream over the overlay (.fips) is data-plane @@ -661,6 +682,9 @@ if ! compose up -d; then echo " FAIL compose up failed" exit 1 fi +# Phase 7 measures bloom-filter warmup from node start, not from its own +# start, so the clock has to be taken here. +MESH_UP_AT=$SECONDS # Optional netem: applied via `docker exec ... tc qdisc` on each # container's eth0 — host bridge qdisc does NOT shape inter-container @@ -948,43 +972,80 @@ echo "" # .estimated_mesh_size) should converge to the true node count across # versions. A mixed-version bloom/tree-encoding divergence shows up as a # node that never produces an in-band estimate (or returns null). Strict -# band = [0.75N, 1.25N]; polled up to MESH_SIZE_TIMEOUT (the estimate -# converges over minutes and is transiently jittery). +# band = [0.75N, 1.25N]. +# +# The verdict is taken on a settled estimate rather than on a node's +# first tolerable reading. Two windows enforce that. Nothing sampled +# before MESH_SIZE_WARMUP has elapsed since the mesh came up counts at +# all, because the estimate is documented as unreliable during warmup; +# after it, a node is credited only once it has held the band unbroken +# for MESH_SIZE_SETTLE. Every node is re-polled every round and any +# out-of-band reading restarts that node's settle clock, so a node +# cannot be credited for a sample it has since contradicted. echo "Phase 7: Mesh-size estimate convergence (strict ±25% of true N=$NUM_NODES)" PASSED=0; FAILED=0 ms_lo="$(awk -v n="$NUM_NODES" 'BEGIN{printf "%.2f", 0.75*n}')" ms_hi="$(awk -v n="$NUM_NODES" 'BEGIN{printf "%.2f", 1.25*n}')" -echo " band [$ms_lo, $ms_hi], poll up to ${MESH_SIZE_TIMEOUT}s" -declare -A MS_EST MS_OK -ms_deadline=$(( SECONDS + MESH_SIZE_TIMEOUT )) +declare -A MS_EST MS_OK MS_INBAND_SINCE +ms_accept_after=$(( MESH_UP_AT + MESH_SIZE_WARMUP )) +echo " band [$ms_lo, $ms_hi], warmup ends $(( ms_accept_after - SECONDS ))s from now, then ${MESH_SIZE_SETTLE}s settled, poll up to ${MESH_SIZE_TIMEOUT}s beyond that" + +# Poll through the warmup as well. Nothing here is asserted on — it is +# the trajectory the phase has never recorded, and it is what an +# undercount that never recovers would show up in. +while [ "$SECONDS" -lt "$ms_accept_after" ]; do + ms_line="" + for n in "${NODES[@]}"; do + MS_EST[$n]="$(mesh_estimate "$n")" + ms_line+=" $n=${MS_EST[$n]}" + done + echo " warmup, $(( ms_accept_after - SECONDS ))s to go:$ms_line" + sleep 30 +done + +ms_deadline=$(( SECONDS + MESH_SIZE_SETTLE + MESH_SIZE_TIMEOUT )) while :; do all_ok=1 for n in "${NODES[@]}"; do - [ "${MS_OK[$n]:-0}" = "1" ] && continue - est="$(docker exec "${CONTAINER[$n]}" fipsctl show status 2>/dev/null \ - | python3 -c "import sys,json; v=json.load(sys.stdin).get('estimated_mesh_size'); print(v if v is not None else 'null')" 2>/dev/null || echo null)" + est="$(mesh_estimate "$n")" MS_EST[$n]="$est" if [ "$est" != "null" ] && awk "BEGIN{exit !($est>=$ms_lo && $est<=$ms_hi)}"; then + [ -z "${MS_INBAND_SINCE[$n]:-}" ] && MS_INBAND_SINCE[$n]="$SECONDS" + else + unset "MS_INBAND_SINCE[$n]" + fi + if [ -n "${MS_INBAND_SINCE[$n]:-}" ] \ + && [ $(( SECONDS - ${MS_INBAND_SINCE[$n]} )) -ge "$MESH_SIZE_SETTLE" ]; then MS_OK[$n]=1 else + MS_OK[$n]=0 all_ok=0 fi done [ "$all_ok" = "1" ] && break [ "$SECONDS" -ge "$ms_deadline" ] && break - sleep 3 + sleep "$MESH_SIZE_POLL" done for n in "${NODES[@]}"; do s="${SLOT_OF[$n]}"; u="$(echo "$s" | tr '[:lower:]' '[:upper:]')" if [ "${MS_OK[$n]:-0}" = "1" ]; then - echo " PASS $n [$u]: estimated_mesh_size=${MS_EST[$n]}" + echo " PASS $n [$u]: estimated_mesh_size=${MS_EST[$n]} (held band ${MESH_SIZE_SETTLE}s+ after warmup)" PASSED=$((PASSED + 1)) else - echo " FAIL $n [$u]: estimated_mesh_size=${MS_EST[$n]:-null} (outside [$ms_lo,$ms_hi] after ${MESH_SIZE_TIMEOUT}s)" + if [ -n "${MS_INBAND_SINCE[$n]:-}" ]; then + ms_why="held band only $(( SECONDS - ${MS_INBAND_SINCE[$n]} ))s of the ${MESH_SIZE_SETTLE}s settle window" + else + ms_why="outside [$ms_lo,$ms_hi]" + fi + echo " FAIL $n [$u]: estimated_mesh_size=${MS_EST[$n]:-null} ($ms_why)" FAILED=$((FAILED + 1)) - INTEROP_FAILURES+=("[mesh-size] node $n ($u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}): estimate=${MS_EST[$n]:-null} outside [$ms_lo,$ms_hi]") + INTEROP_FAILURES+=("[mesh-size] node $n ($u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}): estimate=${MS_EST[$n]:-null} $ms_why") fi done +# The band is per-node, so a green says every node was plausible, not +# that the nodes agreed. Print the spread so nobody has to infer it. +ms_spread="$(printf '%s\n' "${MS_EST[@]}" | sort -n | awk '/^[0-9]+$/{ if (lo=="") lo=$1; hi=$1 } END{ if (lo=="") print "no numeric estimates"; else if (lo==hi) print "all nodes agree on " lo; else print "nodes disagree: " lo " to " hi }')" +echo " NOTE: final estimates — $ms_spread (agreement is reported, not asserted)" phase_result "Mesh-size estimate convergence" echo ""