diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f05b0e..6be1685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Coord cache invalidation made surgical at parent-position-change + and root-change sites. Replaces the previous unconditional + `CoordCache::clear()` calls with two targeted methods: + `invalidate_via_node(node_addr)` (drops entries whose cached + ancestry contains the changed node, used at parent-switch / + become-root / loop-detection sites) and `invalidate_other_roots` + (drops entries from a different tree, used at root-change sites). + The previous global flush left `find_next_hop` returning `None` + for every non-direct-peer destination after every parent switch + until the cache passively re-warmed; surgical invalidation + preserves entries that remain correct across the topology change. + Peer-removal retains the original "no invalidation" behavior + (`find_next_hop` already recomputes against the current peer set + every call, and Discovery handles "no route" on demand). +- Rekey integration test (`testing/static/scripts/rekey-test.sh`): + Phase 1, Phase 3, and Phase 5 strict per-pair pings now retry up + to 4 attempts (configurable via `MAX_PING_ATTEMPTS` / + `PING_RETRY_DELAY`). Under low-level packet loss (1% per + direction), single-shot 20-pair ping_all misses with probability + ~33% per phase from ICMP noise alone, masking the routing-state + signal the asserts target. The 4-attempt retry brings that floor + to ~3.2e-6 per phase. The `wait_for_full_baseline` convergence + loop itself stays single-shot — retries there would conflate + transient ping loss with still-converging routing state. Test + scaffold only; no daemon code changes. - Apply ±15s symmetric jitter per session to the FMP and FSP rekey timer trigger. Eliminates the steady-state dual-initiation race in symmetric-start meshes; previously the smaller-NodeAddr diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index 6465b39..28ca83c 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -157,6 +157,23 @@ LOG_EVENT_POLL_INTERVAL=1 TIMEOUT=5 CONVERGENCE_PING_TIMEOUT=1 +# Strict-ping retry policy for the per-phase ping_all asserts. Under 1% +# i.i.d. packet loss (the lab condition surfaced by ISSUE-2026-0028) a +# single-shot ping fails at roughly 2% per directed pair, so a 20-pair +# strict assert misses with probability ~1 - (0.98)^20 ≈ 33%, which is +# below the per-pair loss-math floor but well above the routing-state +# signal we want to measure. Retrying each failing pair up to +# MAX_PING_ATTEMPTS-1 additional times with ~PING_RETRY_DELAY between +# attempts drops the loss-math floor to ~(0.02)^MAX_PING_ATTEMPTS per +# pair, making any residual failure attributable to a non-loss +# mechanism rather than ICMP noise. Applied to Phase 1 (final strict +# ping_all after wait_for_full_baseline converges) and Phase 3 / Phase +# 5 (post-rekey strict asserts). The wait_for_full_baseline convergence +# loop itself stays single-shot — its job is to detect when the mesh +# first sees a fully clean 20-pair batch, and retries inside the loop +# would conflate "transient ping loss" with "still converging." +MAX_PING_ATTEMPTS=4 +PING_RETRY_DELAY=1 PASSED=0 FAILED=0 TOTAL_PASSED=0 @@ -181,25 +198,43 @@ ping_one() { local label="$3" local quiet="${4:-}" local ping_timeout="${5:-$TIMEOUT}" + local max_attempts="${6:-1}" - if output=$(docker exec "fips-$from" ping6 -c 1 -W "$ping_timeout" "${to_npub}.fips" 2>&1); then - local rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2) - if [ -z "$quiet" ]; then - echo " $label ... OK (${rtt:-?}ms)" + local attempt=1 + local output rtt + while (( attempt <= max_attempts )); do + if (( attempt > 1 )); then + sleep "$PING_RETRY_DELAY" fi - PASSED=$((PASSED + 1)) - else - if [ -z "$quiet" ]; then + if output=$(docker exec "fips-$from" ping6 -c 1 -W "$ping_timeout" "${to_npub}.fips" 2>&1); then + rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2) + if [ -z "$quiet" ]; then + if (( attempt == 1 )); then + echo " $label ... OK (${rtt:-?}ms)" + else + echo " $label ... OK (${rtt:-?}ms, attempt $attempt)" + fi + fi + PASSED=$((PASSED + 1)) + return + fi + attempt=$((attempt + 1)) + done + if [ -z "$quiet" ]; then + if (( max_attempts > 1 )); then + echo " $label ... FAIL (after $max_attempts attempts)" + else echo " $label ... FAIL" fi - FAILED=$((FAILED + 1)) fi + FAILED=$((FAILED + 1)) } # Run all 20 directed pairs ping_all() { local quiet="${1:-}" local ping_timeout="${2:-$TIMEOUT}" + local max_attempts="${3:-1}" PASSED=0 FAILED=0 for i in 0 1 2 3 4; do @@ -209,7 +244,7 @@ ping_all() { for j in 0 1 2 3 4; do [ "$i" -eq "$j" ] && continue ping_one "node-${LABELS[$i],,}" "${NPUBS[$j]}" \ - "${LABELS[$i]} → ${LABELS[$j]}" "$quiet" "$ping_timeout" + "${LABELS[$i]} → ${LABELS[$j]}" "$quiet" "$ping_timeout" "$max_attempts" done done } @@ -332,7 +367,7 @@ echo "" echo "Phase 1: Pre-rekey connectivity (waiting for convergence)" wait_for_peers fips-node-a 2 "$BASELINE_CONVERGENCE_TIMEOUT" || true if wait_for_full_baseline "$BASELINE_CONVERGENCE_TIMEOUT"; then - ping_all + ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS" phase_result "Pre-rekey baseline (all 20 pairs)" else echo " Best observed baseline before timeout: $PASSED/$((PASSED + FAILED)) passed" @@ -359,7 +394,7 @@ echo "" # Verify connectivity after first rekey (strict — no failures allowed) echo "Phase 3: Post-rekey connectivity (settling ${REKEY_SETTLE}s)" sleep "$REKEY_SETTLE" -ping_all +ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS" phase_result "Post-first-rekey (all 20 pairs)" echo "" @@ -370,7 +405,7 @@ sleep "$SECOND_REKEY_WAIT" # Verify connectivity after second rekey (back-to-back) echo "Phase 5: Post-second-rekey connectivity (settling ${REKEY_SETTLE}s)" sleep "$REKEY_SETTLE" -ping_all +ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS" phase_result "Post-second-rekey (all 20 pairs)" echo ""