From ffd78440a8f5070c5d04d0a7e909285d2e62a446 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Tue, 26 May 2026 02:50:51 +0000 Subject: [PATCH] node: periodically re-broadcast TreeAnnounce on no-change in check_periodic_parent_reeval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the eventually-consistent gap in spanning-tree state distribution. Every existing send_tree_announce_to_all call site gates on a local state-change event (parent switch, self-root promotion, ancestry change, peer promotion, parent loss). Once a partition latches — for example a parent-switch announce stranded in the brief cross-init handshake swap window, where the announce arrives on a session-index whose decrypt-worker entry has been unregistered — neither side's state changes again, so neither side ever re-broadcasts. The existing 60 s check_periodic_parent_reeval was a re-evaluation, not a re-broadcast: it short-circuited silently on no-change. Production-side healing depended on incidental link churn; lab harnesses with stable docker-bridge links had no equivalent path. Add a final else branch that fires send_tree_announce_to_all unconditionally on the no-change path, alongside the existing switch and self-promote arms. Receivers coalesce by sequence comparison (ParentDeclaration::is_fresher_than) and short-circuit at the `if !updated` gate in handle_tree_announce; same-sequence repeats drop silently with no cascade. The per-peer 500 ms rate-limiter is well below this 60 s cadence and does not suppress the heartbeat broadcast. The fix is a general protocol-robustness improvement: it addresses any in-flight TreeAnnounce loss class, not only the specific cross-init swap-window drop site. testing/static/scripts/rekey-test.sh BASELINE_CONVERGENCE_TIMEOUT 60 -> 65 so a partition healed by the periodic broadcast at T+60 lands inside the convergence window. wait_for_full_baseline early-exits on PASS, so successful reps see no extra wall-clock. --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ src/node/tree.rs | 16 ++++++++++++++++ testing/static/scripts/rekey-test.sh | 10 ++++++++-- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a276ce5..d0d87b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Spanning-tree state distribution is now eventually-consistent. + Previously every `send_tree_announce_to_all` call site fired only + on a local state-change event (parent switch, self-root promotion, + ancestry change, peer promotion, parent loss). Once a partition + latched — for example, a parent-switch announce lost in transit + via the brief cross-init handshake swap window where one peer's + outbound session is about to become the loser session and the + receiver has no matching decrypt-worker entry — no node's state + changed again, so no node ever re-broadcast. The existing 60-second + `check_periodic_parent_reeval` short-circuited silently on no-change + (it was a re-evaluation, not a re-broadcast), and production-side + healing depended on incidental link churn (NAT keepalive refresh, + MMP timeout, peer re-promotion after a transport blip). The + function now ends with an unconditional `send_tree_announce_to_all` + on the no-change branch, alongside the existing switch and + self-promote arms; receivers coalesce by sequence comparison + (`ParentDeclaration::is_fresher_than`) and short-circuit at the + `if !updated` gate in `handle_tree_announce`, so same-sequence + repeats drop silently with no cascade. The per-peer 500 ms + rate-limiter is well below this 60-second cadence and does not + suppress the heartbeat broadcast. `BASELINE_CONVERGENCE_TIMEOUT` + in `testing/static/scripts/rekey-test.sh` is bumped from 60 to 65 + so any partition healed by the periodic broadcast at T+60 lands + inside the convergence window; `wait_for_full_baseline` early-exits + on PASS, so successful reps see no extra wall-clock. + - `rx_loop` tick-arm stall under convergence-phase mesh pressure is eliminated. Previously, the tick body's per-peer `check_*` loops (heartbeats, bloom announces, MMP reports, tree announces) diff --git a/src/node/tree.rs b/src/node/tree.rs index 336c8b0..c5f42e1 100644 --- a/src/node/tree.rs +++ b/src/node/tree.rs @@ -495,6 +495,22 @@ impl Node { self.send_tree_announce_to_all().await; let all_peers: Vec = self.peers.keys().copied().collect(); self.bloom_state.mark_all_updates_needed(all_peers); + } else { + // Periodic re-broadcast on no-change: makes TreeAnnounce + // distribution eventually-consistent. Receivers coalesce + // by sequence via ParentDeclaration::is_fresher_than and + // short-circuit at the `if !updated` gate in + // handle_tree_announce; the per-peer 500 ms rate-limiter + // never blocks at this 60 s cadence. Closes the cross-init + // in-flight loss recovery gap where the swap window can + // strand one side's announce on a session-index the other + // side cannot decrypt. + trace!( + seq = self.tree_state.my_declaration().sequence(), + root = %self.tree_state.root(), + "Periodic TreeAnnounce re-broadcast (no state change)" + ); + self.send_tree_announce_to_all().await; } } diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index 0e26a99..9b1507f 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -145,8 +145,14 @@ fi # ── Full test ───────────────────────────────────────────────────────── trap 'echo ""; echo "Test interrupted"; exit 130' INT -# Wait times derived from rekey timer -BASELINE_CONVERGENCE_TIMEOUT=60 +# Wait times derived from rekey timer. +# BASELINE_CONVERGENCE_TIMEOUT must cover one full daemon +# node.tree.reeval_interval_secs (default 60) plus a small margin +# so any partition that only heals via the periodic TreeAnnounce +# re-broadcast lands inside the convergence window. wait_for_full_baseline +# early-exits on PASS, so successful reps are unaffected by the +# extra headroom. +BASELINE_CONVERGENCE_TIMEOUT=65 REKEY_SETTLE=12 # > DRAIN_WINDOW_SECS (10) so post-rekey samples are off the old session # First FMP rekey should follow shortly after the 35s interval once the mesh is # fully converged. Keep this bounded to preserve a meaningful scheduling check