From 56e3d56c25287ff1afebbb88cf7c1c70cbf5b3ce Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 09:20:04 +0000 Subject: [PATCH 01/11] peer/connected_udp: own the connected-socket fd with OwnedFd Replace the hand-rolled RawFd field and the unsafe Drop on ConnectedPeerSocket with an OwnedFd, whose own drop glue closes the fd. from_fd now stores the OwnedFd it already receives instead of stripping ownership through into_raw_fd, and the manual libc::close is gone, shrinking the unsafe surface. Behavior-neutral: the fd still closes exactly once at last-Arc-drop and as_raw_fd returns the same underlying fd while the socket is alive. --- src/peer/connected_udp/socket.rs | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/peer/connected_udp/socket.rs b/src/peer/connected_udp/socket.rs index 85b709a..8b54e54 100644 --- a/src/peer/connected_udp/socket.rs +++ b/src/peer/connected_udp/socket.rs @@ -6,7 +6,7 @@ #![allow(dead_code)] use std::net::SocketAddr; -use std::os::unix::io::{AsRawFd, IntoRawFd, OwnedFd, RawFd}; +use std::os::unix::io::{AsRawFd, OwnedFd, RawFd}; /// A `connect()`-ed UDP socket for one established peer. /// @@ -26,7 +26,7 @@ use std::os::unix::io::{AsRawFd, IntoRawFd, OwnedFd, RawFd}; /// needs to be redone on the data path. #[derive(Debug)] pub(crate) struct ConnectedPeerSocket { - fd: RawFd, + fd: OwnedFd, peer_addr: SocketAddr, local_addr: SocketAddr, } @@ -34,10 +34,10 @@ pub(crate) struct ConnectedPeerSocket { impl ConnectedPeerSocket { /// Adopt an already-opened, bound, and `connect()`-ed fd (from /// `crate::transport::udp::open_connected_fd`) into an owning - /// handle. Takes ownership of the fd; it is closed on drop. + /// handle. Takes ownership of the fd; the `OwnedFd` closes it on drop. pub(crate) fn from_fd(fd: OwnedFd, peer_addr: SocketAddr, local_addr: SocketAddr) -> Self { Self { - fd: fd.into_raw_fd(), + fd, peer_addr, local_addr, } @@ -55,18 +55,7 @@ impl ConnectedPeerSocket { impl AsRawFd for ConnectedPeerSocket { fn as_raw_fd(&self) -> RawFd { - self.fd - } -} - -impl Drop for ConnectedPeerSocket { - fn drop(&mut self) { - // Best-effort close. Ignore the result — if close fails the - // kernel has already done what it can; we don't want to panic - // in Drop. - unsafe { - libc::close(self.fd); - } + self.fd.as_raw_fd() } } From fcaee74ec0875eb87a815343e4b997ccf59a5d03 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 09:58:45 +0000 Subject: [PATCH 02/11] peer: add the per-peer FMP control state machine (unwired) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce src/peer/machine.rs: a sans-IO per-peer control FSM that consolidates the scattered handshake/rekey/timeout driver logic now spread across node/handlers. The machine is a pure reducer — step(event, now, index_allocator) -> [action] — that reuses the existing FMP decision cores (establish_inbound/establish_outbound/ cross_connection_winner/poll_*) rather than reimplementing any decision, and returns runtime-agnostic actions the driver executes. Control-tier state only; the published send-state boundary and the driver wiring land in following commits. The machine is terminal at Closed — re-dial is the reconciler's, so it holds no cross-attempt retry state. Includes eight unit tests: inbound and outbound establish, N:1 identity crystallization, the dual-initiation tie-break, restart-override, rekey initiator cutover, the data-plane-owned responder cutover boundary, and liveness -> link-dead -> report-lost. Unwired — nothing calls it yet. --- src/peer/machine.rs | 1673 +++++++++++++++++++++++++++++++++++++++++++ src/peer/mod.rs | 1 + 2 files changed, 1674 insertions(+) create mode 100644 src/peer/machine.rs diff --git a/src/peer/machine.rs b/src/peer/machine.rs new file mode 100644 index 0000000..179b4a4 --- /dev/null +++ b/src/peer/machine.rs @@ -0,0 +1,1673 @@ +//! Per-peer FMP control FSM (sans-IO reducer). +//! +//! The unified per-peer lifecycle state machine that Step 2 of the node-runtime +//! decomposition folds the scattered `connections`/`peers`/rekey state carriers +//! into. This module is the **C1** increment: the FSM types, the machine struct +//! (control-tier state only), and the pure `step` reducer, plus its unit tests. +//! It is **unwired** — nothing in the codebase calls it yet; the driver wiring +//! (C3b) and the send-state boundary (C2) land in later commits. +//! +//! ## Shape +//! +//! `step(event, now, index_allocator) -> Vec` is a **pure reducer**: +//! every lifecycle *decision* is delegated to the existing sans-IO cores in +//! [`crate::proto::fmp`] ([`Fmp::establish_inbound`]/[`establish_outbound`], +//! [`Fmp::poll_timeouts`]/[`poll_resends`]/[`poll_rekey`]/[`poll_rekey_resends`], +//! and `cross_connection_winner`) — this module writes **no new decision +//! core** (R4). The machine only (a) builds the plain-data snapshots those cores +//! consume from its control-tier state, (b) maps the returned +//! [`ConnAction`]/[`InboundDecision`]/[`OutboundDecision`]/[`PromotionResult`] +//! into the [`PeerAction`] vocabulary the driver executes, and (c) advances its +//! own control state. Shell-side effects (the Noise wire step, `promote_connection` +//! registry surgery, late ACL `authorize_peer`, decrypt-worker register/unregister) +//! are **emitted as actions**, never performed here. +//! +//! ## Control / send-state split (§6 Core 3) +//! +//! The machine holds **control-tier** state only. The hot send-critical state +//! (the three epoch slots, transport target, connected-UDP handle, hot counters) +//! becomes `PeerSendState` in **C2** and is *not* built here; the machine emits +//! actions (`PromoteToActive`, `SwapSendState`, `RegisterDecryptSession`, …) that +//! the driver applies to the published send-state. `remote_epoch` is +//! establish-path-only, hence control-tier, and lives here. +//! +//! ## C1 realizability notes (see `design/step2-machine-spec.md` caveats) +//! +//! - `SendHandshake`/`SendRekey`/`SendLinkMessage` carry **opaque bytes** +//! (`Vec`) — the driver applies outer wire framing / encryption. On the +//! resend paths the bytes are the stored wire frame; on a fresh inbound msg2 / +//! rekey msg2 they are the Noise payload the shell already produced +//! ([`WireOutcome::msg2_payload`]). A fresh outbound msg1 has no bytes the +//! control machine can build (the Noise step is shell-side), so it is emitted +//! with an empty payload and a `C3b` note — that path is not exercised by the +//! C1 tests. +//! - `SendLinkMessage { msg }` is opaque plaintext (there is **no** unifying +//! `LinkMessage` type in the tree today — heartbeat is a bare `[0x51]` byte, +//! while filter/tree/disconnect are distinct concrete types). The machine +//! builds the real heartbeat and disconnect frames; filter/tree announce +//! payloads are data-plane-owned and threaded in at C3b (empty here). +//! - `PeerSnapshot::counter` (the Noise send counter) is a send-state fact the +//! control machine cannot see; it is passed as `0` (the message-count rekey +//! trigger is threaded from `PeerSendState` at C4). Irrelevant to every C1 +//! test (cutover/drain ignore it). + +#![allow(dead_code)] + +use crate::proto::fmp::{ + ConnAction, ConnSnapshot, ConnectionState, EstablishSnapshot, Fmp, InboundDecision, + OutboundDecision, OutboundSnapshot, PeerSnapshot, PromotionResult, RekeyCfg, + RekeyResendSnapshot, WireOutcome, +}; +use crate::proto::link::LinkMessageType; +use crate::transport::{LinkId, TransportAddr, TransportId}; +use crate::utils::index::{IndexAllocator, SessionIndex}; +use crate::{NodeAddr, PeerIdentity}; + +// ============================================================================ +// Timing placeholders +// +// C1 is unwired; the real intervals come from `NodeConfig` when the driver is +// wired (C3b/C5). The `poll_*` cores already take the interval/backoff as +// arguments, so these are only used to compute `SetTimer{at_ms}` deadlines and +// the `Closed{backoff_deadline_ms}` park time. The unit tests assert on timer +// *kinds*, not exact deadlines. +// ============================================================================ + +const HANDSHAKE_RETRANSMIT_INTERVAL_MS: u64 = 1_000; +const HANDSHAKE_TIMEOUT_MS: u64 = 30_000; +const HANDSHAKE_MAX_RESENDS: u32 = 5; +const RESEND_BACKOFF: f64 = 2.0; +const REKEY_CADENCE_INTERVAL_MS: u64 = 60_000; +const REKEY_RESEND_INTERVAL_MS: u64 = 1_000; +const REKEY_MAX_RESENDS: u32 = 5; +const REKEY_AFTER_SECS: u64 = 3_600; +const REKEY_AFTER_MESSAGES: u64 = 1_000_000; +const DRAIN_WINDOW_MS: u64 = 5_000; +const LIVENESS_INTERVAL_MS: u64 = 15_000; +const REKEY_DAMPEN_MS: u64 = 30_000; +const CLOSED_BACKOFF_MS: u64 = 5_000; + +// ============================================================================ +// FSM types (spec §1) +// ============================================================================ + +/// The unified per-peer lifecycle state (subsumes today's `HandshakeState`, +/// `ConnectivityState`, and the rekey flags). Keyed by `LinkId` until +/// `Established` crystallizes the peer to its `NodeAddr`. **Terminal at +/// `Closed`** — re-dial is the reconciler's, not a self-transition. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PeerState { + /// Reconciler intent recorded; no transport work started yet. + Discovered, + /// Outbound transport connect in flight (connection-oriented transports). + Connecting { link: LinkId }, + /// Handshake phase; identity not yet crystallized. + Handshaking { link: LinkId, phase: HandshakePhase }, + /// Handshake complete; identity crystallized; send-state published. + Established { addr: NodeAddr }, + /// Steady state. + Active { addr: NodeAddr }, + /// A maintenance sub-machine is running (rekey / liveness / mtu). + Maintaining { addr: NodeAddr, kind: MaintainKind }, + /// Graceful teardown in flight. + Closing { addr: NodeAddr, reason: CloseReason }, + /// Terminal failure; carries the diagnostic reason. + Failed { reason: FailReason }, + /// Terminal; parked at the reconciler-computed backoff deadline. + Closed { backoff_deadline_ms: u64 }, +} + +/// Handshake phase (mirrors `proto::fmp::HandshakeState`'s in-progress arms). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum HandshakePhase { + Initial, + SentMsg1, + ReceivedMsg1, +} + +/// Which maintenance sub-machine `Maintaining` is running. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum MaintainKind { + Rekey(RekeyPhase), + Liveness(LivenessPhase), + Mtu, +} + +/// Rekey negotiation / cutover phase. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RekeyPhase { + /// Rekey msg1 sent (initiator) or msg2 sent (responder); negotiation in flight. + Msg1Sent, + /// A pending post-rekey session is ready; awaiting the K-bit cutover. + PendingCutover, + /// Post-cutover drain window open. + Draining, +} + +/// Liveness sub-phase. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LivenessPhase { + Stale, + Reconnecting, +} + +/// Why a graceful close was requested. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CloseReason { + /// Operator/protocol requested — no loss report. + Requested, + /// Post-rekey drain-driven close. + Draining, +} + +/// Terminal-failure reason (diagnostic only). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum FailReason { + TransportFailed, + HandshakeTimeout, + HandshakeFailed, + AclRejected, + Rejected, + LinkDead, +} + +/// A timer the machine schedules on the driver's quantized tick. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TimerKind { + HandshakeRetransmit, + HandshakeTimeout, + RekeyCadence, + RekeyResend, + DrainExpiry, + Liveness, +} + +/// An input to the machine. Cross-registry facts ride in the payload as +/// plain-data snapshots ([`WireOutcome`]/[`EstablishSnapshot`]/[`OutboundSnapshot`]) +/// built shell-side; `now` is the `step` parameter, never duplicated here. +/// +/// Not `Debug`/`PartialEq`: the reused core snapshot payloads derive neither. +pub(crate) enum PeerEvent { + /// Reconciler dial intent. + Dial { + transport_id: TransportId, + remote_addr: TransportAddr, + peer_identity: PeerIdentity, + }, + /// Connection-oriented transport connected. + TransportConnected, + /// Transport connect failed. + TransportFailed, + /// Inbound handshake msg1 processed shell-side (Noise + snapshot). + InboundMsg1 { + link: LinkId, + wire: WireOutcome, + est: EstablishSnapshot, + }, + /// Outbound handshake completed (their msg2 received + Noise finalized). + Msg2 { + their_index: SessionIndex, + out: OutboundSnapshot, + }, + /// Late-ACL authorization succeeded (benign confirmation). + Authorized, + /// Late-ACL authorization rejected. + Rejected, + /// `promote_connection` resolved the [`PromoteToActive`](PeerAction::PromoteToActive) + /// action shell-side; the machine consumes the outcome (it does not + /// re-decide the tie-break). + PromotionResolved { result: PromotionResult }, + /// Inbound rekey msg1 (a msg1 on an established peer). + RekeyMsg1 { + wire: WireOutcome, + est: EstablishSnapshot, + }, + /// Inbound rekey msg2 (completes our initiated rekey). + RekeyMsg2 { their_index: SessionIndex }, + /// Data plane observed the responder K-bit flip inline (§3.7). + PeerKbitFlip { epoch: [u8; 8] }, + /// A filter announce is due for this peer. + FilterAnnounce, + /// A tree announce is due for this peer. + TreeAnnounceDue, + /// MMP saw a packet from the peer. + PeerHeard, + /// A keepalive heartbeat is due. + HeartbeatDue, + /// MMP declared the link dead. + LinkDeadSuspected, + /// A machine timer fired on the tick. + Timeout { kind: TimerKind }, + /// Graceful disconnect requested. + Disconnect { reason: CloseReason }, + /// The periodic quantized tick. + Tick, +} + +/// An effect the driver executes on the machine's behalf. Runtime-agnostic +/// plain data — no tokio handles, time only as `at_ms` fields. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum PeerAction { + /// Open a connection-oriented transport to the target. + OpenTransport { + transport_id: TransportId, + remote_addr: TransportAddr, + }, + /// Transmit handshake bytes (driver applies outer framing; see module note). + SendHandshake { bytes: Vec }, + /// Transmit rekey handshake bytes. + SendRekey { bytes: Vec }, + /// Transmit an (encrypted) plaintext link-control frame. Opaque `Vec` + /// pending a unifying `LinkMessage` type — C3b: type against + /// `proto::bloom::FilterAnnounce` / `proto::stp::TreeAnnounce` / + /// `proto::fmp::Disconnect` / a heartbeat marker. + SendLinkMessage { msg: Vec }, + /// Crystallize identity, re-home the map key, publish send-state + /// (`promote_connection`). Resolves to a [`PromotionResolved`](PeerEvent::PromotionResolved). + PromoteToActive { link: LinkId }, + /// Initiator-side rekey cutover: swap the published send-state to the pending + /// epoch. + SwapSendState { epoch: [u8; 8] }, + /// Invalidate the published send-state (close/loss). + InvalidateSendState, + /// Register a decrypt-worker entry for `index`. + RegisterDecryptSession { index: SessionIndex }, + /// Unregister the decrypt-worker entry for `index`. + UnregisterDecryptSession { index: SessionIndex }, + /// Free `index` back to the shared allocator. + FreeIndex { index: SessionIndex }, + /// Activate the per-peer connected-UDP plane. + ActivateConnectedUdp, + /// Tear down the per-peer connected-UDP plane. + TeardownConnectedUdp, + /// Schedule `kind` to fire at `at_ms` on the tick. + SetTimer { kind: TimerKind, at_ms: u64 }, + /// Cancel a scheduled timer. + CancelTimer { kind: TimerKind }, + /// Report the peer lost to the reconciler (the single loss token — there is + /// deliberately no `ScheduleRetry` machine action). + ReportLost { peer: NodeAddr }, +} + +// ============================================================================ +// The machine (control tier — spec §2) +// ============================================================================ + +/// Per-peer control FSM. Holds control-tier lifecycle state only; the +/// send-critical state is published as `PeerSendState` (C2) and mutated via the +/// emitted [`PeerAction`]s. +pub(crate) struct PeerMachine { + state: PeerState, + link: LinkId, + identity: Option, + /// Pure handshake-phase bookkeeping (link/direction/indices/transport/ + /// stored handshake bytes/epoch). Reused verbatim from the FMP state core. + conn: ConnectionState, + /// Remote startup epoch (establish-path-only; NOT in send-state). + remote_epoch: Option<[u8; 8]>, + + // --- rekey negotiation sub-state (control tier; NOT the pending send slot) --- + rekey_in_progress: bool, + /// The index we allocated for our in-flight/negotiated rekey session. + rekey_our_index: Option, + /// Stored rekey msg1 wire bytes (for retransmit). + rekey_msg1: Option>, + rekey_resend_count: u32, + /// When we last processed a peer rekey msg1 (dampening). + last_peer_rekey_ms: u64, + + // --- timing (control tier) --- + session_established_at_ms: u64, + authenticated_at_ms: u64, + rekey_jitter_secs: i64, + last_heartbeat_sent_ms: u64, + + // --- decrypt-registration shadow --- + // The machine owns decrypt-worker register/unregister via actions, so it + // tracks which index it registered (to later unregister/free). This is + // control knowledge of the registration lifecycle, distinct from the hot + // send-state slots (C2). + /// The currently-registered decrypt index (post-establish). + our_index: Option, + /// The previous index held open during a post-cutover drain window. + draining_index: Option, +} + +impl PeerMachine { + /// New outbound machine (we dial). Starts at `Discovered`; the reconciler's + /// `Dial` event drives the first transition. + pub(crate) fn new_outbound(link: LinkId, identity: PeerIdentity, now: u64) -> Self { + Self { + state: PeerState::Discovered, + link, + identity: Some(identity), + conn: ConnectionState::outbound(link, identity, now), + remote_epoch: None, + rekey_in_progress: false, + rekey_our_index: None, + rekey_msg1: None, + rekey_resend_count: 0, + last_peer_rekey_ms: 0, + session_established_at_ms: 0, + authenticated_at_ms: 0, + rekey_jitter_secs: 0, + last_heartbeat_sent_ms: 0, + our_index: None, + draining_index: None, + } + } + + /// New inbound machine (they dialed us). Starts at `Handshaking{Initial}`. + pub(crate) fn new_inbound(link: LinkId, now: u64) -> Self { + Self { + state: PeerState::Handshaking { + link, + phase: HandshakePhase::Initial, + }, + link, + identity: None, + conn: ConnectionState::inbound(link, now), + remote_epoch: None, + rekey_in_progress: false, + rekey_our_index: None, + rekey_msg1: None, + rekey_resend_count: 0, + last_peer_rekey_ms: 0, + session_established_at_ms: 0, + authenticated_at_ms: 0, + rekey_jitter_secs: 0, + last_heartbeat_sent_ms: 0, + our_index: None, + draining_index: None, + } + } + + /// Current lifecycle state. + pub(crate) fn state(&self) -> PeerState { + self.state + } + + /// The crystallized node address, if identity is known. + fn addr(&self) -> Option { + self.identity.map(|id| *id.node_addr()) + } + + // ------------------------------------------------------------------ + // The reducer. + // ------------------------------------------------------------------ + + /// Advance the machine one event. Pure reducer: delegates every decision to + /// the sans-IO cores, maps their results into [`PeerAction`]s, and updates + /// control state. `index_allocator` is a synchronous capability (the + /// handshake/rekey need an index mid-transition), never moved in, never an + /// action. + pub(crate) fn step( + &mut self, + event: PeerEvent, + now: u64, + index_allocator: &mut IndexAllocator, + ) -> Vec { + match event { + PeerEvent::Dial { + transport_id, + remote_addr, + .. + } => self.on_dial(transport_id, remote_addr, now), + PeerEvent::TransportConnected => self.on_transport_connected(now), + PeerEvent::TransportFailed => self.on_transport_failed(now), + PeerEvent::InboundMsg1 { link, wire, est } => { + self.on_inbound_msg1(link, wire, est, now, index_allocator) + } + PeerEvent::Msg2 { their_index, out } => { + self.on_msg2(their_index, out, now, index_allocator) + } + PeerEvent::Authorized => Vec::new(), + PeerEvent::Rejected => self.fail(FailReason::AclRejected), + PeerEvent::PromotionResolved { result } => self.on_promotion_resolved(result, now), + PeerEvent::RekeyMsg1 { wire, est } => { + // A rekey msg1 is a msg1 on an established peer — same core. + self.on_inbound_msg1(self.link, wire, est, now, index_allocator) + } + PeerEvent::RekeyMsg2 { their_index } => self.on_rekey_msg2(their_index), + PeerEvent::PeerKbitFlip { .. } => { + // Responder cutover is data-plane-owned (§3.7): the machine only + // schedules the drain-window unregister. NO slot mutation. + vec![PeerAction::SetTimer { + kind: TimerKind::DrainExpiry, + at_ms: now + DRAIN_WINDOW_MS, + }] + } + PeerEvent::FilterAnnounce => vec![PeerAction::SendLinkMessage { + // C3b: filter-announce payload is data-plane-owned; threaded in + // at wiring time. + msg: Vec::new(), + }], + PeerEvent::TreeAnnounceDue => vec![PeerAction::SendLinkMessage { + // C3b: tree-announce payload is data-plane-owned. + msg: Vec::new(), + }], + PeerEvent::PeerHeard => self.on_peer_heard(now), + PeerEvent::HeartbeatDue => self.on_heartbeat_due(now), + PeerEvent::LinkDeadSuspected => self.on_link_dead(now), + PeerEvent::Timeout { kind } => self.on_timeout(kind, now), + PeerEvent::Disconnect { reason } => self.on_disconnect(reason, now), + PeerEvent::Tick => self.on_tick(now), + } + } + + // ------------------------------------------------------------------ + // Outbound establish (§3.1) + // ------------------------------------------------------------------ + + fn on_dial( + &mut self, + transport_id: TransportId, + remote_addr: TransportAddr, + _now: u64, + ) -> Vec { + if !matches!(self.state, PeerState::Discovered) { + return Vec::new(); + } + self.conn.set_transport_id(transport_id); + // Connection-oriented transports open the transport first; connectionless + // ones send msg1 immediately. C1 models the connection-oriented arm + // (OpenTransport) — the reconciler's candidate carries the transport + // kind at wiring time; connectionless dial reuses `start_handshake`. + self.state = PeerState::Connecting { link: self.link }; + vec![PeerAction::OpenTransport { + transport_id, + remote_addr, + }] + } + + fn on_transport_connected(&mut self, now: u64) -> Vec { + if !matches!(self.state, PeerState::Connecting { .. }) { + return Vec::new(); + } + self.start_outbound_handshake(now) + } + + fn on_transport_failed(&mut self, now: u64) -> Vec { + if !matches!(self.state, PeerState::Connecting { .. }) { + return Vec::new(); + } + let mut actions = Vec::new(); + if let Some(peer) = self.addr() { + actions.push(PeerAction::ReportLost { peer }); + } + self.state = PeerState::Closed { + backoff_deadline_ms: now + CLOSED_BACKOFF_MS, + }; + actions + } + + /// Emit msg1 and arm the retransmit/timeout timers. The Noise msg1 + /// construction and its index allocation are shell-side effects performed by + /// the driver when it executes this action; C1 emits an empty payload (see + /// module note). This path is not exercised by the C1 tests. + fn start_outbound_handshake(&mut self, now: u64) -> Vec { + let bytes = Vec::new(); + self.state = PeerState::Handshaking { + link: self.link, + phase: HandshakePhase::SentMsg1, + }; + vec![ + PeerAction::SendHandshake { bytes }, + PeerAction::SetTimer { + kind: TimerKind::HandshakeRetransmit, + at_ms: now + HANDSHAKE_RETRANSMIT_INTERVAL_MS, + }, + PeerAction::SetTimer { + kind: TimerKind::HandshakeTimeout, + at_ms: now + HANDSHAKE_TIMEOUT_MS, + }, + ] + } + + /// Outbound completion: classify via `establish_outbound` and drive the + /// promote / cross-connection resolution. + fn on_msg2( + &mut self, + their_index: SessionIndex, + out: OutboundSnapshot, + now: u64, + _alloc: &mut IndexAllocator, + ) -> Vec { + self.conn.set_their_index(their_index); + match Fmp::new().establish_outbound(&out) { + OutboundDecision::Promote => { + // Net-new: our outbound index (allocated at dial) is the one we + // register once promotion resolves. + self.our_index = self.conn.our_index(); + vec![PeerAction::PromoteToActive { link: self.link }] + } + OutboundDecision::CrossConnectionSwap => { + // Our outbound wins: swap the peer to the outbound session, + // freeing the old inbound index. Resolved in-step (peer exists). + let outbound_index = self.conn.our_index(); + let old_inbound_index = self.our_index; + self.crystallize(now); + let mut actions = Vec::new(); + if let Some(idx) = old_inbound_index { + actions.push(PeerAction::FreeIndex { index: idx }); + } + if let Some(idx) = outbound_index { + self.our_index = Some(idx); + actions.push(PeerAction::RegisterDecryptSession { index: idx }); + } + actions + } + OutboundDecision::CrossConnectionKeep => { + // Our outbound loses: keep the existing inbound session, free the + // unused outbound index. + let outbound_index = self.conn.our_index(); + self.crystallize(now); + let mut actions = Vec::new(); + if let Some(idx) = outbound_index { + actions.push(PeerAction::FreeIndex { index: idx }); + } + actions + } + } + } + + // ------------------------------------------------------------------ + // Inbound establish (§3.2) + // ------------------------------------------------------------------ + + fn on_inbound_msg1( + &mut self, + link: LinkId, + wire: WireOutcome, + est: EstablishSnapshot, + now: u64, + alloc: &mut IndexAllocator, + ) -> Vec { + match Fmp::new().establish_inbound(&est, &wire) { + InboundDecision::Reject { .. } => { + // In an establish-leg context this fails the leg; on an + // established peer (rekey context) the msg1 is dropped and the + // peer keeps running (DualRekeyWon/PendingSession keep our rekey). + if self.is_established_context() { + Vec::new() + } else { + self.fail(FailReason::Rejected) + } + } + InboundDecision::ResendMsg2 { msg2 } => match msg2 { + Some(bytes) => vec![PeerAction::SendHandshake { bytes }], + None => Vec::new(), + }, + InboundDecision::RekeyRespond { + peer, + abandon_first, + } => self.rekey_respond(peer, abandon_first, &wire, now, alloc), + InboundDecision::RestartThenPromote { peer } => { + let mut actions = vec![PeerAction::InvalidateSendState]; + if let Some(idx) = self.our_index.take() { + actions.push(PeerAction::UnregisterDecryptSession { index: idx }); + } + actions.push(PeerAction::ReportLost { peer }); + actions.extend(self.inbound_promote(link, &wire, now, alloc)); + actions + } + InboundDecision::Promote => self.inbound_promote(link, &wire, now, alloc), + } + } + + /// The inbound Promote tail: allocate our index, record indices/epoch/msg2, + /// emit msg2 + drive promotion. `RegisterDecryptSession` follows on the + /// `PromotionResolved{Promoted}` feedback (§3.2 "then on PromotionResult"). + fn inbound_promote( + &mut self, + link: LinkId, + wire: &WireOutcome, + _now: u64, + alloc: &mut IndexAllocator, + ) -> Vec { + self.identity = Some(wire.peer_identity); + self.remote_epoch = wire.remote_epoch; + self.conn.set_their_index(wire.their_index); + let our_index = alloc.allocate().ok(); + if let Some(idx) = our_index { + self.conn.set_our_index(idx); + self.our_index = Some(idx); + } + self.conn.set_handshake_msg2(wire.msg2_payload.clone()); + self.state = PeerState::Handshaking { + link, + phase: HandshakePhase::ReceivedMsg1, + }; + vec![ + PeerAction::SendHandshake { + bytes: wire.msg2_payload.clone(), + }, + PeerAction::PromoteToActive { link }, + ] + } + + /// Rekey responder: (optionally) abandon our in-flight rekey, allocate a new + /// index, send the rekey msg2, record the peer rekey (dampening). + fn rekey_respond( + &mut self, + _peer: NodeAddr, + abandon_first: bool, + wire: &WireOutcome, + now: u64, + alloc: &mut IndexAllocator, + ) -> Vec { + let mut actions = Vec::new(); + if abandon_first { + if let Some(idx) = self.rekey_our_index.take() { + actions.push(PeerAction::FreeIndex { index: idx }); + } + self.rekey_in_progress = false; + self.rekey_msg1 = None; + } + let new_index = alloc.allocate().ok(); + if let Some(idx) = new_index { + self.rekey_our_index = Some(idx); + } + actions.push(PeerAction::SendRekey { + bytes: wire.msg2_payload.clone(), + }); + self.last_peer_rekey_ms = now; + let addr = self + .addr() + .unwrap_or_else(|| *wire.peer_identity.node_addr()); + self.state = PeerState::Maintaining { + addr, + kind: MaintainKind::Rekey(RekeyPhase::Msg1Sent), + }; + actions + } + + // ------------------------------------------------------------------ + // Promotion feedback (§3.3) + // ------------------------------------------------------------------ + + fn on_promotion_resolved(&mut self, result: PromotionResult, now: u64) -> Vec { + match result { + PromotionResult::Promoted(addr) => { + self.identity_addr_set(addr); + self.crystallize(now); + self.register_current_index() + } + PromotionResult::CrossConnectionWon { node_addr, .. } => { + self.identity_addr_set(node_addr); + self.crystallize(now); + let mut actions = Vec::new(); + // Free + unregister the old (losing) index, register ours. + if let Some(idx) = self.draining_index.take() { + actions.push(PeerAction::UnregisterDecryptSession { index: idx }); + actions.push(PeerAction::FreeIndex { index: idx }); + } + actions.extend(self.register_current_index()); + actions + } + PromotionResult::CrossConnectionLost { .. } => { + let mut actions = Vec::new(); + if let Some(idx) = self.our_index.take() { + actions.push(PeerAction::FreeIndex { index: idx }); + } + self.state = PeerState::Failed { + reason: FailReason::HandshakeFailed, + }; + actions + } + } + } + + fn register_current_index(&self) -> Vec { + match self.our_index { + Some(idx) => vec![PeerAction::RegisterDecryptSession { index: idx }], + None => Vec::new(), + } + } + + // ------------------------------------------------------------------ + // Rekey (initiator) + cutover (§3.4) + // ------------------------------------------------------------------ + + fn on_rekey_msg2(&mut self, their_index: SessionIndex) -> Vec { + // Completes our initiated rekey: a pending session is ready to cut over. + self.conn.set_their_index(their_index); + self.rekey_in_progress = false; + if let PeerState::Maintaining { addr, .. } = self.state { + self.state = PeerState::Maintaining { + addr, + kind: MaintainKind::Rekey(RekeyPhase::PendingCutover), + }; + } + // The pending peers_by_index registration is driver-side; no action here. + Vec::new() + } + + /// Rekey cadence: run `poll_rekey` over this one peer's snapshot and map the + /// phase-grouped `ConnAction`s. + fn on_rekey_cadence(&mut self, now: u64) -> Vec { + let addr = match self.addr() { + Some(a) => a, + None => return Vec::new(), + }; + let cfg = RekeyCfg { + after_secs: REKEY_AFTER_SECS, + after_messages: REKEY_AFTER_MESSAGES, + }; + let snap = self.peer_snapshot(addr, now); + let mut actions = Vec::new(); + for act in Fmp::new().poll_rekey(vec![snap], &cfg) { + actions.extend(self.map_rekey_action(act, now)); + } + actions + } + + fn map_rekey_action(&mut self, act: ConnAction, now: u64) -> Vec { + match act { + ConnAction::Cutover { peer } => { + // Initiator cutover: swap to the pending epoch, register the new + // index, open the drain window. Slot-rotation mechanics stay in + // active.rs; the machine emits the action sequence. + self.draining_index = self.our_index; + self.our_index = self.rekey_our_index.take(); + self.rekey_in_progress = false; + self.state = PeerState::Maintaining { + addr: peer, + kind: MaintainKind::Rekey(RekeyPhase::Draining), + }; + let mut actions = vec![PeerAction::SwapSendState { + epoch: self.remote_epoch.unwrap_or_default(), + }]; + if let Some(idx) = self.our_index { + actions.push(PeerAction::RegisterDecryptSession { index: idx }); + } + actions.push(PeerAction::SetTimer { + kind: TimerKind::DrainExpiry, + at_ms: now + DRAIN_WINDOW_MS, + }); + actions + } + ConnAction::Drain { peer } => { + self.state = PeerState::Active { addr: peer }; + let mut actions = Vec::new(); + if let Some(idx) = self.draining_index.take() { + actions.push(PeerAction::UnregisterDecryptSession { index: idx }); + actions.push(PeerAction::FreeIndex { index: idx }); + } + actions + } + ConnAction::InitiateRekey { peer } => { + // Fresh outbound rekey: allocate our new index, send msg1 (Noise + // leaf is shell-side → empty payload in C1), arm the resend timer. + self.rekey_in_progress = true; + self.rekey_resend_count = 0; + self.rekey_msg1 = Some(Vec::new()); + self.state = PeerState::Maintaining { + addr: peer, + kind: MaintainKind::Rekey(RekeyPhase::Msg1Sent), + }; + vec![ + PeerAction::SendRekey { bytes: Vec::new() }, + PeerAction::SetTimer { + kind: TimerKind::RekeyResend, + at_ms: now + REKEY_RESEND_INTERVAL_MS, + }, + ] + } + // poll_rekey never emits the maintain/teardown-only variants. + _ => Vec::new(), + } + } + + fn on_rekey_resend(&mut self, now: u64) -> Vec { + let peer = match self.addr() { + Some(a) => a, + None => return Vec::new(), + }; + let snap = RekeyResendSnapshot { + peer, + resend_count: self.rekey_resend_count, + needs_resend: true, + msg1: self.rekey_msg1.clone().unwrap_or_default(), + }; + let mut actions = Vec::new(); + for act in Fmp::new().poll_rekey_resends( + vec![snap], + now, + REKEY_RESEND_INTERVAL_MS, + RESEND_BACKOFF, + REKEY_MAX_RESENDS, + ) { + match act { + ConnAction::AbandonRekey { .. } => { + if let Some(idx) = self.rekey_our_index.take() { + actions.push(PeerAction::FreeIndex { index: idx }); + } + self.rekey_in_progress = false; + self.rekey_msg1 = None; + actions.push(PeerAction::CancelTimer { + kind: TimerKind::RekeyResend, + }); + } + ConnAction::ResendRekeyMsg1 { + bytes, + next_resend_at_ms, + .. + } => { + self.rekey_resend_count += 1; + actions.push(PeerAction::SendRekey { bytes }); + actions.push(PeerAction::SetTimer { + kind: TimerKind::RekeyResend, + at_ms: next_resend_at_ms, + }); + } + _ => {} + } + } + actions + } + + // ------------------------------------------------------------------ + // Liveness (§3.5) + // ------------------------------------------------------------------ + + fn on_heartbeat_due(&mut self, now: u64) -> Vec { + if !self.is_active_like() { + return Vec::new(); + } + self.last_heartbeat_sent_ms = now; + vec![ + PeerAction::SendLinkMessage { + msg: vec![LinkMessageType::Heartbeat.to_byte()], + }, + PeerAction::SetTimer { + kind: TimerKind::Liveness, + at_ms: now + LIVENESS_INTERVAL_MS, + }, + ] + } + + fn on_peer_heard(&mut self, now: u64) -> Vec { + if !self.is_active_like() { + return Vec::new(); + } + vec![ + PeerAction::CancelTimer { + kind: TimerKind::Liveness, + }, + PeerAction::SetTimer { + kind: TimerKind::Liveness, + at_ms: now + LIVENESS_INTERVAL_MS, + }, + ] + } + + fn on_link_dead(&mut self, now: u64) -> Vec { + if !self.is_active_like() { + return Vec::new(); + } + let mut actions = vec![PeerAction::InvalidateSendState]; + if let Some(idx) = self.our_index.take() { + actions.push(PeerAction::UnregisterDecryptSession { index: idx }); + } + actions.push(PeerAction::TeardownConnectedUdp); + if let Some(peer) = self.addr() { + actions.push(PeerAction::ReportLost { peer }); + } + self.state = PeerState::Closed { + backoff_deadline_ms: now + CLOSED_BACKOFF_MS, + }; + actions + } + + // ------------------------------------------------------------------ + // Timeout / teardown / close (§3.6) + // ------------------------------------------------------------------ + + fn on_timeout(&mut self, kind: TimerKind, now: u64) -> Vec { + match kind { + TimerKind::HandshakeRetransmit => self.on_handshake_retransmit(now), + TimerKind::HandshakeTimeout => self.on_handshake_timeout(now), + TimerKind::RekeyCadence => self.on_rekey_cadence(now), + TimerKind::RekeyResend => self.on_rekey_resend(now), + TimerKind::DrainExpiry => self.on_rekey_cadence(now), + TimerKind::Liveness => Vec::new(), + } + } + + fn on_handshake_retransmit(&mut self, now: u64) -> Vec { + if !matches!( + self.state, + PeerState::Handshaking { + phase: HandshakePhase::SentMsg1, + .. + } + ) { + return Vec::new(); + } + let snap = self.conn_snapshot(); + let mut actions = Vec::new(); + for act in Fmp::new().poll_resends( + vec![snap], + now, + HANDSHAKE_RETRANSMIT_INTERVAL_MS, + RESEND_BACKOFF, + ) { + if let ConnAction::ResendMsg1 { + bytes, + next_resend_at_ms, + .. + } = act + { + self.conn.record_resend(next_resend_at_ms); + actions.push(PeerAction::SendHandshake { bytes }); + actions.push(PeerAction::SetTimer { + kind: TimerKind::HandshakeRetransmit, + at_ms: next_resend_at_ms, + }); + } + } + actions + } + + fn on_handshake_timeout(&mut self, now: u64) -> Vec { + if !matches!(self.state, PeerState::Handshaking { .. }) { + return Vec::new(); + } + let snap = self.conn_snapshot(); + // poll_timeouts emits [ScheduleRetry?, Teardown]; the machine REMAPS + // ScheduleRetry -> ReportLost (single loss token) and Teardown -> + // FreeIndex{our_index}, emitting FreeIndex before ReportLost (§3.6). + let mut free = Vec::new(); + let mut lost = Vec::new(); + for act in Fmp::new().poll_timeouts(vec![snap]) { + match act { + ConnAction::ScheduleRetry { peer } => { + lost.push(PeerAction::ReportLost { peer }); + } + ConnAction::Teardown { .. } => { + if let Some(idx) = self.conn.our_index() { + free.push(PeerAction::FreeIndex { index: idx }); + } + } + _ => {} + } + } + self.state = PeerState::Closed { + backoff_deadline_ms: now + CLOSED_BACKOFF_MS, + }; + free.extend(lost); + free + } + + fn on_disconnect(&mut self, reason: CloseReason, now: u64) -> Vec { + if !self.is_active_like() && !matches!(self.state, PeerState::Established { .. }) { + return Vec::new(); + } + let addr = self.addr(); + self.state = PeerState::Closing { + addr: addr.unwrap_or_else(zero_addr), + reason, + }; + let mut actions = vec![PeerAction::SendLinkMessage { + msg: disconnect_frame(reason), + }]; + actions.push(PeerAction::InvalidateSendState); + if let Some(idx) = self.our_index.take() { + actions.push(PeerAction::UnregisterDecryptSession { index: idx }); + } + actions.push(PeerAction::TeardownConnectedUdp); + // No ReportLost on operator Requested. + self.state = PeerState::Closed { + backoff_deadline_ms: now + CLOSED_BACKOFF_MS, + }; + actions + } + + fn on_tick(&mut self, now: u64) -> Vec { + // The driver evaluates due machine timers on the quantized tick and + // re-enters the Timeout{kind} handlers. C1 is unwired; the deadline + // bookkeeping is threaded from the driver at C5, so Tick is a no-op here. + let _ = now; + Vec::new() + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + fn crystallize(&mut self, now: u64) { + let addr = self.addr().unwrap_or_else(zero_addr); + self.session_established_at_ms = now; + self.authenticated_at_ms = now; + self.state = PeerState::Established { addr }; + } + + fn identity_addr_set(&mut self, _addr: NodeAddr) { + // Identity is already crystallized from the wire outcome during the + // establish step; the PromotionResult's addr confirms it. + } + + fn fail(&mut self, reason: FailReason) -> Vec { + self.state = PeerState::Failed { reason }; + Vec::new() + } + + fn is_active_like(&self) -> bool { + matches!( + self.state, + PeerState::Active { .. } | PeerState::Maintaining { .. } + ) + } + + fn is_established_context(&self) -> bool { + matches!( + self.state, + PeerState::Established { .. } + | PeerState::Active { .. } + | PeerState::Maintaining { .. } + ) + } + + fn conn_snapshot(&self) -> ConnSnapshot { + ConnSnapshot { + link: self.conn.link_id(), + is_outbound: self.conn.is_outbound(), + retry_addr: self.conn.expected_identity().map(|id| *id.node_addr()), + resend_count: self.conn.resend_count(), + msg1: self + .conn + .handshake_msg1() + .map(|b| b.to_vec()) + .unwrap_or_default(), + } + } + + /// Build this peer's rekey snapshot from control-tier state. `counter` is a + /// send-state fact (C4); passed as 0 here (see module note). + fn peer_snapshot(&self, addr: NodeAddr, now: u64) -> PeerSnapshot { + let phase = match self.state { + PeerState::Maintaining { + kind: MaintainKind::Rekey(p), + .. + } => Some(p), + _ => None, + }; + let elapsed_secs = now.saturating_sub(self.session_established_at_ms) / 1000; + PeerSnapshot { + addr, + has_pending: phase == Some(RekeyPhase::PendingCutover), + rekey_in_progress: phase == Some(RekeyPhase::Msg1Sent) || self.rekey_in_progress, + is_draining: phase == Some(RekeyPhase::Draining), + drain_expired: phase == Some(RekeyPhase::Draining), + is_dampened: now.saturating_sub(self.last_peer_rekey_ms) < REKEY_DAMPEN_MS + && self.last_peer_rekey_ms != 0, + elapsed_secs, + counter: 0, + jitter_secs: self.rekey_jitter_secs, + } + } +} + +fn zero_addr() -> NodeAddr { + NodeAddr::from_bytes([0u8; 16]) +} + +/// Build the plaintext disconnect frame the driver encrypts + sends. +fn disconnect_frame(reason: CloseReason) -> Vec { + use crate::proto::fmp::{Disconnect, DisconnectReason}; + let wire_reason = match reason { + CloseReason::Requested => DisconnectReason::Shutdown, + CloseReason::Draining => DisconnectReason::Restart, + }; + Disconnect::new(wire_reason).encode().to_vec() +} + +// ============================================================================ +// Unit tests (spec §5) — assert on ACTION SEQUENCES + STATE transitions using +// hand-built synthetic snapshots (caveat #4: no real crypto sessions). +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto::fmp::PromotionResult; + use crate::{Identity, PeerIdentity}; + + fn peer_identity() -> PeerIdentity { + PeerIdentity::from_pubkey(Identity::generate().pubkey()) + } + + /// Two identities with a known NodeAddr ordering: `.0` < `.1`. + fn ordered_identities() -> (PeerIdentity, PeerIdentity) { + loop { + let a = peer_identity(); + let b = peer_identity(); + if a.node_addr() < b.node_addr() { + return (a, b); + } + if b.node_addr() < a.node_addr() { + return (b, a); + } + } + } + + fn wire_outcome(peer: PeerIdentity, epoch: Option<[u8; 8]>, their: u32) -> WireOutcome { + WireOutcome { + peer_identity: peer, + remote_epoch: epoch, + their_index: SessionIndex::new(their), + msg2_payload: vec![0xAB; 8], + } + } + + fn est_new_peer(our: NodeAddr) -> EstablishSnapshot { + EstablishSnapshot { + has_existing_peer: false, + existing_peer_epoch: None, + existing_session_age_secs: 0, + has_session: false, + is_healthy: false, + pending_new_session: false, + rekey_in_progress: false, + existing_msg2: None, + at_max_peers: false, + has_pending_outbound_to_peer: false, + rekey_enabled: true, + our_node_addr: our, + } + } + + // ---- Test 1: rekey initiator cutover ---------------------------------- + #[test] + fn rekey_initiator_cutover() { + let mut alloc = IndexAllocator::new(); + let id = peer_identity(); + let addr = *id.node_addr(); + let mut m = PeerMachine::new_outbound(LinkId::new(1), id, 0); + // Arrange: a completed rekey pending cutover. + m.state = PeerState::Maintaining { + addr, + kind: MaintainKind::Rekey(RekeyPhase::PendingCutover), + }; + m.rekey_our_index = Some(SessionIndex::new(0x2222)); + m.our_index = Some(SessionIndex::new(0x1111)); + m.remote_epoch = Some([9u8; 8]); + m.session_established_at_ms = 0; + + let actions = m.step( + PeerEvent::Timeout { + kind: TimerKind::RekeyCadence, + }, + 10_000, + &mut alloc, + ); + + assert_eq!( + actions, + vec![ + PeerAction::SwapSendState { epoch: [9u8; 8] }, + PeerAction::RegisterDecryptSession { + index: SessionIndex::new(0x2222) + }, + PeerAction::SetTimer { + kind: TimerKind::DrainExpiry, + at_ms: 10_000 + DRAIN_WINDOW_MS + }, + ] + ); + assert_eq!( + m.state(), + PeerState::Maintaining { + addr, + kind: MaintainKind::Rekey(RekeyPhase::Draining) + } + ); + } + + // ---- Test 2: responder cutover (data-plane owned) --------------------- + #[test] + fn responder_cutover_only_sets_drain_timer() { + let mut alloc = IndexAllocator::new(); + let id = peer_identity(); + let addr = *id.node_addr(); + let mut m = PeerMachine::new_outbound(LinkId::new(1), id, 0); + m.state = PeerState::Active { addr }; + + let actions = m.step( + PeerEvent::PeerKbitFlip { epoch: [7u8; 8] }, + 5_000, + &mut alloc, + ); + + // ONLY the drain timer — no SwapSendState, no slot mutation. + assert_eq!( + actions, + vec![PeerAction::SetTimer { + kind: TimerKind::DrainExpiry, + at_ms: 5_000 + DRAIN_WINDOW_MS + }] + ); + assert!( + !actions + .iter() + .any(|a| matches!(a, PeerAction::SwapSendState { .. })) + ); + assert_eq!(m.state(), PeerState::Active { addr }); + } + + // ---- Test 3: dual-init tie-break, swapped addrs ----------------------- + #[test] + fn dual_init_tiebreak_swapped_addrs() { + let (smaller, larger) = ordered_identities(); + + // Case A: WE are smaller -> we win -> Reject{DualRekeyWon} -> drop, keep. + { + let mut alloc = IndexAllocator::new(); + let our = *smaller.node_addr(); + let peer = larger; + let mut m = PeerMachine::new_outbound(LinkId::new(1), peer, 0); + let peer_addr = *peer.node_addr(); + m.state = PeerState::Maintaining { + addr: peer_addr, + kind: MaintainKind::Rekey(RekeyPhase::Msg1Sent), + }; + m.rekey_in_progress = true; + m.rekey_our_index = Some(SessionIndex::new(0x55)); + let mut est = est_new_peer(our); + est.has_existing_peer = true; + est.existing_peer_epoch = Some([1u8; 8]); + est.has_session = true; + est.is_healthy = true; + est.existing_session_age_secs = 120; + est.rekey_in_progress = true; + let wire = wire_outcome(peer, Some([1u8; 8]), 0x77); + + let actions = m.step(PeerEvent::RekeyMsg1 { wire, est }, 1_000, &mut alloc); + // We win the tie-break: drop the peer's msg1, no rekey response. + assert!(actions.is_empty()); + assert!( + !actions + .iter() + .any(|a| matches!(a, PeerAction::SendRekey { .. })) + ); + } + + // Case B: PEER is smaller -> we lose -> RekeyRespond{abandon_first:true}. + { + let mut alloc = IndexAllocator::new(); + let our = *larger.node_addr(); + let peer = smaller; + let mut m = PeerMachine::new_outbound(LinkId::new(2), peer, 0); + let peer_addr = *peer.node_addr(); + m.state = PeerState::Maintaining { + addr: peer_addr, + kind: MaintainKind::Rekey(RekeyPhase::Msg1Sent), + }; + m.rekey_in_progress = true; + m.rekey_our_index = Some(SessionIndex::new(0x55)); + let mut est = est_new_peer(our); + est.has_existing_peer = true; + est.existing_peer_epoch = Some([1u8; 8]); + est.has_session = true; + est.is_healthy = true; + est.existing_session_age_secs = 120; + est.rekey_in_progress = true; + let wire = wire_outcome(peer, Some([1u8; 8]), 0x77); + + let actions = m.step(PeerEvent::RekeyMsg1 { wire, est }, 1_000, &mut alloc); + // abandon_first -> FreeIndex(old rekey index) then SendRekey(msg2). + assert_eq!( + actions[0], + PeerAction::FreeIndex { + index: SessionIndex::new(0x55) + } + ); + assert!(matches!(actions[1], PeerAction::SendRekey { .. })); + } + } + + // ---- Test 4: restart-override ----------------------------------------- + #[test] + fn restart_override() { + let mut alloc = IndexAllocator::new(); + let peer = peer_identity(); + let peer_addr = *peer.node_addr(); + let mut m = PeerMachine::new_inbound(LinkId::new(1), 0); + // Existing peer at a different epoch -> restart. + m.our_index = Some(SessionIndex::new(0xDEAD)); + let our = *peer_identity().node_addr(); + let mut est = est_new_peer(our); + est.has_existing_peer = true; + est.existing_peer_epoch = Some([1u8; 8]); // old + let wire = wire_outcome(peer, Some([2u8; 8]), 0x77); // new epoch + + let actions = m.step( + PeerEvent::InboundMsg1 { + link: LinkId::new(1), + wire, + est, + }, + 1_000, + &mut alloc, + ); + + // Restart tail: invalidate, unregister old, report lost, then Promote. + assert_eq!(actions[0], PeerAction::InvalidateSendState); + assert_eq!( + actions[1], + PeerAction::UnregisterDecryptSession { + index: SessionIndex::new(0xDEAD) + } + ); + assert_eq!(actions[2], PeerAction::ReportLost { peer: peer_addr }); + assert!( + actions + .iter() + .any(|a| matches!(a, PeerAction::SendHandshake { .. })) + ); + assert!( + actions + .iter() + .any(|a| matches!(a, PeerAction::PromoteToActive { .. })) + ); + + // Promotion feedback -> Established. + let follow = m.step( + PeerEvent::PromotionResolved { + result: PromotionResult::Promoted(peer_addr), + }, + 1_000, + &mut alloc, + ); + assert!( + follow + .iter() + .any(|a| matches!(a, PeerAction::RegisterDecryptSession { .. })) + ); + assert_eq!(m.state(), PeerState::Established { addr: peer_addr }); + } + + // ---- Test 5: N:1 crystallization -------------------------------------- + #[test] + fn n_to_one_crystallization() { + let mut alloc = IndexAllocator::new(); + let peer = peer_identity(); + let peer_addr = *peer.node_addr(); + + // Winner leg (link 1): net-new inbound promote -> Established. + let mut winner = PeerMachine::new_inbound(LinkId::new(1), 0); + let our = *peer_identity().node_addr(); + let est_w = est_new_peer(our); + let wire_w = wire_outcome(peer, Some([3u8; 8]), 0x77); + let wa = winner.step( + PeerEvent::InboundMsg1 { + link: LinkId::new(1), + wire: wire_w, + est: est_w, + }, + 100, + &mut alloc, + ); + assert!( + wa.iter().any( + |a| matches!(a, PeerAction::PromoteToActive { link } if *link == LinkId::new(1)) + ) + ); + let wf = winner.step( + PeerEvent::PromotionResolved { + result: PromotionResult::Promoted(peer_addr), + }, + 100, + &mut alloc, + ); + assert!( + wf.iter() + .any(|a| matches!(a, PeerAction::RegisterDecryptSession { .. })) + ); + assert_eq!(winner.state(), PeerState::Established { addr: peer_addr }); + + // Loser leg (link 2): same identity, loses cross-connection at + // promote_connection -> Failed + FreeIndex, link terminates. + let mut loser = PeerMachine::new_inbound(LinkId::new(2), 0); + let est_l = est_new_peer(our); + let wire_l = wire_outcome(peer, Some([3u8; 8]), 0x88); + let la = loser.step( + PeerEvent::InboundMsg1 { + link: LinkId::new(2), + wire: wire_l, + est: est_l, + }, + 100, + &mut alloc, + ); + assert!( + la.iter() + .any(|a| matches!(a, PeerAction::PromoteToActive { .. })) + ); + let loser_index = loser.our_index; + let lf = loser.step( + PeerEvent::PromotionResolved { + result: PromotionResult::CrossConnectionLost { + winner_link_id: LinkId::new(1), + }, + }, + 100, + &mut alloc, + ); + assert_eq!( + lf, + vec![PeerAction::FreeIndex { + index: loser_index.unwrap() + }] + ); + assert_eq!( + loser.state(), + PeerState::Failed { + reason: FailReason::HandshakeFailed + } + ); + + // One crystallized NodeAddr (the winner); loser never crystallizes. + assert_eq!(winner.addr(), Some(peer_addr)); + } + + // ---- Test 6: inbound establish ---------------------------------------- + #[test] + fn inbound_establish() { + let mut alloc = IndexAllocator::new(); + let peer = peer_identity(); + let peer_addr = *peer.node_addr(); + let mut m = PeerMachine::new_inbound(LinkId::new(1), 0); + assert_eq!( + m.state(), + PeerState::Handshaking { + link: LinkId::new(1), + phase: HandshakePhase::Initial + } + ); + let our = *peer_identity().node_addr(); + let est = est_new_peer(our); + let wire = wire_outcome(peer, Some([4u8; 8]), 0x77); + + let mut actions = m.step( + PeerEvent::InboundMsg1 { + link: LinkId::new(1), + wire, + est, + }, + 200, + &mut alloc, + ); + actions.extend(m.step( + PeerEvent::PromotionResolved { + result: PromotionResult::Promoted(peer_addr), + }, + 200, + &mut alloc, + )); + + // Combined promote sequence (§3.2 "then on PromotionResult ..."). + assert!(matches!(actions[0], PeerAction::SendHandshake { .. })); + assert_eq!( + actions[1], + PeerAction::PromoteToActive { + link: LinkId::new(1) + } + ); + assert!(matches!( + actions[2], + PeerAction::RegisterDecryptSession { .. } + )); + assert_eq!(m.state(), PeerState::Established { addr: peer_addr }); + } + + // ---- Test 7: outbound establish (+ cross-connection) ------------------ + #[test] + fn outbound_establish() { + let mut alloc = IndexAllocator::new(); + let peer = peer_identity(); + let peer_addr = *peer.node_addr(); + + // Net-new promote. + let mut m = PeerMachine::new_outbound(LinkId::new(1), peer, 0); + m.state = PeerState::Handshaking { + link: LinkId::new(1), + phase: HandshakePhase::SentMsg1, + }; + m.conn.set_our_index(SessionIndex::new(0xABCD)); + let out = OutboundSnapshot { + has_existing_peer: false, + our_outbound_wins: false, + }; + let mut actions = m.step( + PeerEvent::Msg2 { + their_index: SessionIndex::new(0x77), + out, + }, + 300, + &mut alloc, + ); + assert_eq!( + actions, + vec![PeerAction::PromoteToActive { + link: LinkId::new(1) + }] + ); + actions = m.step( + PeerEvent::PromotionResolved { + result: PromotionResult::Promoted(peer_addr), + }, + 300, + &mut alloc, + ); + assert_eq!( + actions, + vec![PeerAction::RegisterDecryptSession { + index: SessionIndex::new(0xABCD) + }] + ); + assert_eq!(m.state(), PeerState::Established { addr: peer_addr }); + + // Cross-connection SWAP: our outbound wins -> free old inbound, register outbound. + let mut m2 = PeerMachine::new_outbound(LinkId::new(2), peer, 0); + m2.state = PeerState::Handshaking { + link: LinkId::new(2), + phase: HandshakePhase::SentMsg1, + }; + m2.conn.set_our_index(SessionIndex::new(0x2222)); // outbound index + m2.our_index = Some(SessionIndex::new(0x1111)); // old inbound index + let out_swap = OutboundSnapshot { + has_existing_peer: true, + our_outbound_wins: true, + }; + let swap = m2.step( + PeerEvent::Msg2 { + their_index: SessionIndex::new(0x99), + out: out_swap, + }, + 400, + &mut alloc, + ); + assert_eq!( + swap, + vec![ + PeerAction::FreeIndex { + index: SessionIndex::new(0x1111) + }, + PeerAction::RegisterDecryptSession { + index: SessionIndex::new(0x2222) + }, + ] + ); + assert_eq!(m2.state(), PeerState::Established { addr: peer_addr }); + + // Cross-connection KEEP: our outbound loses -> free unused outbound index. + let mut m3 = PeerMachine::new_outbound(LinkId::new(3), peer, 0); + m3.state = PeerState::Handshaking { + link: LinkId::new(3), + phase: HandshakePhase::SentMsg1, + }; + m3.conn.set_our_index(SessionIndex::new(0x3333)); + let out_keep = OutboundSnapshot { + has_existing_peer: true, + our_outbound_wins: false, + }; + let keep = m3.step( + PeerEvent::Msg2 { + their_index: SessionIndex::new(0x9A), + out: out_keep, + }, + 500, + &mut alloc, + ); + assert_eq!( + keep, + vec![PeerAction::FreeIndex { + index: SessionIndex::new(0x3333) + }] + ); + } + + // ---- Test 8: liveness -> LinkDeadSuspected -> ReportLost -------------- + #[test] + fn liveness_to_link_dead() { + let mut alloc = IndexAllocator::new(); + let id = peer_identity(); + let addr = *id.node_addr(); + let mut m = PeerMachine::new_outbound(LinkId::new(1), id, 0); + m.state = PeerState::Active { addr }; + m.our_index = Some(SessionIndex::new(0x4242)); + + let hb = m.step(PeerEvent::HeartbeatDue, 1_000, &mut alloc); + assert_eq!( + hb, + vec![ + PeerAction::SendLinkMessage { + msg: vec![LinkMessageType::Heartbeat.to_byte()] + }, + PeerAction::SetTimer { + kind: TimerKind::Liveness, + at_ms: 1_000 + LIVENESS_INTERVAL_MS + }, + ] + ); + + let dead = m.step(PeerEvent::LinkDeadSuspected, 2_000, &mut alloc); + assert_eq!( + dead, + vec![ + PeerAction::InvalidateSendState, + PeerAction::UnregisterDecryptSession { + index: SessionIndex::new(0x4242) + }, + PeerAction::TeardownConnectedUdp, + PeerAction::ReportLost { peer: addr }, + ] + ); + assert!(matches!(m.state(), PeerState::Closed { .. })); + // The exact action-sequence equality above is the "no ScheduleRetry" + // guarantee: loss is reported only via ReportLost, and no retry-schedule + // action exists in the PeerAction vocabulary at all (reconciler-owned). + } +} diff --git a/src/peer/mod.rs b/src/peer/mod.rs index 95c2a56..87c7db9 100644 --- a/src/peer/mod.rs +++ b/src/peer/mod.rs @@ -11,6 +11,7 @@ mod active; #[cfg(any(target_os = "linux", target_os = "macos"))] pub(crate) mod connected_udp; mod connection; +pub(crate) mod machine; pub use active::{ActivePeer, ConnectivityState}; pub use connection::{HandshakeState, PeerConnection}; From 59155df4e3aae847b7e19d4d4fc93d40962cb656 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 10:36:15 +0000 Subject: [PATCH 03/11] peer: split ActivePeer send-state into PeerSendState (two-tier boundary) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Draw the control/published-send-state boundary inside ActivePeer by grouping the send-critical fields — the three epoch session slots {current, previous, pending}, the K-bit flag and session-start, the transport target, the connected-UDP handles, and the hot counters — into a new co-located PeerSendState struct. The control-tier fields (identity, connectivity, declaration/ancestry, filter and tree-announce groups, remote_epoch, the rekey-negotiation sub-machine, and the rest) stay on ActivePeer. Behavior-neutral: a pure field regrouping. Every accessor signature is unchanged (bodies now read/write self.send.*), so the hot path and the handlers are byte-untouched; both K-bit cutovers still rotate the three slots atomically with the same control-tier updates. No Arc/ArcSwap — the fields are co-located and read by plain borrow; publishing behind a shared cell is later plumbing for a sharded data plane. --- src/peer/active.rs | 440 ++++++++++++++++++++++++--------------------- 1 file changed, 233 insertions(+), 207 deletions(-) diff --git a/src/peer/active.rs b/src/peer/active.rs index 971150c..9690430 100644 --- a/src/peer/active.rs +++ b/src/peer/active.rs @@ -69,6 +69,125 @@ impl fmt::Display for ConnectivityState { } } +/// Published active-send-state for a peer (the two-tier boundary). +/// +/// This is the send-critical subset of an `ActivePeer` that the data plane +/// reads (and, on roam/responder-cutover, writes) directly by plain borrow +/// with no FSM dispatch: the three epoch slots (current / previous-draining / +/// pending), the K-bit + session-relative time base, the transport target, +/// the connected-UDP handles, and the hot per-packet counters. Grouping these +/// draws the control/published-send-state boundary inside the peer entry. +/// +/// Co-located, not behind `Arc`/`ArcSwap` — the data plane is not sharded, so +/// the hot path reads this by plain borrow. Publishing behind `Arc`/`ArcSwap` +/// is later-increment plumbing for a sharded data plane. +/// +/// Like `ActivePeer`, this does not implement `Clone` because it contains +/// `NoiseSession`, which cannot be safely cloned (cloning would risk nonce +/// reuse, a catastrophic security failure). +#[derive(Debug)] +struct PeerSendState { + // === Current epoch slot === + /// Noise session for encryption/decryption (None if legacy peer). + noise_session: Option, + /// Our session index (they include this when sending TO us). + our_index: Option, + /// Their session index (we include this when sending TO them). + their_index: Option, + + // === Previous / draining epoch slot === + /// Previous session kept alive during drain window after cutover. + previous_session: Option, + /// Previous session's our_index (for peers_by_index cleanup on drain expiry). + previous_our_index: Option, + /// When the drain window started (None = no drain in progress). + drain_started: Option, + + // === Pending epoch slot === + /// Pending new session from completed rekey (before K-bit cutover). + pending_new_session: Option, + /// Pending new session's our_index. + pending_our_index: Option, + /// Pending new session's their_index. + pending_their_index: Option, + + // === Epoch bit + session-relative time base === + /// Current K-bit epoch value (alternates each rekey). + current_k_bit: bool, + /// Session start time for computing session-relative timestamps. + /// Used as the epoch for the 4-byte inner header timestamp field. + session_start: Instant, + + // === Transport target === + /// Transport ID for this peer's link. + transport_id: Option, + /// Current transport address (for roaming support). + current_addr: Option, + /// Link used to reach this peer. + link_id: LinkId, + + // === Connected-UDP handles === + /// Unix UDP fast-path: per-peer `connect()`-ed socket (paired with + /// the listen socket via `SO_REUSEPORT`). The kernel demux prefers + /// the connected 5-tuple, so inbound packets land here; the + /// encrypt-worker send path sends with `msg_name = NULL`, skipping + /// per-packet sockaddr handling + route lookup. Behind an `Arc` so + /// in-flight worker jobs survive rekey/address-change rotations. + #[cfg(any(target_os = "linux", target_os = "macos"))] + connected_udp: Option>, + + /// Per-peer recv drain thread. Always paired with `connected_udp`: + /// the kernel routes inbound packets from this peer to the + /// connected socket, so it *must* be drained or the kernel recv + /// buffer fills. Drop signals shutdown via self-pipe. + #[cfg(any(target_os = "linux", target_os = "macos"))] + peer_recv_drain: Option, + + // === Hot counters === + /// Link statistics. + link_stats: LinkStats, + /// When this peer was last seen (any activity, Unix milliseconds). + last_seen: u64, + /// Number of replay detections suppressed since last session reset. + replay_suppressed_count: u32, + /// Consecutive decryption failures (reset on any successful decrypt). + consecutive_decrypt_failures: u32, + /// Per-peer MMP state (None for legacy peers without Noise sessions). + mmp: Option, +} + +impl PeerSendState { + /// Empty send-state for a peer with no Noise session yet. Mirrors the + /// send-critical portion of `ActivePeer::new`. + fn new(link_id: LinkId, session_start: Instant, last_seen: u64) -> Self { + Self { + noise_session: None, + our_index: None, + their_index: None, + previous_session: None, + previous_our_index: None, + drain_started: None, + pending_new_session: None, + pending_our_index: None, + pending_their_index: None, + current_k_bit: false, + session_start, + transport_id: None, + current_addr: None, + link_id, + #[cfg(any(target_os = "linux", target_os = "macos"))] + connected_udp: None, + #[cfg(any(target_os = "linux", target_os = "macos"))] + peer_recv_drain: None, + link_stats: LinkStats::new(), + last_seen, + replay_suppressed_count: 0, + consecutive_decrypt_failures: 0, + mmp: None, + } + } +} + /// A fully authenticated remote FIPS node. /// /// Created only after successful Noise KK handshake. The identity is @@ -84,23 +203,9 @@ pub struct ActivePeer { identity: PeerIdentity, // === Connection === - /// Link used to reach this peer. - link_id: LinkId, /// Current connectivity state. connectivity: ConnectivityState, - // === Session (Wire Protocol) === - /// Noise session for encryption/decryption (None if legacy peer). - noise_session: Option, - /// Our session index (they include this when sending TO us). - our_index: Option, - /// Their session index (we include this when sending TO them). - their_index: Option, - /// Transport ID for this peer's link. - transport_id: Option, - /// Current transport address (for roaming support). - current_addr: Option, - // === Spanning Tree === /// Their latest parent declaration. declaration: Option, @@ -125,27 +230,14 @@ pub struct ActivePeer { /// Whether we owe them a filter update. pending_filter_update: bool, - // === Timing === - /// Session start time for computing session-relative timestamps. - /// Used as the epoch for the 4-byte inner header timestamp field. - session_start: Instant, - // === Statistics === - /// Link statistics. - link_stats: LinkStats, /// When this peer was authenticated (Unix milliseconds). authenticated_at: u64, - /// When this peer was last seen (any activity, Unix milliseconds). - last_seen: u64, // === Epoch (Restart Detection) === /// Remote peer's startup epoch (from handshake). Used to detect restarts. remote_epoch: Option<[u8; 8]>, - // === MMP === - /// Per-peer MMP state (None for legacy peers without Noise sessions). - mmp: Option, - // === Heartbeat === /// When we last sent a heartbeat to this peer. last_heartbeat_sent: Option, @@ -155,12 +247,6 @@ pub struct ActivePeer { /// Cleared after the handshake timeout window. handshake_msg2: Option>, - // === Replay Detection Suppression === - /// Number of replay detections suppressed since last session reset. - replay_suppressed_count: u32, - /// Consecutive decryption failures (reset on any successful decrypt). - consecutive_decrypt_failures: u32, - // === Rekey (Key Rotation) === /// When the current Noise session was established (for rekey timer). session_established_at: Instant, @@ -170,20 +256,6 @@ pub struct ActivePeer { /// dual-initiation in symmetric-start meshes; mean interval is /// preserved. rekey_jitter_secs: i64, - /// Current K-bit epoch value (alternates each rekey). - current_k_bit: bool, - /// Previous session kept alive during drain window after cutover. - previous_session: Option, - /// Previous session's our_index (for peers_by_index cleanup on drain expiry). - previous_our_index: Option, - /// When the drain window started (None = no drain in progress). - drain_started: Option, - /// Pending new session from completed rekey (before K-bit cutover). - pending_new_session: Option, - /// Pending new session's our_index. - pending_our_index: Option, - /// Pending new session's their_index. - pending_their_index: Option, /// Whether a rekey is currently in progress (handshake sent, not yet complete). rekey_in_progress: bool, /// When we last received a rekey msg1 from this peer (dampening). @@ -199,21 +271,10 @@ pub struct ActivePeer { /// In-progress rekey: number of msg1 retransmissions performed so far. rekey_msg1_resend_count: u32, - /// Unix UDP fast-path: per-peer `connect()`-ed socket (paired with - /// the listen socket via `SO_REUSEPORT`). The kernel demux prefers - /// the connected 5-tuple, so inbound packets land here; the - /// encrypt-worker send path sends with `msg_name = NULL`, skipping - /// per-packet sockaddr handling + route lookup. Behind an `Arc` so - /// in-flight worker jobs survive rekey/address-change rotations. - #[cfg(any(target_os = "linux", target_os = "macos"))] - connected_udp: Option>, - - /// Per-peer recv drain thread. Always paired with `connected_udp`: - /// the kernel routes inbound packets from this peer to the - /// connected socket, so it *must* be drained or the kernel recv - /// buffer fills. Drop signals shutdown via self-pipe. - #[cfg(any(target_os = "linux", target_os = "macos"))] - peer_recv_drain: Option, + // === Published active-send-state (two-tier boundary) === + /// The send-critical subset read (and, on roam/responder-cutover, written) + /// directly by the data plane. See `PeerSendState`. + send: PeerSendState, } impl ActivePeer { @@ -225,13 +286,7 @@ impl ActivePeer { let now = Instant::now(); Self { identity, - link_id, connectivity: ConnectivityState::Connected, - noise_session: None, - our_index: None, - their_index: None, - transport_id: None, - current_addr: None, declaration: None, ancestry: None, tree_announce_min_interval_ms: 500, @@ -241,25 +296,12 @@ impl ActivePeer { filter_sequence: 0, filter_received_at: 0, pending_filter_update: true, // Send filter on new connection - session_start: now, - link_stats: LinkStats::new(), authenticated_at, - last_seen: authenticated_at, remote_epoch: None, - mmp: None, last_heartbeat_sent: None, handshake_msg2: None, - replay_suppressed_count: 0, - consecutive_decrypt_failures: 0, session_established_at: now, rekey_jitter_secs: draw_rekey_jitter(), - current_k_bit: false, - previous_session: None, - previous_our_index: None, - drain_started: None, - pending_new_session: None, - pending_our_index: None, - pending_their_index: None, rekey_in_progress: false, last_peer_rekey: None, rekey_handshake: None, @@ -267,10 +309,7 @@ impl ActivePeer { rekey_msg1: None, rekey_msg1_next_resend: 0, rekey_msg1_resend_count: 0, - #[cfg(any(target_os = "linux", target_os = "macos"))] - connected_udp: None, - #[cfg(any(target_os = "linux", target_os = "macos"))] - peer_recv_drain: None, + send: PeerSendState::new(link_id, now, authenticated_at), } } @@ -285,7 +324,7 @@ impl ActivePeer { link_stats: LinkStats, ) -> Self { let mut peer = Self::new(identity, link_id, authenticated_at); - peer.link_stats = link_stats; + peer.send.link_stats = link_stats; peer } @@ -309,15 +348,22 @@ impl ActivePeer { remote_epoch: Option<[u8; 8]>, ) -> Self { let now = Instant::now(); + let mut send = PeerSendState::new(link_id, now, authenticated_at); + send.noise_session = Some(noise_session); + send.our_index = Some(our_index); + send.their_index = Some(their_index); + send.transport_id = Some(transport_id); + send.current_addr = Some(current_addr); + send.link_stats = link_stats; + send.mmp = Some(MmpPeerState::new( + mmp_config.mode, + mmp_config.log_interval_secs, + mmp_config.owd_window_size, + is_initiator, + )); Self { identity, - link_id, connectivity: ConnectivityState::Connected, - noise_session: Some(noise_session), - our_index: Some(our_index), - their_index: Some(their_index), - transport_id: Some(transport_id), - current_addr: Some(current_addr), declaration: None, ancestry: None, tree_announce_min_interval_ms: 500, @@ -327,30 +373,12 @@ impl ActivePeer { filter_sequence: 0, filter_received_at: 0, pending_filter_update: true, - session_start: now, - link_stats, authenticated_at, - last_seen: authenticated_at, remote_epoch, - mmp: Some(MmpPeerState::new( - mmp_config.mode, - mmp_config.log_interval_secs, - mmp_config.owd_window_size, - is_initiator, - )), last_heartbeat_sent: None, handshake_msg2: None, - replay_suppressed_count: 0, - consecutive_decrypt_failures: 0, session_established_at: now, rekey_jitter_secs: draw_rekey_jitter(), - current_k_bit: false, - previous_session: None, - previous_our_index: None, - drain_started: None, - pending_new_session: None, - pending_our_index: None, - pending_their_index: None, rekey_in_progress: false, last_peer_rekey: None, rekey_handshake: None, @@ -358,10 +386,7 @@ impl ActivePeer { rekey_msg1: None, rekey_msg1_next_resend: 0, rekey_msg1_resend_count: 0, - #[cfg(any(target_os = "linux", target_os = "macos"))] - connected_udp: None, - #[cfg(any(target_os = "linux", target_os = "macos"))] - peer_recv_drain: None, + send, } } @@ -374,7 +399,7 @@ impl ActivePeer { pub(crate) fn connected_udp( &self, ) -> Option> { - self.connected_udp.clone() + self.send.connected_udp.clone() } /// Install a per-peer `connect()`-ed UDP socket with its paired @@ -388,10 +413,10 @@ impl ActivePeer { ) { // Drop the old drain BEFORE the old socket so its last fd // reference is released cleanly. - self.peer_recv_drain = None; - self.connected_udp = None; - self.connected_udp = Some(socket); - self.peer_recv_drain = Some(drain); + self.send.peer_recv_drain = None; + self.send.connected_udp = None; + self.send.connected_udp = Some(socket); + self.send.peer_recv_drain = Some(drain); } /// Clear the per-peer connected UDP socket + drain. The drain @@ -401,8 +426,8 @@ impl ActivePeer { #[cfg(any(target_os = "linux", target_os = "macos"))] #[allow(dead_code)] // called from session-deregister + rekey follow-up pub(crate) fn clear_connected_udp(&mut self) { - self.peer_recv_drain = None; - self.connected_udp = None; + self.send.peer_recv_drain = None; + self.send.connected_udp = None; } // === Identity Accessors === @@ -436,7 +461,7 @@ impl ActivePeer { /// Get the link ID. pub fn link_id(&self) -> LinkId { - self.link_id + self.send.link_id } /// Get the connectivity state. @@ -463,34 +488,34 @@ impl ActivePeer { /// Check if this peer has a Noise session. pub fn has_session(&self) -> bool { - self.noise_session.is_some() + self.send.noise_session.is_some() } /// Get the Noise session, if present. pub fn noise_session(&self) -> Option<&NoiseSession> { - self.noise_session.as_ref() + self.send.noise_session.as_ref() } /// Get mutable access to the Noise session. pub fn noise_session_mut(&mut self) -> Option<&mut NoiseSession> { - self.noise_session.as_mut() + self.send.noise_session.as_mut() } /// Get our session index (they use this to send TO us). pub fn our_index(&self) -> Option { - self.our_index + self.send.our_index } /// Get their session index (we use this to send TO them). pub fn their_index(&self) -> Option { - self.their_index + self.send.their_index } /// Update their session index (used during cross-connection resolution /// when the losing node keeps its inbound session but needs the peer's /// outbound index). pub fn set_their_index(&mut self, index: SessionIndex) { - self.their_index = Some(index); + self.send.their_index = Some(index); } /// Replace the Noise session and indices during cross-connection resolution. @@ -509,21 +534,21 @@ impl ActivePeer { new_their_index: SessionIndex, ) -> Option { self.reset_replay_suppressed(); - let old_our_index = self.our_index; - self.noise_session = Some(new_session); - self.our_index = Some(new_our_index); - self.their_index = Some(new_their_index); + let old_our_index = self.send.our_index; + self.send.noise_session = Some(new_session); + self.send.our_index = Some(new_our_index); + self.send.their_index = Some(new_their_index); old_our_index } /// Get the transport ID for this peer. pub fn transport_id(&self) -> Option { - self.transport_id + self.send.transport_id } /// Get the current transport address. pub fn current_addr(&self) -> Option<&TransportAddr> { - self.current_addr.as_ref() + self.send.current_addr.as_ref() } /// Update the current address (for roaming support). @@ -533,10 +558,10 @@ impl ActivePeer { /// use this to invalidate per-peer `connect(2)`-ed UDP sockets whose /// 5-tuple just went stale. pub fn set_current_addr(&mut self, transport_id: TransportId, addr: TransportAddr) -> bool { - let changed = - self.transport_id != Some(transport_id) || self.current_addr.as_ref() != Some(&addr); - self.transport_id = Some(transport_id); - self.current_addr = Some(addr); + let changed = self.send.transport_id != Some(transport_id) + || self.send.current_addr.as_ref() != Some(&addr); + self.send.transport_id = Some(transport_id); + self.send.current_addr = Some(addr); changed } @@ -561,38 +586,38 @@ impl ActivePeer { /// Increment replay suppression counter. Returns the new count. pub fn increment_replay_suppressed(&mut self) -> u32 { - self.replay_suppressed_count += 1; - self.replay_suppressed_count + self.send.replay_suppressed_count += 1; + self.send.replay_suppressed_count } /// Reset replay suppression counter, returning previous count. pub fn reset_replay_suppressed(&mut self) -> u32 { - let count = self.replay_suppressed_count; - self.replay_suppressed_count = 0; + let count = self.send.replay_suppressed_count; + self.send.replay_suppressed_count = 0; count } /// Current replay suppression count. pub fn replay_suppressed_count(&self) -> u32 { - self.replay_suppressed_count + self.send.replay_suppressed_count } // === Decryption Failure Tracking === /// Increment consecutive decryption failure counter, returning new count. pub fn increment_decrypt_failures(&mut self) -> u32 { - self.consecutive_decrypt_failures += 1; - self.consecutive_decrypt_failures + self.send.consecutive_decrypt_failures += 1; + self.send.consecutive_decrypt_failures } /// Reset consecutive decryption failure counter. pub fn reset_decrypt_failures(&mut self) { - self.consecutive_decrypt_failures = 0; + self.send.consecutive_decrypt_failures = 0; } /// Current consecutive decryption failure count. pub fn consecutive_decrypt_failures(&self) -> u32 { - self.consecutive_decrypt_failures + self.send.consecutive_decrypt_failures } // === Epoch Accessors === @@ -663,24 +688,24 @@ impl ActivePeer { /// Get link statistics. pub fn link_stats(&self) -> &LinkStats { - &self.link_stats + &self.send.link_stats } /// Get mutable link statistics. pub fn link_stats_mut(&mut self) -> &mut LinkStats { - &mut self.link_stats + &mut self.send.link_stats } // === MMP Accessors === /// Get MMP state (None for legacy peers without sessions). pub fn mmp(&self) -> Option<&MmpPeerState> { - self.mmp.as_ref() + self.send.mmp.as_ref() } /// Get mutable MMP state. pub fn mmp_mut(&mut self) -> Option<&mut MmpPeerState> { - self.mmp.as_mut() + self.send.mmp.as_mut() } /// Link cost for routing decisions. @@ -716,12 +741,12 @@ impl ActivePeer { /// When this peer was last seen. pub fn last_seen(&self) -> u64 { - self.last_seen + self.send.last_seen } /// Time since last activity. pub fn idle_time(&self, current_time_ms: u64) -> u64 { - current_time_ms.saturating_sub(self.last_seen) + current_time_ms.saturating_sub(self.send.last_seen) } /// Connection duration since authentication. @@ -734,12 +759,12 @@ impl ActivePeer { /// Returns milliseconds since session establishment, truncated to u32. /// Wraps at ~49.7 days which is acceptable for session-relative timing. pub fn session_elapsed_ms(&self) -> u32 { - self.session_start.elapsed().as_millis() as u32 + self.send.session_start.elapsed().as_millis() as u32 } /// When this peer's session started (for link-dead fallback timing). pub fn session_start(&self) -> Instant { - self.session_start + self.send.session_start } // === Heartbeat === @@ -758,7 +783,7 @@ impl ActivePeer { /// Update last seen timestamp. pub fn touch(&mut self, current_time_ms: u64) { - self.last_seen = current_time_ms; + self.send.last_seen = current_time_ms; // If we were stale, receiving traffic makes us connected again if self.connectivity == ConnectivityState::Stale { self.connectivity = ConnectivityState::Connected; @@ -785,12 +810,12 @@ impl ActivePeer { /// Mark peer as connected (e.g., after successful reconnect). pub fn mark_connected(&mut self, current_time_ms: u64) { self.connectivity = ConnectivityState::Connected; - self.last_seen = current_time_ms; + self.send.last_seen = current_time_ms; } /// Update the link ID (e.g., on reconnect). pub fn set_link_id(&mut self, link_id: LinkId) { - self.link_id = link_id; + self.send.link_id = link_id; } // === Tree Updates === @@ -804,7 +829,7 @@ impl ActivePeer { ) { self.declaration = Some(declaration); self.ancestry = Some(ancestry); - self.last_seen = current_time_ms; + self.send.last_seen = current_time_ms; } /// Clear peer's tree position. @@ -858,7 +883,7 @@ impl ActivePeer { self.inbound_filter = Some(filter); self.filter_sequence = sequence; self.filter_received_at = current_time_ms; - self.last_seen = current_time_ms; + self.send.last_seen = current_time_ms; } /// Clear peer's inbound filter. @@ -908,7 +933,7 @@ impl ActivePeer { mode, ..MmpConfig::default() }; - self.mmp = Some(MmpPeerState::new( + self.send.mmp = Some(MmpPeerState::new( config.mode, config.log_interval_secs, config.owd_window_size, @@ -923,7 +948,8 @@ impl ActivePeer { /// is compiled out of release builds. #[cfg(test)] pub(crate) fn test_backdate_session_start(&mut self, age: std::time::Duration) { - self.session_start = self + self.send.session_start = self + .send .session_start .checked_sub(age) .unwrap_or_else(Instant::now); @@ -941,7 +967,7 @@ impl ActivePeer { /// Current K-bit epoch value. pub fn current_k_bit(&self) -> bool { - self.current_k_bit + self.send.current_k_bit } /// Whether a rekey is currently in progress. @@ -969,38 +995,38 @@ impl ActivePeer { /// Get the pending new session's our_index. pub fn pending_our_index(&self) -> Option { - self.pending_our_index + self.send.pending_our_index } /// Get the pending new session's their_index. pub fn pending_their_index(&self) -> Option { - self.pending_their_index + self.send.pending_their_index } /// Get the previous session's our_index (during drain). pub fn previous_our_index(&self) -> Option { - self.previous_our_index + self.send.previous_our_index } /// Get the previous session for decryption fallback. pub fn previous_session(&self) -> Option<&NoiseSession> { - self.previous_session.as_ref() + self.send.previous_session.as_ref() } /// Get mutable access to the previous session for decryption. pub fn previous_session_mut(&mut self) -> Option<&mut NoiseSession> { - self.previous_session.as_mut() + self.send.previous_session.as_mut() } /// Get the pending new session (completed rekey, not yet cut over). pub fn pending_new_session(&self) -> Option<&NoiseSession> { - self.pending_new_session.as_ref() + self.send.pending_new_session.as_ref() } /// Mutable access to the pending new session, for trial-decrypt of an /// inbound frame before promoting it on a peer K-bit flip. pub fn pending_new_session_mut(&mut self) -> Option<&mut NoiseSession> { - self.pending_new_session.as_mut() + self.send.pending_new_session.as_mut() } /// Store a completed rekey session and its indices. @@ -1013,9 +1039,9 @@ impl ActivePeer { our_index: SessionIndex, their_index: SessionIndex, ) { - self.pending_new_session = Some(session); - self.pending_our_index = Some(our_index); - self.pending_their_index = Some(their_index); + self.send.pending_new_session = Some(session); + self.send.pending_our_index = Some(our_index); + self.send.pending_their_index = Some(their_index); self.rekey_in_progress = false; // Clear initiator handshake state (index now lives in pending_our_index) self.rekey_our_index = None; @@ -1031,24 +1057,24 @@ impl ActivePeer { /// flips the K-bit. Returns the old our_index that should remain in peers_by_index /// during the drain window. pub fn cutover_to_new_session(&mut self) -> Option { - let new_session = self.pending_new_session.take()?; - let new_our_index = self.pending_our_index.take(); - let new_their_index = self.pending_their_index.take(); + let new_session = self.send.pending_new_session.take()?; + let new_our_index = self.send.pending_our_index.take(); + let new_their_index = self.send.pending_their_index.take(); // Demote current to previous - self.previous_session = self.noise_session.take(); - self.previous_our_index = self.our_index; - self.drain_started = Some(Instant::now()); + self.send.previous_session = self.send.noise_session.take(); + self.send.previous_our_index = self.send.our_index; + self.send.drain_started = Some(Instant::now()); // Promote pending to current - self.noise_session = Some(new_session); - self.our_index = new_our_index; - self.their_index = new_their_index; + self.send.noise_session = Some(new_session); + self.send.our_index = new_our_index; + self.send.their_index = new_their_index; // Flip K-bit and reset timing - self.current_k_bit = !self.current_k_bit; + self.send.current_k_bit = !self.send.current_k_bit; self.session_established_at = Instant::now(); - self.session_start = Instant::now(); + self.send.session_start = Instant::now(); self.rekey_in_progress = false; self.rekey_msg1_resend_count = 0; self.rekey_jitter_secs = draw_rekey_jitter(); @@ -1056,11 +1082,11 @@ impl ActivePeer { // Reset MMP counters to avoid metric discontinuity let now_ms = crate::time::mono_ms(); - if let Some(mmp) = &mut self.mmp { + if let Some(mmp) = &mut self.send.mmp { mmp.reset_for_rekey(now_ms); } - self.previous_our_index + self.send.previous_our_index } /// Handle receiving a K-bit flip from the peer (responder side). @@ -1068,24 +1094,24 @@ impl ActivePeer { /// Promotes pending_new_session to current, demotes current to previous. /// Returns the old our_index for drain tracking. pub fn handle_peer_kbit_flip(&mut self) -> Option { - let new_session = self.pending_new_session.take()?; - let new_our_index = self.pending_our_index.take(); - let new_their_index = self.pending_their_index.take(); + let new_session = self.send.pending_new_session.take()?; + let new_our_index = self.send.pending_our_index.take(); + let new_their_index = self.send.pending_their_index.take(); // Demote current to previous - self.previous_session = self.noise_session.take(); - self.previous_our_index = self.our_index; - self.drain_started = Some(Instant::now()); + self.send.previous_session = self.send.noise_session.take(); + self.send.previous_our_index = self.send.our_index; + self.send.drain_started = Some(Instant::now()); // Promote pending to current - self.noise_session = Some(new_session); - self.our_index = new_our_index; - self.their_index = new_their_index; + self.send.noise_session = Some(new_session); + self.send.our_index = new_our_index; + self.send.their_index = new_their_index; // Match peer's K-bit - self.current_k_bit = !self.current_k_bit; + self.send.current_k_bit = !self.send.current_k_bit; self.session_established_at = Instant::now(); - self.session_start = Instant::now(); + self.send.session_start = Instant::now(); self.rekey_in_progress = false; self.rekey_msg1_resend_count = 0; self.rekey_jitter_secs = draw_rekey_jitter(); @@ -1093,16 +1119,16 @@ impl ActivePeer { // Reset MMP counters to avoid metric discontinuity let now_ms = crate::time::mono_ms(); - if let Some(mmp) = &mut self.mmp { + if let Some(mmp) = &mut self.send.mmp { mmp.reset_for_rekey(now_ms); } - self.previous_our_index + self.send.previous_our_index } /// Check if the drain window has expired. pub fn drain_expired(&self, drain_secs: u64) -> bool { - match self.drain_started { + match self.send.drain_started { Some(t) => t.elapsed().as_secs() >= drain_secs, None => false, } @@ -1110,7 +1136,7 @@ impl ActivePeer { /// Whether a drain is in progress. pub fn is_draining(&self) -> bool { - self.drain_started.is_some() + self.send.drain_started.is_some() } /// Complete the drain: drop previous session and free its index. @@ -1118,9 +1144,9 @@ impl ActivePeer { /// Returns the previous our_index so the caller can remove it from /// peers_by_index and free it from the IndexAllocator. pub fn complete_drain(&mut self) -> Option { - self.previous_session = None; - self.drain_started = None; - self.previous_our_index.take() + self.send.previous_session = None; + self.send.drain_started = None; + self.send.previous_our_index.take() } /// Abandon an in-progress rekey. @@ -1136,9 +1162,9 @@ impl ActivePeer { self.rekey_in_progress = false; // Return whichever index needs freeing self.rekey_our_index.take().or_else(|| { - self.pending_new_session = None; - self.pending_their_index = None; - self.pending_our_index.take() + self.send.pending_new_session = None; + self.send.pending_their_index = None; + self.send.pending_our_index.take() }) } From 4a0584a5e9a2a189d3416e24b28f44319335567a Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 11:11:20 +0000 Subject: [PATCH 04/11] node/dataplane: add the per-peer machine home and action executor (unwired) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Node.peer_machines (a LinkId-keyed map from the stable link handle to the per-peer control machine) as the home for the machines, and a new dataplane/peer_actions.rs holding execute_peer_actions / advance_peer_machine: the executor that maps each PeerAction the machine emits to its shell call — frame and send a handshake via build_msg2, drive promote_connection and feed the PromotionResult back through the machine, tear a peer down via remove_active_peer, free session indices, report loss via note_link_dead. Actions for the rekey, connected-UDP, and timer paths are stubbed with notes for the commits that fold those mechanisms in. Unwired: nothing drives the machine yet — no live handler path calls the executor and peer_machines is never populated — so this is behavior-neutral; the inbound and outbound establish paths still run their existing inline logic. The executor is cut over path-by-path in the following commits. --- src/node/dataplane/mod.rs | 1 + src/node/dataplane/peer_actions.rs | 216 +++++++++++++++++++++++++++++ src/node/mod.rs | 14 ++ 3 files changed, 231 insertions(+) create mode 100644 src/node/dataplane/peer_actions.rs diff --git a/src/node/dataplane/mod.rs b/src/node/dataplane/mod.rs index 48ec010..e0643a1 100644 --- a/src/node/dataplane/mod.rs +++ b/src/node/dataplane/mod.rs @@ -12,4 +12,5 @@ pub(crate) mod connected_udp; mod dispatch; mod encrypted; mod forwarding; +mod peer_actions; mod rx_loop; diff --git a/src/node/dataplane/peer_actions.rs b/src/node/dataplane/peer_actions.rs new file mode 100644 index 0000000..88f00c3 --- /dev/null +++ b/src/node/dataplane/peer_actions.rs @@ -0,0 +1,216 @@ +//! Executor for the per-peer control machine's [`PeerAction`]s (Step 2 / C3). +//! +//! The per-peer FSM in [`crate::peer::machine`] is a sans-IO reducer: it decides +//! *what* must happen and returns a `Vec`; this module is the *doing* +//! half — the thin driver that maps each action onto the exact shell call it +//! stands for (`build_msg2` + `transport.send`, `promote_connection`, +//! `remove_active_peer`, `index_allocator.free`, `note_link_dead`, …). +//! +//! ## C3-1 skeleton (SHADOW-ONLY) +//! +//! This is the **C3-1** increment: the machine home (`Node.peer_machines`), the +//! executor, and the disjoint-borrow advance helper. It is **unwired** — the live +//! `handle_msg1`/`handle_msg2` path does not drive it yet, so every method here is +//! `#[allow(dead_code)]`. The inbound cutover (`handle_msg1` → `step(InboundMsg1)`) +//! lands in **C3-2**, the outbound cutover (`handle_msg2` / dial) in **C3-3**. +//! +//! Arms the C3 ladder does not yet exercise are inert stubs carrying the sub-commit +//! that realizes them (`OpenTransport`→C3-3, `SendRekey`/`SwapSendState`→C4, +//! `SendLinkMessage`→C4/C5, `SetTimer`/`CancelTimer`→C5 inert, connected-UDP→C6). +//! `RegisterDecryptSession` is a deliberate no-op — see its arm for the C3-2 note. + +use crate::node::Node; +use crate::peer::machine::{PeerAction, PeerEvent}; +use crate::proto::fmp::wire::build_msg2; +use crate::transport::{LinkId, TransportAddr, TransportId}; +use crate::utils::index::SessionIndex; +use crate::{NodeAddr, PeerIdentity}; +use std::collections::VecDeque; + +/// Ambient shell facts a [`PeerAction`] executor needs that the machine's +/// runtime-agnostic action payloads deliberately omit (verified identity, +/// transport target, the msg2 framing indices, the promotion timestamp). +/// +/// Unlike a machine event/action payload this is **executor-side**, so it may +/// hold real values resolved from the wire context (cf. `handle_msg1`'s +/// `wire`/`packet` locals and `drive_promote_to_active`'s ambient args). It is +/// built fresh per driven step by the caller at cutover time (C3-2/C3-3). +#[allow(dead_code)] +pub(in crate::node) struct PeerActionCtx { + /// The authenticated peer identity (GAP-1: `PromoteToActive` / + /// `InvalidateSendState` resolve their `NodeAddr` from this). + pub(in crate::node) verified_identity: PeerIdentity, + /// The transport the exchange is happening over (msg2 send target, decrypt + /// cache-key transport half). + pub(in crate::node) transport_id: TransportId, + /// The peer's wire address (msg2 send target). + pub(in crate::node) remote_addr: TransportAddr, + /// Our session index for this exchange (GAP-2: msg2 framing sender_idx). + pub(in crate::node) our_index: Option, + /// The peer's session index for this exchange (GAP-2: msg2 framing + /// receiver_idx). + pub(in crate::node) their_index: Option, + /// The wire timestamp driving this step (promotion ts / loss-report clock). + pub(in crate::node) now_ms: u64, +} + +impl Node { + /// Advance the machine for `link` by one event and execute the resulting + /// actions. + /// + /// The borrow structure the whole seam turns on (spec risk #8): the machine + /// needs `&mut IndexAllocator` as a synchronous capability *while it is + /// itself borrowed mutably out of `peer_machines`*. `peer_machines` and + /// `index_allocator` are **distinct `Node` fields**, so the collect below is + /// a disjoint two-field borrow the checker accepts; once the actions are + /// collected both borrows drop and the executor runs against `&mut self`. + #[allow(dead_code)] + pub(in crate::node) async fn advance_peer_machine( + &mut self, + link: LinkId, + event: PeerEvent, + now: u64, + ambient: &PeerActionCtx, + ) { + let actions = match self.peer_machines.get_mut(&link) { + // Disjoint field borrow: `self.peer_machines` (the map entry) and + // `self.index_allocator` (the capability) are separate fields. + Some(machine) => machine.step(event, now, &mut self.index_allocator), + None => return, + }; + self.execute_peer_actions(link, ambient, actions).await; + } + + /// Map each [`PeerAction`] onto its shell call (spec's executor table). + /// + /// `PromoteToActive` feeds its [`PromotionResult`](crate::proto::fmp::PromotionResult) + /// back into the machine (GAP-1) and appends the follow-up actions to the same + /// worklist — a queue rather than self-recursion so the async executor stays a + /// single flat future (no boxing) and the emitted order is preserved (the + /// establish sequences always end in `PromoteToActive`, so its follow-ups run + /// after any siblings). + #[allow(dead_code)] + pub(in crate::node) async fn execute_peer_actions( + &mut self, + link: LinkId, + ambient: &PeerActionCtx, + actions: Vec, + ) { + let _ = link; + let mut queue: VecDeque = actions.into(); + while let Some(action) = queue.pop_front() { + match action { + PeerAction::OpenTransport { .. } => { + // C3-3: outbound dial (`initiate_connection`, + // `lifecycle/mod.rs:470`). Outbound establish is not cut over + // until C3-3; inert in the C3-1 skeleton. + } + PeerAction::SendHandshake { bytes } => { + // GAP-2: the machine payload is the UNFRAMED Noise msg2 payload; + // frame it with our/their index (mirrors `handshake.rs:472`'s + // `build_msg2(our_index, their_index, &payload)`) before the + // wire send. A fresh-outbound msg1 (empty payload → build msg1 + // from indices) is framed differently and lands in C3-3. + if let (Some(sender_idx), Some(receiver_idx)) = + (ambient.our_index, ambient.their_index) + { + let frame = build_msg2(sender_idx, receiver_idx, &bytes); + if let Some(transport) = self.transports.get(&ambient.transport_id) { + let _ = transport.send(&ambient.remote_addr, &frame).await; + } + } + } + PeerAction::SendRekey { .. } => { + // C4: rekey msg2 framing (`build_msg2(our_new_index, …)`, + // `handshake.rs:365`) + send. Rekey fold is out of C3 scope. + } + PeerAction::SendLinkMessage { .. } => { + // C4/C5: encrypt + send a link-control frame (heartbeat / filter + // / tree / disconnect). Data-plane-owned; out of C3 scope. + } + PeerAction::PromoteToActive { link: promote_link } => { + // GAP-1: ambient supplies the verified identity + promotion ts + // that `promote_connection` needs (cf. `drive_promote_to_active`). + match self.promote_connection( + promote_link, + ambient.verified_identity, + ambient.now_ms, + ) { + Ok(result) => { + // Feed the outcome back into the machine and fold the + // follow-up actions (RegisterDecryptSession, cross-conn + // frees) into the worklist. Disjoint field borrow again. + let follow = match self.peer_machines.get_mut(&promote_link) { + Some(machine) => machine.step( + PeerEvent::PromotionResolved { result }, + ambient.now_ms, + &mut self.index_allocator, + ), + None => Vec::new(), + }; + queue.extend(follow); + } + Err(_e) => { + // C3-2 realizes the promotion-failure cleanup tail + // (`handle_msg1:587` / `handle_msg2:1005`). + } + } + } + PeerAction::SwapSendState { .. } => { + // C4: initiator cutover (`active.rs:1033` + // `cutover_to_new_session`). + } + PeerAction::InvalidateSendState => { + // GAP-4 (biggest): the FULL teardown. `remove_active_peer` + // (`dispatch.rs:107`) frees the four index slots + // (current/rekey/pending/previous), drops `peers_by_index`, + // unregisters the decrypt worker, removes the FSP `sessions` + // entry and `pending_tun_packets`. The machine emits NO + // `FreeIndex` for those slots, so there is no double-free. + self.remove_active_peer(ambient.verified_identity.node_addr()); + } + PeerAction::RegisterDecryptSession { index } => { + let _ = index; + // C3-2 (HALT-reported): the decrypt-worker registration still + // runs INSIDE `promote_connection` (`handshake.rs:1193/1305`), + // which is the single source of truth for its ~40 direct + // `promote_connection` callers (unit/integration tests) and the + // two live handlers. Relocating it out (GAP-3) would perturb the + // live promote path, so C3-1 keeps it there and drives this + // action as a no-op; the relocation lands with the inbound + // cutover in C3-2. + } + PeerAction::UnregisterDecryptSession { index } => { + // Executor supplies `transport_id` from ambient; keyed by + // (tid, index) like `remove_active_peer` / the rekey drain path. + #[cfg(unix)] + self.unregister_decrypt_worker_session((ambient.transport_id, index.as_u32())); + #[cfg(not(unix))] + let _ = index; + } + PeerAction::FreeIndex { index } => { + let _ = self.index_allocator.free(index); + } + PeerAction::ActivateConnectedUdp | PeerAction::TeardownConnectedUdp => { + // C6: connected-UDP plane ownership (`connected_udp.rs`). + } + PeerAction::SetTimer { .. } | PeerAction::CancelTimer { .. } => { + // C5: timers become actions on the existing quantized tick. + // INERT in C3 — the legacy tick timers still run, so driving + // these would double-schedule (spec risk #7). + } + PeerAction::ReportLost { peer } => { + // The single loss token → the reconciler reflex (`driver.rs:48`). + self.report_peer_lost(peer, ambient.now_ms); + } + } + } + } + + /// `ReportLost` → `note_link_dead` (kept as a named seam so the ambient clock + /// source is explicit and C5 can thread the reconciler-computed backoff). + #[allow(dead_code)] + fn report_peer_lost(&mut self, peer: NodeAddr, now_ms: u64) { + self.note_link_dead(peer, now_ms); + } +} diff --git a/src/node/mod.rs b/src/node/mod.rs index 937645d..b724c1f 100644 --- a/src/node/mod.rs +++ b/src/node/mod.rs @@ -37,6 +37,7 @@ use self::reloadable::Reloadable; pub(crate) const REKEY_JITTER_SECS: i64 = 15; use crate::cache::CoordCache; use crate::node::session::SessionEntry; +use crate::peer::machine::PeerMachine; use crate::peer::{ActivePeer, PeerConnection}; use crate::proto::bloom::{BloomFilter, BloomState}; use crate::proto::fmp::Fmp; @@ -352,6 +353,17 @@ pub struct Node { /// Indexed by LinkId since we don't know the peer's identity yet. connections: HashMap, + // === Per-Peer Control Machines (Step 2 / C3) === + /// Per-peer lifecycle control FSMs, keyed by the stable `LinkId` that spans + /// the handshake→active lifetime. A NEW parallel structure introduced by the + /// node-runtime decomposition: `connections`/`peers` stay byte-unchanged (hot + /// path pristine) and are cut over to this machine home path-by-path. Unwired + /// in C3-1 — the executor (`dataplane/peer_actions.rs`) and advance helper + /// exist but the live `handle_msg1`/`handle_msg2` path does not drive them yet; + /// the inbound cutover lands in C3-2. + #[allow(dead_code)] + peer_machines: HashMap, + // === Peers (Active Phase) === /// Authenticated peers. /// Indexed by NodeAddr (verified identity). @@ -618,6 +630,7 @@ impl Node { child_exit_tx: None, child_exit_rx: None, connections: HashMap::new(), + peer_machines: HashMap::new(), peers: HashMap::new(), sessions: HashMap::new(), identity_cache: HashMap::new(), @@ -765,6 +778,7 @@ impl Node { child_exit_tx: None, child_exit_rx: None, connections: HashMap::new(), + peer_machines: HashMap::new(), peers: HashMap::new(), sessions: HashMap::new(), identity_cache: HashMap::new(), From 0bf031dd3224ee2cc75e0a944767821b5293c7bc Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 12:05:17 +0000 Subject: [PATCH 05/11] node: drive net-new inbound establish through the per-peer machine Cut the net-new inbound handshake (a fresh msg1 that promotes to a new peer, plus the at-capacity reject) over from the inline handle_msg1 logic to the per-peer control machine. handle_msg1 still classifies via establish_inbound and still owns the Noise wire step, the late ACL check, and the promote_connection registry surgery; for the net-new path it now builds the machine, steps it, and executes the returned actions. Authorization is interposed between two machine steps so the session index is allocated only after the ACL check passes: a rejected or unauthorized msg1 consumes no index, matching the prior order exactly. The msg2 wire bytes, the index-allocation sequence, and the reject metrics are all byte-neutral. Restart, resend, rekey-respond, and the other reject arms stay inline unchanged; they move to the machine once outbound establish is cut over and every promoted peer has a machine. Also fills in the executor's send-failure and promote-failure cleanup so a mid-establish error tears the leg down and frees its index exactly as before. --- src/node/dataplane/mod.rs | 2 + src/node/dataplane/peer_actions.rs | 43 +++++- src/node/handlers/handshake.rs | 190 ++++++++++++++++++++++- src/peer/machine.rs | 234 +++++++++++++++++++++++------ 4 files changed, 412 insertions(+), 57 deletions(-) diff --git a/src/node/dataplane/mod.rs b/src/node/dataplane/mod.rs index e0643a1..bcf5d61 100644 --- a/src/node/dataplane/mod.rs +++ b/src/node/dataplane/mod.rs @@ -14,3 +14,5 @@ mod encrypted; mod forwarding; mod peer_actions; mod rx_loop; + +pub(in crate::node) use peer_actions::PeerActionCtx; diff --git a/src/node/dataplane/peer_actions.rs b/src/node/dataplane/peer_actions.rs index 88f00c3..40d1891 100644 --- a/src/node/dataplane/peer_actions.rs +++ b/src/node/dataplane/peer_actions.rs @@ -20,6 +20,7 @@ //! `RegisterDecryptSession` is a deliberate no-op — see its arm for the C3-2 note. use crate::node::Node; +use crate::node::reject::{HandshakeReject, RejectReason}; use crate::peer::machine::{PeerAction, PeerEvent}; use crate::proto::fmp::wire::build_msg2; use crate::transport::{LinkId, TransportAddr, TransportId}; @@ -96,7 +97,6 @@ impl Node { ambient: &PeerActionCtx, actions: Vec, ) { - let _ = link; let mut queue: VecDeque = actions.into(); while let Some(action) = queue.pop_front() { match action { @@ -115,8 +115,30 @@ impl Node { (ambient.our_index, ambient.their_index) { let frame = build_msg2(sender_idx, receiver_idx, &bytes); - if let Some(transport) = self.transports.get(&ambient.transport_id) { - let _ = transport.send(&ambient.remote_addr, &frame).await; + // GAP-5: surface the send Result. A missing transport skips + // the send and continues (mirrors `handle_msg1`'s + // `if let Some(transport)` guard); a send *error* runs the + // pre-refactor msg2-send-failure cleanup (`handle_msg1` + // L494-503) and ABORTS the remaining queue so the queued + // `PromoteToActive` never runs. + let send_err = match self.transports.get(&ambient.transport_id) { + Some(transport) => { + transport.send(&ambient.remote_addr, &frame).await.is_err() + } + None => false, + }; + if send_err { + self.connections.remove(&link); + self.links.remove(&link); + self.addr_to_link + .remove(&(ambient.transport_id, ambient.remote_addr.clone())); + if let Some(idx) = ambient.our_index { + let _ = self.index_allocator.free(idx); + } + self.peer_machines.remove(&link); + self.stats_mut() + .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); + return; } } } @@ -151,8 +173,19 @@ impl Node { queue.extend(follow); } Err(_e) => { - // C3-2 realizes the promotion-failure cleanup tail - // (`handle_msg1:587` / `handle_msg2:1005`). + // GAP-4: promotion failed. `promote_connection` already + // removed `connections[link]`; mirror the pre-refactor + // cleanup (`handle_msg1` L587-591): drop the link + + // reverse map, free our index, discard the machine, and + // record the reject. The queue is drained (PromoteToActive + // is the last establish action), so no explicit abort. + self.remove_link(&promote_link); + if let Some(idx) = ambient.our_index { + let _ = self.index_allocator.free(idx); + } + self.peer_machines.remove(&promote_link); + self.stats_mut() + .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); } } } diff --git a/src/node/handlers/handshake.rs b/src/node/handlers/handshake.rs index 8b6875f..9aff408 100644 --- a/src/node/handlers/handshake.rs +++ b/src/node/handlers/handshake.rs @@ -3,8 +3,10 @@ use crate::NodeAddr; use crate::PeerIdentity; use crate::node::acl::PeerAclContext; +use crate::node::dataplane::PeerActionCtx; use crate::node::reject::{HandshakeReject, RejectReason}; use crate::node::{Node, NodeError}; +use crate::peer::machine::{FailReason, HandshakePhase, PeerEvent, PeerMachine, PeerState}; use crate::peer::{ActivePeer, PeerConnection}; use crate::proto::fmp::wire::{Msg1Header, Msg2Header, build_msg2}; use crate::proto::fmp::{ @@ -272,13 +274,47 @@ impl Node { // shared authorize → allocate → send-msg2 → promote tail; the other // variants complete the rate-limiter and return here. match self.fmp.establish_inbound(&est, &wire) { - InboundDecision::Reject { reason } => { + InboundDecision::Reject { + reason: InboundReject::AtMaxPeers, + } => { + // C3-2a net-new arm: drive the reject through the machine to + // prove realization A — a net-new msg1 at the max-peers cap + // reaches `Failed{Rejected}` with the index allocator untouched + // (no allocate before the reject). The transient machine is + // discarded (never inserted into `peer_machines`); `conn`/ + // `link_id` were never inserted into the registry either. + debug!( + peer = %self.peer_display_name(&peer_node_addr), + max = self.max_peers(), + "Silent-dropping Msg1 at max_peers cap (early gate; no Msg2 sent)" + ); + let mut machine = PeerMachine::new_inbound(link_id, packet.timestamp_ms); + let _ = machine.step( + PeerEvent::InboundMsg1 { + link: link_id, + wire, + est, + }, + packet.timestamp_ms, + &mut self.index_allocator, + ); + debug_assert!(matches!( + machine.state(), + PeerState::Failed { + reason: FailReason::Rejected + } + )); + self.msg1_rate_limiter.complete_handshake(); + self.stats_mut() + .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); + return; + } + InboundDecision::Reject { + reason: reason @ (InboundReject::PendingSession | InboundReject::DualRekeyWon), + } => { + // Existing-peer rekey rejects — still inline (C4). Byte-unchanged + // from the pre-refactor shared reject tail. match reason { - InboundReject::AtMaxPeers => debug!( - peer = %self.peer_display_name(&peer_node_addr), - max = self.max_peers(), - "Silent-dropping Msg1 at max_peers cap (early gate; no Msg2 sent)" - ), InboundReject::PendingSession => debug!( peer = %self.peer_display_name(&peer_node_addr), "Rekey msg1 received but already have pending session, dropping" @@ -287,6 +323,7 @@ impl Node { peer = %self.peer_display_name(&peer_node_addr), "Dual rekey initiation: we win (smaller addr), dropping their msg1" ), + InboundReject::AtMaxPeers => unreachable!(), } // `conn`/`link_id` were never inserted into the registry, so the // local drop suffices — no cleanup needed. @@ -419,7 +456,146 @@ impl Node { .unwrap_or(0); self.note_link_dead(peer, now_ms); } - InboundDecision::Promote => {} + InboundDecision::Promote => { + // === C3-2a: net-new inbound establish, driven by the machine. === + // Realization A (two-phase authorize): Phase 1 classifies with no + // allocation; the shell interposes the late-ACL gate here; Phase 2 + // allocates the single index and emits [SendHandshake, + // PromoteToActive]. A rejected/unauthorized msg1 therefore + // allocates NO index — matching the pre-refactor + // authorize-before-allocate ordering exactly. + + // Keep the shell's own copies of the msg2 framing inputs before + // the machine event consumes `wire` (WireOutcome is not Clone). + let msg2_payload = wire.msg2_payload.clone(); + let their_index = wire.their_index; + + let mut machine = PeerMachine::new_inbound(link_id, packet.timestamp_ms); + + // Phase 1: classify (no allocation, no actions for a net-new leg). + let phase1 = machine.step( + PeerEvent::InboundMsg1 { + link: link_id, + wire, + est, + }, + packet.timestamp_ms, + &mut self.index_allocator, + ); + debug_assert!(phase1.is_empty()); + debug_assert!(matches!( + machine.state(), + PeerState::Handshaking { + phase: HandshakePhase::ReceivedMsg1, + .. + } + )); + + // Shell interposition: late-ACL authorize BEFORE any allocation. + if self + .authorize_peer( + &peer_identity, + PeerAclContext::InboundHandshake, + packet.transport_id, + &packet.remote_addr, + ) + .is_err() + { + let _ = machine.step( + PeerEvent::Rejected, + packet.timestamp_ms, + &mut self.index_allocator, + ); + self.msg1_rate_limiter.complete_handshake(); + self.stats_mut() + .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); + return; + } + + // Phase 2: allocate our index + emit [SendHandshake, PromoteToActive]. + let promote_actions = machine.step( + PeerEvent::Authorized, + packet.timestamp_ms, + &mut self.index_allocator, + ); + let our_index = match machine.our_index() { + Some(idx) => idx, + None => { + // Allocation exhausted in Phase 2 (mirrors the pre-refactor + // allocate-failure path): no msg2, no promote. + self.msg1_rate_limiter.complete_handshake(); + self.stats_mut() + .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); + return; + } + }; + + // Shell registry surgery (Option A1), in the pre-refactor order: + // set indices on the shell connection, insert link / reverse map / + // connection, then build + store the framed msg2. + conn.set_our_index(our_index); + conn.set_their_index(their_index); + let link = Link::connectionless( + link_id, + packet.transport_id, + packet.remote_addr.clone(), + LinkDirection::Inbound, + Duration::from_millis(self.config().node.base_rtt_ms), + ); + self.links.insert(link_id, link); + self.addr_to_link.insert(addr_key, link_id); + self.connections.insert(link_id, conn); + let wire_msg2 = build_msg2(our_index, their_index, &msg2_payload); + if let Some(conn) = self.connections.get_mut(&link_id) { + conn.set_handshake_msg2(wire_msg2.clone()); + } + + // Register the machine (Promote tail only — discarded on every + // reject/resend/rekey arm per the insertion discipline). + self.peer_machines.insert(link_id, machine); + + // Execute [SendHandshake, PromoteToActive]. The executor frames + + // sends msg2 (bytes identical to `wire_msg2`), promotes via + // `promote_connection`, feeds PromotionResolved back, and runs the + // inert RegisterDecryptSession (R2 — register stays in + // `promote_connection`). Its send-failure / promote-failure arms + // run the pre-refactor cleanup and remove the machine, leaving it + // absent (not Established). + let ambient = PeerActionCtx { + verified_identity: peer_identity, + transport_id: packet.transport_id, + remote_addr: packet.remote_addr.clone(), + our_index: Some(our_index), + their_index: Some(their_index), + now_ms: packet.timestamp_ms, + }; + self.execute_peer_actions(link_id, &ambient, promote_actions) + .await; + + // Post-`Promoted` shell tail (byte-identical to the pre-refactor + // Promoted arm), reached only when promotion succeeded (the machine + // is now Established); a send/promote failure removed the machine + // and already cleaned up. + if matches!( + self.peer_machines.get(&link_id).map(|m| m.state()), + Some(PeerState::Established { .. }) + ) { + // Store msg2 on peer for resend on duplicate msg1 + if let Some(peer) = self.peers.get_mut(&peer_node_addr) { + peer.set_handshake_msg2(wire_msg2.clone()); + } + // Send initial tree announce to new peer + if let Err(e) = self.send_tree_announce_to_peer(&peer_node_addr).await { + debug!(peer = %self.peer_display_name(&peer_node_addr), error = %e, "Failed to send initial TreeAnnounce"); + } + // Schedule filter announce (sent on next tick via debounce) + self.bloom_state.mark_update_needed(peer_node_addr); + self.reset_lookup_backoff(); + } + + self.msg1_rate_limiter.complete_handshake(); + return; + } } if self diff --git a/src/peer/machine.rs b/src/peer/machine.rs index 179b4a4..2b5e885 100644 --- a/src/peer/machine.rs +++ b/src/peer/machine.rs @@ -305,6 +305,10 @@ pub(crate) struct PeerMachine { conn: ConnectionState, /// Remote startup epoch (establish-path-only; NOT in send-state). remote_epoch: Option<[u8; 8]>, + /// Inbound two-phase authorize (realization A): the opaque Noise msg2 + /// payload stashed in Phase 1 (`InboundMsg1`) and emitted in Phase 2 + /// (`on_authorized`), so a rejected/unauthorized msg1 allocates no index. + pending_msg2_payload: Option>, // --- rekey negotiation sub-state (control tier; NOT the pending send slot) --- rekey_in_progress: bool, @@ -343,6 +347,7 @@ impl PeerMachine { identity: Some(identity), conn: ConnectionState::outbound(link, identity, now), remote_epoch: None, + pending_msg2_payload: None, rekey_in_progress: false, rekey_our_index: None, rekey_msg1: None, @@ -368,6 +373,7 @@ impl PeerMachine { identity: None, conn: ConnectionState::inbound(link, now), remote_epoch: None, + pending_msg2_payload: None, rekey_in_progress: false, rekey_our_index: None, rekey_msg1: None, @@ -387,6 +393,14 @@ impl PeerMachine { self.state } + /// The index we allocated for this peer's inbound session, once Phase 2 + /// (`on_authorized`) has run. `None` before allocation (and after a + /// rejected/unauthorized msg1). The inbound cutover reads this to perform + /// the shell registry surgery with the machine-owned index (Option A1). + pub(crate) fn our_index(&self) -> Option { + self.our_index + } + /// The crystallized node address, if identity is known. fn addr(&self) -> Option { self.identity.map(|id| *id.node_addr()) @@ -421,7 +435,7 @@ impl PeerMachine { PeerEvent::Msg2 { their_index, out } => { self.on_msg2(their_index, out, now, index_allocator) } - PeerEvent::Authorized => Vec::new(), + PeerEvent::Authorized => self.on_authorized(now, index_allocator), PeerEvent::Rejected => self.fail(FailReason::AclRejected), PeerEvent::PromotionResolved { result } => self.on_promotion_resolved(result, now), PeerEvent::RekeyMsg1 { wire, est } => { @@ -608,40 +622,67 @@ impl PeerMachine { actions.push(PeerAction::UnregisterDecryptSession { index: idx }); } actions.push(PeerAction::ReportLost { peer }); - actions.extend(self.inbound_promote(link, &wire, now, alloc)); + actions.extend(self.inbound_classify(link, &wire)); actions } - InboundDecision::Promote => self.inbound_promote(link, &wire, now, alloc), + InboundDecision::Promote => self.inbound_classify(link, &wire), } } - /// The inbound Promote tail: allocate our index, record indices/epoch/msg2, - /// emit msg2 + drive promotion. `RegisterDecryptSession` follows on the - /// `PromotionResolved{Promoted}` feedback (§3.2 "then on PromotionResult"). - fn inbound_promote( - &mut self, - link: LinkId, - wire: &WireOutcome, - _now: u64, - alloc: &mut IndexAllocator, - ) -> Vec { + /// Inbound **Phase 1** (realization A): classify the fresh leg *without* + /// allocating an index. Records identity/epoch/their-index and stashes the + /// opaque msg2 payload, parking at `Handshaking{ReceivedMsg1}` — the + /// "awaiting Authorized" marker. The index allocation and the msg2/promote + /// emission happen in Phase 2 ([`Self::on_authorized`]) only after the + /// shell's late-ACL gate passes, so a rejected/unauthorized msg1 allocates + /// nothing (preserving the pre-refactor global index-allocation sequence). + fn inbound_classify(&mut self, link: LinkId, wire: &WireOutcome) -> Vec { self.identity = Some(wire.peer_identity); self.remote_epoch = wire.remote_epoch; self.conn.set_their_index(wire.their_index); - let our_index = alloc.allocate().ok(); - if let Some(idx) = our_index { - self.conn.set_our_index(idx); - self.our_index = Some(idx); - } - self.conn.set_handshake_msg2(wire.msg2_payload.clone()); + self.pending_msg2_payload = Some(wire.msg2_payload.clone()); self.state = PeerState::Handshaking { link, phase: HandshakePhase::ReceivedMsg1, }; + Vec::new() + } + + /// Inbound **Phase 2** (realization A): the late-ACL gate passed shell-side. + /// Allocate our index NOW — the single inbound allocation point — record it + /// on `conn`, and emit the msg2 send + promotion. `RegisterDecryptSession` + /// follows on the `PromotionResolved{Promoted}` feedback (§3.2). Guarded to + /// the inbound `ReceivedMsg1` phase so the benign outbound `Authorized` + /// confirmation stays a no-op (state `Handshaking{SentMsg1}` and every other + /// state fall through to `Vec::new()`). + fn on_authorized(&mut self, _now: u64, alloc: &mut IndexAllocator) -> Vec { + if !matches!( + self.state, + PeerState::Handshaking { + phase: HandshakePhase::ReceivedMsg1, + .. + } + ) { + return Vec::new(); + } + let our_index = match alloc.allocate() { + Ok(idx) => idx, + Err(_) => { + // Allocation exhausted: no index, no msg2, no promote. The shell + // records the reject + completes the rate-limiter bracket + // (mirrors the pre-refactor `handle_msg1` allocate-failure path). + self.state = PeerState::Failed { + reason: FailReason::Rejected, + }; + return Vec::new(); + } + }; + self.conn.set_our_index(our_index); + self.our_index = Some(our_index); + let bytes = self.pending_msg2_payload.take().unwrap_or_default(); + let link = self.link; vec![ - PeerAction::SendHandshake { - bytes: wire.msg2_payload.clone(), - }, + PeerAction::SendHandshake { bytes }, PeerAction::PromoteToActive { link }, ] } @@ -1352,22 +1393,37 @@ mod tests { &mut alloc, ); - // Restart tail: invalidate, unregister old, report lost, then Promote. - assert_eq!(actions[0], PeerAction::InvalidateSendState); + // Phase 1: restart tail only (invalidate, unregister old, report lost), + // then park at ReceivedMsg1 — no index allocated yet (realization A). assert_eq!( - actions[1], - PeerAction::UnregisterDecryptSession { - index: SessionIndex::new(0xDEAD) - } + actions, + vec![ + PeerAction::InvalidateSendState, + PeerAction::UnregisterDecryptSession { + index: SessionIndex::new(0xDEAD) + }, + PeerAction::ReportLost { peer: peer_addr }, + ] ); - assert_eq!(actions[2], PeerAction::ReportLost { peer: peer_addr }); + assert!(matches!( + m.state(), + PeerState::Handshaking { + phase: HandshakePhase::ReceivedMsg1, + .. + } + )); + assert_eq!(m.our_index(), None); + assert_eq!(alloc.count(), 0); + + // Phase 2: late-ACL gate passed -> allocate + Promote tail. + let promote = m.step(PeerEvent::Authorized, 1_000, &mut alloc); assert!( - actions + promote .iter() .any(|a| matches!(a, PeerAction::SendHandshake { .. })) ); assert!( - actions + promote .iter() .any(|a| matches!(a, PeerAction::PromoteToActive { .. })) ); @@ -1400,7 +1456,7 @@ mod tests { let our = *peer_identity().node_addr(); let est_w = est_new_peer(our); let wire_w = wire_outcome(peer, Some([3u8; 8]), 0x77); - let wa = winner.step( + let wp1 = winner.step( PeerEvent::InboundMsg1 { link: LinkId::new(1), wire: wire_w, @@ -1409,6 +1465,8 @@ mod tests { 100, &mut alloc, ); + assert!(wp1.is_empty()); // Phase 1 classifies without emitting. + let wa = winner.step(PeerEvent::Authorized, 100, &mut alloc); assert!( wa.iter().any( |a| matches!(a, PeerAction::PromoteToActive { link } if *link == LinkId::new(1)) @@ -1432,7 +1490,7 @@ mod tests { let mut loser = PeerMachine::new_inbound(LinkId::new(2), 0); let est_l = est_new_peer(our); let wire_l = wire_outcome(peer, Some([3u8; 8]), 0x88); - let la = loser.step( + let lp1 = loser.step( PeerEvent::InboundMsg1 { link: LinkId::new(2), wire: wire_l, @@ -1441,6 +1499,8 @@ mod tests { 100, &mut alloc, ); + assert!(lp1.is_empty()); // Phase 1 classifies without emitting. + let la = loser.step(PeerEvent::Authorized, 100, &mut alloc); assert!( la.iter() .any(|a| matches!(a, PeerAction::PromoteToActive { .. })) @@ -1490,7 +1550,9 @@ mod tests { let est = est_new_peer(our); let wire = wire_outcome(peer, Some([4u8; 8]), 0x77); - let mut actions = m.step( + // Phase 1 (InboundMsg1): classify only — no actions, no allocation, + // parked at ReceivedMsg1 (realization A). + let phase1 = m.step( PeerEvent::InboundMsg1 { link: LinkId::new(1), wire, @@ -1499,29 +1561,111 @@ mod tests { 200, &mut alloc, ); - actions.extend(m.step( + assert!(phase1.is_empty()); + assert_eq!( + m.state(), + PeerState::Handshaking { + link: LinkId::new(1), + phase: HandshakePhase::ReceivedMsg1 + } + ); + assert_eq!(m.our_index(), None); + assert_eq!(alloc.count(), 0); // allocator untouched pre-authorize + + // Phase 2 (Authorized): allocate + [SendHandshake, PromoteToActive]. + let phase2 = m.step(PeerEvent::Authorized, 200, &mut alloc); + assert!(matches!(phase2[0], PeerAction::SendHandshake { .. })); + assert_eq!( + phase2[1], + PeerAction::PromoteToActive { + link: LinkId::new(1) + } + ); + assert!(m.our_index().is_some()); + assert_eq!(alloc.count(), 1); // exactly one index allocated + + // Phase 3 (PromotionResolved{Promoted}): register + Established. + let phase3 = m.step( PeerEvent::PromotionResolved { result: PromotionResult::Promoted(peer_addr), }, 200, &mut alloc, - )); - - // Combined promote sequence (§3.2 "then on PromotionResult ..."). - assert!(matches!(actions[0], PeerAction::SendHandshake { .. })); - assert_eq!( - actions[1], - PeerAction::PromoteToActive { - link: LinkId::new(1) - } ); assert!(matches!( - actions[2], + phase3[0], PeerAction::RegisterDecryptSession { .. } )); assert_eq!(m.state(), PeerState::Established { addr: peer_addr }); } + // ---- Test 6b: inbound late-ACL rejected -> no allocation -------------- + #[test] + fn inbound_authorize_rejected_no_alloc() { + let mut alloc = IndexAllocator::new(); + let peer = peer_identity(); + let mut m = PeerMachine::new_inbound(LinkId::new(1), 0); + let our = *peer_identity().node_addr(); + let est = est_new_peer(our); + let wire = wire_outcome(peer, Some([4u8; 8]), 0x77); + + // Phase 1 classifies (no alloc). + let phase1 = m.step( + PeerEvent::InboundMsg1 { + link: LinkId::new(1), + wire, + est, + }, + 200, + &mut alloc, + ); + assert!(phase1.is_empty()); + assert_eq!(alloc.count(), 0); + + // Late-ACL rejects -> Failed{AclRejected}, still no allocation. + let rej = m.step(PeerEvent::Rejected, 200, &mut alloc); + assert!(rej.is_empty()); + assert_eq!( + m.state(), + PeerState::Failed { + reason: FailReason::AclRejected + } + ); + assert_eq!(alloc.count(), 0); + assert_eq!(m.our_index(), None); + } + + // ---- Test 6c: inbound reject at max_peers -> no allocation ------------ + #[test] + fn inbound_at_max_peers_reject_no_alloc() { + let mut alloc = IndexAllocator::new(); + let peer = peer_identity(); + let mut m = PeerMachine::new_inbound(LinkId::new(1), 0); + let our = *peer_identity().node_addr(); + let mut est = est_new_peer(our); + est.at_max_peers = true; + let wire = wire_outcome(peer, Some([4u8; 8]), 0x77); + + let actions = m.step( + PeerEvent::InboundMsg1 { + link: LinkId::new(1), + wire, + est, + }, + 200, + &mut alloc, + ); + assert!(actions.is_empty()); + assert_eq!( + m.state(), + PeerState::Failed { + reason: FailReason::Rejected + } + ); + assert_eq!(alloc.count(), 0); + assert_eq!(m.our_index(), None); + } + // ---- Test 7: outbound establish (+ cross-connection) ------------------ #[test] fn outbound_establish() { From c80a7fdea50888c53beedcf0aa124360c4475b9b Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 12:46:23 +0000 Subject: [PATCH 06/11] node: drive restart inbound establish through the per-peer machine Cut the restart handshake (an inbound msg1 from a peer that reconnected with a new epoch) over to the per-peer control machine, completing the inbound establish cutover. The old peer is torn down and the fresh leg promotes through the same two-step authorize-then-allocate path as the net-new case: the machine's first step emits the old-peer teardown (invalidate send-state, report loss), the shell interposes the ACL check, and the second step allocates the new index and sends msg2. The old index is freed before the new one is allocated and the msg2 wire bytes are unchanged, so the sequence stays byte-neutral. With restart driven through the machine, the shared inline establish tail that only the restart arm reached is deleted. Also bounds the peer_machines map (remove_active_peer now drops the peer's machine entry) and restores the msg2-send, promote, and index-allocation failure warnings the cutover had dropped. --- src/node/dataplane/dispatch.rs | 9 + src/node/dataplane/peer_actions.rs | 17 +- src/node/handlers/handshake.rs | 380 +++++++++++++++-------------- 3 files changed, 215 insertions(+), 191 deletions(-) diff --git a/src/node/dataplane/dispatch.rs b/src/node/dataplane/dispatch.rs index 4d8bdd3..5bee263 100644 --- a/src/node/dataplane/dispatch.rs +++ b/src/node/dataplane/dispatch.rs @@ -187,6 +187,15 @@ impl Node { // Remove link and address mapping self.remove_link(&link_id); + // Bound `peer_machines` (C3-2b follow-up #2): drop this peer's machine + // entry, keyed by the `link_id` derived above BEFORE the `peers` removal. + // This cleans up the OLD peer's machine on an inbound restart and prevents + // unbounded growth on the establish success path. NEUTRAL: nothing on the + // live path reads `peer_machines` except the establish executor, which only + // ever touches the in-flight establish's (distinct) `link_id`; no reader + // depends on a stale entry, so removal changes no behavior — it only bounds + // the map. + self.peer_machines.remove(&link_id); if let Some(transport_id) = transport_id { self.cleanup_bootstrap_transport_if_unused(transport_id); } diff --git a/src/node/dataplane/peer_actions.rs b/src/node/dataplane/peer_actions.rs index 40d1891..f1f8aab 100644 --- a/src/node/dataplane/peer_actions.rs +++ b/src/node/dataplane/peer_actions.rs @@ -27,6 +27,7 @@ use crate::transport::{LinkId, TransportAddr, TransportId}; use crate::utils::index::SessionIndex; use crate::{NodeAddr, PeerIdentity}; use std::collections::VecDeque; +use tracing::warn; /// Ambient shell facts a [`PeerAction`] executor needs that the machine's /// runtime-agnostic action payloads deliberately omit (verified identity, @@ -123,11 +124,15 @@ impl Node { // `PromoteToActive` never runs. let send_err = match self.transports.get(&ambient.transport_id) { Some(transport) => { - transport.send(&ambient.remote_addr, &frame).await.is_err() + transport.send(&ambient.remote_addr, &frame).await.err() } - None => false, + None => None, }; - if send_err { + if let Some(e) = send_err { + // Restored pre-refactor msg2-send-failure warn! + // (`handle_msg1` L665): the send error text is surfaced + // at the executor point where the failure is now handled. + warn!(link_id = %link, error = %e, "Failed to send msg2"); self.connections.remove(&link); self.links.remove(&link); self.addr_to_link @@ -172,13 +177,17 @@ impl Node { }; queue.extend(follow); } - Err(_e) => { + Err(e) => { // GAP-4: promotion failed. `promote_connection` already // removed `connections[link]`; mirror the pre-refactor // cleanup (`handle_msg1` L587-591): drop the link + // reverse map, free our index, discard the machine, and // record the reject. The queue is drained (PromoteToActive // is the last establish action), so no explicit abort. + // + // Restored pre-refactor promote-failure warn! + // (`handle_msg1` L757). + warn!(link_id = %promote_link, error = %e, "Failed to promote inbound connection"); self.remove_link(&promote_link); if let Some(idx) = ambient.our_index { let _ = self.index_allocator.free(idx); diff --git a/src/node/handlers/handshake.rs b/src/node/handlers/handshake.rs index 9aff408..ba1b7f0 100644 --- a/src/node/handlers/handshake.rs +++ b/src/node/handlers/handshake.rs @@ -307,7 +307,6 @@ impl Node { self.msg1_rate_limiter.complete_handshake(); self.stats_mut() .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); - return; } InboundDecision::Reject { reason: reason @ (InboundReject::PendingSession | InboundReject::DualRekeyWon), @@ -330,7 +329,6 @@ impl Node { self.msg1_rate_limiter.complete_handshake(); self.stats_mut() .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); - return; } InboundDecision::ResendMsg2 { msg2 } => { if let Some(msg2) = msg2.as_deref() @@ -349,7 +347,6 @@ impl Node { } } self.msg1_rate_limiter.complete_handshake(); - return; } InboundDecision::RekeyRespond { peer, @@ -439,22 +436,201 @@ impl Node { // as rekeys (not new connections). The temporary `conn`/`link_id` // were never inserted into the registry, so no cleanup is needed. self.msg1_rate_limiter.complete_handshake(); - return; } InboundDecision::RestartThenPromote { peer } => { - // Epoch mismatch — peer restarted. Tear down the stale session - // and schedule a reconnect, then fall through to promote the - // fresh handshake as a new connection. + // === C3-2b: restart inbound establish, driven by the machine. === + // Epoch mismatch — the peer restarted. The fresh leg is promoted + // exactly like a net-new inbound (realization A two-phase + // authorize); the OLD peer's teardown is the machine's Phase-1 + // `[InvalidateSendState, ReportLost{peer}]`: + // InvalidateSendState → remove_active_peer(old): frees the four + // index slots + `peers_by_index` + decrypt unregister + FSP + // `sessions` + `pending_tun_packets` (GAP-4). The fresh leg's + // `our_index` is None, so the machine emits NO + // UnregisterDecryptSession (N1). + // ReportLost{peer} → note_link_dead(old): reconnect backoff. + // These execute BEFORE authorize/allocate, preserving the + // pre-refactor order exactly (remove_active_peer → note_link_dead → + // authorize → allocate → send msg2 → promote). `peer` here equals + // `peer_identity.node_addr()` (see `establish_inbound`), so the + // executor's `InvalidateSendState` + // (`ambient.verified_identity.node_addr()`) targets the same addr + // as the pre-refactor `remove_active_peer(&peer)`. debug!( peer = %self.peer_display_name(&peer), "Peer restart detected (epoch mismatch), removing stale session" ); - self.remove_active_peer(&peer); - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - self.note_link_dead(peer, now_ms); + + // Keep the shell's own copies of the msg2 framing inputs before + // the machine event consumes `wire` (WireOutcome is not Clone). + let msg2_payload = wire.msg2_payload.clone(); + let their_index = wire.their_index; + + let mut machine = PeerMachine::new_inbound(link_id, packet.timestamp_ms); + + // Phase 1: classify + emit the old-peer teardown. For a restart the + // fresh leg has `our_index == None`, so the emitted sequence is + // exactly [InvalidateSendState, ReportLost{peer}]; the machine then + // parks at Handshaking{ReceivedMsg1} (no allocation — realization A). + let phase1 = machine.step( + PeerEvent::InboundMsg1 { + link: link_id, + wire, + est, + }, + packet.timestamp_ms, + &mut self.index_allocator, + ); + debug_assert!(matches!( + machine.state(), + PeerState::Handshaking { + phase: HandshakePhase::ReceivedMsg1, + .. + } + )); + + // Execute the Phase-1 teardown, in emitted order + // (InvalidateSendState before ReportLost, both before + // authorize/alloc). N2 CLOCK NOTE — INTENTIONAL DIVERGENCE: the + // pre-refactor arm timestamped `note_link_dead` with + // `SystemTime::now()` wall-clock; routing `ReportLost` through the + // executor uses `ambient.now_ms == packet.timestamp_ms`. This is an + // accepted sub-millisecond reconnect-backoff timing shift — NOT + // on-wire, NOT index/metrics — see design/step2-c3-2-blueprint.md + // N2. The machine is not yet in `peer_machines`, but these two + // actions do not touch the map, so executing them here is safe. + let teardown_ctx = PeerActionCtx { + verified_identity: peer_identity, + transport_id: packet.transport_id, + remote_addr: packet.remote_addr.clone(), + our_index: None, + their_index: Some(their_index), + now_ms: packet.timestamp_ms, + }; + self.execute_peer_actions(link_id, &teardown_ctx, phase1) + .await; + + // Shell interposition: late-ACL authorize BEFORE any allocation. + if self + .authorize_peer( + &peer_identity, + PeerAclContext::InboundHandshake, + packet.transport_id, + &packet.remote_addr, + ) + .is_err() + { + let _ = machine.step( + PeerEvent::Rejected, + packet.timestamp_ms, + &mut self.index_allocator, + ); + self.msg1_rate_limiter.complete_handshake(); + self.stats_mut() + .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); + return; + } + + // Phase 2: allocate our index + emit [SendHandshake, PromoteToActive]. + let promote_actions = machine.step( + PeerEvent::Authorized, + packet.timestamp_ms, + &mut self.index_allocator, + ); + let our_index = match machine.our_index() { + Some(idx) => idx, + None => { + // Allocation exhausted in Phase 2 (mirrors the pre-refactor + // allocate-failure path): no msg2, no promote. The old peer + // has already been torn down above — identical to the + // pre-refactor arm, which also removed the stale peer before + // hitting the shared allocate-failure return. + warn!("Failed to allocate session index for inbound"); + self.msg1_rate_limiter.complete_handshake(); + self.stats_mut() + .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); + return; + } + }; + + // Shell registry surgery (Option A1), in the pre-refactor order: + // set indices on the shell connection, insert link / reverse map / + // connection, then build + store the framed msg2. The old index was + // already freed by `remove_active_peer` above, BEFORE this fresh + // allocation — matching the pre-refactor allocation sequence. + conn.set_our_index(our_index); + conn.set_their_index(their_index); + let link = Link::connectionless( + link_id, + packet.transport_id, + packet.remote_addr.clone(), + LinkDirection::Inbound, + Duration::from_millis(self.config().node.base_rtt_ms), + ); + self.links.insert(link_id, link); + self.addr_to_link.insert(addr_key, link_id); + self.connections.insert(link_id, conn); + let wire_msg2 = build_msg2(our_index, their_index, &msg2_payload); + if let Some(conn) = self.connections.get_mut(&link_id) { + conn.set_handshake_msg2(wire_msg2.clone()); + } + + // Register the machine (Promote/Restart tail only). + self.peer_machines.insert(link_id, machine); + + // Execute [SendHandshake, PromoteToActive]. Because the old peer was + // removed in Phase 1, `promote_connection`'s cross-connection branch + // (`peers.get(addr)`) cannot fire, so it always returns `Promoted`; + // the defensive `PromotionResolved{CrossConnectionWon/Lost}` + // follow-ups are unreachable here (see the post-tail note). + let ambient = PeerActionCtx { + verified_identity: peer_identity, + transport_id: packet.transport_id, + remote_addr: packet.remote_addr.clone(), + our_index: Some(our_index), + their_index: Some(their_index), + now_ms: packet.timestamp_ms, + }; + self.execute_peer_actions(link_id, &ambient, promote_actions) + .await; + + // Post-`Promoted` shell tail (byte-identical to the pre-refactor + // Promoted arm), reached only when promotion succeeded (machine now + // Established); a send/promote failure removed the machine and + // already cleaned up. + // + // DEFENSIVE CROSS-CONNECTION (risk #5): the machine's + // `PromotionResolved{CrossConnectionWon/Lost}` follow-ups run the + // index-level cleanup generically in the executor, but the loser- + // link surgery (close_connection → remove_link → addr_to_link) is + // NOT reproduced here — it is UNREACHABLE on the driven restart + // path: Phase-1 `remove_active_peer` removed `peers[addr]`, so + // `promote_connection` returns `Promoted`. The full cross-connection + // link surgery lands in C3-3 (blueprint risk #5 / N3); the + // debug_assert below catches any regression that reaches a non- + // Established, non-absent state. + debug_assert!(matches!( + self.peer_machines.get(&link_id).map(|m| m.state()), + Some(PeerState::Established { .. }) | None + )); + if matches!( + self.peer_machines.get(&link_id).map(|m| m.state()), + Some(PeerState::Established { .. }) + ) { + // Store msg2 on peer for resend on duplicate msg1 + if let Some(peer) = self.peers.get_mut(&peer_node_addr) { + peer.set_handshake_msg2(wire_msg2.clone()); + } + // Send initial tree announce to new peer + if let Err(e) = self.send_tree_announce_to_peer(&peer_node_addr).await { + debug!(peer = %self.peer_display_name(&peer_node_addr), error = %e, "Failed to send initial TreeAnnounce"); + } + // Schedule filter announce (sent on next tick via debounce) + self.bloom_state.mark_update_needed(peer_node_addr); + self.reset_lookup_backoff(); + } + + self.msg1_rate_limiter.complete_handshake(); } InboundDecision::Promote => { // === C3-2a: net-new inbound establish, driven by the machine. === @@ -522,7 +698,10 @@ impl Node { Some(idx) => idx, None => { // Allocation exhausted in Phase 2 (mirrors the pre-refactor - // allocate-failure path): no msg2, no promote. + // allocate-failure path): no msg2, no promote. The concrete + // allocator error is consumed inside `on_authorized`, so the + // restored warn! carries the pre-refactor message text only. + warn!("Failed to allocate session index for inbound"); self.msg1_rate_limiter.complete_handshake(); self.stats_mut() .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); @@ -594,181 +773,8 @@ impl Node { } self.msg1_rate_limiter.complete_handshake(); - return; } } - - if self - .authorize_peer( - &wire.peer_identity, - PeerAclContext::InboundHandshake, - packet.transport_id, - &packet.remote_addr, - ) - .is_err() - { - self.msg1_rate_limiter.complete_handshake(); - self.stats_mut() - .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); - return; - } - - // Note: we don't early-return if peer is already in self.peers here. - // promote_connection handles cross-connection resolution via tie-breaker. - - // Allocate our session index - let our_index = match self.index_allocator.allocate() { - Ok(idx) => idx, - Err(e) => { - self.msg1_rate_limiter.complete_handshake(); - warn!(error = %e, "Failed to allocate session index for inbound"); - self.stats_mut() - .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); - return; - } - }; - - conn.set_our_index(our_index); - conn.set_their_index(wire.their_index); - - // Create link - let link = Link::connectionless( - link_id, - packet.transport_id, - packet.remote_addr.clone(), - LinkDirection::Inbound, - Duration::from_millis(self.config().node.base_rtt_ms), - ); - - self.links.insert(link_id, link); - self.addr_to_link.insert(addr_key, link_id); - self.connections.insert(link_id, conn); - - // Build and send msg2 response, storing for potential resend - let wire_msg2 = build_msg2(our_index, wire.their_index, &wire.msg2_payload); - if let Some(conn) = self.connections.get_mut(&link_id) { - conn.set_handshake_msg2(wire_msg2.clone()); - } - - if let Some(transport) = self.transports.get(&packet.transport_id) { - match transport.send(&packet.remote_addr, &wire_msg2).await { - Ok(bytes) => { - debug!( - link_id = %link_id, - our_index = %our_index, - their_index = %wire.their_index, - bytes, - "Sent msg2 response" - ); - } - Err(e) => { - warn!( - link_id = %link_id, - error = %e, - "Failed to send msg2" - ); - // Clean up on failure - self.connections.remove(&link_id); - self.links.remove(&link_id); - self.addr_to_link - .remove(&(packet.transport_id, packet.remote_addr)); - let _ = self.index_allocator.free(our_index); - self.msg1_rate_limiter.complete_handshake(); - self.stats_mut() - .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); - return; - } - } - } - - // Responder handshake is complete after receive_handshake_init (Noise IK - // pattern: responder processes msg1 and generates msg2 in one step). - // Promote the connection to active peer now. - let promote = ConnAction::PromoteToActive { link: link_id }; - match self.drive_promote_to_active(promote, wire.peer_identity, packet.timestamp_ms) { - Ok(result) => { - match result { - PromotionResult::Promoted(node_addr) => { - // Store msg2 on peer for resend on duplicate msg1 - if let Some(peer) = self.peers.get_mut(&node_addr) { - peer.set_handshake_msg2(wire_msg2.clone()); - } - // Promotion is logged once by `promote_connection` - // ("Connection promoted to active peer"); no separate - // inbound-path line. - // Send initial tree announce to new peer - if let Err(e) = self.send_tree_announce_to_peer(&node_addr).await { - debug!(peer = %self.peer_display_name(&node_addr), error = %e, "Failed to send initial TreeAnnounce"); - } - // Schedule filter announce (sent on next tick via debounce) - self.bloom_state.mark_update_needed(node_addr); - self.reset_lookup_backoff(); - } - PromotionResult::CrossConnectionWon { - loser_link_id, - node_addr, - } => { - // Store msg2 on peer for resend on duplicate msg1 - if let Some(peer) = self.peers.get_mut(&node_addr) { - peer.set_handshake_msg2(wire_msg2.clone()); - } - // Close the losing TCP connection (no-op for connectionless) - if let Some(loser_link) = self.links.get(&loser_link_id) { - let loser_tid = loser_link.transport_id(); - let loser_addr = loser_link.remote_addr().clone(); - if let Some(transport) = self.transports.get(&loser_tid) { - transport.close_connection(&loser_addr).await; - } - } - // Clean up the losing connection's link - self.remove_link(&loser_link_id); - debug!( - peer = %self.peer_display_name(&node_addr), - loser_link_id = %loser_link_id, - "Inbound cross-connection won, loser link cleaned up" - ); - // Send initial tree announce to peer (new or reconnected) - if let Err(e) = self.send_tree_announce_to_peer(&node_addr).await { - debug!(peer = %self.peer_display_name(&node_addr), error = %e, "Failed to send initial TreeAnnounce"); - } - // Schedule filter announce (sent on next tick via debounce) - self.bloom_state.mark_update_needed(node_addr); - self.reset_lookup_backoff(); - } - PromotionResult::CrossConnectionLost { winner_link_id } => { - // Close the losing TCP connection (no-op for connectionless) - if let Some(transport) = self.transports.get(&packet.transport_id) { - transport.close_connection(&packet.remote_addr).await; - } - // This connection lost — clean up its link - self.remove_link(&link_id); - // Restore addr_to_link for the winner's link - self.addr_to_link.insert( - (packet.transport_id, packet.remote_addr.clone()), - winner_link_id, - ); - debug!( - winner_link_id = %winner_link_id, - "Inbound cross-connection lost, keeping existing" - ); - } - } - } - Err(e) => { - warn!( - link_id = %link_id, - error = %e, - "Failed to promote inbound connection" - ); - // Clean up on promotion failure - self.remove_link(&link_id); - let _ = self.index_allocator.free(our_index); - self.stats_mut() - .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); - } - } - - self.msg1_rate_limiter.complete_handshake(); } /// Find stored msg2 bytes for a given link (pre- or post-promotion). From e9112cc1bb2c854f5d0c77bcef5d670e241ee346 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 13:42:08 +0000 Subject: [PATCH 07/11] node: drive net-new outbound establish through the per-peer machine Cut the net-new outbound handshake completion (a received msg2 that promotes a fresh outbound leg to a new peer) over to the per-peer control machine, mirroring the inbound cutover. handle_msg2 still runs the msg2 prologue, the ACL check, and the cross-connection swap/keep arms inline; for the net-new promote it now builds a transient machine, steps it, and drives promote_connection through the executor. The session index was already allocated at dial, so there is no two-phase authorize here. The wire, index sequence, and peer registry state after promote are byte-neutral. To keep the promote-failure path neutral, the executor's cleanup now distinguishes inbound from outbound: an outbound promote failure records the reject only, matching the prior handler, rather than the inbound path's link and index teardown. Cross-connection swap/keep, rekey-msg2, and the dial path stay inline; the loser-link surgery for the currently unreachable driven cross-connection case lands with the register relocation next. --- src/node/dataplane/peer_actions.rs | 64 +++++++--- src/node/handlers/handshake.rs | 184 +++++++++++++++++------------ src/proto/fmp/core.rs | 7 ++ 3 files changed, 162 insertions(+), 93 deletions(-) diff --git a/src/node/dataplane/peer_actions.rs b/src/node/dataplane/peer_actions.rs index f1f8aab..c3c3593 100644 --- a/src/node/dataplane/peer_actions.rs +++ b/src/node/dataplane/peer_actions.rs @@ -54,6 +54,14 @@ pub(in crate::node) struct PeerActionCtx { pub(in crate::node) their_index: Option, /// The wire timestamp driving this step (promotion ts / loss-report clock). pub(in crate::node) now_ms: u64, + /// Establish direction for this exchange. Discriminates the + /// `PromoteToActive` failure cleanup: the pre-refactor inbound + /// (`handle_msg1`) and outbound (`handle_msg2`) promote-Err arms were NOT + /// byte-identical, so the executor must reproduce each. `false` = inbound + /// (drop link + reverse map + free index), `true` = outbound (record the + /// reject only; leave the dead link/`addr_to_link` for the stale-connection + /// reaper, matching old `handle_msg2`). + pub(in crate::node) is_outbound: bool, } impl Node { @@ -179,22 +187,48 @@ impl Node { } Err(e) => { // GAP-4: promotion failed. `promote_connection` already - // removed `connections[link]`; mirror the pre-refactor - // cleanup (`handle_msg1` L587-591): drop the link + - // reverse map, free our index, discard the machine, and - // record the reject. The queue is drained (PromoteToActive - // is the last establish action), so no explicit abort. - // - // Restored pre-refactor promote-failure warn! - // (`handle_msg1` L757). - warn!(link_id = %promote_link, error = %e, "Failed to promote inbound connection"); - self.remove_link(&promote_link); - if let Some(idx) = ambient.our_index { - let _ = self.index_allocator.free(idx); + // removed `connections[link]` and (on error) handled its + // own index internally. The pre-refactor inbound and + // outbound promote-Err arms were NOT byte-identical, so + // discriminate on `ambient.is_outbound`. The queue is + // drained (PromoteToActive is the last establish action), + // so no explicit abort. + if ambient.is_outbound { + // OLD outbound (`handle_msg2` promote-Err): warn + + // record_reject ONLY. NO `remove_link`, NO + // `index_allocator.free`, NO `addr_to_link` removal — + // the dead link/addr_to_link/pending_outbound were + // left for the 30s stale-connection reaper + // (`promote_connection` already handled + // `connections[link]`/its index on error). Restored + // pre-refactor outbound warn! ("Failed to promote + // connection"). + // + // The transient outbound machine was inserted BEFORE + // execute (Model A); it is additive C3-1 state that + // did not exist pre-refactor, so removing the just- + // inserted machine on failure is neutral vs old and + // prevents a leak. + warn!(link_id = %promote_link, error = %e, "Failed to promote connection"); + self.stats_mut().record_reject(RejectReason::Handshake( + HandshakeReject::BadState, + )); + self.peer_machines.remove(&promote_link); + } else { + // OLD inbound (`handle_msg1` L587-591): drop the link + // + reverse map, free our index, discard the machine, + // and record the reject. Restored pre-refactor inbound + // promote-failure warn! (`handle_msg1` L757). + warn!(link_id = %promote_link, error = %e, "Failed to promote inbound connection"); + self.remove_link(&promote_link); + if let Some(idx) = ambient.our_index { + let _ = self.index_allocator.free(idx); + } + self.peer_machines.remove(&promote_link); + self.stats_mut().record_reject(RejectReason::Handshake( + HandshakeReject::BadState, + )); } - self.peer_machines.remove(&promote_link); - self.stats_mut() - .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); } } } diff --git a/src/node/handlers/handshake.rs b/src/node/handlers/handshake.rs index ba1b7f0..953beb7 100644 --- a/src/node/handlers/handshake.rs +++ b/src/node/handlers/handshake.rs @@ -6,7 +6,9 @@ use crate::node::acl::PeerAclContext; use crate::node::dataplane::PeerActionCtx; use crate::node::reject::{HandshakeReject, RejectReason}; use crate::node::{Node, NodeError}; -use crate::peer::machine::{FailReason, HandshakePhase, PeerEvent, PeerMachine, PeerState}; +use crate::peer::machine::{ + FailReason, HandshakePhase, PeerAction, PeerEvent, PeerMachine, PeerState, +}; use crate::peer::{ActivePeer, PeerConnection}; use crate::proto::fmp::wire::{Msg1Header, Msg2Header, build_msg2}; use crate::proto::fmp::{ @@ -506,6 +508,7 @@ impl Node { our_index: None, their_index: Some(their_index), now_ms: packet.timestamp_ms, + is_outbound: false, }; self.execute_peer_actions(link_id, &teardown_ctx, phase1) .await; @@ -590,6 +593,7 @@ impl Node { our_index: Some(our_index), their_index: Some(their_index), now_ms: packet.timestamp_ms, + is_outbound: false, }; self.execute_peer_actions(link_id, &ambient, promote_actions) .await; @@ -747,6 +751,7 @@ impl Node { our_index: Some(our_index), their_index: Some(their_index), now_ms: packet.timestamp_ms, + is_outbound: false, }; self.execute_peer_actions(link_id, &ambient, promote_actions) .await; @@ -1114,85 +1119,102 @@ impl Node { return; } - // Normal path: promote to active peer - let promote = ConnAction::PromoteToActive { link: link_id }; - match self.drive_promote_to_active(promote, peer_identity, packet.timestamp_ms) { - Ok(result) => { - // Clean up pending_outbound - self.pending_outbound.remove(&key); + // === C3-3a: net-new outbound establish, driven by the machine. === + // ONLY the `establish_outbound == Promote` arm is cut over here. The + // Swap/Keep cross-connection arms and the rekey-msg2 completion branch + // above STAY INLINE (§0): they mutate an existing already-promoted peer + // via `replace_session` with no PeerAction, so the machine's C1 Swap/Keep + // arms cannot be neutral until `PeerSendState` expresses `replace_session`. + // + // This arm is `has_existing_peer == false` only, so `promote_connection` + // always hits its else branch and returns `Promoted`; the defensive + // `CrossConnectionWon/Lost` follow-ups are UNREACHABLE here (their + // loser-link surgery lands in C3-3b). Direct analog of the C3-2a inbound + // net-new arm — no ordering constraint, lowest risk. + // + // Model A: build a TRANSIENT outbound machine, step `Msg2 → + // [PromoteToActive]`, execute it (→ `promote_connection` → + // `PromotionResolved{Promoted}` → inert `RegisterDecryptSession`, R2), and + // insert into `peer_machines` only on the Promoted (Established) tail. The + // outbound `our_index` was allocated at DIAL (unchanged), the outbound + // promote sends nothing on the wire, and `promote_connection` frees + // nothing new — so the index sequence, `peers`/`peers_by_index`/ + // `addr_to_link` state, and metrics are byte-identical to the pre-refactor + // Promoted arm. `pending_outbound` lifecycle stays shell-side (removed on + // the Established tail, exactly where the pre-refactor Ok arm removed it); + // the machine never touches it. + let mut machine = PeerMachine::new_outbound(link_id, peer_identity, packet.timestamp_ms); - match result { - PromotionResult::Promoted(node_addr) => { - info!( - peer = %self.peer_display_name(&node_addr), - "Peer promoted to active" - ); - // Send initial tree announce to new peer - if let Err(e) = self.send_tree_announce_to_peer(&node_addr).await { - debug!(peer = %self.peer_display_name(&node_addr), error = %e, "Failed to send initial TreeAnnounce"); - } - // Schedule filter announce (sent on next tick via debounce) - self.bloom_state.mark_update_needed(node_addr); - self.reset_lookup_backoff(); - } - PromotionResult::CrossConnectionWon { - loser_link_id, - node_addr, - } => { - // Close the losing TCP connection (no-op for connectionless) - if let Some(loser_link) = self.links.get(&loser_link_id) { - let loser_tid = loser_link.transport_id(); - let loser_addr = loser_link.remote_addr().clone(); - if let Some(transport) = self.transports.get(&loser_tid) { - transport.close_connection(&loser_addr).await; - } - } - // Clean up the losing connection's link - self.remove_link(&loser_link_id); - // Ensure addr_to_link points to the winning link - self.addr_to_link - .insert((packet.transport_id, packet.remote_addr.clone()), link_id); - debug!( - peer = %self.peer_display_name(&node_addr), - loser_link_id = %loser_link_id, - "Outbound cross-connection won, loser link cleaned up" - ); - // Send initial tree announce to peer (new or reconnected) - if let Err(e) = self.send_tree_announce_to_peer(&node_addr).await { - debug!(peer = %self.peer_display_name(&node_addr), error = %e, "Failed to send initial TreeAnnounce"); - } - // Schedule filter announce (sent on next tick via debounce) - self.bloom_state.mark_update_needed(node_addr); - self.reset_lookup_backoff(); - } - PromotionResult::CrossConnectionLost { winner_link_id } => { - // Close the losing TCP connection (no-op for connectionless) - if let Some(transport) = self.transports.get(&packet.transport_id) { - transport.close_connection(&packet.remote_addr).await; - } - // This connection lost — clean up its link - self.remove_link(&link_id); - // Ensure addr_to_link points to the winner's link - self.addr_to_link.insert( - (packet.transport_id, packet.remote_addr.clone()), - winner_link_id, - ); - debug!( - winner_link_id = %winner_link_id, - "Outbound cross-connection lost, keeping existing" - ); - } - } - } - Err(e) => { - warn!( - link_id = %link_id, - error = %e, - "Failed to promote connection" - ); - self.stats_mut() - .record_reject(RejectReason::Handshake(HandshakeReject::BadState)); + // Step `Msg2 → [PromoteToActive]`. The machine re-runs the pure + // `establish_outbound` on the snapshot (a harmless second pure call, as in + // C3-2a); `has_existing_peer == false` reproduces the `Promote` decision. + let promote_actions = machine.step( + PeerEvent::Msg2 { + their_index: header.sender_idx, + out: out_snap, + }, + packet.timestamp_ms, + &mut self.index_allocator, + ); + debug_assert_eq!( + promote_actions, + vec![PeerAction::PromoteToActive { link: link_id }] + ); + + // Register the machine (Promote tail only — the Swap/Keep/rekey arms above + // all returned without inserting). Inserted BEFORE execute so the + // executor's `PromoteToActive` arm can feed `PromotionResolved` back into + // it via the `peer_machines` lookup. The outbound `link_id` was allocated + // at dial and the dial path never inserts a machine (Model A), so this + // cannot collide with an existing entry. + self.peer_machines.insert(link_id, machine); + + // Execute `[PromoteToActive]`. The executor calls `promote_connection` + // (identical to the pre-refactor `drive_promote_to_active`), feeds + // `PromotionResolved{Promoted}` back, and runs the inert + // `RegisterDecryptSession` (R2 — register stays in `promote_connection`). + // A promote failure (e.g. `MaxPeersExceeded` if peers filled between dial + // and msg2) runs the executor's Err cleanup and removes the machine, + // leaving it absent (not Established). + let ambient = PeerActionCtx { + verified_identity: peer_identity, + transport_id: packet.transport_id, + remote_addr: packet.remote_addr.clone(), + our_index, + their_index: Some(header.sender_idx), + now_ms: packet.timestamp_ms, + is_outbound: true, + }; + self.execute_peer_actions(link_id, &ambient, promote_actions) + .await; + + // Post-`Promoted` shell tail (byte-identical to the pre-refactor Promoted + // arm), reached only when promotion succeeded (machine now Established). + // `pending_outbound.remove` runs here — exactly where the pre-refactor Ok + // arm removed it, before the TreeAnnounce/bloom/backoff tail. A promote + // failure removed the machine and skips the whole tail (the pre-refactor + // Err arm likewise left `pending_outbound` in place and only recorded the + // reject, which the executor's Err arm already did). + debug_assert!(matches!( + self.peer_machines.get(&link_id).map(|m| m.state()), + Some(PeerState::Established { .. }) | None + )); + if matches!( + self.peer_machines.get(&link_id).map(|m| m.state()), + Some(PeerState::Established { .. }) + ) { + self.pending_outbound.remove(&key); + info!( + peer = %self.peer_display_name(&peer_node_addr), + "Peer promoted to active" + ); + // Send initial tree announce to new peer + if let Err(e) = self.send_tree_announce_to_peer(&peer_node_addr).await { + debug!(peer = %self.peer_display_name(&peer_node_addr), error = %e, "Failed to send initial TreeAnnounce"); } + // Schedule filter announce (sent on next tick via debounce) + self.bloom_state.mark_update_needed(peer_node_addr); + self.reset_lookup_backoff(); } } @@ -1206,6 +1228,12 @@ impl Node { /// [`PromotionResult`] so the caller can drive the site-specific /// post-promotion tail (TreeAnnounce, bloom mark, discovery-backoff reset, /// loser-link cleanup). + /// + // C3-3a cut the last live caller (the inline outbound `Promote` arm) over to + // the executor's `PromoteToActive` path, so this is now unused. Its caller + // census / retirement is C3-3b (blueprint § C3-3b); kept here (allowed) until + // then so the diff stays scoped to the outbound Promote cutover. + #[allow(dead_code)] fn drive_promote_to_active( &mut self, action: ConnAction, diff --git a/src/proto/fmp/core.rs b/src/proto/fmp/core.rs index 5fa27d9..44d2cf4 100644 --- a/src/proto/fmp/core.rs +++ b/src/proto/fmp/core.rs @@ -319,6 +319,13 @@ pub(crate) enum ConnAction { /// promotion (resolving the verified identity and promotion timestamp from /// the ambient wire context) and then runs the post-promotion tail /// (TreeAnnounce, bloom mark, discovery-backoff reset, loser-link cleanup). + // + // C3-3a cut the inline outbound `Promote` arm — the last constructor of this + // variant — over to the machine's `PeerAction::PromoteToActive` seam. The only + // remaining reference is the (now dead-code-allowed) `drive_promote_to_active` + // matcher; both are retired together in C3-3b, so this variant is allowed + // until then to keep the C3-3a diff scoped to the outbound Promote cutover. + #[allow(dead_code)] PromoteToActive { link: LinkId }, } From 800cfb23e3f3f32714e5bdebb483ea209bd9706a Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 14:20:56 +0000 Subject: [PATCH 08/11] node: relocate decrypt-session registration to the establish executor Move register_decrypt_worker_session out of promote_connection into the executor's PromoteToActive handler, gated on a promoted or cross-connection-won result. Every live promote now flows through that one executor path, so registration still fires exactly once at the same synchronous point; the direct test callers of promote_connection spawn no worker pool, so the call was already a no-op for them. Add the cross-connection loser-link teardown (close the losing transport, remove its link, re-point addr_to_link at the winner) to the executor as a guarded follow-up. It is unreachable on the current driven establish paths, which only promote net-new peers, and asserts so, but keeps the executor complete for when that case is driven. Remove the now-dead drive_promote_to_active and ConnAction::PromoteToActive. --- src/node/dataplane/peer_actions.rs | 113 ++++++++++++++++++++++++++--- src/node/handlers/handshake.rs | 73 ++++++------------- src/proto/fmp/core.rs | 15 ---- 3 files changed, 125 insertions(+), 76 deletions(-) diff --git a/src/node/dataplane/peer_actions.rs b/src/node/dataplane/peer_actions.rs index c3c3593..65097da 100644 --- a/src/node/dataplane/peer_actions.rs +++ b/src/node/dataplane/peer_actions.rs @@ -22,6 +22,7 @@ use crate::node::Node; use crate::node::reject::{HandshakeReject, RejectReason}; use crate::peer::machine::{PeerAction, PeerEvent}; +use crate::proto::fmp::PromotionResult; use crate::proto::fmp::wire::build_msg2; use crate::transport::{LinkId, TransportAddr, TransportId}; use crate::utils::index::SessionIndex; @@ -35,7 +36,7 @@ use tracing::warn; /// /// Unlike a machine event/action payload this is **executor-side**, so it may /// hold real values resolved from the wire context (cf. `handle_msg1`'s -/// `wire`/`packet` locals and `drive_promote_to_active`'s ambient args). It is +/// `wire`/`packet` locals and `promote_connection`'s ambient args). It is /// built fresh per driven step by the caller at cutover time (C3-2/C3-3). #[allow(dead_code)] pub(in crate::node) struct PeerActionCtx { @@ -165,15 +166,40 @@ impl Node { } PeerAction::PromoteToActive { link: promote_link } => { // GAP-1: ambient supplies the verified identity + promotion ts - // that `promote_connection` needs (cf. `drive_promote_to_active`). + // that `promote_connection` needs (resolved from the wire ctx). match self.promote_connection( promote_link, ambient.verified_identity, ambient.now_ms, ) { Ok(result) => { + // R1 (C3-3b): the decrypt-worker registration relocated + // OUT of `promote_connection` into THIS single executor + // arm — the one live caller of `promote_connection` (both + // the inbound `handle_msg1` and outbound `handle_msg2` + // net-new establish paths reach it here). Register iff the + // promotion actually created or replaced a peer + // (`Promoted | CrossConnectionWon`), NEVER on + // `CrossConnectionLost`. Run synchronously right after + // `promote_connection` returns, before feeding + // `PromotionResolved` and before any await — the exact + // synchronous point (and Promoted/Won gating) of the + // pre-refactor in-`promote_connection` call. No-op when + // the worker pool isn't spawned (`register_...` early- + // returns), so the direct `promote_connection` test + // callers (which bypass this executor) are unaffected. + #[cfg(unix)] + match result { + PromotionResult::Promoted(node_addr) + | PromotionResult::CrossConnectionWon { node_addr, .. } => { + self.register_decrypt_worker_session(&node_addr); + } + PromotionResult::CrossConnectionLost { .. } => {} + } + // Feed the outcome back into the machine and fold the - // follow-up actions (RegisterDecryptSession, cross-conn + // follow-up actions (RegisterDecryptSession — now a + // redundant no-op, see its arm — and the cross-conn index // frees) into the worklist. Disjoint field borrow again. let follow = match self.peer_machines.get_mut(&promote_link) { Some(machine) => machine.step( @@ -184,6 +210,70 @@ impl Node { None => Vec::new(), }; queue.extend(follow); + + // Defensive cross-connection loser-link surgery (C3-3b). + // LINK-ONLY: close the losing transport connection, drop + // its link, and re-point `addr_to_link`, reproducing the + // pre-refactor inline `handle_msg2`/`handle_msg1` per-arm + // order EXACTLY. The index-plane frees/unregisters are + // owned by the machine's `PromotionResolved{Won/Lost}` + // follow-up (queued just above), so NOTHING here touches + // an index — no double-free. + // + // UNREACHABLE on every current driven path: the inbound + // and outbound net-new establish arms only route to the + // machine when no promoted peer exists for the node_addr + // (and `RestartThenPromote` removes the old peer first), + // so `promote_connection` always returns `Promoted`. The + // `debug_assert!(false, ..)` catches any future path that + // drives a cross-connection through the executor without + // the matching send-state handling. + match result { + PromotionResult::CrossConnectionWon { loser_link_id, .. } => { + debug_assert!( + false, + "executor CrossConnectionWon is unreachable on \ + driven net-new establish paths" + ); + // Close the losing transport connection (no-op for + // connectionless) via the LOSER link's own + // transport/addr, then drop the losing link. + if let Some(loser_link) = self.links.get(&loser_link_id) { + let loser_tid = loser_link.transport_id(); + let loser_addr = loser_link.remote_addr().clone(); + if let Some(transport) = self.transports.get(&loser_tid) { + transport.close_connection(&loser_addr).await; + } + } + self.remove_link(&loser_link_id); + // Point `addr_to_link` at the winning (current) + // link. + self.addr_to_link.insert( + (ambient.transport_id, ambient.remote_addr.clone()), + promote_link, + ); + } + PromotionResult::CrossConnectionLost { winner_link_id } => { + debug_assert!( + false, + "executor CrossConnectionLost is unreachable on \ + driven net-new establish paths" + ); + // Close this (losing) connection, drop its link, + // and restore `addr_to_link` to the winner. + if let Some(transport) = + self.transports.get(&ambient.transport_id) + { + transport.close_connection(&ambient.remote_addr).await; + } + self.remove_link(&promote_link); + self.addr_to_link.insert( + (ambient.transport_id, ambient.remote_addr.clone()), + winner_link_id, + ); + } + PromotionResult::Promoted(_) => {} + } } Err(e) => { // GAP-4: promotion failed. `promote_connection` already @@ -247,14 +337,15 @@ impl Node { } PeerAction::RegisterDecryptSession { index } => { let _ = index; - // C3-2 (HALT-reported): the decrypt-worker registration still - // runs INSIDE `promote_connection` (`handshake.rs:1193/1305`), - // which is the single source of truth for its ~40 direct - // `promote_connection` callers (unit/integration tests) and the - // two live handlers. Relocating it out (GAP-3) would perturb the - // live promote path, so C3-1 keeps it there and drives this - // action as a no-op; the relocation lands with the inbound - // cutover in C3-2. + // No-op by design. C3-3b (R1-a) relocated the decrypt-worker + // registration into the `PromoteToActive` Ok arm above, gated on + // the returned `PromotionResult`, so it runs once per live + // promote (Promoted/Won) at the pre-refactor synchronous point. + // This machine-emitted action is now redundant with that arm; + // kept as an inert no-op (rather than removing the emission) so + // the machine's action sequence and its unit tests stay + // unchanged. The keyed-by-NodeAddr register does not need the + // machine's `index` payload. } PeerAction::UnregisterDecryptSession { index } => { // Executor supplies `transport_id` from ambient; keyed by diff --git a/src/node/handlers/handshake.rs b/src/node/handlers/handshake.rs index 953beb7..acae1d3 100644 --- a/src/node/handlers/handshake.rs +++ b/src/node/handlers/handshake.rs @@ -12,7 +12,7 @@ use crate::peer::machine::{ use crate::peer::{ActivePeer, PeerConnection}; use crate::proto::fmp::wire::{Msg1Header, Msg2Header, build_msg2}; use crate::proto::fmp::{ - ConnAction, EstablishSnapshot, EstablishView, InboundDecision, InboundReject, OutboundDecision, + EstablishSnapshot, EstablishView, InboundDecision, InboundReject, OutboundDecision, OutboundSnapshot, PromotionResult, WireOutcome, cross_connection_winner, }; use crate::transport::{Link, LinkDirection, LinkId, ReceivedPacket}; @@ -1169,13 +1169,14 @@ impl Node { // cannot collide with an existing entry. self.peer_machines.insert(link_id, machine); - // Execute `[PromoteToActive]`. The executor calls `promote_connection` - // (identical to the pre-refactor `drive_promote_to_active`), feeds - // `PromotionResolved{Promoted}` back, and runs the inert - // `RegisterDecryptSession` (R2 — register stays in `promote_connection`). - // A promote failure (e.g. `MaxPeersExceeded` if peers filled between dial - // and msg2) runs the executor's Err cleanup and removes the machine, - // leaving it absent (not Established). + // Execute `[PromoteToActive]`. The executor calls `promote_connection`, + // feeds `PromotionResolved{Promoted}` back, registers the decrypt-worker + // session (R1 — C3-3b relocated the register into the executor's + // `PromoteToActive` Ok arm, gated on the result), and runs the now-inert + // `RegisterDecryptSession` follow-up. A promote failure (e.g. + // `MaxPeersExceeded` if peers filled between dial and msg2) runs the + // executor's Err cleanup and removes the machine, leaving it absent (not + // Established). let ambient = PeerActionCtx { verified_identity: peer_identity, transport_id: packet.transport_id, @@ -1218,36 +1219,6 @@ impl Node { } } - /// Execute a [`ConnAction::PromoteToActive`] from the establish machine. - /// - /// The decision to promote is made by the establish handlers (and, from the - /// establish-core stage on, the pure decision in `proto::fmp`); this is the - /// executor half of the seam. It runs the promotion through - /// [`Self::promote_connection`], resolving the verified identity and - /// promotion timestamp from the ambient wire context, and returns the - /// [`PromotionResult`] so the caller can drive the site-specific - /// post-promotion tail (TreeAnnounce, bloom mark, discovery-backoff reset, - /// loser-link cleanup). - /// - // C3-3a cut the last live caller (the inline outbound `Promote` arm) over to - // the executor's `PromoteToActive` path, so this is now unused. Its caller - // census / retirement is C3-3b (blueprint § C3-3b); kept here (allowed) until - // then so the diff stays scoped to the outbound Promote cutover. - #[allow(dead_code)] - fn drive_promote_to_active( - &mut self, - action: ConnAction, - verified_identity: PeerIdentity, - current_time_ms: u64, - ) -> Result { - match action { - ConnAction::PromoteToActive { link } => { - self.promote_connection(link, verified_identity, current_time_ms) - } - _ => unreachable!("drive_promote_to_active requires a PromoteToActive action"), - } - } - /// Promote a connection to active peer after successful authentication. /// /// Handles cross-connection detection and resolution using tie-breaker rules. @@ -1396,11 +1367,12 @@ impl Node { "Cross-connection resolved: this connection won" ); - // Hand the FMP recv cipher + replay window to the - // decrypt shard worker. (Same as normal-promotion tail - // below.) - #[cfg(unix)] - self.register_decrypt_worker_session(&peer_node_addr); + // R1 (C3-3b): the decrypt-worker registration is no longer done + // here — it relocated OUT of `promote_connection` into the single + // executor `PromoteToActive` Ok arm (`peer_actions.rs`), gated on + // the returned `PromotionResult` (`Promoted | CrossConnectionWon`). + // The executor runs it synchronously right after this call returns, + // before any await, so the live establish behaviour is unchanged. Ok(PromotionResult::CrossConnectionWon { loser_link_id, @@ -1506,13 +1478,14 @@ impl Node { "Connection promoted to active peer" ); - // Hand the FMP recv cipher + replay window to the - // decrypt shard worker. From this point on the worker - // is the sole authority on FMP replay protection for - // this session. No-op when the worker pool isn't - // spawned (unit-test path or `FIPS_DECRYPT_WORKERS=0`). - #[cfg(unix)] - self.register_decrypt_worker_session(&peer_node_addr); + // R1 (C3-3b): the decrypt-worker registration relocated OUT of + // `promote_connection` into the single executor `PromoteToActive` Ok + // arm (`peer_actions.rs`), gated on the returned `PromotionResult` + // (`Promoted | CrossConnectionWon`, never `CrossConnectionLost`). The + // executor runs it synchronously right after this call returns, before + // any await — same point, same effect as the pre-refactor in-place call + // (no-op when the worker pool isn't spawned; unit-test path or + // `FIPS_DECRYPT_WORKERS=0`). Ok(PromotionResult::Promoted(peer_node_addr)) } diff --git a/src/proto/fmp/core.rs b/src/proto/fmp/core.rs index 44d2cf4..a3d1e07 100644 --- a/src/proto/fmp/core.rs +++ b/src/proto/fmp/core.rs @@ -312,21 +312,6 @@ pub(crate) enum ConnAction { bytes: Vec, next_resend_at_ms: u64, }, - /// Promote the completed handshake connection on `link` to an active peer - /// (`promote_connection`): moves the Noise session out of the - /// `PeerConnection`, resolves cross-connection precedence via the - /// tie-breaker, and installs the `ActivePeer`. The shell executes the - /// promotion (resolving the verified identity and promotion timestamp from - /// the ambient wire context) and then runs the post-promotion tail - /// (TreeAnnounce, bloom mark, discovery-backoff reset, loser-link cleanup). - // - // C3-3a cut the inline outbound `Promote` arm — the last constructor of this - // variant — over to the machine's `PeerAction::PromoteToActive` seam. The only - // remaining reference is the (now dead-code-allowed) `drive_promote_to_active` - // matcher; both are retired together in C3-3b, so this variant is allowed - // until then to keep the C3-3a diff scoped to the outbound Promote cutover. - #[allow(dead_code)] - PromoteToActive { link: LinkId }, } /// Read-only view of FMP connection/peer state the lifecycle core needs. From 0ebd1b44c01f6c249a7a1f22fc0898443d5c32b6 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 14:51:11 +0000 Subject: [PATCH 09/11] node: add the initiator-cutover and drain executor actions (unwired) Prepare the establish executor to drive FMP rekey by activating the SwapSendState action (initiator K-bit cutover via cutover_to_new_session, with the gated decrypt-worker re-registration) and adding a CompleteDrain action (erase the drained previous session: free its index, drop its peers_by_index entry, unregister its decrypt session). Both reproduce the current inline rekey.rs cutover and drain bodies. The machine's drain mapping now emits CompleteDrain, using the real drained index rather than a shadow copy. Unwired: nothing drives the machine's rekey path yet (live rekey still runs inline), so these arms are unreachable and the change is behavior-neutral. The cadence fold that routes cutover and drain through the machine follows. --- src/node/dataplane/peer_actions.rs | 71 ++++++++++++++++++++++++++++-- src/peer/machine.rs | 33 +++++++++++--- 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/src/node/dataplane/peer_actions.rs b/src/node/dataplane/peer_actions.rs index 65097da..b879ce1 100644 --- a/src/node/dataplane/peer_actions.rs +++ b/src/node/dataplane/peer_actions.rs @@ -28,7 +28,7 @@ use crate::transport::{LinkId, TransportAddr, TransportId}; use crate::utils::index::SessionIndex; use crate::{NodeAddr, PeerIdentity}; use std::collections::VecDeque; -use tracing::warn; +use tracing::{debug, trace, warn}; /// Ambient shell facts a [`PeerAction`] executor needs that the machine's /// runtime-agnostic action payloads deliberately omit (verified identity, @@ -323,8 +323,73 @@ impl Node { } } PeerAction::SwapSendState { .. } => { - // C4: initiator cutover (`active.rs:1033` - // `cutover_to_new_session`). + // C4-0: initiator cutover. Reproduces the `ConnAction::Cutover` + // body in `handlers/rekey.rs:53-88` EXACTLY. `addr` is resolved + // from the ambient verified identity (as `InvalidateSendState` + // does). The decrypt re-register folds HERE, gated on + // `did_cutover` — the generic `RegisterDecryptSession` arm stays a + // no-op so a promote never double-registers. Shadow-only until the + // cadence fold routes here (C4-1). + let node_addr = *ambient.verified_identity.node_addr(); + let did_cutover = if let Some(peer) = self.peers.get_mut(&node_addr) { + if let Some(_old_our_index) = peer.cutover_to_new_session() { + // New index was pre-registered in peers_by_index + // during msg2 handling (handshake.rs). + debug_assert!( + peer.transport_id().is_some() + && peer.our_index().is_some() + && self.peers_by_index.contains_key(&( + peer.transport_id().unwrap(), + peer.our_index().unwrap().as_u32() + )), + "peers_by_index should contain pre-registered new index after cutover" + ); + debug!( + peer = %self.peer_display_name(&node_addr), + "Rekey cutover complete (initiator), K-bit flipped" + ); + true + } else { + false + } + } else { + false + }; + // Re-register the new session with the decrypt worker — the + // cache_key (transport_id, our_index) just changed, so the + // old worker entry is stale and every packet on the new + // session would miss the worker's HashMap lookup. + #[cfg(unix)] + if did_cutover { + self.register_decrypt_worker_session(&node_addr); + } + #[cfg(not(unix))] + let _ = did_cutover; + } + PeerAction::CompleteDrain { peer: node_addr } => { + // C4-0: initiator drain completion. Reproduces the + // `ConnAction::Drain` body in `handlers/rekey.rs:90-111` EXACTLY. + // Extract the real previous index + transport_id under the peer + // borrow, drop the borrow, then run the cache_key cleanup (which + // takes &mut self for unregister_decrypt_worker_session). + // Shadow-only until the cadence fold routes here (C4-1). + let drained = self.peers.get_mut(&node_addr).and_then(|peer| { + peer.complete_drain().map(|idx| (idx, peer.transport_id())) + }); + if let Some((old_our_index, transport_id)) = drained { + if let Some(tid) = transport_id { + let cache_key = (tid, old_our_index.as_u32()); + self.peers_by_index.remove(&cache_key); + #[cfg(unix)] + self.unregister_decrypt_worker_session(cache_key); + } + let _ = self.index_allocator.free(old_our_index); + trace!( + peer = %self.peer_display_name(&node_addr), + old_index = %old_our_index, + "Drain complete, previous session erased" + ); + } } PeerAction::InvalidateSendState => { // GAP-4 (biggest): the FULL teardown. `remove_active_peer` diff --git a/src/peer/machine.rs b/src/peer/machine.rs index 2b5e885..61b358a 100644 --- a/src/peer/machine.rs +++ b/src/peer/machine.rs @@ -268,6 +268,11 @@ pub(crate) enum PeerAction { /// Initiator-side rekey cutover: swap the published send-state to the pending /// epoch. SwapSendState { epoch: [u8; 8] }, + /// Complete an initiator-side rekey drain: retire the previous session slot + /// (drop its `peers_by_index`/decrypt-worker entry, free its index). The + /// executor reads the REAL previous index from `ActivePeer::complete_drain` + /// (not a machine-shadow index, which can drift). + CompleteDrain { peer: NodeAddr }, /// Invalidate the published send-state (close/loss). InvalidateSendState, /// Register a decrypt-worker entry for `index`. @@ -829,13 +834,12 @@ impl PeerMachine { actions } ConnAction::Drain { peer } => { + // The executor reads the real previous_our_index from + // `ActivePeer::complete_drain` and does the peers_by_index / + // decrypt-worker / index-free cleanup, replacing the old + // shadow-index emission (which could drift from the real index). self.state = PeerState::Active { addr: peer }; - let mut actions = Vec::new(); - if let Some(idx) = self.draining_index.take() { - actions.push(PeerAction::UnregisterDecryptSession { index: idx }); - actions.push(PeerAction::FreeIndex { index: idx }); - } - actions + vec![PeerAction::CompleteDrain { peer }] } ConnAction::InitiateRekey { peer } => { // Fresh outbound rekey: allocate our new index, send msg1 (Noise @@ -1264,6 +1268,23 @@ mod tests { kind: MaintainKind::Rekey(RekeyPhase::Draining) } ); + + // A second cadence tick from the (expired) drain window completes the + // drain: the machine now emits the single `CompleteDrain` send-state + // write (executor reads the real previous index) instead of the old + // shadow-index `[UnregisterDecryptSession, FreeIndex]` pair. + let drain_actions = m.step( + PeerEvent::Timeout { + kind: TimerKind::RekeyCadence, + }, + 20_000, + &mut alloc, + ); + assert_eq!( + drain_actions, + vec![PeerAction::CompleteDrain { peer: addr }] + ); + assert_eq!(m.state(), PeerState::Active { addr }); } // ---- Test 2: responder cutover (data-plane owned) --------------------- From e05b868cf8e80460a33753732612ed292230f784 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 16:20:45 +0000 Subject: [PATCH 10/11] node: drive rekey cadence cutover and drain through the peer machine Route each Cutover and Drain that the shell-side batch poll_rekey decides through the per-peer machine and the executor, replacing the inline effect bodies in check_rekey. The batch decision and its per-peer snapshots stay shell-side and byte-unchanged: poll_rekey phase-groups all cutovers, then all drains, then all initiations across the peer set, and that ordering governs the shared index allocator's free-then-allocate sequence that appears on the wire, so the machine only consumes the already-decided actions (a new RekeyConsume event) without re-deciding. InitiateRekey stays inline (its Noise msg1 build is a shell-side leaf) with a RekeyInitiated observation feeding the machine so its control state stays coherent for the next tick's cutover. The cutover and drain logs, which relocated into the executor in the prior commit, are pinned back to the fips::node::handlers::rekey tracing target so they stay visible under the operator's module log filter. Also clears the machine's shadow draining_index on drain so a later cross-connection resolution cannot double-free the already-freed index. --- src/node/dataplane/peer_actions.rs | 10 ++ src/node/handlers/rekey.rs | 230 ++++++++++++++++++++++------- src/peer/machine.rs | 137 +++++++++++++++++ 3 files changed, 320 insertions(+), 57 deletions(-) diff --git a/src/node/dataplane/peer_actions.rs b/src/node/dataplane/peer_actions.rs index b879ce1..61216e4 100644 --- a/src/node/dataplane/peer_actions.rs +++ b/src/node/dataplane/peer_actions.rs @@ -345,6 +345,12 @@ impl Node { "peers_by_index should contain pre-registered new index after cutover" ); debug!( + // Pin the target to the pre-refactor module: this + // cutover log relocated from handlers/rekey.rs into + // the executor, but operators (and the test harness) + // filter it under fips::node::handlers::rekey. Keeping + // the target preserves the observable log contract. + target: "fips::node::handlers::rekey", peer = %self.peer_display_name(&node_addr), "Rekey cutover complete (initiator), K-bit flipped" ); @@ -385,6 +391,10 @@ impl Node { } let _ = self.index_allocator.free(old_our_index); trace!( + // Pin to the pre-refactor module (see the cutover log + // above) so the relocated drain log stays visible under + // the operator's fips::node::handlers::rekey filter. + target: "fips::node::handlers::rekey", peer = %self.peer_display_name(&node_addr), old_index = %old_our_index, "Drain complete, previous session erased" diff --git a/src/node/handlers/rekey.rs b/src/node/handlers/rekey.rs index b8b309a..57e8f7d 100644 --- a/src/node/handlers/rekey.rs +++ b/src/node/handlers/rekey.rs @@ -7,13 +7,16 @@ use crate::NodeAddr; use crate::node::Node; +use crate::node::dataplane::PeerActionCtx; use crate::noise::HandshakeState; +use crate::peer::machine::PeerEvent; use crate::proto::fmp::wire::build_msg1; use crate::proto::fmp::{ConnAction, LifecycleView, PeerSnapshot, RekeyCfg, RekeyResendSnapshot}; use crate::proto::fsp::{ FspAction, RekeyMsg3ResendSnapshot, SessionSetup, SessionSnapshot, cutover_timer_elapsed, }; use crate::proto::link::SessionDatagram; +use crate::transport::{TransportAddr, TransportId}; use tracing::{debug, trace, warn}; /// Keep previous session alive for this long after cutover. @@ -46,72 +49,36 @@ impl Node { // The shell snapshots each healthy peer's rekey ages/flags (every clock // read resolved here); the core decides cutover/drain/trigger with no // clock, phase-grouped to preserve the pre-refactor execution order. + // The batch `poll_rekey` + snapshots STAY SHELL-SIDE and BYTE-UNCHANGED + // (Finding B): the cross-peer phase-grouping (all Cutover → all Drain → + // all InitiateRekey) governs the shared `index_allocator` free-then-alloc + // SEQUENCE that appears on the wire. The machine must NOT re-poll; it + // CONSUMES each decided `ConnAction` in the same order the batch returned. let snapshots = self.rekey_peers(); for action in self.fmp.poll_rekey(snapshots, &cfg) { match action { - // Execute cutover for initiator side. + // Initiator cutover: route the decided action through the peer + // machine + executor (C4-1). The executor's `SwapSendState` arm + // reproduces the pre-refactor cutover body EXACTLY. ConnAction::Cutover { peer: node_addr } => { - let did_cutover = if let Some(peer) = self.peers.get_mut(&node_addr) { - if let Some(_old_our_index) = peer.cutover_to_new_session() { - // New index was pre-registered in peers_by_index - // during msg2 handling (handshake.rs). - debug_assert!( - peer.transport_id().is_some() - && peer.our_index().is_some() - && self.peers_by_index.contains_key(&( - peer.transport_id().unwrap(), - peer.our_index().unwrap().as_u32() - )), - "peers_by_index should contain pre-registered new index after cutover" - ); - debug!( - peer = %self.peer_display_name(&node_addr), - "Rekey cutover complete (initiator), K-bit flipped" - ); - true - } else { - false - } - } else { - false - }; - // Re-register the new session with the decrypt worker — the - // cache_key (transport_id, our_index) just changed, so the - // old worker entry is stale and every packet on the new - // session would miss the worker's HashMap lookup. - #[cfg(unix)] - if did_cutover { - self.register_decrypt_worker_session(&node_addr); - } - #[cfg(not(unix))] - let _ = did_cutover; + self.route_rekey_cadence(node_addr, ConnAction::Cutover { peer: node_addr }) + .await; } - // Execute drain completion. + // Drain completion: route through the machine + executor. The + // executor's `CompleteDrain` arm reads the REAL previous index + // from `complete_drain()` and frees it at the same point the old + // inline body did (index-order preserving). ConnAction::Drain { peer: node_addr } => { - // Extract the old index and transport_id under the peer - // borrow, then drop the borrow so the cache_key cleanup - // below can take &mut self for unregister_decrypt_worker_session. - let drained = self.peers.get_mut(&node_addr).and_then(|peer| { - peer.complete_drain().map(|idx| (idx, peer.transport_id())) - }); - if let Some((old_our_index, transport_id)) = drained { - if let Some(tid) = transport_id { - let cache_key = (tid, old_our_index.as_u32()); - self.peers_by_index.remove(&cache_key); - #[cfg(unix)] - self.unregister_decrypt_worker_session(cache_key); - } - let _ = self.index_allocator.free(old_our_index); - trace!( - peer = %self.peer_display_name(&node_addr), - old_index = %old_our_index, - "Drain complete, previous session erased" - ); - } + self.route_rekey_cadence(node_addr, ConnAction::Drain { peer: node_addr }) + .await; } - // Initiate a new rekey. + // Initiate a new rekey: STAYS INLINE (the Noise msg1 build + + // index allocation are a shell-side leaf, byte-unchanged). Feed + // the machine a `RekeyInitiated` observation afterward so its + // control state stays coherent for the next tick's Cutover/Drain. ConnAction::InitiateRekey { peer: node_addr } => { self.initiate_rekey(&node_addr).await; + self.observe_rekey_initiated(&node_addr); } #[allow(unreachable_patterns)] _ => {} @@ -119,6 +86,155 @@ impl Node { } } + /// Route a cadence-decided `Cutover`/`Drain` `ConnAction` through the peer + /// machine + executor (C4-1). The shell already decided (batch `poll_rekey`); + /// the machine consumes via [`PeerEvent::RekeyConsume`] WITHOUT re-polling, + /// preserving the phase order. The `SwapSendState`/`CompleteDrain` executor + /// arms reproduce the pre-refactor inline effect bodies exactly. + /// + /// Finding A: an established peer always has a `peer_machine`. If the peer + /// vanished between snapshot and effect, the old inline body was a no-op, so + /// we do nothing; if the machine is absent (impossible per Finding A) we fall + /// back to the byte-identical inline body under a `debug_assert`. + async fn route_rekey_cadence(&mut self, node_addr: NodeAddr, action: ConnAction) { + let link = match self.peers.get(&node_addr) { + Some(peer) => peer.link_id(), + None => return, + }; + if !self.peer_machines.contains_key(&link) { + debug_assert!( + false, + "peer machine present for every established rekey peer (Finding A)" + ); + match action { + ConnAction::Cutover { peer } => self.cutover_peer_inline(&peer), + ConnAction::Drain { peer } => self.drain_peer_inline(&peer), + _ => {} + } + return; + } + let ambient = self.rekey_cadence_ctx(&node_addr); + self.advance_peer_machine( + link, + PeerEvent::RekeyConsume { action }, + Self::now_ms(), + &ambient, + ) + .await; + } + + /// Feed the machine the `RekeyInitiated` observation after the inline + /// `initiate_rekey` (C4-1). The obs emits no action, so there is no executor + /// pass — a bare `step` keeps the machine's control state coherent. + fn observe_rekey_initiated(&mut self, node_addr: &NodeAddr) { + let link = match self.peers.get(node_addr) { + Some(peer) => peer.link_id(), + None => return, + }; + if let Some(machine) = self.peer_machines.get_mut(&link) { + let acts = machine.step( + PeerEvent::RekeyInitiated, + Self::now_ms(), + &mut self.index_allocator, + ); + debug_assert!(acts.is_empty(), "RekeyInitiated is a pure observation"); + } else { + debug_assert!( + false, + "peer machine present for every established rekey peer (Finding A)" + ); + } + } + + /// Ambient shell facts for the routed cadence Cutover/Drain step. Only + /// `verified_identity` is read by the `SwapSendState`/`CompleteDrain` + /// executor arms — `SwapSendState` resolves its `NodeAddr` from it (so it must + /// equal `node_addr`), and `CompleteDrain` carries its peer in the action + /// payload. The transport/index/direction fields are unused by these two arms + /// (they matter only to `PromoteToActive`, never emitted on this path) and are + /// populated best-effort for coherence. + fn rekey_cadence_ctx(&self, node_addr: &NodeAddr) -> PeerActionCtx { + let peer = &self.peers[node_addr]; + PeerActionCtx { + verified_identity: *peer.identity(), + transport_id: peer.transport_id().unwrap_or_else(|| TransportId::new(0)), + remote_addr: peer + .current_addr() + .cloned() + .unwrap_or_else(|| TransportAddr::new(Vec::new())), + our_index: peer.our_index(), + their_index: peer.their_index(), + now_ms: Self::now_ms(), + is_outbound: false, + } + } + + /// Pre-refactor initiator cutover body, retained as the release fallback for + /// the (Finding-A-impossible) missing-machine case. Byte-identical to the old + /// inline `ConnAction::Cutover` arm and to the executor's `SwapSendState` arm. + fn cutover_peer_inline(&mut self, node_addr: &NodeAddr) { + let did_cutover = if let Some(peer) = self.peers.get_mut(node_addr) { + if let Some(_old_our_index) = peer.cutover_to_new_session() { + // New index was pre-registered in peers_by_index during msg2 + // handling (handshake.rs). + debug_assert!( + peer.transport_id().is_some() + && peer.our_index().is_some() + && self.peers_by_index.contains_key(&( + peer.transport_id().unwrap(), + peer.our_index().unwrap().as_u32() + )), + "peers_by_index should contain pre-registered new index after cutover" + ); + debug!( + peer = %self.peer_display_name(node_addr), + "Rekey cutover complete (initiator), K-bit flipped" + ); + true + } else { + false + } + } else { + false + }; + // Re-register the new session with the decrypt worker — the cache_key + // (transport_id, our_index) just changed, so the old worker entry is + // stale and every packet on the new session would miss the lookup. + #[cfg(unix)] + if did_cutover { + self.register_decrypt_worker_session(node_addr); + } + #[cfg(not(unix))] + let _ = did_cutover; + } + + /// Pre-refactor drain-completion body, retained as the release fallback for + /// the (Finding-A-impossible) missing-machine case. Byte-identical to the old + /// inline `ConnAction::Drain` arm and to the executor's `CompleteDrain` arm. + fn drain_peer_inline(&mut self, node_addr: &NodeAddr) { + // Extract the old index and transport_id under the peer borrow, then drop + // the borrow so the cache_key cleanup below can take &mut self for + // unregister_decrypt_worker_session. + let drained = self + .peers + .get_mut(node_addr) + .and_then(|peer| peer.complete_drain().map(|idx| (idx, peer.transport_id()))); + if let Some((old_our_index, transport_id)) = drained { + if let Some(tid) = transport_id { + let cache_key = (tid, old_our_index.as_u32()); + self.peers_by_index.remove(&cache_key); + #[cfg(unix)] + self.unregister_decrypt_worker_session(cache_key); + } + let _ = self.index_allocator.free(old_our_index); + trace!( + peer = %self.peer_display_name(node_addr), + old_index = %old_our_index, + "Drain complete, previous session erased" + ); + } + } + /// Snapshot every healthy peer with a session for the rekey decision, /// pre-computing its monotonic ages and timer predicates so the pure core /// applies the thresholds without reading a clock (see [`PeerSnapshot`]). diff --git a/src/peer/machine.rs b/src/peer/machine.rs index 61b358a..b7f3bf4 100644 --- a/src/peer/machine.rs +++ b/src/peer/machine.rs @@ -224,6 +224,19 @@ pub(crate) enum PeerEvent { }, /// Inbound rekey msg2 (completes our initiated rekey). RekeyMsg2 { their_index: SessionIndex }, + /// A cadence-decided rekey `ConnAction` to CONSUME (C4-1). The shell ran the + /// batch `poll_rekey` across the whole peer set (phase-grouped, index-order + /// preserving — Finding B) and routes each decided action here; the machine + /// applies the control-tier transition + emits the send-state write + /// (`SwapSendState`/`CompleteDrain`) WITHOUT re-polling. Carries only + /// `Cutover`/`Drain` on the C4-1 path (`InitiateRekey` stays inline shell-side + /// with a [`RekeyInitiated`](PeerEvent::RekeyInitiated) observation). + RekeyConsume { action: ConnAction }, + /// OBSERVATION: the shell initiated an outbound rekey inline (the Noise msg1 + /// leaf + index allocation are shell-side). Advances the control state to + /// `Maintaining{Rekey(Msg1Sent)}` so the next tick's `Cutover`/`Drain` consume + /// transitions from a coherent phase. Emits no action. + RekeyInitiated, /// Data plane observed the responder K-bit flip inline (§3.7). PeerKbitFlip { epoch: [u8; 8] }, /// A filter announce is due for this peer. @@ -448,6 +461,8 @@ impl PeerMachine { self.on_inbound_msg1(self.link, wire, est, now, index_allocator) } PeerEvent::RekeyMsg2 { their_index } => self.on_rekey_msg2(their_index), + PeerEvent::RekeyConsume { action } => self.map_rekey_action(action, now), + PeerEvent::RekeyInitiated => self.on_rekey_initiated(), PeerEvent::PeerKbitFlip { .. } => { // Responder cutover is data-plane-owned (§3.7): the machine only // schedules the drain-window unregister. NO slot mutation. @@ -789,6 +804,31 @@ impl PeerMachine { Vec::new() } + /// OBS (C4-1): the shell ran `initiate_rekey` inline — the Noise msg1 leaf, + /// the index allocation, the wire send, and the `set_rekey_state` on the + /// `ActivePeer` all happened shell-side. This is a pure observation that + /// advances the machine's control state to `Maintaining{Rekey(Msg1Sent)}` so + /// the subsequent cadence `Cutover`/`Drain` consume transitions from a + /// coherent phase. Emits NO action (nothing left to do). No-op unless the peer + /// is in an established-like state (defensive; the shell only initiates on + /// healthy established peers). + fn on_rekey_initiated(&mut self) -> Vec { + let addr = match self.addr() { + Some(a) => a, + None => return Vec::new(), + }; + if !self.is_established_context() { + return Vec::new(); + } + self.rekey_in_progress = true; + self.rekey_resend_count = 0; + self.state = PeerState::Maintaining { + addr, + kind: MaintainKind::Rekey(RekeyPhase::Msg1Sent), + }; + Vec::new() + } + /// Rekey cadence: run `poll_rekey` over this one peer's snapshot and map the /// phase-grouped `ConnAction`s. fn on_rekey_cadence(&mut self, now: u64) -> Vec { @@ -838,6 +878,14 @@ impl PeerMachine { // `ActivePeer::complete_drain` and does the peers_by_index / // decrypt-worker / index-free cleanup, replacing the old // shadow-index emission (which could drift from the real index). + // + // Clear the shadow `draining_index` set by the Cutover arm: the + // real previous index is now retired by `CompleteDrain`, so a + // leftover `Some(stale)` would double-free if a later + // `CrossConnectionWon` consumed it in `on_promotion_resolved` + // (C4-0 latent item 1). Post-rekey cross-connection promotion is + // not a live path, but clearing here removes the hazard outright. + self.draining_index = None; self.state = PeerState::Active { addr: peer }; vec![PeerAction::CompleteDrain { peer }] } @@ -1835,4 +1883,93 @@ mod tests { // guarantee: loss is reported only via ReportLost, and no retry-schedule // action exists in the PeerAction vocabulary at all (reconciler-owned). } + + // ---- Test 9: cadence CONSUME (C4-1) ----------------------------------- + // The shell polls the batch `poll_rekey` and routes each decided ConnAction + // as `RekeyConsume` — the machine maps it WITHOUT re-polling, yielding the + // same action sequence + transition as the machine-driven cadence (Test 1), + // and the Drain consume clears the shadow `draining_index`. + #[test] + fn rekey_consume_cutover_then_drain() { + let mut alloc = IndexAllocator::new(); + let id = peer_identity(); + let addr = *id.node_addr(); + let mut m = PeerMachine::new_outbound(LinkId::new(1), id, 0); + m.state = PeerState::Maintaining { + addr, + kind: MaintainKind::Rekey(RekeyPhase::PendingCutover), + }; + m.rekey_our_index = Some(SessionIndex::new(0x2222)); + m.our_index = Some(SessionIndex::new(0x1111)); + m.remote_epoch = Some([9u8; 8]); + + // Consume the shell-decided Cutover: identical sequence to Test 1. + let cut = m.step( + PeerEvent::RekeyConsume { + action: ConnAction::Cutover { peer: addr }, + }, + 10_000, + &mut alloc, + ); + assert_eq!( + cut, + vec![ + PeerAction::SwapSendState { epoch: [9u8; 8] }, + PeerAction::RegisterDecryptSession { + index: SessionIndex::new(0x2222) + }, + PeerAction::SetTimer { + kind: TimerKind::DrainExpiry, + at_ms: 10_000 + DRAIN_WINDOW_MS + }, + ] + ); + assert_eq!( + m.state(), + PeerState::Maintaining { + addr, + kind: MaintainKind::Rekey(RekeyPhase::Draining) + } + ); + // Cutover stashed the old index in the drain shadow. + assert_eq!(m.draining_index, Some(SessionIndex::new(0x1111))); + + // Consume the shell-decided Drain: single CompleteDrain, Active, and the + // shadow drain index is CLEARED (double-free guard, C4-0 latent item 1). + let drain = m.step( + PeerEvent::RekeyConsume { + action: ConnAction::Drain { peer: addr }, + }, + 20_000, + &mut alloc, + ); + assert_eq!(drain, vec![PeerAction::CompleteDrain { peer: addr }]); + assert_eq!(m.state(), PeerState::Active { addr }); + assert_eq!(m.draining_index, None); + } + + // ---- Test 10: RekeyInitiated observation (C4-1) ----------------------- + // The shell ran `initiate_rekey` inline; the obs advances control state to + // Msg1Sent and emits nothing. + #[test] + fn rekey_initiated_observation() { + let mut alloc = IndexAllocator::new(); + let id = peer_identity(); + let addr = *id.node_addr(); + let mut m = PeerMachine::new_outbound(LinkId::new(1), id, 0); + m.state = PeerState::Established { addr }; + + let acts = m.step(PeerEvent::RekeyInitiated, 5_000, &mut alloc); + assert!(acts.is_empty()); + assert_eq!( + m.state(), + PeerState::Maintaining { + addr, + kind: MaintainKind::Rekey(RekeyPhase::Msg1Sent) + } + ); + assert!(m.rekey_in_progress); + // No index allocation happened in the machine (shell-side leaf). + assert_eq!(alloc.count(), 0); + } } From 5d5da69a5b6645ffc06b43a28ab9b64290bde33b Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Mon, 13 Jul 2026 17:08:35 +0000 Subject: [PATCH 11/11] node: drive the link-dead peer reap through the per-peer machine Route each link-dead peer that the tick sweep's plan_heartbeats decides to reap through the per-peer machine and executor, replacing the inline reap body in check_link_heartbeats. The batch decision, the liveness snapshots (read from the hot-path-written receive clock), and the heartbeat-send arm stay shell-side and byte-unchanged; the machine only consumes the decided LinkDeadSuspected, tearing the peer down via remove_active_peer and reporting the loss to the reconciler exactly as before, on the same tick with the same wall-clock timestamp. The reap log stays shell-side. The machine's link-dead handler no longer emits a decrypt-session unregister keyed by its shadow index (the full peer teardown already unregisters the real index; the shadow could have drifted to a reused index), and its guard now covers the Established state a freshly promoted peer sits in. Handshake-timeout, retransmit, and stale-connection cleanup stay inline: they act on pre-promotion legs that have no machine, and the loss reflex they use differs from the link-dead one. --- src/node/handlers/mmp.rs | 69 ++++++++++++++++++++++++++++++++++++++-- src/peer/machine.rs | 27 ++++++++++------ 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/src/node/handlers/mmp.rs b/src/node/handlers/mmp.rs index e1f9866..43fe932 100644 --- a/src/node/handlers/mmp.rs +++ b/src/node/handlers/mmp.rs @@ -6,14 +6,17 @@ use crate::NodeAddr; use crate::node::Node; +use crate::node::dataplane::PeerActionCtx; use crate::node::reject::{MmpReject, RejectReason, TreeReject}; use crate::node::tree::sign_declaration; +use crate::peer::machine::PeerEvent; use crate::proto::link::LinkMessageType; use crate::proto::mmp::{ LinkReportKind, LinkReportSnapshot, MmpAction, PeerLivenessSnapshot, ReceiverReport, RrLog, SenderReport, }; use crate::proto::stp::ParentEval; +use crate::transport::{TransportAddr, TransportId}; use std::time::{Duration, Instant}; use tracing::{debug, info, trace, warn}; @@ -500,13 +503,15 @@ impl Node { for action in actions { match action { MmpAction::ReapPeer { peer } => { + // Log SHELL-SIDE before routing so the reap keeps the + // `fips::node::handlers::mmp` tracing target (no relocation into + // the executor, no target pin needed). debug!( peer = %self.peer_display_name(&peer), timeout_secs = self.config().node.link_dead_timeout_secs, "Removing peer: link dead timeout" ); - self.remove_active_peer(&peer); - self.note_link_dead(peer, now_ms); + self.route_link_dead(peer, now_ms).await; } MmpAction::Heartbeat { peer } => { if let Some(p) = self.peers.get_mut(&peer) { @@ -526,4 +531,64 @@ impl Node { } } } + + /// Route a link-dead liveness reap through the peer machine + executor + /// (C5-1). Mirrors [`route_rekey_cadence`](Node::route_rekey_cadence): the + /// shell already decided (the tick sweep's `plan_heartbeats` batch emitted + /// this `ReapPeer` in phase order), so the machine only CONSUMES the decision + /// via [`PeerEvent::LinkDeadSuspected`]. The resulting executor arms + /// (`InvalidateSendState` → `remove_active_peer`, `ReportLost` → + /// `note_link_dead`) reproduce the pre-refactor inline reap body exactly. + /// + /// Finding A: an established peer always has a `peer_machine`. If the peer + /// vanished between snapshot and effect, the old inline body was already a + /// no-op, so we return; if the machine is absent (impossible per Finding A) + /// we fall back to the byte-identical inline body under a `debug_assert`. + /// + /// `now_ms` is the sweep's hoisted wall-clock ms (the same value the old reap + /// fed `note_link_dead`); it flows to the executor `ReportLost` arm via + /// `ambient.now_ms`. + async fn route_link_dead(&mut self, node_addr: NodeAddr, now_ms: u64) { + let link = match self.peers.get(&node_addr) { + Some(peer) => peer.link_id(), + None => return, + }; + if !self.peer_machines.contains_key(&link) { + debug_assert!( + false, + "peer machine present for every established peer (Finding A)" + ); + self.remove_active_peer(&node_addr); + self.note_link_dead(node_addr, now_ms); + return; + } + let ambient = self.link_dead_ctx(&node_addr, now_ms); + self.advance_peer_machine(link, PeerEvent::LinkDeadSuspected, Self::now_ms(), &ambient) + .await; + } + + /// Ambient shell facts for the routed liveness reap. Mirrors + /// [`rekey_cadence_ctx`](Node::rekey_cadence_ctx). The executor reads only + /// `verified_identity` (`InvalidateSendState` → `remove_active_peer` resolves + /// its `NodeAddr` from it, so it must equal `node_addr`) and `now_ms` + /// (`ReportLost` → `note_link_dead`, the wall-clock reconnect basis). The + /// transport/index/direction fields are unused by these two arms and are + /// populated best-effort for coherence. `now_ms` is threaded in (rather than + /// re-read) so the value fed to `note_link_dead` is byte-identical to the old + /// reap's hoisted wall-clock for every peer in the sweep. + fn link_dead_ctx(&self, node_addr: &NodeAddr, now_ms: u64) -> PeerActionCtx { + let peer = &self.peers[node_addr]; + PeerActionCtx { + verified_identity: *peer.identity(), + transport_id: peer.transport_id().unwrap_or_else(|| TransportId::new(0)), + remote_addr: peer + .current_addr() + .cloned() + .unwrap_or_else(|| TransportAddr::new(Vec::new())), + our_index: peer.our_index(), + their_index: peer.their_index(), + now_ms, + is_outbound: false, + } + } } diff --git a/src/peer/machine.rs b/src/peer/machine.rs index b7f3bf4..ea91952 100644 --- a/src/peer/machine.rs +++ b/src/peer/machine.rs @@ -996,14 +996,26 @@ impl PeerMachine { } fn on_link_dead(&mut self, now: u64) -> Vec { - if !self.is_active_like() { + // Guard the full established set (Established | Active | Maintaining), not + // just `is_active_like()`: a peer that never rekeyed stays parked in + // `Established` (the machine reaches `Active` only via a rekey `Drain`), + // yet the pre-refactor liveness reap tore down EVERY dead established peer. + // A too-narrow `is_active_like()` guard here would silently skip the common + // (never-rekeyed) reap target. Mirrors `on_disconnect`'s guard. + if !self.is_established_context() { return Vec::new(); } - let mut actions = vec![PeerAction::InvalidateSendState]; - if let Some(idx) = self.our_index.take() { - actions.push(PeerAction::UnregisterDecryptSession { index: idx }); - } - actions.push(PeerAction::TeardownConnectedUdp); + // `InvalidateSendState` maps to the executor's `remove_active_peer`, which + // unregisters the decrypt worker by the REAL current index. The machine's + // shadow `our_index` is deliberately NOT used to unregister here (C5-0): it + // can drift to a reused index and wrongly unregister ANOTHER peer's worker + // session. `TeardownConnectedUdp` is inert (C6 — the old reap had no + // connected-UDP teardown, so inert is neutral); `ReportLost` drives the + // loss reflex (`note_link_dead`). + let mut actions = vec![ + PeerAction::InvalidateSendState, + PeerAction::TeardownConnectedUdp, + ]; if let Some(peer) = self.addr() { actions.push(PeerAction::ReportLost { peer }); } @@ -1871,9 +1883,6 @@ mod tests { dead, vec![ PeerAction::InvalidateSendState, - PeerAction::UnregisterDecryptSession { - index: SessionIndex::new(0x4242) - }, PeerAction::TeardownConnectedUdp, PeerAction::ReportLost { peer: addr }, ]