diff --git a/src/node/handlers/mmp.rs b/src/node/handlers/mmp.rs index b505b4e..f9941ae 100644 --- a/src/node/handlers/mmp.rs +++ b/src/node/handlers/mmp.rs @@ -521,6 +521,7 @@ impl Node { let now = Instant::now(); let heartbeat_interval = Duration::from_secs(self.config.node.heartbeat_interval_secs); let dead_timeout = Duration::from_secs(self.config.node.link_dead_timeout_secs); + let max_resends = self.config.node.rate_limit.handshake_max_resends; let heartbeat_msg = [LinkMessageType::Heartbeat.to_byte()]; // Collect heartbeats to send and dead peers to remove @@ -530,7 +531,7 @@ impl Node { for (node_addr, peer) in self.peers.iter() { // Check liveness via MMP receiver last_recv_time. // Fall back to session_start for peers that never sent data. - let is_dead = if let Some(mmp) = peer.mmp() { + let time_dead = if let Some(mmp) = peer.mmp() { let reference_time = mmp .receiver .last_recv_time() @@ -539,6 +540,17 @@ impl Node { } else { false }; + + // Suppress teardown while an FMP rekey is genuinely in flight with + // budget left: a rekey-handshake link is not silent. The msg1 + // resend cap guarantees this terminates (abandon on exhaustion or + // cutover on completion clears `rekey_in_progress`), so a truly + // dead link is reaped on the next cycle. + let rekey_active = peer.rekey_in_progress() + && peer.rekey_msg1_resend_count() < max_resends + && peer.rekey_msg1().is_some(); + + let is_dead = time_dead && !rekey_active; if is_dead { dead_peers.push(*node_addr); continue; diff --git a/src/node/handlers/rekey.rs b/src/node/handlers/rekey.rs index 5f033c1..4460230 100644 --- a/src/node/handlers/rekey.rs +++ b/src/node/handlers/rekey.rs @@ -235,19 +235,37 @@ impl Node { } let interval_ms = self.config.node.rate_limit.handshake_resend_interval_ms; + let backoff = self.config.node.rate_limit.handshake_resend_backoff; + let max_resends = self.config.node.rate_limit.handshake_max_resends; // Collect peers needing action let mut to_resend: Vec<(NodeAddr, Vec)> = Vec::new(); + let mut to_abandon: Vec = Vec::new(); for (node_addr, peer) in &self.peers { if !peer.rekey_in_progress() || peer.rekey_msg1().is_none() { continue; } + if peer.rekey_msg1_resend_count() >= max_resends { + to_abandon.push(*node_addr); + continue; + } if peer.needs_msg1_resend(now_ms) { to_resend.push((*node_addr, peer.rekey_msg1().unwrap().to_vec())); } } + // Abandon rekey cycles that exhausted their retransmission budget. + for node_addr in to_abandon { + if let Some(peer) = self.peers.get_mut(&node_addr) { + peer.abandon_rekey(); + } + warn!( + peer = %self.peer_display_name(&node_addr), + "FMP rekey aborted: msg1 unconfirmed after max retransmissions, abandoning cycle" + ); + } + for (node_addr, msg1_bytes) in to_resend { let (transport_id, remote_addr) = match self.peers.get(&node_addr) { Some(p) => match (p.transport_id(), p.current_addr()) { @@ -263,12 +281,13 @@ impl Node { false }; - if sent { - if let Some(peer) = self.peers.get_mut(&node_addr) { - peer.set_msg1_next_resend(now_ms + interval_ms); - } + if sent && let Some(peer) = self.peers.get_mut(&node_addr) { + let count = peer.rekey_msg1_resend_count() + 1; + let next = now_ms + (interval_ms as f64 * backoff.powi(count as i32)) as u64; + peer.record_rekey_msg1_resend(next); trace!( peer = %self.peer_display_name(&node_addr), + resend = count, "Resent rekey msg1" ); } diff --git a/src/node/tests/heartbeat.rs b/src/node/tests/heartbeat.rs new file mode 100644 index 0000000..4976ed3 --- /dev/null +++ b/src/node/tests/heartbeat.rs @@ -0,0 +1,114 @@ +//! Link-dead heartbeat rekey-awareness integration tests. +//! +//! `check_link_heartbeats()` reaps a peer after the link-dead timeout, +//! but suppresses teardown while an FMP rekey is genuinely in flight with +//! its msg1 retransmission budget unexhausted. These tests drive a real +//! two-node UDP peering, inject rekey state on the peer, and verify the +//! suppress / resume / regression behaviors. `link_dead_timeout_secs` is +//! set to 0 so the elapsed-time predicate is always satisfied and the only +//! variable is the rekey-active guard. + +use super::spanning_tree::*; +use super::*; +use crate::Identity; +use crate::noise::HandshakeState; +use crate::utils::index::SessionIndex; + +/// Arm a real (initiator) FMP rekey on the peer the given node holds for +/// `peer_addr`, so the msg1 resend budget can be exercised. +fn arm_rekey(node: &mut crate::node::Node, peer_addr: &NodeAddr) { + let remote = Identity::generate(); + let local = Identity::generate(); + let hs = HandshakeState::new_initiator(local.keypair(), remote.pubkey_full()); + let peer = node.get_peer_mut(peer_addr).expect("peer present"); + peer.set_rekey_state(hs, SessionIndex::new(7), vec![0xAB; 64], 0); +} + +/// A peer past the link-dead timeout is NOT reaped while an FMP rekey is in +/// progress with its msg1 budget unexhausted. +#[tokio::test] +async fn heartbeat_suppressed_during_rekey() { + let mut nodes = run_tree_test(2, &[(0, 1)], false).await; + verify_tree_convergence(&nodes); + + let addr_1 = *nodes[1].node.node_addr(); + assert!(nodes[0].node.get_peer(&addr_1).is_some()); + + // Force every link to read as dead on elapsed time alone. + nodes[0].node.config.node.link_dead_timeout_secs = 0; + + // Arm a rekey with budget left (count 0 < max_resends default 5). + arm_rekey(&mut nodes[0].node, &addr_1); + assert!(nodes[0].node.get_peer(&addr_1).unwrap().rekey_in_progress()); + + nodes[0].node.check_link_heartbeats().await; + + assert!( + nodes[0].node.get_peer(&addr_1).is_some(), + "peer reaped despite an in-flight rekey with budget remaining" + ); + + cleanup_nodes(&mut nodes).await; +} + +/// Once the msg1 budget is exhausted the rekey-active guard no longer +/// holds, so a peer past the link-dead timeout IS reaped. +#[tokio::test] +async fn heartbeat_resumes_after_budget_exhausted() { + let mut nodes = run_tree_test(2, &[(0, 1)], false).await; + verify_tree_convergence(&nodes); + + let addr_1 = *nodes[1].node.node_addr(); + assert!(nodes[0].node.get_peer(&addr_1).is_some()); + + nodes[0].node.config.node.link_dead_timeout_secs = 0; + let max_resends = nodes[0].node.config.node.rate_limit.handshake_max_resends; + + arm_rekey(&mut nodes[0].node, &addr_1); + + // Exhaust the budget: count reaches max_resends, guard goes false. + let peer = nodes[0].node.get_peer_mut(&addr_1).unwrap(); + for i in 0..max_resends { + peer.record_rekey_msg1_resend(1000 + i as u64 * 100); + } + assert_eq!( + nodes[0] + .node + .get_peer(&addr_1) + .unwrap() + .rekey_msg1_resend_count(), + max_resends + ); + + nodes[0].node.check_link_heartbeats().await; + + assert!( + nodes[0].node.get_peer(&addr_1).is_none(), + "peer not reaped after its rekey budget was exhausted" + ); + + cleanup_nodes(&mut nodes).await; +} + +/// Regression guard: with no rekey in flight, a peer past the link-dead +/// timeout is reaped exactly as before. +#[tokio::test] +async fn heartbeat_unaffected_without_rekey() { + let mut nodes = run_tree_test(2, &[(0, 1)], false).await; + verify_tree_convergence(&nodes); + + let addr_1 = *nodes[1].node.node_addr(); + assert!(nodes[0].node.get_peer(&addr_1).is_some()); + assert!(!nodes[0].node.get_peer(&addr_1).unwrap().rekey_in_progress()); + + nodes[0].node.config.node.link_dead_timeout_secs = 0; + + nodes[0].node.check_link_heartbeats().await; + + assert!( + nodes[0].node.get_peer(&addr_1).is_none(), + "dead peer with no rekey in flight should be reaped" + ); + + cleanup_nodes(&mut nodes).await; +} diff --git a/src/node/tests/mod.rs b/src/node/tests/mod.rs index dbd1892..78aa4af 100644 --- a/src/node/tests/mod.rs +++ b/src/node/tests/mod.rs @@ -17,6 +17,7 @@ mod discovery; mod ethernet; mod forwarding; mod handshake; +mod heartbeat; mod routing; mod session; mod spanning_tree; diff --git a/src/peer/active.rs b/src/peer/active.rs index 2ef0c2d..3a32593 100644 --- a/src/peer/active.rs +++ b/src/peer/active.rs @@ -195,6 +195,8 @@ pub struct ActivePeer { rekey_msg1: Option>, /// In-progress rekey: next resend timestamp (Unix ms). rekey_msg1_next_resend: u64, + /// In-progress rekey: number of msg1 retransmissions performed so far. + rekey_msg1_resend_count: u32, } impl ActivePeer { @@ -247,6 +249,7 @@ impl ActivePeer { rekey_our_index: None, rekey_msg1: None, rekey_msg1_next_resend: 0, + rekey_msg1_resend_count: 0, } } @@ -328,6 +331,7 @@ impl ActivePeer { rekey_our_index: None, rekey_msg1: None, rekey_msg1_next_resend: 0, + rekey_msg1_resend_count: 0, } } @@ -885,6 +889,7 @@ impl ActivePeer { self.rekey_handshake = None; self.rekey_msg1 = None; self.rekey_msg1_next_resend = 0; + self.rekey_msg1_resend_count = 0; } /// Cut over to the pending new session (initiator side). @@ -912,6 +917,7 @@ impl ActivePeer { self.session_established_at = Instant::now(); self.session_start = Instant::now(); self.rekey_in_progress = false; + self.rekey_msg1_resend_count = 0; self.rekey_jitter_secs = draw_rekey_jitter(); self.reset_replay_suppressed(); @@ -948,6 +954,7 @@ impl ActivePeer { self.session_established_at = Instant::now(); self.session_start = Instant::now(); self.rekey_in_progress = false; + self.rekey_msg1_resend_count = 0; self.rekey_jitter_secs = draw_rekey_jitter(); self.reset_replay_suppressed(); @@ -992,6 +999,7 @@ impl ActivePeer { self.rekey_handshake = None; self.rekey_msg1 = None; self.rekey_msg1_next_resend = 0; + self.rekey_msg1_resend_count = 0; self.rekey_in_progress = false; // Return whichever index needs freeing self.rekey_our_index.take().or_else(|| { @@ -1015,6 +1023,7 @@ impl ActivePeer { self.rekey_our_index = Some(our_index); self.rekey_msg1 = Some(wire_msg1); self.rekey_msg1_next_resend = next_resend_ms; + self.rekey_msg1_resend_count = 0; self.rekey_in_progress = true; } @@ -1043,6 +1052,7 @@ impl ActivePeer { // Clear msg1 resend state self.rekey_msg1 = None; self.rekey_msg1_next_resend = 0; + self.rekey_msg1_resend_count = 0; Ok(session) } @@ -1061,6 +1071,17 @@ impl ActivePeer { pub fn set_msg1_next_resend(&mut self, next_ms: u64) { self.rekey_msg1_next_resend = next_ms; } + + /// Number of rekey msg1 retransmissions performed so far. + pub fn rekey_msg1_resend_count(&self) -> u32 { + self.rekey_msg1_resend_count + } + + /// Record a rekey msg1 retransmission and schedule the next one. + pub fn record_rekey_msg1_resend(&mut self, next_ms: u64) { + self.rekey_msg1_resend_count += 1; + self.rekey_msg1_next_resend = next_ms; + } } #[cfg(test)] @@ -1319,4 +1340,52 @@ mod tests { n ); } + + /// Put a peer into a rekey-in-progress state with a real (initiator) + /// handshake so the msg1 resend budget can be exercised. + fn arm_rekey(peer: &mut ActivePeer) { + let remote = Identity::generate(); + let local = Identity::generate(); + let hs = NoiseHandshakeState::new_initiator(local.keypair(), remote.pubkey_full()); + peer.set_rekey_state(hs, SessionIndex::new(7), vec![0xAB; 64], 0); + } + + #[test] + fn rekey_msg1_resend_count_increments_and_caps() { + let identity = make_peer_identity(); + let mut peer = ActivePeer::new(identity, LinkId::new(1), 1000); + arm_rekey(&mut peer); + + assert!(peer.rekey_in_progress()); + assert_eq!(peer.rekey_msg1_resend_count(), 0); + assert!(peer.rekey_msg1().is_some()); + + // The driver records one resend per call; the count tracks them. + let max_resends: u32 = 5; + for i in 0..max_resends { + peer.record_rekey_msg1_resend(1000 + i as u64 * 100); + assert_eq!(peer.rekey_msg1_resend_count(), i + 1); + } + assert_eq!(peer.rekey_msg1_resend_count(), max_resends); + } + + #[test] + fn rekey_msg1_budget_exhaustion_abandons_cleanly() { + let identity = make_peer_identity(); + let mut peer = ActivePeer::new(identity, LinkId::new(1), 1000); + arm_rekey(&mut peer); + + // Simulate the driver exhausting its budget. + let max_resends: u32 = 5; + for i in 0..max_resends { + peer.record_rekey_msg1_resend(1000 + i as u64 * 100); + } + assert_eq!(peer.rekey_msg1_resend_count(), max_resends); + + // Budget exhausted -> abandon: state clears and the counter resets. + peer.abandon_rekey(); + assert!(!peer.rekey_in_progress()); + assert!(peer.rekey_msg1().is_none()); + assert_eq!(peer.rekey_msg1_resend_count(), 0); + } }