Auto-reconnect after MMP peer removal, directed outbound configs, sim improvements

Auto-reconnect:
- Add per-peer auto_reconnect config (default true) to PeerConfig
- schedule_reconnect() feeds removed peers back into retry system with
  unlimited retries and exponential backoff after MMP dead timeout
- RetryState gains reconnect flag to distinguish startup retries
  (max_retries-limited) from auto-reconnect (unlimited)

Retry re-fire fix:
- process_pending_retries() now pushes retry_after_ms past the handshake
  timeout window after successful initiate_peer_connection(), preventing
  retries from firing every tick with no backoff

Chaos sim improvements:
- Directed outbound configs: BFS spanning tree + lower-ID-first assignment
  eliminates dual-connect race conditions in simulation
- Save runner log (runner.log) alongside per-node logs for event correlation
- Increase churn-20 traffic aggressiveness and node churn (max_down_nodes
  3→5, traffic interval min 0s, duration max 90s, concurrent flows 5→10)

src/node/handlers/mmp.rs

@@ -418,7 +418,12 @@ impl Node {
             }
         }
 
-        // Remove dead peers
+        // Remove dead peers and schedule auto-reconnect
+        let now_ms = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .map(|d| d.as_millis() as u64)
+            .unwrap_or(0);
+
         for addr in &dead_peers {
             warn!(
                 peer = %self.peer_display_name(addr),
@@ -426,6 +431,7 @@ impl Node {
                 "Removing peer: link dead timeout"
             );
             self.remove_active_peer(addr);
+            self.schedule_reconnect(*addr, now_ms);
         }
 
         // Send heartbeats (skip peers we just removed)

src/node/handlers/timeout.rs

@@ -51,7 +51,7 @@ impl Node {
                 if conn.is_outbound()
                     && let Some(identity) = conn.expected_identity()
                 {
-                    self.schedule_retry(*identity.node_addr(), now_ms);
+                    self.schedule_retry(*identity.node_addr(), now_ms, false);
                 }
             }
             self.cleanup_stale_connection(link_id, now_ms);

src/node/retry.rs

@@ -22,6 +22,9 @@ pub struct RetryState {
 
     /// Timestamp (Unix ms) when the next retry should be attempted.
     pub retry_after_ms: u64,
+
+    /// Whether this is an auto-reconnect (unlimited retries, ignores max_retries).
+    pub reconnect: bool,
 }
 
 impl RetryState {
@@ -31,6 +34,7 @@ impl RetryState {
             peer_config,
             retry_count: 0,
             retry_after_ms: 0,
+            reconnect: false,
         }
     }
 
@@ -48,12 +52,18 @@ impl Node {
     /// Schedule a retry for a failed outbound connection, if applicable.
     ///
     /// Only schedules if the peer is an auto-connect peer and max retries
-    /// have not been exhausted. Does nothing if the peer is already connected
-    /// or has a connection in progress.
-    pub(super) fn schedule_retry(&mut self, node_addr: NodeAddr, now_ms: u64) {
+    /// have not been exhausted (unless `reconnect` is true, which retries
+    /// indefinitely). Does nothing if the peer is already connected or has
+    /// a connection in progress.
+    pub(super) fn schedule_retry(
+        &mut self,
+        node_addr: NodeAddr,
+        now_ms: u64,
+        reconnect: bool,
+    ) {
         let retry_cfg = &self.config.node.retry;
         let max_retries = retry_cfg.max_retries;
-        if max_retries == 0 {
+        if max_retries == 0 && !reconnect {
             return;
         }
 
@@ -69,7 +79,7 @@ impl Node {
         if let Some(state) = self.retry_pending.get_mut(&node_addr) {
             // Already tracking — increment
             state.retry_count += 1;
-            if state.retry_count > max_retries {
+            if !state.reconnect && state.retry_count > max_retries {
                 info!(
                     peer = %peer_name,
                     attempts = state.retry_count,
@@ -83,6 +93,7 @@ impl Node {
             debug!(
                 peer = %peer_name,
                 retry = state.retry_count,
+                reconnect = state.reconnect,
                 delay_secs = delay / 1000,
                 "Scheduling connection retry"
             );
@@ -101,10 +112,12 @@ impl Node {
             if let Some(pc) = peer_config {
                 let mut state = RetryState::new(pc);
                 state.retry_count = 1;
+                state.reconnect = reconnect;
                 let delay = state.backoff_ms(base_interval_ms, max_backoff_ms);
                 state.retry_after_ms = now_ms + delay;
                 debug!(
                     peer = %self.peer_display_name(&node_addr),
+                    reconnect = reconnect,
                     delay_secs = delay / 1000,
                     "First connection attempt failed, scheduling retry"
                 );
@@ -114,6 +127,51 @@ impl Node {
         }
     }
 
+    /// Schedule auto-reconnect for a peer removed by MMP dead timeout.
+    ///
+    /// Looks up the peer in auto-connect config and checks `auto_reconnect`.
+    /// If enabled, feeds the peer into the retry system with unlimited retries.
+    pub(super) fn schedule_reconnect(&mut self, node_addr: NodeAddr, now_ms: u64) {
+        // Find peer in auto-connect config
+        let peer_config = self
+            .config
+            .auto_connect_peers()
+            .find(|pc| {
+                PeerIdentity::from_npub(&pc.npub)
+                    .map(|id| *id.node_addr() == node_addr)
+                    .unwrap_or(false)
+            })
+            .cloned();
+
+        let Some(pc) = peer_config else {
+            return; // Not an auto-connect peer, no reconnect
+        };
+
+        if !pc.auto_reconnect {
+            debug!(
+                peer = %self.peer_display_name(&node_addr),
+                "Auto-reconnect disabled for peer, skipping"
+            );
+            return;
+        }
+
+        let base_interval_ms = self.config.node.retry.base_interval_secs * 1000;
+        let max_backoff_ms = self.config.node.retry.max_backoff_secs * 1000;
+
+        let mut state = RetryState::new(pc);
+        state.reconnect = true;
+        let delay = state.backoff_ms(base_interval_ms, max_backoff_ms);
+        state.retry_after_ms = now_ms + delay;
+
+        info!(
+            peer = %self.peer_display_name(&node_addr),
+            delay_secs = delay / 1000,
+            "Scheduling auto-reconnect after link-dead removal"
+        );
+
+        self.retry_pending.insert(node_addr, state);
+    }
+
     /// Process pending retries whose time has arrived.
     ///
     /// For each due retry, initiates a fresh connection attempt. The retry
@@ -155,12 +213,21 @@ impl Node {
 
             match self.initiate_peer_connection(&peer_config).await {
                 Ok(()) => {
+                    // Push retry_after_ms past the handshake timeout window so
+                    // we don't re-fire on the next tick. If the handshake
+                    // succeeds, promote_connection() clears retry_pending. If
+                    // it times out, check_timeouts() calls schedule_retry()
+                    // which bumps the counter and applies proper backoff.
+                    let hs_timeout_ms =
+                        self.config.node.rate_limit.handshake_timeout_secs * 1000;
+                    if let Some(state) = self.retry_pending.get_mut(&node_addr) {
+                        state.retry_after_ms = now_ms + hs_timeout_ms;
+                    }
                     debug!(
                         peer = %self.peer_display_name(&node_addr),
-                        "Retry connection initiated"
+                        "Retry connection initiated, suppressing re-fire for {}s",
+                        self.config.node.rate_limit.handshake_timeout_secs,
                     );
-                    // Don't remove from retry_pending — wait for promotion
-                    // (success) or next timeout (failure triggers schedule_retry again)
                 }
                 Err(e) => {
                     warn!(
@@ -169,7 +236,8 @@ impl Node {
                         "Retry connection initiation failed"
                     );
                     // Immediate failure counts as an attempt — schedule next retry
-                    self.schedule_retry(node_addr, now_ms);
+                    // (reconnect flag is preserved on existing retry_pending entry)
+                    self.schedule_retry(node_addr, now_ms, false);
                 }
             }
         }
@@ -189,6 +257,7 @@ mod tests {
             peer_config: PeerConfig::default(),
             retry_count: 0,
             retry_after_ms: 0,
+            reconnect: false,
         };
         // base = 5000ms
         assert_eq!(state.backoff_ms(5000, TEST_MAX_BACKOFF_MS), 5000); // 5s * 2^0
@@ -224,6 +293,7 @@ mod tests {
             peer_config: PeerConfig::default(),
             retry_count: 20, // 2^20 * 5000 would be huge
             retry_after_ms: 0,
+            reconnect: false,
         };
         assert_eq!(state.backoff_ms(5000, TEST_MAX_BACKOFF_MS), TEST_MAX_BACKOFF_MS);
     }
@@ -234,6 +304,7 @@ mod tests {
             peer_config: PeerConfig::default(),
             retry_count: 3,
             retry_after_ms: 0,
+            reconnect: false,
         };
         assert_eq!(state.backoff_ms(0, TEST_MAX_BACKOFF_MS), 0);
     }

src/node/tests/unit.rs

@@ -566,7 +566,7 @@ fn test_schedule_retry_creates_entry() {
 
     assert!(node.retry_pending.is_empty());
 
-    node.schedule_retry(peer_node_addr, 1000);
+    node.schedule_retry(peer_node_addr, 1000, false);
 
     assert_eq!(node.retry_pending.len(), 1);
     let state = node.retry_pending.get(&peer_node_addr).unwrap();
@@ -593,11 +593,11 @@ fn test_schedule_retry_increments() {
     let mut node = Node::new(config).unwrap();
 
     // First failure
-    node.schedule_retry(peer_node_addr, 1000);
+    node.schedule_retry(peer_node_addr, 1000, false);
     assert_eq!(node.retry_pending.get(&peer_node_addr).unwrap().retry_count, 1);
 
     // Second failure
-    node.schedule_retry(peer_node_addr, 11_000);
+    node.schedule_retry(peer_node_addr, 11_000, false);
     let state = node.retry_pending.get(&peer_node_addr).unwrap();
     assert_eq!(state.retry_count, 2);
     // backoff_ms(5000) with retry_count=2 = 5000 * 4 = 20000
@@ -622,14 +622,14 @@ fn test_schedule_retry_max_retries_exhausted() {
     let mut node = Node::new(config).unwrap();
 
     // Attempts 1 and 2 should schedule retries
-    node.schedule_retry(peer_node_addr, 1000);
+    node.schedule_retry(peer_node_addr, 1000, false);
     assert!(node.retry_pending.contains_key(&peer_node_addr));
 
-    node.schedule_retry(peer_node_addr, 2000);
+    node.schedule_retry(peer_node_addr, 2000, false);
     assert!(node.retry_pending.contains_key(&peer_node_addr));
 
     // Attempt 3 exceeds max_retries=2, should remove entry
-    node.schedule_retry(peer_node_addr, 3000);
+    node.schedule_retry(peer_node_addr, 3000, false);
     assert!(
         !node.retry_pending.contains_key(&peer_node_addr),
         "Should be removed after max retries exhausted"
@@ -653,7 +653,7 @@ fn test_schedule_retry_disabled() {
 
     let mut node = Node::new(config).unwrap();
 
-    node.schedule_retry(peer_node_addr, 1000);
+    node.schedule_retry(peer_node_addr, 1000, false);
     assert!(
         node.retry_pending.is_empty(),
         "No retry should be scheduled when max_retries=0"
@@ -669,7 +669,7 @@ fn test_schedule_retry_ignores_non_autoconnect() {
     // No peers configured at all
     let mut node = make_node();
 
-    node.schedule_retry(peer_node_addr, 1000);
+    node.schedule_retry(peer_node_addr, 1000, false);
     assert!(
         node.retry_pending.is_empty(),
         "No retry for unconfigured peer"
@@ -691,7 +691,7 @@ fn test_schedule_retry_skips_connected_peer() {
     assert_eq!(node.peer_count(), 1);
 
     // Scheduling a retry for an already-connected peer should be a no-op
-    node.schedule_retry(node_addr, 3000);
+    node.schedule_retry(node_addr, 3000, false);
     assert!(
         node.retry_pending.is_empty(),
         "No retry for already-connected peer"