Merge branch 'refactor-node' into refactor-node-next

This commit is contained in:
Johnathan Corgan
2026-07-13 00:50:40 +00:00
16 changed files with 2213 additions and 545 deletions
+14 -17
View File
@@ -8,7 +8,7 @@ use fips::config::{IdentitySource, resolve_identity};
use fips::version;
use fips::{Config, Node};
use std::path::PathBuf;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, info};
use tracing_subscriber::{EnvFilter, fmt};
/// FIPS mesh network daemon
@@ -157,26 +157,23 @@ async fn run_daemon(
info!("FIPS running");
// Run the RX event loop until shutdown signal.
// stop() drops the packet channel, causing run_rx_loop to exit.
tokio::select! {
result = node.run_rx_loop() => {
match result {
Ok(()) => info!("RX loop exited"),
Err(e) => error!("RX loop error: {}", e),
}
}
_ = shutdown_signal => {
info!("Shutdown signal received");
}
// Serve until the shutdown signal, then drain in place before returning.
// The rx loop observes the signal directly, so its channels are never
// destructively cancelled — they live in the loop's locals across serve and
// drain, and are dropped only on clean exit (after which teardown does not
// need them). On the signal the loop broadcasts a shutdown Disconnect and
// waits (bounded by node.drain_timeout_secs) for peers to clear.
match node.run_rx_loop_with_shutdown(shutdown_signal).await {
Ok(()) => info!("RX loop exited"),
Err(e) => error!("RX loop error: {}", e),
}
info!("FIPS shutting down");
// Stop the node (shuts down transports, TUN, I/O threads)
if let Err(e) = node.stop().await {
warn!("Error during shutdown: {}", e);
}
// Close the drain window (if the loop drained) and tear down. A drained
// loop tears down without re-broadcasting; a loop that exited some other
// way falls back to the immediate stop().
node.finish_shutdown().await;
info!("FIPS shutdown complete");
}
+44
View File
@@ -1045,6 +1045,19 @@ pub struct NodeConfig {
#[serde(default = "NodeConfig::default_link_dead_timeout_secs")]
pub link_dead_timeout_secs: u64,
/// Graceful-shutdown drain deadline in seconds (`node.drain_timeout_secs`).
/// The bounded `Draining` phase broadcasts a shutdown `Disconnect` and then
/// waits up to this long for peers to clear before tearing down, early-
/// exiting as soon as all peers are gone. `None` selects the 2-second
/// default (see [`NodeConfig::drain_timeout`]).
///
/// Kept `Option` deliberately: `NodeConfig` has no `deny_unknown_fields`, so
/// a naive non-`Option` add with a `default` fn would silently rewrite the
/// value into deployed configs on the next serialize. The `Option` +
/// `skip_serializing_if` keeps absent configs absent.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub drain_timeout_secs: Option<u64>,
/// Resource limits (`node.limits.*`).
#[serde(default)]
pub limits: LimitsConfig,
@@ -1127,6 +1140,7 @@ impl Default for NodeConfig {
base_rtt_ms: 100,
heartbeat_interval_secs: 10,
link_dead_timeout_secs: 30,
drain_timeout_secs: None,
limits: LimitsConfig::default(),
rate_limit: RateLimitConfig::default(),
retry: RetryConfig::default(),
@@ -1177,6 +1191,14 @@ impl NodeConfig {
fn default_link_dead_timeout_secs() -> u64 {
30
}
/// Graceful-shutdown drain deadline as a `Duration`.
///
/// Returns the configured `drain_timeout_secs`, or the 2-second default
/// when unset. Used by the daemon's bounded `Draining` phase.
pub fn drain_timeout(&self) -> std::time::Duration {
std::time::Duration::from_secs(self.drain_timeout_secs.unwrap_or(2))
}
}
#[cfg(test)]
@@ -1213,6 +1235,28 @@ owd_window_size: 48
assert_eq!(config.owd_window_size, DEFAULT_OWD_WINDOW_SIZE);
}
#[test]
fn test_drain_timeout_default_and_override() {
// Unset → the 2-second default.
let c = NodeConfig::default();
assert_eq!(c.drain_timeout_secs, None);
assert_eq!(c.drain_timeout(), std::time::Duration::from_secs(2));
// Explicit override is honored.
let c2 = NodeConfig {
drain_timeout_secs: Some(10),
..NodeConfig::default()
};
assert_eq!(c2.drain_timeout(), std::time::Duration::from_secs(10));
// A zero override is a valid (immediate) drain, not the default.
let c3 = NodeConfig {
drain_timeout_secs: Some(0),
..NodeConfig::default()
};
assert_eq!(c3.drain_timeout(), std::time::Duration::from_secs(0));
}
#[test]
fn test_ecn_config_defaults() {
let c = EcnConfig::default();
+3 -3
View File
@@ -166,7 +166,7 @@ impl Node {
#[cfg(unix)]
{
let cache_key = (packet.transport_id, header.receiver_idx.as_u32());
if let Some(workers) = self.decrypt_workers.as_ref().cloned()
if let Some(workers) = self.supervisor.decrypt_workers.as_ref().cloned()
&& self.decrypt_registered_sessions.contains(&cache_key)
{
let job = crate::node::decrypt_worker::DecryptJob {
@@ -460,7 +460,7 @@ impl Node {
/// black-hole the session.
#[cfg(unix)]
pub(in crate::node) fn register_decrypt_worker_session(&mut self, node_addr: &crate::NodeAddr) {
let Some(workers) = self.decrypt_workers.as_ref().cloned() else {
let Some(workers) = self.supervisor.decrypt_workers.as_ref().cloned() else {
return;
};
let (cache_key, state) = {
@@ -501,7 +501,7 @@ impl Node {
&mut self,
cache_key: (crate::transport::TransportId, u32),
) {
if let Some(workers) = self.decrypt_workers.as_ref() {
if let Some(workers) = self.supervisor.decrypt_workers.as_ref() {
workers.unregister_session(cache_key);
}
self.decrypt_registered_sessions.remove(&cache_key);
+62 -2
View File
@@ -44,12 +44,42 @@ impl Node {
/// This method takes ownership of the packet_rx channel and runs
/// until the channel is closed (typically when stop() is called).
pub async fn run_rx_loop(&mut self) -> Result<(), NodeError> {
// No shutdown observer → today's infinite loop, byte-identical. All
// existing callers/tests use this; `pending()` never fires, so the
// shutdown/deadline arms below stay permanently disabled.
self.run_rx_loop_with_shutdown(std::future::pending()).await
}
/// The rx event loop, which serves until `shutdown` fires and then drains
/// **in place** before returning.
///
/// The channel receivers are moved into this frame's locals and live across
/// both serve and drain, so — unlike a `select!`-cancelled loop — they are
/// never destructively dropped mid-flight; they are released only on clean
/// exit, after which teardown does not need them.
///
/// - While serving (`drain_deadline == None`) the loop is behaviorally
/// identical to before: the shutdown arm, the deadline arm, and the
/// peers-empty early-exit are all guarded off, so the hot per-packet path
/// and the `biased` order of the real arms are unchanged.
/// - When `shutdown` fires, the loop calls [`Node::enter_drain`] once
/// (broadcast Disconnect, gate the reconciler off) and arms the bounded
/// deadline, then keeps servicing inbound/tick/peer-removal until all
/// peers clear or the deadline elapses, then returns. The caller
/// ([`Node::finish_shutdown`]) closes the window and tears down.
pub async fn run_rx_loop_with_shutdown(
&mut self,
shutdown: impl std::future::Future<Output = ()>,
) -> Result<(), NodeError> {
tokio::pin!(shutdown);
// `None` = serving; `Some(deadline)` = draining (bounded window).
let mut drain_deadline: Option<tokio::time::Instant> = None;
let mut packet_rx = self.packet_rx.take().ok_or(NodeError::NotStarted)?;
// Take the TUN outbound receiver, or create a dummy channel that never
// produces messages (when TUN is disabled). Holding the sender prevents
// the channel from closing.
let (mut tun_outbound_rx, _tun_guard) = match self.tun_outbound_rx.take() {
let (mut tun_outbound_rx, _tun_guard) = match self.supervisor.tun_outbound_rx.take() {
Some(rx) => (rx, None),
None => {
let (tx, rx) = tokio::sync::mpsc::channel(1);
@@ -59,7 +89,7 @@ impl Node {
// Take the DNS identity receiver, or create a dummy channel (when DNS
// is disabled). Same pattern as TUN outbound.
let (mut dns_identity_rx, _dns_guard) = match self.dns_identity_rx.take() {
let (mut dns_identity_rx, _dns_guard) = match self.supervisor.dns_identity_rx.take() {
Some(rx) => (rx, None),
None => {
let (tx, rx) = tokio::sync::mpsc::channel(1);
@@ -124,6 +154,13 @@ impl Node {
crate::perf_profile::maybe_spawn_reporter();
loop {
// Bounded drain mode: break as soon as all peers have cleared. In
// normal mode (`None`) this short-circuits before touching
// `self.peers`, so the loop is byte-identical.
if drain_deadline.is_some() && self.peers.is_empty() {
info!("Drain complete: all peers cleared, ending drain loop");
break;
}
tokio::select! {
biased;
// Decrypt-worker fallback drains FIRST. Under sustained
@@ -288,6 +325,27 @@ impl Node {
#[cfg(any(target_os = "linux", target_os = "macos"))]
self.activate_connected_udp_sessions().await;
}
// Shutdown signal → enter the bounded drain in place, ONCE.
// Gated on `is_none()` so it only fires while serving; after
// entering drain the arm is disabled (the completed signal is
// never polled again) and the deadline arm below bounds the
// window. Placed after the real arms so their `biased` priority
// is unchanged, and inert while serving with `pending()`.
_ = &mut shutdown, if drain_deadline.is_none() => {
self.enter_drain().await;
drain_deadline =
Some(tokio::time::Instant::now() + self.config().node.drain_timeout());
}
// Bounded drain deadline (drain mode only). Placed LAST so the
// `biased` priority of the normal arms is unchanged, and gated
// on `is_some()` so in normal mode the branch is disabled — the
// future is created but never polled and never fires.
_ = tokio::time::sleep_until(
drain_deadline.unwrap_or_else(tokio::time::Instant::now)
), if drain_deadline.is_some() => {
info!("Drain deadline elapsed, ending drain loop");
break;
}
}
}
@@ -323,9 +381,11 @@ impl Node {
// discovery-layer cooldown to the long protocol-mismatch
// window and emit a single WARN per fresh observation.
if self
.supervisor
.nostr_rendezvous
.is_bootstrap_transport(&packet.transport_id)
&& let Some(npub) = self
.supervisor
.nostr_rendezvous
.bootstrap_transport_npub(&packet.transport_id)
.cloned()
+4 -4
View File
@@ -365,7 +365,7 @@ impl Node {
mark_ipv6_ecn_ce(&mut packet);
self.metrics().congestion.ce_received.inc();
}
if let Some(tun_tx) = &self.tun_tx {
if let Some(tun_tx) = &self.supervisor.tun_tx {
if let Err(e) = tun_tx.send(packet) {
debug!(error = %e, "Failed to deliver decompressed IPv6 packet to TUN");
}
@@ -1916,7 +1916,7 @@ impl Node {
send: PipelinedSend<'_>,
) -> Result<bool, NodeError> {
let dest_addr = send.dest_addr;
let Some(workers) = self.encrypt_workers.as_ref().cloned() else {
let Some(workers) = self.supervisor.encrypt_workers.as_ref().cloned() else {
return Ok(false);
};
@@ -2528,7 +2528,7 @@ impl Node {
let our_ipv6 = FipsAddress::from_node_addr(self.node_addr()).to_ipv6();
if let Some(response) =
build_dest_unreachable(original_packet, DestUnreachableCode::NoRoute, our_ipv6)
&& let Some(tun_tx) = &self.tun_tx
&& let Some(tun_tx) = &self.supervisor.tun_tx
{
let _ = tun_tx.send(response);
}
@@ -2565,7 +2565,7 @@ impl Node {
// SAFETY: slice is exactly 16 bytes; length validated above (>= 40)
let dest_addr = Ipv6Addr::from(<[u8; 16]>::try_from(&original_packet[24..40]).unwrap());
if let Some(response) = build_packet_too_big(original_packet, mtu, dest_addr)
&& let Some(tun_tx) = &self.tun_tx
&& let Some(tun_tx) = &self.supervisor.tun_tx
{
debug!(
original_src = %src_addr,
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+55 -97
View File
@@ -75,14 +75,13 @@ use crate::transport::{
};
use crate::upper::hosts::HostMap;
use crate::upper::icmp_rate_limit::IcmpRateLimiter;
use crate::upper::tun::{TunError, TunOutboundRx, TunState, TunTx};
use crate::upper::tun::{TunError, TunState, TunTx};
use crate::utils::index::IndexAllocator;
use crate::{Config, ConfigError, Identity, IdentityError, NodeAddr, PeerIdentity, TreeCoordinate};
use rand::Rng;
use std::collections::{BTreeSet, HashMap, VecDeque};
use std::fmt;
use std::sync::Arc;
use std::thread::JoinHandle;
use thiserror::Error;
/// Errors related to node operations.
@@ -172,6 +171,9 @@ pub enum NodeError {
#[error("bootstrap handoff failed: {0}")]
BootstrapHandoff(String),
#[error("node start failed: no operational transports")]
NoOperationalTransports,
}
/// Node operational state.
@@ -181,8 +183,23 @@ pub enum NodeState {
Created,
/// Starting up (initializing transports).
Starting,
/// Fully operational.
/// Fully operational — every configured child came up (design doc §9.1).
Running,
/// Operational but degraded (design doc §9.1): ≥1 transport is up and the
/// node is serving, but one or more configured optional children (a
/// transport beyond the first, Nostr, mDNS, TUN, DNS, or a worker pool)
/// failed to start. Still operational — a degraded node serves traffic.
Degraded,
/// Bounded graceful drain in progress (design doc §6): a shutdown
/// `Disconnect` has been broadcast and the node is waiting for peers to
/// clear (bounded by `node.drain_timeout_secs`) before teardown. Not
/// operational; the daemon drain path advances to `Stopping` via the
/// supervisor's `DrainDeadlineElapsed`, never through `stop()`.
Draining,
/// Start failed fatally: zero transports came up (design doc §9.1). The
/// driver tears down any children that did come up and `start()` returns
/// an error. Not operational and not restartable in-process.
Failed,
/// Shutting down.
Stopping,
/// Stopped.
@@ -190,19 +207,23 @@ pub enum NodeState {
}
impl NodeState {
/// Check if node is operational.
/// Check if node is operational. A `Degraded` node is operational — it is
/// serving, just missing an optional child (design doc §9.1).
pub fn is_operational(&self) -> bool {
matches!(self, NodeState::Running)
matches!(self, NodeState::Running | NodeState::Degraded)
}
/// Check if node can be started.
/// Check if node can be started. A `Failed` node is not restartable
/// in-process (design doc §9.1) — only a fresh `Created` or a cleanly
/// `Stopped` node can start.
pub fn can_start(&self) -> bool {
matches!(self, NodeState::Created | NodeState::Stopped)
}
/// Check if node can be stopped.
/// Check if node can be stopped. Both `Running` and `Degraded` nodes are
/// operational and can be stopped or drained.
pub fn can_stop(&self) -> bool {
matches!(self, NodeState::Running)
matches!(self, NodeState::Running | NodeState::Degraded)
}
}
@@ -212,6 +233,9 @@ impl fmt::Display for NodeState {
NodeState::Created => "created",
NodeState::Starting => "starting",
NodeState::Running => "running",
NodeState::Degraded => "degraded",
NodeState::Draining => "draining",
NodeState::Failed => "failed",
NodeState::Stopping => "stopping",
NodeState::Stopped => "stopped",
};
@@ -287,9 +311,13 @@ pub struct Node {
/// `update_peers`; readers reach it through the accessors.
context: Arc<context::NodeContext>,
// === State ===
/// Node operational state.
state: NodeState,
// === Lifecycle Supervisor ===
/// Owner of the lifecycle-managed substrate handles (packet-send channel,
/// TUN plumbing, DNS task, Nostr/LAN rendezvous, encrypt/decrypt worker
/// pools) plus the published `NodeState` and the sans-IO supervisor FSM
/// that authors their spawn/teardown ordering. Reached via
/// `self.supervisor.*`.
supervisor: lifecycle::supervisor::Supervisor,
// === Spanning Tree ===
/// Local spanning tree state.
@@ -320,8 +348,6 @@ pub struct Node {
addr_to_link: HashMap<AddrKey, LinkId>,
// === Packet Channel ===
/// Packet sender for transports.
packet_tx: Option<PacketTx>,
/// Packet receiver (for event loop).
packet_rx: Option<PacketRx>,
@@ -398,24 +424,6 @@ pub struct Node {
tun_state: TunState,
/// TUN interface name (for cleanup).
tun_name: Option<String>,
/// TUN packet sender channel.
tun_tx: Option<TunTx>,
/// Receiver for outbound packets from the TUN reader.
tun_outbound_rx: Option<TunOutboundRx>,
/// TUN reader thread handle.
tun_reader_handle: Option<JoinHandle<()>>,
/// TUN writer thread handle.
tun_writer_handle: Option<JoinHandle<()>>,
/// Shutdown pipe: writing to this fd unblocks the TUN reader thread on macOS.
/// On Linux, deleting the interface via netlink serves the same purpose.
#[cfg(target_os = "macos")]
tun_shutdown_fd: Option<std::os::unix::io::RawFd>,
// === DNS Responder ===
/// Receiver for resolved identities from the DNS responder.
dns_identity_rx: Option<crate::upper::dns::DnsIdentityRx>,
/// DNS responder task handle.
dns_task: Option<tokio::task::JoinHandle<()>>,
// === Index-Based Session Dispatch ===
/// Allocator for session indices.
@@ -464,17 +472,6 @@ pub struct Node {
/// are exhausted.
retry_pending: HashMap<NodeAddr, retry::RetryState>,
/// Node-side driver state for the Nostr overlay peer-rendezvous
/// subsystem: the engine handle, its startup timestamp, the one-shot
/// startup-sweep latch, and the per-peer bootstrap-transport bookkeeping
/// adopted from NAT-traversal handoffs.
nostr_rendezvous: crate::nostr::RendezvousDriver,
/// mDNS / DNS-SD responder + browser for local-link peer discovery.
/// Identity is unverified at this layer — the Noise XX handshake
/// initiated against an mDNS-observed endpoint is what proves the
/// peer holds the matching private key.
lan_rendezvous: Option<Arc<crate::mdns::LanRendezvous>>,
// === Periodic Parent Re-evaluation ===
/// Timestamp of last periodic parent re-evaluation (for pacing).
last_parent_reeval: Option<std::time::Instant>,
@@ -510,20 +507,6 @@ pub struct Node {
/// published through a lock-free snapshot for the display path.
host_map: reloadable::HostMapReloadable,
/// Off-task FMP-encrypt + UDP-send worker pool. Unix-only —
/// the worker issues direct sendmmsg(2) / sendmsg+UDP_GSO calls
/// on raw fds via `AsRawFd`. None on Windows or when the worker
/// pool failed to spawn.
#[cfg(unix)]
pub(crate) encrypt_workers: Option<encrypt_worker::EncryptWorkerPool>,
/// Off-task FMP decrypt worker pool — receiver-side mirror of
/// `encrypt_workers`. Workers are shards: each owns its session
/// state directly in a thread-local `HashMap` (no `RwLock`,
/// no `Mutex` per packet). Hash-by-cache-key dispatch.
#[cfg(unix)]
pub(crate) decrypt_workers: Option<decrypt_worker::DecryptWorkerPool>,
/// Sessions whose recv cipher + replay window have been handed
/// off to a decrypt shard worker. Lookup gate on the hot receive
/// path: if the cache-key is in here, dispatch to worker; else
@@ -634,7 +617,7 @@ impl Node {
Ok(Self {
context,
state: NodeState::Created,
supervisor: lifecycle::supervisor::Supervisor::new(),
tree_state,
bloom_state,
coord_cache,
@@ -642,7 +625,6 @@ impl Node {
transport_drops: HashMap::new(),
links: HashMap::new(),
addr_to_link: HashMap::new(),
packet_tx: None,
packet_rx: None,
connections: HashMap::new(),
peers: HashMap::new(),
@@ -665,14 +647,6 @@ impl Node {
)),
tun_state,
tun_name: None,
tun_tx: None,
tun_outbound_rx: None,
tun_reader_handle: None,
tun_writer_handle: None,
#[cfg(target_os = "macos")]
tun_shutdown_fd: None,
dns_identity_rx: None,
dns_task: None,
index_allocator: IndexAllocator::new(),
peers_by_index: HashMap::new(),
pending_outbound: HashMap::new(),
@@ -692,8 +666,6 @@ impl Node {
),
pending_connects: Vec::new(),
retry_pending: HashMap::new(),
nostr_rendezvous: crate::nostr::RendezvousDriver::default(),
lan_rendezvous: None,
last_parent_reeval: None,
last_congestion_log: None,
estimated_mesh_size: None,
@@ -704,10 +676,6 @@ impl Node {
host_map,
path_mtu_lookup: Arc::new(std::sync::RwLock::new(HashMap::new())),
#[cfg(unix)]
encrypt_workers: None,
#[cfg(unix)]
decrypt_workers: None,
#[cfg(unix)]
decrypt_registered_sessions: std::collections::HashSet::new(),
#[cfg(unix)]
decrypt_fallback_rx: Some(decrypt_fallback_rx),
@@ -797,7 +765,7 @@ impl Node {
Ok(Self {
context,
state: NodeState::Created,
supervisor: lifecycle::supervisor::Supervisor::new(),
tree_state,
bloom_state,
coord_cache,
@@ -805,7 +773,6 @@ impl Node {
transport_drops: HashMap::new(),
links: HashMap::new(),
addr_to_link: HashMap::new(),
packet_tx: None,
packet_rx: None,
connections: HashMap::new(),
peers: HashMap::new(),
@@ -828,14 +795,6 @@ impl Node {
)),
tun_state,
tun_name: None,
tun_tx: None,
tun_outbound_rx: None,
tun_reader_handle: None,
tun_writer_handle: None,
#[cfg(target_os = "macos")]
tun_shutdown_fd: None,
dns_identity_rx: None,
dns_task: None,
index_allocator: IndexAllocator::new(),
peers_by_index: HashMap::new(),
pending_outbound: HashMap::new(),
@@ -852,8 +811,6 @@ impl Node {
lookup: Lookup::new(LookupBackoff::new(), LookupForwardRateLimiter::new()),
pending_connects: Vec::new(),
retry_pending: HashMap::new(),
nostr_rendezvous: crate::nostr::RendezvousDriver::default(),
lan_rendezvous: None,
last_parent_reeval: None,
last_congestion_log: None,
estimated_mesh_size: None,
@@ -864,10 +821,6 @@ impl Node {
host_map,
path_mtu_lookup: Arc::new(std::sync::RwLock::new(HashMap::new())),
#[cfg(unix)]
encrypt_workers: None,
#[cfg(unix)]
decrypt_workers: None,
#[cfg(unix)]
decrypt_registered_sessions: std::collections::HashSet::new(),
#[cfg(unix)]
decrypt_fallback_rx: Some(decrypt_fallback_rx),
@@ -1234,7 +1187,7 @@ impl Node {
/// Get the node state.
pub fn state(&self) -> NodeState {
self.state
self.supervisor.state
}
/// Get the node uptime.
@@ -1244,7 +1197,7 @@ impl Node {
/// Check if node is operational.
pub fn is_running(&self) -> bool {
self.state.is_operational()
self.supervisor.state.is_operational()
}
/// Check if this is a leaf-only node.
@@ -1560,7 +1513,7 @@ impl Node {
let snapshot = crate::control::snapshot::StatsSnapshot {
history: std::sync::Arc::new(self.stats_history.clone()),
estimated_mesh_size: self.estimated_mesh_size,
state: self.state,
state: self.supervisor.state,
tun_state: self.tun_state,
tun_name: self.tun_name.clone(),
effective_ipv6_mtu: self.effective_ipv6_mtu(),
@@ -2308,7 +2261,11 @@ impl Node {
}
pub(crate) fn cleanup_bootstrap_transport_if_unused(&mut self, transport_id: TransportId) {
if !self.nostr_rendezvous.is_bootstrap_transport(&transport_id) {
if !self
.supervisor
.nostr_rendezvous
.is_bootstrap_transport(&transport_id)
{
return;
}
@@ -2338,7 +2295,8 @@ impl Node {
"bootstrap transport has no remaining references; dropping"
);
self.nostr_rendezvous
self.supervisor
.nostr_rendezvous
.remove_bootstrap_transport(&transport_id);
self.transport_drops.remove(&transport_id);
self.transports.remove(&transport_id);
@@ -2415,7 +2373,7 @@ impl Node {
/// Used by control queries (`show_peers` per-peer Nostr-traversal
/// state) to read failure-state without taking shared ownership.
pub fn nostr_rendezvous_handle(&self) -> Option<&crate::nostr::NostrRendezvous> {
self.nostr_rendezvous.engine()
self.supervisor.nostr_rendezvous.engine()
}
/// Iterate over all peer node IDs.
@@ -2706,7 +2664,7 @@ impl Node {
///
/// Returns None if TUN is not active or the node hasn't been started.
pub fn tun_tx(&self) -> Option<&TunTx> {
self.tun_tx.as_ref()
self.supervisor.tun_tx.as_ref()
}
// === Sending ===
@@ -2800,7 +2758,7 @@ impl Node {
{
let send_cipher_opt = session.send_cipher_clone();
if let Some(fmp_cipher) = send_cipher_opt
&& let Some(workers) = self.encrypt_workers.as_ref().cloned()
&& let Some(workers) = self.supervisor.encrypt_workers.as_ref().cloned()
&& let Some(transport) = self.transports.get(&transport_id)
&& let TransportHandle::Udp(udp) = transport
&& let Some(socket) = udp.async_socket()
@@ -3097,7 +3055,7 @@ impl fmt::Debug for Node {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Node")
.field("node_addr", self.node_addr())
.field("state", &self.state)
.field("state", &self.supervisor.state)
.field("is_leaf_only", &self.is_leaf_only())
.field("connections", &self.connection_count())
.field("peers", &self.peer_count())
+2 -2
View File
@@ -282,7 +282,7 @@ impl Node {
// evicts if the relay has nothing, otherwise leaves it. Cheap
// (one Filter fetch with 2s timeout) and bounded by the retry
// backoff cadence.
if let Some(bootstrap) = self.nostr_rendezvous.engine_arc() {
if let Some(bootstrap) = self.supervisor.nostr_rendezvous.engine_arc() {
let _ = bootstrap
.refetch_advert_for_stale_check(&peer_config.npub)
.await;
@@ -318,7 +318,7 @@ impl Node {
// entry expires. Force a re-fetch so the next retry tick
// picks up fresh endpoints.
if matches!(e, NodeError::NoTransportForType(_))
&& let Some(bootstrap) = self.nostr_rendezvous.engine_arc()
&& let Some(bootstrap) = self.supervisor.nostr_rendezvous.engine_arc()
{
let npub = peer_config.npub.clone();
tokio::spawn(async move {
+18 -18
View File
@@ -24,17 +24,17 @@ async fn test_adopted_udp_traversal_completes_handshake() {
let (packet_tx_a, packet_rx_a) = packet_channel(64);
let (packet_tx_b, packet_rx_b) = packet_channel(64);
node_a.packet_tx = Some(packet_tx_a.clone());
node_a.supervisor.packet_tx = Some(packet_tx_a.clone());
node_a.packet_rx = Some(packet_rx_a);
node_a.state = NodeState::Running;
node_a.supervisor.state = NodeState::Running;
let mut transport_b = UdpTransport::new(transport_id_b, None, udp_config, packet_tx_b.clone());
transport_b.start_async().await.unwrap();
let addr_b = transport_b.local_addr().unwrap();
node_b.packet_tx = Some(packet_tx_b.clone());
node_b.supervisor.packet_tx = Some(packet_tx_b.clone());
node_b.packet_rx = Some(packet_rx_b);
node_b.state = NodeState::Running;
node_b.supervisor.state = NodeState::Running;
node_b
.transports
.insert(transport_id_b, TransportHandle::Udp(transport_b));
@@ -107,9 +107,9 @@ async fn test_adopted_udp_traversal_completes_handshake() {
async fn test_failed_adopted_traversal_cleans_up_transport() {
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
node.state = NodeState::Running;
node.supervisor.state = NodeState::Running;
node.index_allocator = IndexAllocator::with_max_attempts(0);
let peer = make_node();
@@ -137,9 +137,9 @@ async fn test_failed_adopted_traversal_cleans_up_transport() {
async fn test_adopted_traversal_skips_already_connected_peer() {
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
node.state = NodeState::Running;
node.supervisor.state = NodeState::Running;
let transport_id = TransportId::new(1);
let link_id = LinkId::new(1);
@@ -196,17 +196,17 @@ async fn test_third_peer_can_handshake_via_adopted_transport_socket() {
let (packet_tx_b, packet_rx_b) = packet_channel(64);
let (packet_tx_c, packet_rx_c) = packet_channel(64);
node_a.packet_tx = Some(packet_tx_a.clone());
node_a.supervisor.packet_tx = Some(packet_tx_a.clone());
node_a.packet_rx = Some(packet_rx_a);
node_a.state = NodeState::Running;
node_a.supervisor.state = NodeState::Running;
node_b.packet_tx = Some(packet_tx_b.clone());
node_b.supervisor.packet_tx = Some(packet_tx_b.clone());
node_b.packet_rx = Some(packet_rx_b);
node_b.state = NodeState::Running;
node_b.supervisor.state = NodeState::Running;
node_c.packet_tx = Some(packet_tx_c.clone());
node_c.supervisor.packet_tx = Some(packet_tx_c.clone());
node_c.packet_rx = Some(packet_rx_c);
node_c.state = NodeState::Running;
node_c.supervisor.state = NodeState::Running;
let mut transport_a = UdpTransport::new(transport_id_a, None, udp_config.clone(), packet_tx_a);
transport_a.start_async().await.unwrap();
@@ -341,9 +341,9 @@ async fn test_adopted_udp_inherits_mtu_from_single_primary_config() {
let mut node = make_node_with(config);
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
node.state = NodeState::Running;
node.supervisor.state = NodeState::Running;
let peer = make_node();
let adopted_socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap();
@@ -391,9 +391,9 @@ async fn test_adopted_udp_inherits_mtu_from_named_primary_config() {
let mut node = make_node_with(config);
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
node.state = NodeState::Running;
node.supervisor.state = NodeState::Running;
let peer = make_node();
let adopted_socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap();
+1 -1
View File
@@ -1217,7 +1217,7 @@ async fn test_check_pending_lookups_default_sequence_unreachable() {
// Inject a TUN sender so `send_icmpv6_dest_unreachable` is observable.
let (tun_tx, tun_rx) = mpsc::channel::<Vec<u8>>();
node.tun_tx = Some(tun_tx);
node.supervisor.tun_tx = Some(tun_tx);
// Build a target identity (the unreachable destination).
let target_identity = Identity::generate();
+2 -2
View File
@@ -310,8 +310,8 @@ async fn test_run_rx_loop_handshake() {
node_b.packet_rx = Some(packet_rx_b);
// Set node state to Running (transports need to be operational)
node_a.state = NodeState::Running;
node_b.state = NodeState::Running;
node_a.supervisor.state = NodeState::Running;
node_b.supervisor.state = NodeState::Running;
// === Phase 1: Node A initiates handshake to Node B ===
+19
View File
@@ -29,6 +29,25 @@ pub(super) fn make_node() -> Node {
make_node_with(Config::new())
}
/// A test node that reaches `Full` health on `start()`.
///
/// A default [`make_node`] configures no transports, so its `start()` now
/// resolves to `NodeState::Failed` (zero transports up, design doc §9.1) and
/// returns `NoOperationalTransports`. Lifecycle-state tests that need a running
/// node build one with a single loopback UDP transport (ephemeral port) as the
/// sole configured child — DNS disabled — so bring-up has exactly one
/// configured child and it comes up (`Full`). Mirrors the udp config in
/// `test_node_start_does_not_wait_for_nostr_relay_startup`.
pub(super) fn make_healthy_node() -> Node {
let mut config = Config::new();
config.transports.udp = crate::config::TransportInstances::Single(crate::config::UdpConfig {
bind_addr: Some("127.0.0.1:0".to_string()),
..Default::default()
});
config.dns.enabled = false;
make_node_with(config)
}
/// Build a test node from an explicit `Config`. Immutable state lives solely in
/// the shared `NodeContext`, built once at construction — there is no
/// post-construction field to poke, so set limits/config on the `Config` here.
+10 -10
View File
@@ -654,7 +654,7 @@ async fn test_session_100_nodes() {
let mut tun_receivers: Vec<mpsc::Receiver<Vec<u8>>> = Vec::with_capacity(NUM_NODES);
for tn in nodes.iter_mut() {
let (tx, rx) = mpsc::channel();
tn.node.tun_tx = Some(tx);
tn.node.supervisor.tun_tx = Some(tx);
tun_receivers.push(rx);
}
@@ -1070,7 +1070,7 @@ async fn test_tun_outbound_established_session() {
// Install TUN receiver on Node 1
let (tun_tx, tun_rx) = std::sync::mpsc::channel();
nodes[1].node.tun_tx = Some(tun_tx);
nodes[1].node.supervisor.tun_tx = Some(tun_tx);
// Build and inject an IPv6 packet
let test_payload = b"data-plane-test-12345";
@@ -1114,7 +1114,7 @@ async fn test_tun_outbound_triggers_session_initiation() {
// Install TUN receiver on Node 1
let (tun_tx, tun_rx) = std::sync::mpsc::channel();
nodes[1].node.tun_tx = Some(tun_tx);
nodes[1].node.supervisor.tun_tx = Some(tun_tx);
// Build and inject an IPv6 packet (identity cache populated at peer promotion)
let test_payload = b"trigger-session-test";
@@ -1167,7 +1167,7 @@ async fn test_tun_outbound_unknown_destination() {
// Install TUN receiver on Node 0 (for ICMPv6 response)
let (tun_tx, tun_rx) = std::sync::mpsc::channel();
nodes[0].node.tun_tx = Some(tun_tx);
nodes[0].node.supervisor.tun_tx = Some(tun_tx);
let src_fips = crate::FipsAddress::from_node_addr(nodes[0].node.node_addr());
@@ -1219,7 +1219,7 @@ async fn test_tun_outbound_3node_forwarded() {
// Install TUN receiver on Node 2
let (tun_tx, tun_rx) = std::sync::mpsc::channel();
nodes[2].node.tun_tx = Some(tun_tx);
nodes[2].node.supervisor.tun_tx = Some(tun_tx);
// Build and inject an IPv6 packet (triggers session initiation to Node 2)
let test_payload = b"forwarded-data-plane";
@@ -1264,7 +1264,7 @@ async fn test_tun_outbound_pending_queue_flush() {
// Install TUN receiver on Node 1
let (tun_tx, tun_rx) = std::sync::mpsc::channel();
nodes[1].node.tun_tx = Some(tun_tx);
nodes[1].node.supervisor.tun_tx = Some(tun_tx);
// Send 5 packets before any session exists
let mut packets = Vec::new();
@@ -1915,7 +1915,7 @@ async fn test_tun_outbound_path_mtu_generates_ptb() {
// Install TUN receiver on source node to capture ICMPv6 PTB
let (tun_tx, tun_rx) = std::sync::mpsc::channel();
nodes[0].node.tun_tx = Some(tun_tx);
nodes[0].node.supervisor.tun_tx = Some(tun_tx);
// Build an IPv6 packet that fits local MTU but exceeds path MTU
let reduced_ipv6_mtu = crate::upper::icmp::effective_ipv6_mtu(reduced_mtu) as usize;
@@ -1972,7 +1972,7 @@ async fn test_tun_outbound_path_mtu_generates_ptb() {
// Verify a packet that fits within path MTU passes through (no PTB)
let (tun_tx2, tun_rx2) = std::sync::mpsc::channel();
nodes[0].node.tun_tx = Some(tun_tx2);
nodes[0].node.supervisor.tun_tx = Some(tun_tx2);
let fitting_payload = vec![0u8; reduced_ipv6_mtu - 41]; // fits within path MTU
let fitting_packet = build_ipv6_packet(&src_fips, &dst_fips, &fitting_payload);
assert!(fitting_packet.len() <= reduced_ipv6_mtu);
@@ -2111,7 +2111,7 @@ async fn test_multihop_pmtud_heterogeneous_mtu() {
// should check PathMtuState and generate ICMPv6 PTB on TUN instead
// of forwarding.
let (tun_tx2, tun_rx2) = std::sync::mpsc::channel();
nodes[0].node.tun_tx = Some(tun_tx2);
nodes[0].node.supervisor.tun_tx = Some(tun_tx2);
nodes[0].node.handle_tun_outbound(ipv6_packet.clone()).await;
@@ -2155,7 +2155,7 @@ async fn test_multihop_pmtud_heterogeneous_mtu() {
// Verify a fitting packet still passes through without PTB
let (tun_tx3, tun_rx3) = std::sync::mpsc::channel();
nodes[0].node.tun_tx = Some(tun_tx3);
nodes[0].node.supervisor.tun_tx = Some(tun_tx3);
let fitting_payload = vec![0xCDu8; 600 - 40]; // 600-byte IPv6 packet, well within 694
let fitting_packet = build_ipv6_packet(&src_fips, &dst_fips, &fitting_payload);
+75 -14
View File
@@ -56,7 +56,7 @@ async fn test_nat_bootstrap_failure_falls_back_to_direct_udp_address() {
let peer_identity = Identity::generate();
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx.clone());
node.supervisor.packet_tx = Some(packet_tx.clone());
node.packet_rx = Some(packet_rx);
let transport_id = TransportId::new(1);
@@ -102,7 +102,7 @@ async fn test_try_peer_addresses_races_all_concrete_udp_candidates() {
let peer_identity = Identity::generate();
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx.clone());
node.supervisor.packet_tx = Some(packet_tx.clone());
node.packet_rx = Some(packet_rx);
let transport_id = TransportId::new(1);
@@ -151,13 +151,16 @@ async fn test_try_peer_addresses_races_all_concrete_udp_candidates() {
#[tokio::test]
async fn test_node_state_transitions() {
let mut node = make_node();
// A transport-less node now resolves to `Failed` on start (design doc
// §9.1), so exercise state transitions with a genuinely healthy node.
let mut node = make_healthy_node();
assert!(!node.is_running());
assert!(node.state().can_start());
node.start().await.unwrap();
assert!(node.is_running());
assert_eq!(node.state(), NodeState::Running);
assert!(!node.state().can_start());
node.stop().await.unwrap();
@@ -165,6 +168,58 @@ async fn test_node_state_transitions() {
assert_eq!(node.state(), NodeState::Stopped);
}
#[tokio::test]
async fn test_transportless_start_fails_and_publishes_failed() {
// The intended behavioral change (design doc §9.1): a node with zero
// transports up cannot serve, so `start()` returns `NoOperationalTransports`
// and leaves the published state at `Failed` (not operational, not
// restartable in-process).
let mut node = make_node();
assert!(node.state().can_start());
let result = node.start().await;
assert!(matches!(result, Err(NodeError::NoOperationalTransports)));
assert_eq!(node.state(), NodeState::Failed);
assert!(!node.is_running());
assert!(!node.state().is_operational());
assert!(!node.state().can_start());
}
#[tokio::test]
async fn test_drain_publishes_draining_state() {
let mut node = make_healthy_node();
node.start().await.unwrap();
assert_eq!(node.state(), NodeState::Running);
assert!(node.state().is_operational());
// Enter the bounded drain in place: publishes the operator-visible
// `Draining` state (not operational) without tearing down.
node.enter_drain().await;
assert_eq!(node.state(), NodeState::Draining);
assert!(!node.state().is_operational());
// `Draining` is neither startable nor externally stoppable; the daemon
// drain finishes via the supervisor's `DrainDeadlineElapsed`, not `stop()`.
assert!(!node.state().can_start());
assert!(!node.state().can_stop());
// Finishing shutdown from `Draining` tears down to `Stopped`.
node.finish_shutdown().await;
assert_eq!(node.state(), NodeState::Stopped);
}
#[tokio::test]
async fn test_immediate_stop_never_publishes_draining() {
let mut node = make_healthy_node();
node.start().await.unwrap();
assert_eq!(node.state(), NodeState::Running);
// The immediate stop() path (used by tests and the stop-now path)
// transitions Running → Stopping → Stopped and never enters `Draining`.
node.stop().await.unwrap();
assert_eq!(node.state(), NodeState::Stopped);
assert_ne!(node.state(), NodeState::Draining);
}
#[tokio::test]
async fn test_node_start_does_not_wait_for_nostr_relay_startup() {
let mut config = Config::new();
@@ -196,7 +251,7 @@ async fn test_node_start_does_not_wait_for_nostr_relay_startup() {
#[tokio::test]
async fn test_node_double_start() {
let mut node = make_node();
let mut node = make_healthy_node();
node.start().await.unwrap();
let result = node.start().await;
@@ -543,7 +598,7 @@ async fn test_node_rx_loop_requires_start() {
#[tokio::test]
async fn test_node_rx_loop_takes_channel() {
let mut node = make_node();
let mut node = make_healthy_node();
node.start().await.unwrap();
// packet_rx should be available after start
@@ -1053,7 +1108,7 @@ async fn update_peers_races_new_alternative_without_dropping_active_peer() {
config.peers = vec![old_peer.clone()];
let mut node = make_node_with(config);
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx.clone());
node.supervisor.packet_tx = Some(packet_tx.clone());
node.packet_rx = Some(packet_rx);
let transport_id = TransportId::new(1);
@@ -1133,7 +1188,9 @@ async fn test_nostr_traversal_failure_skips_connected_peer() {
peer_config: crate::config::PeerConfig::new(peer_identity.npub(), "udp", "127.0.0.1:9"),
reason: "stale traversal failure".to_string(),
});
node.nostr_rendezvous.set_engine(bootstrap.clone());
node.supervisor
.nostr_rendezvous
.set_engine(bootstrap.clone());
node.poll_nostr_rendezvous().await;
@@ -1173,7 +1230,9 @@ async fn test_nostr_traversal_established_skips_connected_peer() {
socket,
),
});
node.nostr_rendezvous.set_engine(bootstrap.clone());
node.supervisor
.nostr_rendezvous
.set_engine(bootstrap.clone());
node.poll_nostr_rendezvous().await;
@@ -1427,7 +1486,7 @@ async fn test_transport_mtu_returns_min_across_operational() {
// iteration order. This is the core ISSUE-2026-0011 regression test.
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
let udp1 = make_udp_transport_with_mtu(1, 1497).await;
@@ -1464,7 +1523,7 @@ async fn test_transport_mtu_min_with_single_operational() {
// operational.
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
let udp = make_udp_transport_with_mtu(1, 1452).await;
@@ -1487,7 +1546,7 @@ async fn test_transport_mtu_min_with_single_operational() {
async fn test_seed_path_mtu_inserts_when_empty() {
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
let udp = make_udp_transport_with_mtu(1, 1452).await;
@@ -1520,7 +1579,7 @@ async fn test_seed_path_mtu_inserts_when_empty() {
async fn test_seed_path_mtu_keeps_tighter_existing_value() {
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
let udp = make_udp_transport_with_mtu(1, 1452).await;
@@ -1560,7 +1619,7 @@ async fn test_seed_path_mtu_keeps_tighter_existing_value() {
async fn test_seed_path_mtu_tightens_looser_existing_value() {
let mut node = make_node();
let (packet_tx, packet_rx) = packet_channel(64);
node.packet_tx = Some(packet_tx);
node.supervisor.packet_tx = Some(packet_tx);
node.packet_rx = Some(packet_rx);
let udp = make_udp_transport_with_mtu(1, 1280).await;
@@ -1730,7 +1789,9 @@ async fn poll_nostr_rendezvous_established_gated_at_capacity() {
socket,
),
});
node.nostr_rendezvous.set_engine(bootstrap.clone());
node.supervisor
.nostr_rendezvous
.set_engine(bootstrap.clone());
let before_peers = node.peer_count();
let before_links = node.link_count();
+7
View File
@@ -661,11 +661,18 @@ DOCKERFILE
# Write a minimal fips.yaml that exercises the new defaults.
# tun.enabled: true so the daemon creates fips0 itself; identity
# persistent so /etc/fips/fips.pub gives us a stable npub to query.
# A UDP transport is configured so at least one transport comes up:
# a node with zero operational transports is Failed and refuses to
# start. This test exercises the .fips DNS responder, not mesh
# connectivity, so any bound transport suffices.
docker exec "$name" bash -c 'cat > /etc/fips/fips.yaml <<EOF
node:
identity:
persistent: true
log_level: debug
transports:
udp:
bind_addr: "0.0.0.0:2121"
tun:
enabled: true
name: fips0