mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 08:14:42 +00:00
node: add bounded graceful-shutdown drain phase
Add an operator-visible Draining phase on daemon shutdown. On the shutdown signal the node broadcasts Disconnect to all peers, then keeps serving for a bounded window - up to node.drain_timeout_secs (default 2s), exiting early once all peers are gone - before tearing down. This lets in-flight traffic settle and peers observe the disconnect before the transports close, rather than the previous immediate teardown. The lifecycle FSM gains a Draining state plus Drain/DrainDeadlineElapsed events; the run loop observes the shutdown signal and transitions to draining in place - one continuous loop, so the channel receivers are never destructively cancelled. The published NodeState gains a Draining variant, visible via control queries during the window. The immediate stop() path used by tests and non-daemon callers is unchanged: it still tears down immediately with no drain wait. The reconciler-gate actions the drain emits are no-ops until the peering reconciler lands and consumes them.
This commit is contained in:
@@ -42,6 +42,36 @@ impl Node {
|
||||
/// This method takes ownership of the packet_rx channel and runs
|
||||
/// until the channel is closed (typically when stop() is called).
|
||||
pub async fn run_rx_loop(&mut self) -> Result<(), NodeError> {
|
||||
// No shutdown observer → today's infinite loop, byte-identical. All
|
||||
// existing callers/tests use this; `pending()` never fires, so the
|
||||
// shutdown/deadline arms below stay permanently disabled.
|
||||
self.run_rx_loop_with_shutdown(std::future::pending()).await
|
||||
}
|
||||
|
||||
/// The rx event loop, which serves until `shutdown` fires and then drains
|
||||
/// **in place** before returning.
|
||||
///
|
||||
/// The channel receivers are moved into this frame's locals and live across
|
||||
/// both serve and drain, so — unlike a `select!`-cancelled loop — they are
|
||||
/// never destructively dropped mid-flight; they are released only on clean
|
||||
/// exit, after which teardown does not need them.
|
||||
///
|
||||
/// - While serving (`drain_deadline == None`) the loop is behaviorally
|
||||
/// identical to before: the shutdown arm, the deadline arm, and the
|
||||
/// peers-empty early-exit are all guarded off, so the hot per-packet path
|
||||
/// and the `biased` order of the real arms are unchanged.
|
||||
/// - When `shutdown` fires, the loop calls [`Node::enter_drain`] once
|
||||
/// (broadcast Disconnect, gate the reconciler off) and arms the bounded
|
||||
/// deadline, then keeps servicing inbound/tick/peer-removal until all
|
||||
/// peers clear or the deadline elapses, then returns. The caller
|
||||
/// ([`Node::finish_shutdown`]) closes the window and tears down.
|
||||
pub async fn run_rx_loop_with_shutdown(
|
||||
&mut self,
|
||||
shutdown: impl std::future::Future<Output = ()>,
|
||||
) -> Result<(), NodeError> {
|
||||
tokio::pin!(shutdown);
|
||||
// `None` = serving; `Some(deadline)` = draining (bounded window).
|
||||
let mut drain_deadline: Option<tokio::time::Instant> = None;
|
||||
let mut packet_rx = self.packet_rx.take().ok_or(NodeError::NotStarted)?;
|
||||
|
||||
// Take the TUN outbound receiver, or create a dummy channel that never
|
||||
@@ -122,6 +152,13 @@ impl Node {
|
||||
crate::perf_profile::maybe_spawn_reporter();
|
||||
|
||||
loop {
|
||||
// Bounded drain mode: break as soon as all peers have cleared. In
|
||||
// normal mode (`None`) this short-circuits before touching
|
||||
// `self.peers`, so the loop is byte-identical.
|
||||
if drain_deadline.is_some() && self.peers.is_empty() {
|
||||
info!("Drain complete: all peers cleared, ending drain loop");
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
// Decrypt-worker fallback drains FIRST. Under sustained
|
||||
@@ -285,6 +322,27 @@ impl Node {
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
self.activate_connected_udp_sessions().await;
|
||||
}
|
||||
// Shutdown signal → enter the bounded drain in place, ONCE.
|
||||
// Gated on `is_none()` so it only fires while serving; after
|
||||
// entering drain the arm is disabled (the completed signal is
|
||||
// never polled again) and the deadline arm below bounds the
|
||||
// window. Placed after the real arms so their `biased` priority
|
||||
// is unchanged, and inert while serving with `pending()`.
|
||||
_ = &mut shutdown, if drain_deadline.is_none() => {
|
||||
self.enter_drain().await;
|
||||
drain_deadline =
|
||||
Some(tokio::time::Instant::now() + self.config().node.drain_timeout());
|
||||
}
|
||||
// Bounded drain deadline (drain mode only). Placed LAST so the
|
||||
// `biased` priority of the normal arms is unchanged, and gated
|
||||
// on `is_some()` so in normal mode the branch is disabled — the
|
||||
// future is created but never polled and never fires.
|
||||
_ = tokio::time::sleep_until(
|
||||
drain_deadline.unwrap_or_else(tokio::time::Instant::now)
|
||||
), if drain_deadline.is_some() => {
|
||||
info!("Drain deadline elapsed, ending drain loop");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+135
-26
@@ -1576,32 +1576,39 @@ impl Node {
|
||||
|
||||
// Reconstruct the supervised up-set from observed runtime presence and
|
||||
// let the FSM author the teardown order (dns → nostr → mdns →
|
||||
// transports (ascending id) → tun). Worker pools are deliberately
|
||||
// absent from the up-set: today's stop() never tears them down.
|
||||
let mut up: Vec<Child> = Vec::new();
|
||||
if self.supervisor.dns_task.is_some() {
|
||||
up.push(Child::Dns);
|
||||
}
|
||||
if self.supervisor.nostr_rendezvous.engine().is_some() {
|
||||
up.push(Child::Nostr);
|
||||
}
|
||||
if self.supervisor.lan_rendezvous.is_some() {
|
||||
up.push(Child::Mdns);
|
||||
}
|
||||
for id in self.transports.keys() {
|
||||
up.push(Child::Transport(*id));
|
||||
}
|
||||
if self.tun_name.is_some() {
|
||||
up.push(Child::Tun);
|
||||
}
|
||||
// transports (ascending id) → tun).
|
||||
let up = self.reconstruct_supervised_up();
|
||||
self.supervisor.fsm = SupervisorFsm::running_with(up);
|
||||
let actions = self.supervisor.fsm.step(Event::Stop);
|
||||
|
||||
// Execute each StopChild as today's verbatim block. Two driver seams
|
||||
// are woven in at their current positions: the shutdown-disconnect
|
||||
// fan-out (after Dns, before Nostr), and dropping the packet channels
|
||||
// (after all transports, before TUN).
|
||||
let mut disconnect_done = false;
|
||||
// Execute the teardown plan. `broadcast_disconnect = true`: the
|
||||
// immediate-stop path owns the single shutdown-Disconnect fan-out,
|
||||
// which the helper emits at the Dns→rest seam.
|
||||
self.execute_teardown(actions, true).await;
|
||||
|
||||
self.supervisor.state = NodeState::Stopped;
|
||||
info!(state = %self.supervisor.state, "Node stopped");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Execute an ordered `StopChild` teardown plan authored by the supervisor
|
||||
/// FSM, reporting each `ChildStopped` back so the machine reaches `Stopped`.
|
||||
///
|
||||
/// This is the teardown body factored out of [`Self::stop`] so the drain
|
||||
/// path can reuse the exact same per-child teardown and channel-drop
|
||||
/// ordering. Two driver seams are woven in at their current positions:
|
||||
///
|
||||
/// - **Seam (a), the shutdown-Disconnect fan-out** (after any Dns teardown,
|
||||
/// before everything else) is gated on `broadcast_disconnect`. The
|
||||
/// immediate `stop()` path passes `true` and emits it here; the drain path
|
||||
/// passes `false` because it already broadcast once at drain entry and
|
||||
/// must not re-broadcast.
|
||||
/// - **Seam (b), dropping the packet channels** (after all transports, before
|
||||
/// TUN) always runs.
|
||||
async fn execute_teardown(&mut self, actions: Vec<Action>, broadcast_disconnect: bool) {
|
||||
// `broadcast_disconnect = false` (drain path) marks the fan-out already
|
||||
// done so neither the in-loop seam nor the trailing seam fires.
|
||||
let mut disconnect_done = !broadcast_disconnect;
|
||||
let mut packet_taken = false;
|
||||
for action in actions {
|
||||
let Action::StopChild(child) = action else {
|
||||
@@ -1723,10 +1730,112 @@ impl Node {
|
||||
self.supervisor.packet_tx.take();
|
||||
self.packet_rx.take();
|
||||
}
|
||||
}
|
||||
|
||||
self.supervisor.state = NodeState::Stopped;
|
||||
info!(state = %self.supervisor.state, "Node stopped");
|
||||
Ok(())
|
||||
/// Reconstruct the supervised up-set from observed runtime presence, so the
|
||||
/// FSM authors the teardown order regardless of how the node reached
|
||||
/// `Running`. Worker pools are deliberately excluded: today's teardown never
|
||||
/// stops them. Shared by [`Self::stop`] and [`Self::enter_drain`].
|
||||
fn reconstruct_supervised_up(&self) -> Vec<Child> {
|
||||
let mut up: Vec<Child> = Vec::new();
|
||||
if self.supervisor.dns_task.is_some() {
|
||||
up.push(Child::Dns);
|
||||
}
|
||||
if self.supervisor.nostr_rendezvous.engine().is_some() {
|
||||
up.push(Child::Nostr);
|
||||
}
|
||||
if self.supervisor.lan_rendezvous.is_some() {
|
||||
up.push(Child::Mdns);
|
||||
}
|
||||
for id in self.transports.keys() {
|
||||
up.push(Child::Transport(*id));
|
||||
}
|
||||
if self.tun_name.is_some() {
|
||||
up.push(Child::Tun);
|
||||
}
|
||||
up
|
||||
}
|
||||
|
||||
/// Enter the bounded graceful drain **in place**, called once by
|
||||
/// [`Self::run_rx_loop_with_shutdown`] when the shutdown signal fires.
|
||||
///
|
||||
/// Seeds the FSM at `Running` from observed presence (same pattern as
|
||||
/// [`Self::stop`]), steps it into `Draining`, and executes the entry
|
||||
/// actions: broadcast a single shutdown `Disconnect`, and no-op the §8
|
||||
/// reconciler-gate actions (the reconciler that consumes them lands in
|
||||
/// Step 1b; the `SetTimer` is likewise a no-op — the bounded wait is the rx
|
||||
/// loop's deadline arm). Teardown is deferred to [`Self::finish_shutdown`].
|
||||
///
|
||||
/// Called from an rx-loop `select!` arm body: the channel receivers are
|
||||
/// already moved into the loop's locals, so borrowing `self` here is sound.
|
||||
pub(in crate::node) async fn enter_drain(&mut self) {
|
||||
let up = self.reconstruct_supervised_up();
|
||||
self.supervisor.fsm = SupervisorFsm::running_with(up);
|
||||
|
||||
let drain_timeout = self.config().node.drain_timeout();
|
||||
// Absolute driver-clock ms, carried into `Draining`/`SetTimer` for
|
||||
// observability; the real bounded wait is the rx loop's deadline arm.
|
||||
let deadline_ms = Self::now_ms().saturating_add(drain_timeout.as_millis() as u64);
|
||||
|
||||
let actions = self.supervisor.fsm.step(Event::Drain { deadline_ms });
|
||||
|
||||
// Publish the operator-visible `Draining` state (a direct write, like
|
||||
// the other `self.state` transitions this milestone uses). The
|
||||
// FSM-owned `PublishState` *action* is not needed for this single
|
||||
// transition; it arrives with the Full/Degraded health split (c), which
|
||||
// a direct write cannot express.
|
||||
self.supervisor.state = NodeState::Draining;
|
||||
info!(state = %self.supervisor.state, "Node draining");
|
||||
|
||||
for action in actions {
|
||||
match action {
|
||||
Action::BroadcastDisconnect => {
|
||||
self.send_disconnect_to_all_peers(DisconnectReason::Shutdown)
|
||||
.await;
|
||||
}
|
||||
Action::SetTimer(_, _) => {
|
||||
// The rx loop owns the bounded wait; the FSM's timer is
|
||||
// carried for observability only. No-op here.
|
||||
}
|
||||
Action::SetPeeringDesired(_) | Action::SuspendReplenish => {
|
||||
// §8 reconciler drain-gate. Documented no-op in this commit:
|
||||
// the homeostatic reconciler that consumes these lands in
|
||||
// Step 1b. Without it there is nothing to reconnect the peers
|
||||
// the drain closes, so the gate is implicitly satisfied.
|
||||
}
|
||||
Action::SpawnChild(_) | Action::StopChild(_) => {
|
||||
// Drain entry never emits child actions; ignore defensively.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
drain_timeout_secs = drain_timeout.as_secs(),
|
||||
peers = self.peers.len(),
|
||||
"Draining: broadcast shutdown Disconnect, waiting for peers to clear"
|
||||
);
|
||||
}
|
||||
|
||||
/// Finish shutdown after [`Self::run_rx_loop_with_shutdown`] returns.
|
||||
///
|
||||
/// Branches on the supervisor's state:
|
||||
/// - if the loop drained (FSM in `Draining`), close the window
|
||||
/// (`DrainDeadlineElapsed`) and tear down **without re-broadcasting** —
|
||||
/// the fan-out already went out at drain entry;
|
||||
/// - otherwise the loop exited some other way (the packet channel closed
|
||||
/// while still `Running` — the degenerate/error path), so fall back to the
|
||||
/// immediate [`Self::stop`] (which broadcasts and tears down).
|
||||
pub async fn finish_shutdown(&mut self) {
|
||||
if self.supervisor.fsm.is_draining() {
|
||||
self.supervisor.state = NodeState::Stopping;
|
||||
info!(state = %self.supervisor.state, "Node stopping (drain complete)");
|
||||
let stop_actions = self.supervisor.fsm.step(Event::DrainDeadlineElapsed);
|
||||
self.execute_teardown(stop_actions, false).await;
|
||||
self.supervisor.state = NodeState::Stopped;
|
||||
info!(state = %self.supervisor.state, "Node stopped");
|
||||
} else if let Err(e) = self.stop().await {
|
||||
warn!(error = %e, "Error during shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
/// Send disconnect notifications to all active peers.
|
||||
|
||||
@@ -22,15 +22,30 @@
|
||||
//! - teardown runs in today's order and, faithfully, does **not** stop the
|
||||
//! encrypt/decrypt worker pools (they are spawned in `start()` but never torn
|
||||
//! down in `stop()`);
|
||||
//! - the machine authors only the `SpawnChild`/`StopChild` *ordering*. The
|
||||
//! `self.state` field writes stay verbatim in the driver at their current
|
||||
//! positions, so the published `NodeState` transitions are byte-for-byte
|
||||
//! unchanged. FSM-owned published state (`PublishState`) is introduced with
|
||||
//! the `Draining`/`Degraded` additions, which need it.
|
||||
//! - the machine authors only the `SpawnChild`/`StopChild` *ordering*, and the
|
||||
//! driver keeps its `self.state` writes at their current positions. The
|
||||
//! behavior-neutral relocation left the published `NodeState` transitions
|
||||
//! byte-for-byte unchanged; the bounded-drain phase below adds exactly one new
|
||||
//! published transition (`Draining`), written directly like the others.
|
||||
//!
|
||||
//! The `Draining` phase and the `Running{Full|Degraded}` health split (design
|
||||
//! doc §6/§9.1) land as the two subsequent, separately-flagged commits and
|
||||
//! extend the [`SupState`], [`Event`], and [`Action`] enums below.
|
||||
//! ## Scope: the bounded `Draining` phase (this commit)
|
||||
//!
|
||||
//! This commit adds the operator-visible bounded-drain additions and nothing
|
||||
//! else: the [`SupState::Draining`] state, the [`Event::Drain`] /
|
||||
//! [`Event::DrainDeadlineElapsed`] events, the drain [`Action`]s
|
||||
//! ([`Action::BroadcastDisconnect`], [`Action::SetTimer`],
|
||||
//! [`Action::SetPeeringDesired`], [`Action::SuspendReplenish`]), and the new
|
||||
//! published [`NodeState::Draining`](crate::node::NodeState::Draining) —
|
||||
//! written directly by the driver at drain entry, exactly like the other
|
||||
//! `self.state` transitions this milestone uses. The existing immediate `Stop`
|
||||
//! path is untouched. `Draining` and `Stop` share a single teardown-plan author
|
||||
//! (`begin_stopping`), so the teardown ordering is defined once.
|
||||
//!
|
||||
//! What is deferred is only the FSM-owned `PublishState` *action* (published
|
||||
//! state authored by the machine rather than the driver): it lands with the
|
||||
//! `Running{Full|Degraded}` health split (design doc §6/§9.1), which needs it
|
||||
//! because a single direct `self.state` write cannot express the health fork.
|
||||
//! The `Draining` published state itself is **not** deferred — it is here.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
@@ -68,9 +83,9 @@ pub(crate) enum Child {
|
||||
/// An input to the supervisor. Results of executing [`Action`]s are fed back as
|
||||
/// `SubstrateUp` / `SubstrateFailed` / `ChildStopped`.
|
||||
///
|
||||
/// Only the events the behavior-neutral rewrite needs are present; `Tick`,
|
||||
/// `ChildExited`, and `DrainDeadlineElapsed` (design doc §6) arrive with the
|
||||
/// `Draining` / `Degraded` commits.
|
||||
/// `Tick` and `ChildExited` (design doc §6) arrive with the `Degraded`/health
|
||||
/// commit; the bounded-drain events (`Drain` / `DrainDeadlineElapsed`) are
|
||||
/// present here.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Event {
|
||||
/// Begin bring-up. `transports` are the ids the driver has already created
|
||||
@@ -106,8 +121,23 @@ pub(crate) enum Event {
|
||||
/// The child that failed to start.
|
||||
child: Child,
|
||||
},
|
||||
/// Begin teardown. Valid from `Running`.
|
||||
/// Begin an immediate teardown (no drain). Valid from `Running`. This is
|
||||
/// the path `node.stop()` uses; unchanged from the behavior-neutral rewrite.
|
||||
Stop,
|
||||
/// Begin a bounded graceful drain. Valid from `Running`. Emits the drain
|
||||
/// entry actions (broadcast Disconnect, arm the deadline timer, gate the
|
||||
/// reconciler off) and moves to `Draining`; the driver then runs the
|
||||
/// bounded drain window before feeding `DrainDeadlineElapsed`.
|
||||
Drain {
|
||||
/// Absolute drain deadline in driver-clock milliseconds, carried into
|
||||
/// `Draining` and the `SetTimer` action for observability. The driver
|
||||
/// owns the actual bounded wait.
|
||||
deadline_ms: u64,
|
||||
},
|
||||
/// The bounded drain window has closed — either the deadline elapsed or all
|
||||
/// peers drained early. Valid from `Draining`; begins the (shared) teardown
|
||||
/// plan, transitioning to `Stopping`.
|
||||
DrainDeadlineElapsed,
|
||||
/// A child the driver was asked to stop has finished stopping.
|
||||
ChildStopped {
|
||||
/// The child that has been torn down.
|
||||
@@ -115,11 +145,33 @@ pub(crate) enum Event {
|
||||
},
|
||||
}
|
||||
|
||||
/// A driver-scheduled timer the supervisor can arm (design doc §6). Only the
|
||||
/// drain deadline exists for now; the handshake/rekey/liveness timers named in
|
||||
/// §8 arrive with later cores.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Timer {
|
||||
/// Fires when the bounded drain window closes. The driver feeds
|
||||
/// [`Event::DrainDeadlineElapsed`] when it elapses (or earlier, when all
|
||||
/// peers have drained).
|
||||
DrainDeadline,
|
||||
}
|
||||
|
||||
/// The reconciler's desired peering set (design doc §8 drain gate). Only
|
||||
/// `Empty` is needed in this commit; the populated variants that the Step-1b
|
||||
/// homeostatic reconciler converges toward land with that core.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum PeeringDesired {
|
||||
/// No peers desired. Entering `Draining` sets this so the reconciler stops
|
||||
/// reconnecting the peers the drain just closed (§8: "Draining switches the
|
||||
/// homeostat off").
|
||||
Empty,
|
||||
}
|
||||
|
||||
/// An effect the driver must perform. The core never performs I/O itself.
|
||||
///
|
||||
/// `PublishState` (design doc §6) is intentionally absent from the
|
||||
/// behavior-neutral rewrite: the driver keeps its verbatim `self.state` writes,
|
||||
/// so no published-state action is needed until `Draining`/`Degraded`.
|
||||
/// `PublishState` (design doc §6) is intentionally absent until the
|
||||
/// `Running{Full|Degraded}` health commit: the driver keeps its verbatim
|
||||
/// `self.state` writes, so no published-state action is needed yet.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Action {
|
||||
/// Bring up this child (the driver performs the spawn / start I/O and
|
||||
@@ -128,6 +180,21 @@ pub(crate) enum Action {
|
||||
/// Tear down this child (the driver performs the stop / join I/O and
|
||||
/// reports `ChildStopped`).
|
||||
StopChild(Child),
|
||||
/// Broadcast a shutdown `Disconnect` to all sendable peers. Emitted once,
|
||||
/// at drain entry; the drain teardown does not re-broadcast.
|
||||
BroadcastDisconnect,
|
||||
/// Arm a driver timer at the given absolute driver-clock milliseconds. In
|
||||
/// this commit only `DrainDeadline` exists; the driver notes the deadline
|
||||
/// and owns the bounded drain wait, so this is a documented no-op beyond
|
||||
/// bookkeeping.
|
||||
SetTimer(Timer, u64),
|
||||
/// Set the reconciler's desired peering set (§8 drain gate). Documented
|
||||
/// **no-op in this commit** — the reconciler that consumes it lands in
|
||||
/// Step 1b; the driver logs/ignores it for now.
|
||||
SetPeeringDesired(PeeringDesired),
|
||||
/// Suspend peer replenishment (§8 drain gate). Documented **no-op in this
|
||||
/// commit** for the same reason as `SetPeeringDesired`.
|
||||
SuspendReplenish,
|
||||
}
|
||||
|
||||
/// Internal supervisor state (design doc §6). Richer than the published
|
||||
@@ -145,6 +212,16 @@ pub(crate) enum SupState {
|
||||
},
|
||||
/// All children resolved; node operational.
|
||||
Running,
|
||||
/// Bounded graceful-drain window (design doc §6/§8). Broadcast Disconnect
|
||||
/// has gone out and the reconciler is gated off (desired peering set
|
||||
/// emptied, replenishment suspended); teardown begins when
|
||||
/// `DrainDeadlineElapsed` arrives. Logically sits between `Running` and
|
||||
/// `Stopping`.
|
||||
Draining {
|
||||
/// Absolute drain deadline in driver-clock milliseconds (carried for
|
||||
/// observability; the driver owns the actual wait).
|
||||
deadline_ms: u64,
|
||||
},
|
||||
/// Tearing children down; `pending` is the set not yet stopped.
|
||||
Stopping {
|
||||
/// Children asked to stop that have not yet reported stopped.
|
||||
@@ -196,6 +273,13 @@ impl SupervisorFsm {
|
||||
&self.state
|
||||
}
|
||||
|
||||
/// Whether the machine is in the bounded-drain window. The driver uses this
|
||||
/// after the rx loop returns to decide between the drain-teardown path and
|
||||
/// the immediate-`stop()` fallback.
|
||||
pub(crate) fn is_draining(&self) -> bool {
|
||||
matches!(self.state, SupState::Draining { .. })
|
||||
}
|
||||
|
||||
/// Advance the machine by one event, returning the effects to perform.
|
||||
pub(crate) fn step(&mut self, event: Event) -> Vec<Action> {
|
||||
match event {
|
||||
@@ -219,6 +303,8 @@ impl SupervisorFsm {
|
||||
Event::SubstrateUp { child } => self.on_substrate_up(child),
|
||||
Event::SubstrateFailed { child } => self.on_substrate_failed(child),
|
||||
Event::Stop => self.on_stop(),
|
||||
Event::Drain { deadline_ms } => self.on_drain(deadline_ms),
|
||||
Event::DrainDeadlineElapsed => self.on_drain_deadline_elapsed(),
|
||||
Event::ChildStopped { child } => self.on_child_stopped(child),
|
||||
}
|
||||
}
|
||||
@@ -306,6 +392,43 @@ impl SupervisorFsm {
|
||||
if !matches!(self.state, SupState::Running) {
|
||||
return Vec::new();
|
||||
}
|
||||
self.begin_stopping()
|
||||
}
|
||||
|
||||
fn on_drain(&mut self, deadline_ms: u64) -> Vec<Action> {
|
||||
// Only a graceful drain from a running node. Inert otherwise (matching
|
||||
// `Stop`'s guard).
|
||||
if !matches!(self.state, SupState::Running) {
|
||||
return Vec::new();
|
||||
}
|
||||
self.state = SupState::Draining { deadline_ms };
|
||||
// Drain entry, in order: broadcast the shutdown Disconnect, arm the
|
||||
// deadline timer, then gate the reconciler off (desired = ∅, suspend
|
||||
// replenishment) so it cannot reconnect the peers the drain just closed
|
||||
// (§8). The up-set is left intact for the eventual teardown plan.
|
||||
vec![
|
||||
Action::BroadcastDisconnect,
|
||||
Action::SetTimer(Timer::DrainDeadline, deadline_ms),
|
||||
Action::SetPeeringDesired(PeeringDesired::Empty),
|
||||
Action::SuspendReplenish,
|
||||
]
|
||||
}
|
||||
|
||||
fn on_drain_deadline_elapsed(&mut self) -> Vec<Action> {
|
||||
// The bounded drain window closed (deadline or all-peers-gone). Author
|
||||
// the same teardown plan `Stop` produces.
|
||||
if !matches!(self.state, SupState::Draining { .. }) {
|
||||
return Vec::new();
|
||||
}
|
||||
self.begin_stopping()
|
||||
}
|
||||
|
||||
/// Author the teardown plan over the current up-set, transition to
|
||||
/// `Stopping`, and return the ordered `StopChild` actions. Shared by the
|
||||
/// immediate `Stop` path ([`Self::on_stop`]) and the drain-window-close path
|
||||
/// ([`Self::on_drain_deadline_elapsed`]) so the teardown ordering is defined
|
||||
/// exactly once.
|
||||
fn begin_stopping(&mut self) -> Vec<Action> {
|
||||
let order = self.teardown_order();
|
||||
self.state = SupState::Stopping {
|
||||
pending: order.iter().copied().collect(),
|
||||
@@ -655,4 +778,101 @@ mod tests {
|
||||
);
|
||||
assert_eq!(s.state(), &SupState::Running);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_from_running_emits_entry_actions_and_enters_draining() {
|
||||
let mut s = SupervisorFsm::running_with([
|
||||
Child::Dns,
|
||||
Child::Nostr,
|
||||
Child::Transport(tid(1)),
|
||||
Child::Tun,
|
||||
]);
|
||||
let actions = s.step(Event::Drain { deadline_ms: 5_000 });
|
||||
// Order matters: broadcast → arm timer → gate reconciler off.
|
||||
assert_eq!(
|
||||
actions,
|
||||
vec![
|
||||
Action::BroadcastDisconnect,
|
||||
Action::SetTimer(Timer::DrainDeadline, 5_000),
|
||||
Action::SetPeeringDesired(PeeringDesired::Empty),
|
||||
Action::SuspendReplenish,
|
||||
]
|
||||
);
|
||||
assert_eq!(s.state(), &SupState::Draining { deadline_ms: 5_000 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_deadline_elapsed_yields_teardown_and_enters_stopping() {
|
||||
let mut s = SupervisorFsm::running_with([
|
||||
Child::Dns,
|
||||
Child::Nostr,
|
||||
Child::Mdns,
|
||||
Child::Transport(tid(2)),
|
||||
Child::Transport(tid(1)),
|
||||
Child::Tun,
|
||||
]);
|
||||
s.step(Event::Drain { deadline_ms: 2_000 });
|
||||
let stops = s.step(Event::DrainDeadlineElapsed);
|
||||
// Same ordering the immediate `Stop` path authors: dns → nostr → mdns →
|
||||
// transports (ascending id) → tun.
|
||||
assert_eq!(
|
||||
stops,
|
||||
vec![
|
||||
Action::StopChild(Child::Dns),
|
||||
Action::StopChild(Child::Nostr),
|
||||
Action::StopChild(Child::Mdns),
|
||||
Action::StopChild(Child::Transport(tid(1))),
|
||||
Action::StopChild(Child::Transport(tid(2))),
|
||||
Action::StopChild(Child::Tun),
|
||||
]
|
||||
);
|
||||
assert!(matches!(s.state(), SupState::Stopping { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_teardown_matches_immediate_stop_teardown() {
|
||||
// The drain path and the immediate-stop path must produce the identical
|
||||
// StopChild plan over the same up-set (single teardown author).
|
||||
let up = [
|
||||
Child::Dns,
|
||||
Child::Nostr,
|
||||
Child::Mdns,
|
||||
Child::Transport(tid(1)),
|
||||
Child::Transport(tid(3)),
|
||||
Child::Tun,
|
||||
];
|
||||
let mut immediate = SupervisorFsm::running_with(up);
|
||||
let stop_plan = immediate.step(Event::Stop);
|
||||
|
||||
let mut drained = SupervisorFsm::running_with(up);
|
||||
drained.step(Event::Drain { deadline_ms: 1_000 });
|
||||
let drain_plan = drained.step(Event::DrainDeadlineElapsed);
|
||||
|
||||
assert_eq!(stop_plan, drain_plan);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_is_inert_from_non_running() {
|
||||
// From `Created`.
|
||||
let mut s = SupervisorFsm::new();
|
||||
assert_eq!(s.step(Event::Drain { deadline_ms: 1_000 }), vec![]);
|
||||
assert_eq!(s.state(), &SupState::Created);
|
||||
|
||||
// From `Stopping` (seed a drain, close its window, then try to drain
|
||||
// again — inert).
|
||||
let mut s2 = SupervisorFsm::running_with([Child::Transport(tid(1))]);
|
||||
s2.step(Event::Drain { deadline_ms: 1_000 });
|
||||
s2.step(Event::DrainDeadlineElapsed);
|
||||
assert!(matches!(s2.state(), SupState::Stopping { .. }));
|
||||
assert_eq!(s2.step(Event::Drain { deadline_ms: 1_000 }), vec![]);
|
||||
assert!(matches!(s2.state(), SupState::Stopping { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_deadline_elapsed_is_inert_outside_draining() {
|
||||
// Inert from `Running` (no drain in progress).
|
||||
let mut s = SupervisorFsm::running_with([Child::Transport(tid(1))]);
|
||||
assert_eq!(s.step(Event::DrainDeadlineElapsed), vec![]);
|
||||
assert_eq!(s.state(), &SupState::Running);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +168,12 @@ pub enum NodeState {
|
||||
Starting,
|
||||
/// Fully operational.
|
||||
Running,
|
||||
/// Bounded graceful drain in progress (design doc §6): a shutdown
|
||||
/// `Disconnect` has been broadcast and the node is waiting for peers to
|
||||
/// clear (bounded by `node.drain_timeout_secs`) before teardown. Not
|
||||
/// operational; the daemon drain path advances to `Stopping` via the
|
||||
/// supervisor's `DrainDeadlineElapsed`, never through `stop()`.
|
||||
Draining,
|
||||
/// Shutting down.
|
||||
Stopping,
|
||||
/// Stopped.
|
||||
@@ -197,6 +203,7 @@ impl fmt::Display for NodeState {
|
||||
NodeState::Created => "created",
|
||||
NodeState::Starting => "starting",
|
||||
NodeState::Running => "running",
|
||||
NodeState::Draining => "draining",
|
||||
NodeState::Stopping => "stopping",
|
||||
NodeState::Stopped => "stopped",
|
||||
};
|
||||
|
||||
@@ -165,6 +165,41 @@ async fn test_node_state_transitions() {
|
||||
assert_eq!(node.state(), NodeState::Stopped);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_drain_publishes_draining_state() {
|
||||
let mut node = make_node();
|
||||
node.start().await.unwrap();
|
||||
assert_eq!(node.state(), NodeState::Running);
|
||||
assert!(node.state().is_operational());
|
||||
|
||||
// Enter the bounded drain in place: publishes the operator-visible
|
||||
// `Draining` state (not operational) without tearing down.
|
||||
node.enter_drain().await;
|
||||
assert_eq!(node.state(), NodeState::Draining);
|
||||
assert!(!node.state().is_operational());
|
||||
// `Draining` is neither startable nor externally stoppable; the daemon
|
||||
// drain finishes via the supervisor's `DrainDeadlineElapsed`, not `stop()`.
|
||||
assert!(!node.state().can_start());
|
||||
assert!(!node.state().can_stop());
|
||||
|
||||
// Finishing shutdown from `Draining` tears down to `Stopped`.
|
||||
node.finish_shutdown().await;
|
||||
assert_eq!(node.state(), NodeState::Stopped);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_immediate_stop_never_publishes_draining() {
|
||||
let mut node = make_node();
|
||||
node.start().await.unwrap();
|
||||
assert_eq!(node.state(), NodeState::Running);
|
||||
|
||||
// The immediate stop() path (used by tests and the stop-now path)
|
||||
// transitions Running → Stopping → Stopped and never enters `Draining`.
|
||||
node.stop().await.unwrap();
|
||||
assert_eq!(node.state(), NodeState::Stopped);
|
||||
assert_ne!(node.state(), NodeState::Draining);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_node_start_does_not_wait_for_nostr_relay_startup() {
|
||||
let mut config = Config::new();
|
||||
|
||||
Reference in New Issue
Block a user