node: extract immutable state into a shared context and atomic metric registry

Store node counters in an atomic metric registry read through &self, and
introduce a shared NodeContext bundle holding the effectively-immutable
fields (config, identity, startup epoch, capability limits). Source the
immutable config and identity reads across the receive hot path, the
handshake/session/mmp/encrypted state machines, and the discovery, tree,
bloom, retry, and lifecycle modules through the context accessors rather
than direct field reads. The Node fields and the context are rebuilt in
lockstep at every mutation site.
This commit is contained in:
Johnathan Corgan
2026-06-02 13:05:03 +00:00
parent 2d0e8de8c8
commit 08b8b3908e
28 changed files with 1140 additions and 957 deletions
+24 -578
View File
@@ -1,4 +1,7 @@
//! Node-level statistics for routing, forwarding, and discovery operations.
//! Node-level statistics for the session, handshake, mmp, and transport
//! families. The forwarding, discovery, tree, bloom, congestion, and error
//! families have migrated to the atomic
//! [`MetricsRegistry`](crate::node::metrics::MetricsRegistry).
//!
//! Unlike `EthernetStats` (which uses `AtomicU64` + `Arc` for cross-task
//! sharing), these counters use plain `u64` because `Node` handlers run
@@ -8,297 +11,9 @@
use serde::Serialize;
use crate::node::reject::{
BloomReject, DiscoveryReject, ForwardingReject, HandshakeReject, MmpReject, RejectReason,
SessionReject, TransportReject, TreeReject,
HandshakeReject, MmpReject, RejectReason, SessionReject, TransportReject,
};
/// Forwarding statistics — packets and bytes for each outcome.
#[derive(Default)]
pub struct ForwardingStats {
pub received_packets: u64,
pub received_bytes: u64,
pub decode_error_packets: u64,
pub decode_error_bytes: u64,
pub ttl_exhausted_packets: u64,
pub ttl_exhausted_bytes: u64,
pub delivered_packets: u64,
pub delivered_bytes: u64,
pub forwarded_packets: u64,
pub forwarded_bytes: u64,
pub drop_no_route_packets: u64,
pub drop_no_route_bytes: u64,
pub drop_mtu_exceeded_packets: u64,
pub drop_mtu_exceeded_bytes: u64,
pub drop_send_error_packets: u64,
pub drop_send_error_bytes: u64,
pub originated_packets: u64,
pub originated_bytes: u64,
}
impl ForwardingStats {
pub fn record_received(&mut self, bytes: usize) {
self.received_packets += 1;
self.received_bytes += bytes as u64;
}
pub fn record_decode_error(&mut self, bytes: usize) {
self.decode_error_packets += 1;
self.decode_error_bytes += bytes as u64;
}
pub fn record_ttl_exhausted(&mut self, bytes: usize) {
self.ttl_exhausted_packets += 1;
self.ttl_exhausted_bytes += bytes as u64;
}
pub fn record_delivered(&mut self, bytes: usize) {
self.delivered_packets += 1;
self.delivered_bytes += bytes as u64;
}
pub fn record_forwarded(&mut self, bytes: usize) {
self.forwarded_packets += 1;
self.forwarded_bytes += bytes as u64;
}
pub fn record_drop_no_route(&mut self, bytes: usize) {
self.drop_no_route_packets += 1;
self.drop_no_route_bytes += bytes as u64;
}
pub fn record_drop_mtu_exceeded(&mut self, bytes: usize) {
self.drop_mtu_exceeded_packets += 1;
self.drop_mtu_exceeded_bytes += bytes as u64;
}
pub fn record_drop_send_error(&mut self, bytes: usize) {
self.drop_send_error_packets += 1;
self.drop_send_error_bytes += bytes as u64;
}
pub fn record_originated(&mut self, bytes: usize) {
self.originated_packets += 1;
self.originated_bytes += bytes as u64;
}
/// Dispatch a typed forwarding rejection to its packet counter.
///
/// The byte-counted side of each outcome is recorded by the
/// existing `record_*` methods at the call site (which know the
/// payload size); `record_reject` only bumps the packet count and
/// is paired with the byte-aware call at the call site while the
/// typed-rejection rollout is in progress. A later change may
/// collapse the two calls into a single typed entry point.
pub(super) fn record_reject(&mut self, reason: ForwardingReject) {
match reason {
ForwardingReject::DecodeError => self.decode_error_packets += 1,
ForwardingReject::TtlExhausted => self.ttl_exhausted_packets += 1,
ForwardingReject::NoRoute => self.drop_no_route_packets += 1,
ForwardingReject::MtuExceeded => self.drop_mtu_exceeded_packets += 1,
ForwardingReject::SendError => self.drop_send_error_packets += 1,
}
}
pub fn snapshot(&self) -> ForwardingStatsSnapshot {
ForwardingStatsSnapshot {
received_packets: self.received_packets,
received_bytes: self.received_bytes,
decode_error_packets: self.decode_error_packets,
decode_error_bytes: self.decode_error_bytes,
ttl_exhausted_packets: self.ttl_exhausted_packets,
ttl_exhausted_bytes: self.ttl_exhausted_bytes,
delivered_packets: self.delivered_packets,
delivered_bytes: self.delivered_bytes,
forwarded_packets: self.forwarded_packets,
forwarded_bytes: self.forwarded_bytes,
drop_no_route_packets: self.drop_no_route_packets,
drop_no_route_bytes: self.drop_no_route_bytes,
drop_mtu_exceeded_packets: self.drop_mtu_exceeded_packets,
drop_mtu_exceeded_bytes: self.drop_mtu_exceeded_bytes,
drop_send_error_packets: self.drop_send_error_packets,
drop_send_error_bytes: self.drop_send_error_bytes,
originated_packets: self.originated_packets,
originated_bytes: self.originated_bytes,
}
}
}
/// Discovery statistics — packet counts for request and response handling.
#[derive(Default)]
pub struct DiscoveryStats {
// Request counters
pub req_received: u64,
pub req_decode_error: u64,
pub req_duplicate: u64,
pub req_dedup_cache_full: u64,
pub req_target_is_us: u64,
pub req_forwarded: u64,
pub req_ttl_exhausted: u64,
pub req_initiated: u64,
pub req_deduplicated: u64,
pub req_backoff_suppressed: u64,
pub req_forward_rate_limited: u64,
pub req_bloom_miss: u64,
pub req_no_tree_peer: u64,
pub req_fallback_forwarded: u64,
// Response counters
pub resp_received: u64,
pub resp_decode_error: u64,
pub resp_forwarded: u64,
pub resp_identity_miss: u64,
pub resp_proof_failed: u64,
pub resp_no_route: u64,
pub resp_accepted: u64,
pub resp_timed_out: u64,
}
impl DiscoveryStats {
pub(super) fn record_reject(&mut self, reason: DiscoveryReject) {
match reason {
DiscoveryReject::ReqDecodeError => self.req_decode_error += 1,
DiscoveryReject::ReqDuplicate => self.req_duplicate += 1,
DiscoveryReject::ReqDedupCacheFull => self.req_dedup_cache_full += 1,
DiscoveryReject::ReqTtlExhausted => self.req_ttl_exhausted += 1,
DiscoveryReject::RespDecodeError => self.resp_decode_error += 1,
DiscoveryReject::RespIdentityMiss => self.resp_identity_miss += 1,
DiscoveryReject::RespProofFailed => self.resp_proof_failed += 1,
DiscoveryReject::RespNoRoute => self.resp_no_route += 1,
}
}
pub fn snapshot(&self) -> DiscoveryStatsSnapshot {
DiscoveryStatsSnapshot {
req_received: self.req_received,
req_decode_error: self.req_decode_error,
req_duplicate: self.req_duplicate,
req_dedup_cache_full: self.req_dedup_cache_full,
req_target_is_us: self.req_target_is_us,
req_forwarded: self.req_forwarded,
req_ttl_exhausted: self.req_ttl_exhausted,
req_initiated: self.req_initiated,
req_deduplicated: self.req_deduplicated,
req_backoff_suppressed: self.req_backoff_suppressed,
req_forward_rate_limited: self.req_forward_rate_limited,
req_bloom_miss: self.req_bloom_miss,
req_no_tree_peer: self.req_no_tree_peer,
req_fallback_forwarded: self.req_fallback_forwarded,
resp_received: self.resp_received,
resp_decode_error: self.resp_decode_error,
resp_forwarded: self.resp_forwarded,
resp_identity_miss: self.resp_identity_miss,
resp_proof_failed: self.resp_proof_failed,
resp_no_route: self.resp_no_route,
resp_accepted: self.resp_accepted,
resp_timed_out: self.resp_timed_out,
}
}
}
/// Spanning tree statistics — announce handling and parent tracking.
#[derive(Default)]
pub struct TreeStats {
// Inbound announce handling
pub received: u64,
pub decode_error: u64,
pub unknown_peer: u64,
pub addr_mismatch: u64,
pub sig_failed: u64,
pub stale: u64,
pub ancestry_invalid: u64,
pub accepted: u64,
pub parent_switched: u64,
pub loop_detected: u64,
pub ancestry_changed: u64,
// Outbound announce sending
pub sent: u64,
pub rate_limited: u64,
pub send_failed: u64,
pub outbound_sign_failed: u64,
// Cumulative events
pub parent_switches: u64,
pub parent_losses: u64,
pub flap_dampened: u64,
}
impl TreeStats {
pub fn snapshot(&self) -> TreeStatsSnapshot {
TreeStatsSnapshot {
received: self.received,
decode_error: self.decode_error,
unknown_peer: self.unknown_peer,
addr_mismatch: self.addr_mismatch,
sig_failed: self.sig_failed,
stale: self.stale,
ancestry_invalid: self.ancestry_invalid,
accepted: self.accepted,
parent_switched: self.parent_switched,
loop_detected: self.loop_detected,
ancestry_changed: self.ancestry_changed,
sent: self.sent,
rate_limited: self.rate_limited,
send_failed: self.send_failed,
outbound_sign_failed: self.outbound_sign_failed,
parent_switches: self.parent_switches,
parent_losses: self.parent_losses,
flap_dampened: self.flap_dampened,
}
}
pub(super) fn record_reject(&mut self, reason: TreeReject) {
match reason {
TreeReject::AncestryInvalid => self.ancestry_invalid += 1,
TreeReject::OutboundSignFailed => self.outbound_sign_failed += 1,
}
}
}
/// Bloom filter statistics — filter announce handling.
#[derive(Default)]
pub struct BloomStats {
// Inbound announce handling
pub received: u64,
pub decode_error: u64,
pub invalid: u64,
pub non_v1: u64,
pub unknown_peer: u64,
pub stale: u64,
pub fill_exceeded: u64,
pub accepted: u64,
// Outbound announce sending
pub sent: u64,
pub debounce_suppressed: u64,
pub send_failed: u64,
}
impl BloomStats {
pub(super) fn record_reject(&mut self, reason: BloomReject) {
match reason {
BloomReject::DecodeError => self.decode_error += 1,
BloomReject::Invalid => self.invalid += 1,
BloomReject::NonV1 => self.non_v1 += 1,
BloomReject::UnknownPeer => self.unknown_peer += 1,
BloomReject::Stale => self.stale += 1,
BloomReject::FillExceeded => self.fill_exceeded += 1,
}
}
pub fn snapshot(&self) -> BloomStatsSnapshot {
BloomStatsSnapshot {
received: self.received,
decode_error: self.decode_error,
invalid: self.invalid,
non_v1: self.non_v1,
unknown_peer: self.unknown_peer,
stale: self.stale,
fill_exceeded: self.fill_exceeded,
accepted: self.accepted,
sent: self.sent,
debounce_suppressed: self.debounce_suppressed,
send_failed: self.send_failed,
}
}
}
/// FSP session statistics — receive-path silent-rejection counters.
///
/// Covers the unknown-session and state-machine-mismatch rejection
@@ -446,77 +161,19 @@ impl TransportStats {
}
}
/// Error signal statistics — counts of each error signal type received.
#[derive(Default)]
pub struct ErrorSignalStats {
pub coords_required: u64,
pub path_broken: u64,
pub mtu_exceeded: u64,
}
impl ErrorSignalStats {
pub fn snapshot(&self) -> ErrorSignalStatsSnapshot {
ErrorSignalStatsSnapshot {
coords_required: self.coords_required,
path_broken: self.path_broken,
mtu_exceeded: self.mtu_exceeded,
}
}
}
/// Congestion event statistics — ECN CE tracking and detection triggers.
#[derive(Default)]
pub struct CongestionStats {
/// Packets forwarded with the CE flag set (incoming CE or locally detected).
pub ce_forwarded: u64,
/// CE-flagged packets received at this node as final destination.
pub ce_received: u64,
/// Number of times detect_congestion() returned true.
pub congestion_detected: u64,
/// Rising-edge transport kernel drop events (not-dropping → dropping).
pub kernel_drop_events: u64,
}
impl CongestionStats {
pub fn record_ce_forwarded(&mut self) {
self.ce_forwarded += 1;
}
pub fn record_ce_received(&mut self) {
self.ce_received += 1;
}
pub fn record_congestion_detected(&mut self) {
self.congestion_detected += 1;
}
pub fn record_kernel_drop_event(&mut self) {
self.kernel_drop_events += 1;
}
pub fn snapshot(&self) -> CongestionStatsSnapshot {
CongestionStatsSnapshot {
ce_forwarded: self.ce_forwarded,
ce_received: self.ce_received,
congestion_detected: self.congestion_detected,
kernel_drop_events: self.kernel_drop_events,
}
}
}
/// Aggregate node statistics.
///
/// Holds only the families that have not migrated to the atomic
/// [`MetricsRegistry`](crate::node::metrics::MetricsRegistry): session,
/// handshake, mmp, and transport. The forwarding, discovery, tree,
/// bloom, congestion, and error families are served exclusively from
/// the registry.
#[derive(Default)]
pub struct NodeStats {
pub forwarding: ForwardingStats,
pub discovery: DiscoveryStats,
pub tree: TreeStats,
pub bloom: BloomStats,
pub session: SessionStats,
pub handshake: HandshakeStats,
pub mmp: MmpStats,
pub transport: TransportStats,
pub errors: ErrorSignalStats,
pub congestion: CongestionStats,
}
impl NodeStats {
@@ -524,37 +181,28 @@ impl NodeStats {
Self::default()
}
pub fn snapshot(&self) -> NodeStatsSnapshot {
NodeStatsSnapshot {
forwarding: self.forwarding.snapshot(),
discovery: self.discovery.snapshot(),
tree: self.tree.snapshot(),
bloom: self.bloom.snapshot(),
session: self.session.snapshot(),
handshake: self.handshake.snapshot(),
mmp: self.mmp.snapshot(),
transport: self.transport.snapshot(),
errors: self.errors.snapshot(),
congestion: self.congestion.snapshot(),
}
}
/// Record a typed rejection from a silent-rejection site.
///
/// Dispatches to the appropriate sub-stats `record_reject` based on
/// the [`RejectReason`] top-level variant. Sub-enums that have not
/// yet had any variants populated still use `match r {}` to keep
/// the dispatch arm exhaustive without dead-code complaints.
/// the [`RejectReason`] top-level variant. Only the families still
/// stored on `NodeStats` (session, handshake, mmp, transport) are
/// routed here; the migrated families are recorded directly on the
/// [`MetricsRegistry`](crate::node::metrics::MetricsRegistry).
pub fn record_reject(&mut self, reason: RejectReason) {
match reason {
RejectReason::Tree(r) => self.tree.record_reject(r),
RejectReason::Bloom(r) => self.bloom.record_reject(r),
RejectReason::Discovery(r) => self.discovery.record_reject(r),
RejectReason::Session(r) => self.session.record_reject(r),
RejectReason::Handshake(r) => self.handshake.record_reject(r),
RejectReason::Forwarding(r) => self.forwarding.record_reject(r),
RejectReason::Transport(r) => self.transport.record_reject(r),
RejectReason::Mmp(r) => self.mmp.record_reject(r),
// The forwarding, discovery, tree, and bloom families are
// recorded directly on the MetricsRegistry and never reach
// this NodeStats dispatch.
RejectReason::Forwarding(_)
| RejectReason::Discovery(_)
| RejectReason::Tree(_)
| RejectReason::Bloom(_) => {
debug_assert!(false, "migrated reject family must use MetricsRegistry");
}
}
}
}
@@ -684,50 +332,10 @@ pub struct CongestionStatsSnapshot {
pub kernel_drop_events: u64,
}
#[derive(Clone, Debug, Default, Serialize)]
pub struct NodeStatsSnapshot {
pub forwarding: ForwardingStatsSnapshot,
pub discovery: DiscoveryStatsSnapshot,
pub tree: TreeStatsSnapshot,
pub bloom: BloomStatsSnapshot,
pub session: SessionStatsSnapshot,
pub handshake: HandshakeStatsSnapshot,
pub mmp: MmpStatsSnapshot,
pub transport: TransportStatsSnapshot,
pub errors: ErrorSignalStatsSnapshot,
pub congestion: CongestionStatsSnapshot,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tree_stats_record_reject_ancestry_invalid() {
let mut stats = TreeStats::default();
stats.record_reject(TreeReject::AncestryInvalid);
stats.record_reject(TreeReject::AncestryInvalid);
assert_eq!(stats.ancestry_invalid, 2);
assert_eq!(stats.outbound_sign_failed, 0);
}
#[test]
fn tree_stats_record_reject_outbound_sign_failed() {
let mut stats = TreeStats::default();
stats.record_reject(TreeReject::OutboundSignFailed);
stats.record_reject(TreeReject::OutboundSignFailed);
assert_eq!(stats.outbound_sign_failed, 2);
assert_eq!(stats.ancestry_invalid, 0);
}
#[test]
fn node_stats_record_reject_dispatches_to_tree() {
let mut stats = NodeStats::new();
stats.record_reject(RejectReason::Tree(TreeReject::OutboundSignFailed));
assert_eq!(stats.tree.outbound_sign_failed, 1);
assert_eq!(stats.tree.ancestry_invalid, 0);
}
#[test]
fn session_stats_record_reject_unknown_session() {
let mut stats = SessionStats::default();
@@ -753,7 +361,6 @@ mod tests {
stats.record_reject(RejectReason::Session(SessionReject::BadState));
assert_eq!(stats.session.unknown_session, 1);
assert_eq!(stats.session.bad_state, 1);
assert_eq!(stats.tree.ancestry_invalid, 0);
}
#[test]
@@ -784,165 +391,6 @@ mod tests {
assert_eq!(stats.handshake.bad_state, 2);
assert_eq!(stats.handshake.unknown_connection, 1);
assert_eq!(stats.session.unknown_session, 0);
assert_eq!(stats.tree.ancestry_invalid, 0);
}
#[test]
fn bloom_stats_record_reject_decode_error() {
let mut s = BloomStats::default();
s.record_reject(BloomReject::DecodeError);
s.record_reject(BloomReject::DecodeError);
assert_eq!(s.decode_error, 2);
assert_eq!(s.invalid, 0);
}
#[test]
fn bloom_stats_record_reject_invalid() {
let mut s = BloomStats::default();
s.record_reject(BloomReject::Invalid);
assert_eq!(s.invalid, 1);
}
#[test]
fn bloom_stats_record_reject_non_v1() {
let mut s = BloomStats::default();
s.record_reject(BloomReject::NonV1);
assert_eq!(s.non_v1, 1);
}
#[test]
fn bloom_stats_record_reject_unknown_peer() {
let mut s = BloomStats::default();
s.record_reject(BloomReject::UnknownPeer);
assert_eq!(s.unknown_peer, 1);
}
#[test]
fn bloom_stats_record_reject_stale() {
let mut s = BloomStats::default();
s.record_reject(BloomReject::Stale);
assert_eq!(s.stale, 1);
}
#[test]
fn bloom_stats_record_reject_fill_exceeded() {
let mut s = BloomStats::default();
s.record_reject(BloomReject::FillExceeded);
assert_eq!(s.fill_exceeded, 1);
}
#[test]
fn node_stats_record_reject_dispatches_to_bloom() {
let mut stats = NodeStats::new();
stats.record_reject(RejectReason::Bloom(BloomReject::DecodeError));
stats.record_reject(RejectReason::Bloom(BloomReject::Stale));
assert_eq!(stats.bloom.decode_error, 1);
assert_eq!(stats.bloom.stale, 1);
assert_eq!(stats.tree.ancestry_invalid, 0);
}
#[test]
fn discovery_stats_record_reject_req_decode_error() {
let mut s = DiscoveryStats::default();
s.record_reject(DiscoveryReject::ReqDecodeError);
assert_eq!(s.req_decode_error, 1);
}
#[test]
fn discovery_stats_record_reject_req_duplicate() {
let mut s = DiscoveryStats::default();
s.record_reject(DiscoveryReject::ReqDuplicate);
assert_eq!(s.req_duplicate, 1);
}
#[test]
fn discovery_stats_record_reject_req_dedup_cache_full() {
let mut s = DiscoveryStats::default();
s.record_reject(DiscoveryReject::ReqDedupCacheFull);
assert_eq!(s.req_dedup_cache_full, 1);
}
#[test]
fn discovery_stats_record_reject_req_ttl_exhausted() {
let mut s = DiscoveryStats::default();
s.record_reject(DiscoveryReject::ReqTtlExhausted);
assert_eq!(s.req_ttl_exhausted, 1);
}
#[test]
fn discovery_stats_record_reject_resp_decode_error() {
let mut s = DiscoveryStats::default();
s.record_reject(DiscoveryReject::RespDecodeError);
assert_eq!(s.resp_decode_error, 1);
}
#[test]
fn discovery_stats_record_reject_resp_identity_miss() {
let mut s = DiscoveryStats::default();
s.record_reject(DiscoveryReject::RespIdentityMiss);
assert_eq!(s.resp_identity_miss, 1);
}
#[test]
fn discovery_stats_record_reject_resp_proof_failed() {
let mut s = DiscoveryStats::default();
s.record_reject(DiscoveryReject::RespProofFailed);
assert_eq!(s.resp_proof_failed, 1);
}
#[test]
fn node_stats_record_reject_dispatches_to_discovery() {
let mut stats = NodeStats::new();
stats.record_reject(RejectReason::Discovery(DiscoveryReject::ReqDecodeError));
stats.record_reject(RejectReason::Discovery(DiscoveryReject::RespProofFailed));
assert_eq!(stats.discovery.req_decode_error, 1);
assert_eq!(stats.discovery.resp_proof_failed, 1);
assert_eq!(stats.tree.ancestry_invalid, 0);
}
#[test]
fn forwarding_stats_record_reject_decode_error() {
let mut s = ForwardingStats::default();
s.record_reject(ForwardingReject::DecodeError);
assert_eq!(s.decode_error_packets, 1);
}
#[test]
fn forwarding_stats_record_reject_ttl_exhausted() {
let mut s = ForwardingStats::default();
s.record_reject(ForwardingReject::TtlExhausted);
assert_eq!(s.ttl_exhausted_packets, 1);
}
#[test]
fn forwarding_stats_record_reject_no_route() {
let mut s = ForwardingStats::default();
s.record_reject(ForwardingReject::NoRoute);
assert_eq!(s.drop_no_route_packets, 1);
}
#[test]
fn forwarding_stats_record_reject_mtu_exceeded() {
let mut s = ForwardingStats::default();
s.record_reject(ForwardingReject::MtuExceeded);
assert_eq!(s.drop_mtu_exceeded_packets, 1);
}
#[test]
fn forwarding_stats_record_reject_send_error() {
let mut s = ForwardingStats::default();
s.record_reject(ForwardingReject::SendError);
assert_eq!(s.drop_send_error_packets, 1);
}
#[test]
fn node_stats_record_reject_dispatches_to_forwarding() {
let mut stats = NodeStats::new();
stats.record_reject(RejectReason::Forwarding(ForwardingReject::NoRoute));
stats.record_reject(RejectReason::Forwarding(ForwardingReject::MtuExceeded));
assert_eq!(stats.forwarding.drop_no_route_packets, 1);
assert_eq!(stats.forwarding.drop_mtu_exceeded_packets, 1);
assert_eq!(stats.tree.ancestry_invalid, 0);
}
#[test]
@@ -969,7 +417,6 @@ mod tests {
stats.record_reject(RejectReason::Mmp(MmpReject::UnknownPeer));
assert_eq!(stats.mmp.decode_error, 1);
assert_eq!(stats.mmp.unknown_peer, 1);
assert_eq!(stats.tree.ancestry_invalid, 0);
}
#[test]
@@ -985,6 +432,5 @@ mod tests {
let mut stats = NodeStats::new();
stats.record_reject(RejectReason::Transport(TransportReject::InboundCapExceeded));
assert_eq!(stats.transport.inbound_cap_exceeded, 1);
assert_eq!(stats.tree.ancestry_invalid, 0);
}
}