proto/routing: sans-IO transit + hop-selection state machine

This commit is contained in:
Johnathan Corgan
2026-07-07 04:12:05 +00:00
parent e03b206f62
commit 9ea57b483a
19 changed files with 1820 additions and 888 deletions
+206 -147
View File
@@ -11,11 +11,9 @@ use crate::node::session_wire::{
FSP_COMMON_PREFIX_SIZE, FSP_HEADER_SIZE, FSP_PHASE_ESTABLISHED, FSP_PHASE_MSG1, FSP_PHASE_MSG2,
FspCommonPrefix, parse_encrypted_coords,
};
use crate::node::{Node, NodeError};
use crate::protocol::{
CoordsRequired, MtuExceeded, PathBroken, SessionAck, SessionDatagram, SessionDatagramRef,
SessionSetup,
};
use crate::node::{Node, NodeError, NodeRoutingView};
use crate::proto::routing::{DropReason, NextHop, RouteAction, RouteOutcome};
use crate::protocol::{SessionAck, SessionDatagram, SessionDatagramRef, SessionSetup};
use std::time::{Duration, Instant};
use tracing::{debug, warn};
@@ -43,123 +41,166 @@ impl Node {
}
};
// TTL enforcement: decrement for forwarding and drop only if the
// received datagram was already exhausted.
if datagram_ref.ttl == 0 {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
debug!(
src = %datagram_ref.src_addr,
dest = %datagram_ref.dest_addr,
"SessionDatagram TTL exhausted, dropping"
);
return;
}
let forwarded_ttl = datagram_ref.ttl - 1;
let my_addr = *self.node_addr();
// Coordinate cache warming from plaintext session-layer headers
self.try_warm_coord_cache_ref(&datagram_ref);
// Local delivery: dispatch to session layer handlers without
// materializing an owned SessionDatagram payload Vec.
if datagram_ref.dest_addr == *self.node_addr() {
self.metrics().forwarding.record_delivered(payload.len());
self.handle_session_payload(
&datagram_ref.src_addr,
datagram_ref.payload,
datagram_ref.path_mtu,
incoming_ce,
)
.await;
return;
// Coordinate cache warming from plaintext session-layer headers. Gated
// on a non-exhausted TTL so a datagram the core will drop as
// TTL-exhausted does not warm the cache, matching the pre-refactor
// ordering (warming ran only after the TTL early-return).
if datagram_ref.ttl != 0 {
self.try_warm_coord_cache_ref(&datagram_ref);
}
let mut datagram = datagram_ref.into_owned();
datagram.ttl = forwarded_ttl;
// Pre-resolve the next hop only for genuine transit packets (TTL > 0
// and not locally destined) so `find_next_hop`'s coord-cache LRU-touch
// side effect keeps the same scope it had inline. Warming above has
// already run, so the resolution observes freshly cached coords.
let next_hop = if datagram_ref.ttl != 0 && datagram_ref.dest_addr != my_addr {
self.resolve_next_hop(&datagram_ref.dest_addr)
} else {
None
};
// Find next hop toward destination
let next_hop_addr = match self.find_next_hop(&datagram.dest_addr) {
Some(peer) => *peer.node_addr(),
None => {
// Read local congestion once and reuse it for both the CE decision
// (via the view) and the congestion metric/log below, keeping
// `detect_congestion` the single source of truth.
let congested = next_hop
.as_ref()
.map(|nh| self.detect_congestion(&nh.addr))
.unwrap_or(false);
// Borrow the routing tables disjointly from `&mut self.routing` for
// the pure decision, then release both before driving the outcome.
let outcome = {
let view = NodeRoutingView {
coord_cache: &self.coord_cache,
peers: &self.peers,
tree_state: &self.tree_state,
congested,
};
self.routing
.route(&datagram_ref, &my_addr, incoming_ce, next_hop, &view)
};
match outcome {
RouteOutcome::Drop {
reason: DropReason::TtlExhausted,
} => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
debug!(
src = %datagram_ref.src_addr,
dest = %datagram_ref.dest_addr,
"SessionDatagram TTL exhausted, dropping"
);
}
RouteOutcome::DeliverLocal => {
// Local delivery: dispatch to session layer handlers without
// materializing an owned SessionDatagram payload Vec.
self.metrics().forwarding.record_delivered(payload.len());
self.handle_session_payload(
&datagram_ref.src_addr,
datagram_ref.payload,
datagram_ref.path_mtu,
incoming_ce,
)
.await;
}
RouteOutcome::NoRoute => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::NoRoute, payload.len());
let original = datagram_ref.into_owned();
debug!(
src = %self.peer_display_name(&datagram.src_addr),
dest = %self.peer_display_name(&datagram.dest_addr),
src = %self.peer_display_name(&original.src_addr),
dest = %self.peer_display_name(&original.dest_addr),
bytes = payload.len(),
"Dropping transit SessionDatagram: no route to destination"
);
self.send_routing_error(&datagram).await;
return;
self.send_routing_error(&original).await;
}
};
RouteOutcome::Forward {
next_hop,
bytes,
outgoing_ce,
} => {
let dest = datagram_ref.dest_addr;
// Apply path_mtu min() from the outgoing link's transport MTU
if let Some(peer) = self.peers.get(&next_hop_addr)
// ECN CE relay: congestion was detected locally above; emit the
// metric and rate-limited log at the transit chokepoint.
if congested {
self.metrics().congestion.congestion_detected.inc();
let now = Instant::now();
let should_log = self
.last_congestion_log
.map(|t| now.duration_since(t) >= Duration::from_secs(5))
.unwrap_or(true);
if should_log {
self.last_congestion_log = Some(now);
debug!(next_hop = %next_hop, "Congestion detected, CE flag set on forwarded packet");
}
}
match self
.send_encrypted_link_message_with_ce(&next_hop, &bytes, outgoing_ce)
.await
{
Err(NodeError::MtuExceeded { mtu, .. }) => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::MtuExceeded, payload.len());
self.send_mtu_exceeded_error(dest, datagram_ref.src_addr, mtu)
.await;
}
Err(e) => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::SendError, payload.len());
debug!(
next_hop = %next_hop,
dest = %dest,
error = %e,
"Failed to forward SessionDatagram"
);
}
Ok(()) => {
self.metrics().forwarding.record_forwarded(bytes.len());
// Classify this transit forward by route class (partition
// of forwarded_packets). Done here, at the data-plane
// chokepoint, so the error-signal routing callers of
// find_next_hop are excluded.
let class = self.classify_forward(&dest, &next_hop);
self.metrics().forwarding.record_route_class(class);
if outgoing_ce {
self.metrics().congestion.ce_forwarded.inc();
}
}
}
}
}
}
/// Resolve the next hop toward `dest` into its address plus the outgoing
/// link's transport MTU. Returns `None` when there is no route.
///
/// The MTU defaults to `u16::MAX` (a no-op min-fold) when the peer's
/// transport is not resolvable, matching the pre-refactor inline behavior
/// where the MTU `if let` chain simply did not fire.
fn resolve_next_hop(&mut self, dest: &NodeAddr) -> Option<NextHop> {
let addr = *self.find_next_hop(dest)?.node_addr();
let link_mtu = if let Some(peer) = self.peers.get(&addr)
&& let Some(tid) = peer.transport_id()
&& let Some(transport) = self.transports.get(&tid)
{
if let Some(addr) = peer.current_addr() {
datagram.path_mtu = datagram.path_mtu.min(transport.link_mtu(addr));
} else {
datagram.path_mtu = datagram.path_mtu.min(transport.mtu());
}
}
// ECN CE relay: propagate incoming CE and detect local congestion
let local_congestion = self.detect_congestion(&next_hop_addr);
let outgoing_ce = incoming_ce || local_congestion;
if local_congestion {
self.metrics().congestion.congestion_detected.inc();
let now = Instant::now();
let should_log = self
.last_congestion_log
.map(|t| now.duration_since(t) >= Duration::from_secs(5))
.unwrap_or(true);
if should_log {
self.last_congestion_log = Some(now);
debug!(next_hop = %next_hop_addr, "Congestion detected, CE flag set on forwarded packet");
}
}
// Forward: re-encode (includes 0x00 type byte) and send
let encoded = datagram.encode();
if let Err(e) = self
.send_encrypted_link_message_with_ce(&next_hop_addr, &encoded, outgoing_ce)
.await
{
match e {
NodeError::MtuExceeded { mtu, .. } => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::MtuExceeded, payload.len());
self.send_mtu_exceeded_error(&datagram, mtu).await;
}
_ => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::SendError, payload.len());
debug!(
next_hop = %next_hop_addr,
dest = %datagram.dest_addr,
error = %e,
"Failed to forward SessionDatagram"
);
}
match peer.current_addr() {
Some(link_addr) => transport.link_mtu(link_addr),
None => transport.mtu(),
}
} else {
self.metrics().forwarding.record_forwarded(encoded.len());
// Classify this transit forward by route class (partition of
// forwarded_packets). Done here, at the data-plane chokepoint, so
// the error-signal routing callers of find_next_hop are excluded.
let class = self.classify_forward(&datagram.dest_addr, &next_hop_addr);
self.metrics().forwarding.record_route_class(class);
if outgoing_ce {
self.metrics().congestion.ce_forwarded.inc();
}
}
u16::MAX
};
Some(NextHop { addr, link_mtu })
}
/// Attempt to warm the coordinate cache from session-layer payload headers.
@@ -260,35 +301,41 @@ impl Node {
/// If we can't route the error back to the source either, drop silently.
/// No cascading errors.
async fn send_routing_error(&mut self, original: &SessionDatagram) {
// Rate limit: one error signal per destination per 100ms
if !self
.routing_error_rate_limiter
.should_send(&original.dest_addr)
{
return;
}
let my_addr = *self.node_addr();
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let default_ttl = self.config().node.session.default_ttl;
let error_payload =
if let Some(coords) = self.coord_cache().get(&original.dest_addr, now_ms) {
let coords = coords.clone();
PathBroken::new(original.dest_addr, my_addr)
.with_last_coords(coords)
.encode()
} else {
CoordsRequired::new(original.dest_addr, my_addr).encode()
// Pure decision: rate-limit gate + PathBroken/CoordsRequired choice +
// error-PDU encode. Borrow the routing tables disjointly from
// `&mut self.routing`, then release them before the reverse-hop lookup.
let action = {
let view = NodeRoutingView {
coord_cache: &self.coord_cache,
peers: &self.peers,
tree_state: &self.tree_state,
congested: false,
};
self.routing.synth_routing_error(
&original.dest_addr,
&original.src_addr,
&my_addr,
&view,
now_ms,
default_ttl,
)
};
let RouteAction::SendError { toward, bytes } = match action {
Some(action) => action,
// Rate limited: drop silently. No cascading errors.
None => return,
};
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
.with_ttl(self.config().node.session.default_ttl);
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
// Resolve the reverse link hop only now, after the gate passed, so
// `find_next_hop`'s coord-cache touch keeps its pre-refactor scope.
let next_hop_addr = match self.find_next_hop(&toward) {
Some(peer) => *peer.node_addr(),
None => {
debug!(
@@ -300,9 +347,8 @@ impl Node {
}
};
let encoded = error_dg.encode();
if let Err(e) = self
.send_encrypted_link_message(&next_hop_addr, &encoded)
.send_encrypted_link_message(&next_hop_addr, &bytes)
.await
{
debug!(
@@ -324,37 +370,50 @@ impl Node {
/// Called when `send_encrypted_link_message()` fails with
/// `NodeError::MtuExceeded` during forwarding. The signal tells the
/// source the bottleneck MTU so it can immediately reduce its path MTU.
async fn send_mtu_exceeded_error(&mut self, original: &SessionDatagram, bottleneck_mtu: u16) {
// Rate limit: reuse routing_error_rate_limiter keyed on dest_addr
if !self
.routing_error_rate_limiter
.should_send(&original.dest_addr)
{
return;
}
///
/// `dest` is the failed datagram's destination (rate-limit key); `toward`
/// is its source, where the signal is routed back.
async fn send_mtu_exceeded_error(
&mut self,
dest: NodeAddr,
toward: NodeAddr,
bottleneck_mtu: u16,
) {
let my_addr = *self.node_addr();
let now_ms = Self::now_ms();
let default_ttl = self.config().node.session.default_ttl;
let error_payload = MtuExceeded::new(original.dest_addr, my_addr, bottleneck_mtu).encode();
// Pure decision: rate-limit gate + MtuExceeded PDU + encode.
let action = self.routing.synth_mtu_exceeded(
&dest,
&toward,
&my_addr,
bottleneck_mtu,
now_ms,
default_ttl,
);
let RouteAction::SendError { toward, bytes } = match action {
Some(action) => action,
// Rate limited: drop silently. No cascading errors.
None => return,
};
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
.with_ttl(self.config().node.session.default_ttl);
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
// Resolve the reverse link hop only now, after the gate passed, so
// `find_next_hop`'s coord-cache touch keeps its pre-refactor scope.
let next_hop_addr = match self.find_next_hop(&toward) {
Some(peer) => *peer.node_addr(),
None => {
debug!(
src = %original.src_addr,
dest = %original.dest_addr,
src = %toward,
dest = %dest,
"Cannot route MtuExceeded signal back to source, dropping"
);
return;
}
};
let encoded = error_dg.encode();
if let Err(e) = self
.send_encrypted_link_message(&next_hop_addr, &encoded)
.send_encrypted_link_message(&next_hop_addr, &bytes)
.await
{
debug!(
@@ -364,8 +423,8 @@ impl Node {
);
} else {
debug!(
original_dest = %original.dest_addr,
error_dest = %original.src_addr,
original_dest = %dest,
error_dest = %toward,
bottleneck_mtu,
"Sent MtuExceeded error signal"
);
+5 -5
View File
@@ -24,14 +24,14 @@ use crate::node::{Node, NodeError};
use crate::noise::{
HandshakeState, XK_HANDSHAKE_MSG1_SIZE, XK_HANDSHAKE_MSG2_SIZE, XK_HANDSHAKE_MSG3_SIZE,
};
use crate::proto::routing::{CoordsRequired, MtuExceeded, PathBroken};
#[cfg(unix)]
use crate::protocol::LinkMessageType;
#[cfg(unix)]
use crate::protocol::SESSION_DATAGRAM_HEADER_SIZE;
use crate::protocol::{
CoordsRequired, FspInnerFlags, MtuExceeded, PathBroken, PathMtuNotification, SessionAck,
SessionDatagram, SessionMessageType, SessionMsg3, SessionReceiverReport, SessionSenderReport,
SessionSetup,
FspInnerFlags, PathMtuNotification, SessionAck, SessionDatagram, SessionMessageType,
SessionMsg3, SessionReceiverReport, SessionSenderReport, SessionSetup,
};
use crate::protocol::{coords_wire_size, encode_coords};
#[cfg(unix)]
@@ -1125,7 +1125,7 @@ impl Node {
// Send standalone CoordsWarmup immediately (rate-limited)
if self
.coords_response_rate_limiter
.should_send(&msg.dest_addr)
.should_send(&msg.dest_addr, Self::now_ms())
{
if let Some(entry) = self.sessions.get(&msg.dest_addr)
&& entry.is_established()
@@ -1186,7 +1186,7 @@ impl Node {
// Send standalone CoordsWarmup immediately (rate-limited)
if self
.coords_response_rate_limiter
.should_send(&msg.dest_addr)
.should_send(&msg.dest_addr, Self::now_ms())
{
if let Some(entry) = self.sessions.get(&msg.dest_addr)
&& entry.is_established()
+4 -37
View File
@@ -90,43 +90,10 @@ pub struct ForwardingMetrics {
}
/// Route class of a transit-forwarded packet, classified from tree
/// coordinates at the forwarding decision point. The six variants
/// partition `forwarded_packets` exactly.
///
/// Two variants are up-and-over forwards (destination not in the chosen
/// peer's subtree); they differ in whether they depend on a child
/// advertising cross-link reach *upward* to its parent:
/// - `TreeDownCross`: the chosen peer is our tree descendant, but the
/// destination is *not* in that child's subtree. The forward only fired
/// because the child advertised cross-link reach upward to us, beyond its
/// own subtree. If children advertised only their subtree upward, this
/// forward would route up instead, so its count measures how much
/// forwarding depends on the upward cross-link advertisement — the
/// dive-to-tree-child cut-through.
/// - `CrosslinkAscend`: the chosen peer is lateral (neither ancestor nor
/// descendant) and the destination is not in its subtree. This is a node
/// using its *own* cross-link, learned via the peer's split-horizon
/// advertisement to its neighbors, so it does not depend on any upward
/// advertisement. Tracked alongside `TreeDownCross` as the lateral
/// up-and-over contrast.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RouteClass {
/// Chosen peer is our ancestor (tree-up).
TreeUp,
/// Chosen peer is our descendant and dest is in its subtree (canonical
/// tree-down).
TreeDown,
/// Chosen peer is our descendant but dest is *not* in its subtree: the
/// dive-to-tree-child cut-through enabled by upward cross-link
/// advertisement.
TreeDownCross,
/// Chosen peer is lateral and dest is in its subtree (subtree entry).
CrosslinkDescend,
/// Chosen peer is lateral and dest is not in its subtree (up-and-over).
CrosslinkAscend,
/// Chosen peer is the destination itself (degenerate direct hop).
DirectPeer,
}
/// coordinates at the forwarding decision point. Defined by the sans-IO
/// routing core and re-exported here for the forwarding-metrics counters
/// ([`ForwardingMetrics::record_route_class`]).
pub(crate) use crate::proto::routing::RouteClass;
impl ForwardingMetrics {
/// Record a received packet of `bytes` payload (packets and bytes).
+99 -141
View File
@@ -18,7 +18,6 @@ mod rate_limit;
pub(crate) mod reject;
mod reloadable;
mod retry;
mod routing_error_rate_limit;
pub(crate) mod session;
pub(crate) mod session_wire;
pub(crate) mod stats;
@@ -30,7 +29,6 @@ pub(crate) mod wire;
use self::rate_limit::HandshakeRateLimiter;
use self::reloadable::Reloadable;
use self::routing_error_rate_limit::RoutingErrorRateLimiter;
/// Half-range of the symmetric jitter applied to the per-session rekey timer.
/// Each session draws an offset uniformly from `[-REKEY_JITTER_SECS,
@@ -47,6 +45,7 @@ use crate::cache::CoordCache;
use crate::node::session::SessionEntry;
use crate::peer::{ActivePeer, PeerConnection};
use crate::proto::discovery::{Discovery, DiscoveryBackoff, DiscoveryForwardRateLimiter};
use crate::proto::routing::{self, Router, RoutingErrorRateLimiter};
#[cfg(unix)]
use crate::transport::ethernet::EthernetTransport;
use crate::transport::nym::NymTransport;
@@ -62,7 +61,7 @@ use crate::upper::hosts::HostMap;
use crate::upper::icmp_rate_limit::IcmpRateLimiter;
use crate::upper::tun::{TunError, TunOutboundRx, TunState, TunTx};
use crate::utils::index::IndexAllocator;
use crate::{Config, ConfigError, Identity, IdentityError, NodeAddr, PeerIdentity};
use crate::{Config, ConfigError, Identity, IdentityError, NodeAddr, PeerIdentity, TreeCoordinate};
use rand::Rng;
use std::collections::{HashMap, HashSet, VecDeque};
use std::fmt;
@@ -415,8 +414,8 @@ pub struct Node {
msg1_rate_limiter: HandshakeRateLimiter,
/// Rate limiter for ICMP Packet Too Big messages.
icmp_rate_limiter: IcmpRateLimiter,
/// Rate limiter for routing error signals (CoordsRequired / PathBroken).
routing_error_rate_limiter: RoutingErrorRateLimiter,
/// Routing-subsystem state (routing error-signal rate limiter).
routing: Router,
/// Rate limiter for source-side CoordsRequired/PathBroken responses.
coords_response_rate_limiter: RoutingErrorRateLimiter,
@@ -658,9 +657,9 @@ impl Node {
pending_outbound: HashMap::new(),
msg1_rate_limiter,
icmp_rate_limiter: IcmpRateLimiter::new(),
routing_error_rate_limiter: RoutingErrorRateLimiter::new(),
coords_response_rate_limiter: RoutingErrorRateLimiter::with_interval(
std::time::Duration::from_millis(coords_response_interval_ms),
routing: Router::new(),
coords_response_rate_limiter: RoutingErrorRateLimiter::with_interval_ms(
coords_response_interval_ms,
),
discovery: Discovery::new(
DiscoveryBackoff::with_params(backoff_base_secs, backoff_max_secs),
@@ -817,9 +816,9 @@ impl Node {
pending_outbound: HashMap::new(),
msg1_rate_limiter,
icmp_rate_limiter: IcmpRateLimiter::new(),
routing_error_rate_limiter: RoutingErrorRateLimiter::new(),
coords_response_rate_limiter: RoutingErrorRateLimiter::with_interval(
std::time::Duration::from_millis(coords_response_interval_ms),
routing: Router::new(),
coords_response_rate_limiter: RoutingErrorRateLimiter::with_interval_ms(
coords_response_interval_ms,
),
discovery: Discovery::new(DiscoveryBackoff::new(), DiscoveryForwardRateLimiter::new()),
pending_connects: Vec::new(),
@@ -2607,11 +2606,27 @@ impl Node {
// 3. Bloom filter candidates — requires dest_coords for loop-free selection.
// If no candidate is strictly closer, fall through to tree routing.
let candidates: Vec<&ActivePeer> = self.destination_in_filters(dest_node_addr);
// The sans-IO core assembles the candidate snapshot over the
// `RoutingView` seam (enumerate peers, apply the bloom `may_reach`
// filter, snapshot each), then picks the winner; the shell supplies
// only the raw per-peer reads.
let candidates = {
let view = NodeRoutingView {
coord_cache: &self.coord_cache,
peers: &self.peers,
tree_state: &self.tree_state,
congested: false,
};
routing::routing_candidates(&view, dest_node_addr)
};
if !candidates.is_empty()
&& let Some(peer) = self.select_best_candidate(&candidates, &dest_coords)
&& let Some(next_hop) = routing::select_best_candidate(
&candidates,
&dest_coords,
self.tree_state.my_coords(),
)
{
return Some(peer);
return self.peers.get(&next_hop);
}
// 4. Greedy tree routing fallback
@@ -2622,142 +2637,29 @@ impl Node {
/// Classify a transit forward by route class from tree coordinates.
///
/// Called at the transit chokepoint after `find_next_hop` returns a peer,
/// so the six classes partition `forwarded_packets` exactly. The branch
/// that `find_next_hop` took (bloom vs greedy-tree) is *not* the route
/// class: a peer can be selected by either, so the cut-through splits
/// (`TreeDownCross`, `CrosslinkAscend`) are decided here from coordinates,
/// not from which branch fired.
///
/// Inputs: our coords (`tree_state.my_coords`), the chosen peer's coords
/// (`tree_state.peer_coords`), and the destination coords (re-read from the
/// coord cache, which `find_next_hop` just touched). Both the tree-down and
/// cross-link branches split on whether the destination is in the chosen
/// peer's subtree; when the dest coords are unavailable that test defaults
/// to "not in subtree", i.e. the up-and-over variant (`TreeDownCross` for a
/// descendant peer, `CrosslinkAscend` for a lateral one).
/// Thin shell adapter over the pure [`routing::classify_forward`]: it
/// pre-resolves the destination coordinates from the coord cache (the
/// sole impurity — a read-only lookup, no LRU touch) and reads our own
/// and the chosen peer's coordinates from tree state, then defers the
/// six-way classification to the sans-IO routing core.
pub(crate) fn classify_forward(
&self,
dest: &NodeAddr,
chosen_peer: &NodeAddr,
) -> metrics::RouteClass {
// Degenerate: the next hop is the destination itself (Branch 2).
if chosen_peer == dest {
return metrics::RouteClass::DirectPeer;
}
let my_addr = self.node_addr();
let my_coords = self.tree_state.my_coords();
// Tree-up: the chosen peer is our ancestor.
if my_coords.has_ancestor(chosen_peer) {
return metrics::RouteClass::TreeUp;
}
// Whether the destination is in the chosen peer's subtree. Both the
// tree-down and cross-link splits below turn on this same test, so it
// is computed once. On the live transit path the dest coords are
// always present here: `find_next_hop` looks them up with an early
// return, so a coord-cache miss yields no next hop to classify (the
// caller signals `CoordsRequired` instead of forwarding). The miss
// branch below is therefore defensive — reachable only by direct
// unit-test calls — and defaults the test to "not in subtree", i.e.
// the up-and-over variant of whichever branch fires (TreeDownCross for
// a descendant peer, CrosslinkAscend for a lateral one), matching the
// original cross-link default-to-ascend.
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let dest_in_peer_subtree = self
.coord_cache
.get(dest, now_ms)
.is_some_and(|dest_coords| dest_coords.has_ancestor(chosen_peer));
// Tree-down: the chosen peer is our descendant (we are its ancestor).
// Split by subtree membership: a dest genuinely below the child is the
// canonical tree-down; a dest *not* below it means we only forwarded
// down because the child advertised cross-link reach upward, beyond its
// own subtree — the dive-to-tree-child cut-through (TreeDownCross).
if let Some(peer_coords) = self.tree_state.peer_coords(chosen_peer)
&& peer_coords.has_ancestor(my_addr)
{
return if dest_in_peer_subtree {
metrics::RouteClass::TreeDown
} else {
metrics::RouteClass::TreeDownCross
};
}
// Cross-link (lateral): split by whether the destination is in the
// chosen peer's subtree. Descend = subtree entry; ascend = up-and-over
// via the node's own cross-link (learned from the peer's split-horizon
// advertisement, independent of any upward advertisement).
if dest_in_peer_subtree {
return metrics::RouteClass::CrosslinkDescend;
}
metrics::RouteClass::CrosslinkAscend
}
/// Select the best peer from a set of bloom filter candidates.
///
/// Uses distance from each candidate's tree coordinates to the destination
/// as the primary metric (after link_cost). Only selects peers that are
/// strictly closer to the destination than we are (self-distance check
/// prevents routing loops).
///
/// Ordering: `(link_cost, distance_to_dest, node_addr)`.
fn select_best_candidate<'a>(
&'a self,
candidates: &[&'a ActivePeer],
dest_coords: &crate::tree::TreeCoordinate,
) -> Option<&'a ActivePeer> {
let my_distance = self.tree_state.my_coords().distance_to(dest_coords);
let mut best: Option<(&ActivePeer, f64, usize)> = None;
for &candidate in candidates {
if !candidate.can_send() {
continue;
}
let cost = candidate.link_cost();
let dist = self
.tree_state
.peer_coords(candidate.node_addr())
.map(|pc| pc.distance_to(dest_coords))
.unwrap_or(usize::MAX);
// Self-distance check: only consider peers strictly closer
// to the destination than we are (prevents routing loops)
if dist >= my_distance {
continue;
}
let dominated = match &best {
None => true,
Some((_, best_cost, best_dist)) => {
cost < *best_cost
|| (cost == *best_cost && dist < *best_dist)
|| (cost == *best_cost
&& dist == *best_dist
&& candidate.node_addr() < best.as_ref().unwrap().0.node_addr())
}
};
if dominated {
best = Some((candidate, cost, dist));
}
}
best.map(|(peer, _, _)| peer)
}
/// Check if a destination is in any peer's bloom filter.
pub fn destination_in_filters(&self, dest: &NodeAddr) -> Vec<&ActivePeer> {
self.peers.values().filter(|p| p.may_reach(dest)).collect()
let dest_coords = self.coord_cache.get(dest, now_ms).cloned();
routing::classify_forward(
dest,
chosen_peer,
self.node_addr(),
self.tree_state.my_coords(),
dest_coords.as_ref(),
self.tree_state.peer_coords(chosen_peer),
)
}
/// Get the TUN packet sender channel.
@@ -3002,6 +2904,62 @@ impl Node {
}
}
/// Shell-side [`routing::RoutingView`] seam over live `Node` state — the sole
/// routing read adapter the shell retains. It hands the sans-IO routing core
/// raw per-peer reads (enumeration plus `may_reach` / `can_send` / `link_cost`
/// / `coords`) so the candidate assembly, selection, and error synthesis all
/// live in `proto::routing::core`; no routing decision or assembly logic
/// remains here.
///
/// Field-narrowed to `coord_cache` + `peers` + `tree_state` (never `&Node`
/// whole) so it borrows disjointly from `&mut self.routing` on the
/// forward/synth path, where the handler also holds the mutable `Router`.
///
/// Two call sites:
/// - `find_next_hop` builds it to assemble bloom candidates via the `peer_*`
/// reads; it never queries `is_congested`, so it leaves `congested` false.
/// - `handle_session_datagram` builds it for `Router::route` / `synth_*`,
/// which read `is_congested` (precomputed once for the resolved next hop)
/// and `cached_coords`.
pub(in crate::node) struct NodeRoutingView<'a> {
pub(in crate::node) coord_cache: &'a CoordCache,
pub(in crate::node) peers: &'a HashMap<NodeAddr, ActivePeer>,
pub(in crate::node) tree_state: &'a TreeState,
pub(in crate::node) congested: bool,
}
impl routing::RoutingView for NodeRoutingView<'_> {
fn is_congested(&self, _next_hop: &NodeAddr) -> bool {
self.congested
}
fn cached_coords(&self, dest: &NodeAddr, now_ms: u64) -> Option<TreeCoordinate> {
self.coord_cache.get(dest, now_ms).cloned()
}
fn peer_addrs(&self) -> Vec<NodeAddr> {
self.peers.keys().copied().collect()
}
fn peer_may_reach(&self, peer: &NodeAddr, dest: &NodeAddr) -> bool {
self.peers.get(peer).is_some_and(|p| p.may_reach(dest))
}
fn peer_can_send(&self, peer: &NodeAddr) -> bool {
self.peers.get(peer).is_some_and(|p| p.can_send())
}
fn peer_link_cost(&self, peer: &NodeAddr) -> f64 {
self.peers
.get(peer)
.map_or(f64::INFINITY, |p| p.link_cost())
}
fn peer_coords(&self, peer: &NodeAddr) -> Option<TreeCoordinate> {
self.tree_state.peer_coords(peer).cloned()
}
}
/// Project an MMP metrics block into the snapshot
/// [`EntityMmp`](crate::control::snapshot::EntityMmp) shared by `show_peers`
/// (link-layer, `path_mtu = None`) and `show_sessions` (session-layer,
-161
View File
@@ -1,161 +0,0 @@
//! Routing error signal rate limiting.
//!
//! Prevents routing error floods (CoordsRequired / PathBroken) by
//! rate-limiting error signals per destination address at transit nodes.
use crate::NodeAddr;
use std::collections::HashMap;
use std::time::{Duration, Instant};
/// Rate limiter for routing error signals (CoordsRequired / PathBroken).
///
/// Tracks the last time a routing error was sent for each destination
/// address and enforces a minimum interval to prevent floods.
pub struct RoutingErrorRateLimiter {
/// Maps destination NodeAddr to the last time we sent an error about it.
last_sent: HashMap<NodeAddr, Instant>,
/// Minimum interval between error signals for the same destination.
min_interval: Duration,
/// Maximum age of entries before cleanup.
max_age: Duration,
}
impl RoutingErrorRateLimiter {
/// Create a new rate limiter.
///
/// Default: max 10 errors/sec per destination (100ms interval).
pub fn new() -> Self {
Self {
last_sent: HashMap::new(),
min_interval: Duration::from_millis(100),
max_age: Duration::from_secs(10),
}
}
/// Create a rate limiter with a custom minimum interval.
pub fn with_interval(min_interval: Duration) -> Self {
Self {
last_sent: HashMap::new(),
min_interval,
max_age: Duration::from_secs(10),
}
}
/// Check if we should send a routing error for this destination.
///
/// Returns true if enough time has passed since the last error for
/// this destination, or if this is the first error. Updates internal
/// state when returning true.
pub fn should_send(&mut self, dest_addr: &NodeAddr) -> bool {
let now = Instant::now();
if let Some(&last) = self.last_sent.get(dest_addr)
&& now.duration_since(last) < self.min_interval
{
return false;
}
self.last_sent.insert(*dest_addr, now);
self.cleanup(now);
true
}
/// Remove entries older than max_age.
fn cleanup(&mut self, now: Instant) {
self.last_sent
.retain(|_, &mut last| now.duration_since(last) < self.max_age);
}
#[cfg(test)]
pub fn len(&self) -> usize {
self.last_sent.len()
}
}
impl Default for RoutingErrorRateLimiter {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::thread;
fn addr(val: u8) -> NodeAddr {
let mut bytes = [0u8; 16];
bytes[0] = val;
NodeAddr::from_bytes(bytes)
}
#[test]
fn test_first_send_allowed() {
let mut limiter = RoutingErrorRateLimiter::new();
assert!(limiter.should_send(&addr(1)));
}
#[test]
fn test_rapid_sends_rate_limited() {
let mut limiter = RoutingErrorRateLimiter::new();
assert!(limiter.should_send(&addr(1)));
assert!(!limiter.should_send(&addr(1)));
assert!(!limiter.should_send(&addr(1)));
}
#[test]
fn test_different_destinations_independent() {
let mut limiter = RoutingErrorRateLimiter::new();
assert!(limiter.should_send(&addr(1)));
assert!(limiter.should_send(&addr(2)));
assert!(!limiter.should_send(&addr(1)));
assert!(!limiter.should_send(&addr(2)));
}
#[test]
fn test_send_allowed_after_interval() {
let mut limiter = RoutingErrorRateLimiter::new();
assert!(limiter.should_send(&addr(1)));
thread::sleep(Duration::from_millis(110));
assert!(limiter.should_send(&addr(1)));
}
#[test]
fn test_cleanup_removes_old_entries() {
let mut limiter = RoutingErrorRateLimiter::new();
assert!(limiter.should_send(&addr(1)));
assert!(limiter.should_send(&addr(2)));
assert_eq!(limiter.len(), 2);
let future = Instant::now() + Duration::from_secs(11);
limiter.cleanup(future);
assert_eq!(limiter.len(), 0);
}
#[test]
fn test_cleanup_preserves_recent_entries() {
let mut limiter = RoutingErrorRateLimiter::new();
assert!(limiter.should_send(&addr(1)));
assert_eq!(limiter.len(), 1);
limiter.cleanup(Instant::now());
assert_eq!(limiter.len(), 1);
}
#[test]
fn test_with_interval_custom_rate() {
let mut limiter = RoutingErrorRateLimiter::with_interval(Duration::from_millis(500));
assert!(limiter.should_send(&addr(1)));
assert!(!limiter.should_send(&addr(1)));
// Still rate-limited after 200ms (would pass with default 100ms)
thread::sleep(Duration::from_millis(200));
assert!(!limiter.should_send(&addr(1)));
// Allowed after 500ms total
thread::sleep(Duration::from_millis(350));
assert!(limiter.should_send(&addr(1)));
}
}