mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 00:04:54 +00:00
proto/routing: sans-IO transit + hop-selection state machine
This commit is contained in:
+206
-147
@@ -11,11 +11,9 @@ use crate::node::session_wire::{
|
||||
FSP_COMMON_PREFIX_SIZE, FSP_HEADER_SIZE, FSP_PHASE_ESTABLISHED, FSP_PHASE_MSG1, FSP_PHASE_MSG2,
|
||||
FspCommonPrefix, parse_encrypted_coords,
|
||||
};
|
||||
use crate::node::{Node, NodeError};
|
||||
use crate::protocol::{
|
||||
CoordsRequired, MtuExceeded, PathBroken, SessionAck, SessionDatagram, SessionDatagramRef,
|
||||
SessionSetup,
|
||||
};
|
||||
use crate::node::{Node, NodeError, NodeRoutingView};
|
||||
use crate::proto::routing::{DropReason, NextHop, RouteAction, RouteOutcome};
|
||||
use crate::protocol::{SessionAck, SessionDatagram, SessionDatagramRef, SessionSetup};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
@@ -43,123 +41,166 @@ impl Node {
|
||||
}
|
||||
};
|
||||
|
||||
// TTL enforcement: decrement for forwarding and drop only if the
|
||||
// received datagram was already exhausted.
|
||||
if datagram_ref.ttl == 0 {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
|
||||
debug!(
|
||||
src = %datagram_ref.src_addr,
|
||||
dest = %datagram_ref.dest_addr,
|
||||
"SessionDatagram TTL exhausted, dropping"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let forwarded_ttl = datagram_ref.ttl - 1;
|
||||
let my_addr = *self.node_addr();
|
||||
|
||||
// Coordinate cache warming from plaintext session-layer headers
|
||||
self.try_warm_coord_cache_ref(&datagram_ref);
|
||||
|
||||
// Local delivery: dispatch to session layer handlers without
|
||||
// materializing an owned SessionDatagram payload Vec.
|
||||
if datagram_ref.dest_addr == *self.node_addr() {
|
||||
self.metrics().forwarding.record_delivered(payload.len());
|
||||
self.handle_session_payload(
|
||||
&datagram_ref.src_addr,
|
||||
datagram_ref.payload,
|
||||
datagram_ref.path_mtu,
|
||||
incoming_ce,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
// Coordinate cache warming from plaintext session-layer headers. Gated
|
||||
// on a non-exhausted TTL so a datagram the core will drop as
|
||||
// TTL-exhausted does not warm the cache, matching the pre-refactor
|
||||
// ordering (warming ran only after the TTL early-return).
|
||||
if datagram_ref.ttl != 0 {
|
||||
self.try_warm_coord_cache_ref(&datagram_ref);
|
||||
}
|
||||
|
||||
let mut datagram = datagram_ref.into_owned();
|
||||
datagram.ttl = forwarded_ttl;
|
||||
// Pre-resolve the next hop only for genuine transit packets (TTL > 0
|
||||
// and not locally destined) so `find_next_hop`'s coord-cache LRU-touch
|
||||
// side effect keeps the same scope it had inline. Warming above has
|
||||
// already run, so the resolution observes freshly cached coords.
|
||||
let next_hop = if datagram_ref.ttl != 0 && datagram_ref.dest_addr != my_addr {
|
||||
self.resolve_next_hop(&datagram_ref.dest_addr)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Find next hop toward destination
|
||||
let next_hop_addr = match self.find_next_hop(&datagram.dest_addr) {
|
||||
Some(peer) => *peer.node_addr(),
|
||||
None => {
|
||||
// Read local congestion once and reuse it for both the CE decision
|
||||
// (via the view) and the congestion metric/log below, keeping
|
||||
// `detect_congestion` the single source of truth.
|
||||
let congested = next_hop
|
||||
.as_ref()
|
||||
.map(|nh| self.detect_congestion(&nh.addr))
|
||||
.unwrap_or(false);
|
||||
|
||||
// Borrow the routing tables disjointly from `&mut self.routing` for
|
||||
// the pure decision, then release both before driving the outcome.
|
||||
let outcome = {
|
||||
let view = NodeRoutingView {
|
||||
coord_cache: &self.coord_cache,
|
||||
peers: &self.peers,
|
||||
tree_state: &self.tree_state,
|
||||
congested,
|
||||
};
|
||||
self.routing
|
||||
.route(&datagram_ref, &my_addr, incoming_ce, next_hop, &view)
|
||||
};
|
||||
|
||||
match outcome {
|
||||
RouteOutcome::Drop {
|
||||
reason: DropReason::TtlExhausted,
|
||||
} => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
|
||||
debug!(
|
||||
src = %datagram_ref.src_addr,
|
||||
dest = %datagram_ref.dest_addr,
|
||||
"SessionDatagram TTL exhausted, dropping"
|
||||
);
|
||||
}
|
||||
RouteOutcome::DeliverLocal => {
|
||||
// Local delivery: dispatch to session layer handlers without
|
||||
// materializing an owned SessionDatagram payload Vec.
|
||||
self.metrics().forwarding.record_delivered(payload.len());
|
||||
self.handle_session_payload(
|
||||
&datagram_ref.src_addr,
|
||||
datagram_ref.payload,
|
||||
datagram_ref.path_mtu,
|
||||
incoming_ce,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
RouteOutcome::NoRoute => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::NoRoute, payload.len());
|
||||
let original = datagram_ref.into_owned();
|
||||
debug!(
|
||||
src = %self.peer_display_name(&datagram.src_addr),
|
||||
dest = %self.peer_display_name(&datagram.dest_addr),
|
||||
src = %self.peer_display_name(&original.src_addr),
|
||||
dest = %self.peer_display_name(&original.dest_addr),
|
||||
bytes = payload.len(),
|
||||
"Dropping transit SessionDatagram: no route to destination"
|
||||
);
|
||||
self.send_routing_error(&datagram).await;
|
||||
return;
|
||||
self.send_routing_error(&original).await;
|
||||
}
|
||||
};
|
||||
RouteOutcome::Forward {
|
||||
next_hop,
|
||||
bytes,
|
||||
outgoing_ce,
|
||||
} => {
|
||||
let dest = datagram_ref.dest_addr;
|
||||
|
||||
// Apply path_mtu min() from the outgoing link's transport MTU
|
||||
if let Some(peer) = self.peers.get(&next_hop_addr)
|
||||
// ECN CE relay: congestion was detected locally above; emit the
|
||||
// metric and rate-limited log at the transit chokepoint.
|
||||
if congested {
|
||||
self.metrics().congestion.congestion_detected.inc();
|
||||
let now = Instant::now();
|
||||
let should_log = self
|
||||
.last_congestion_log
|
||||
.map(|t| now.duration_since(t) >= Duration::from_secs(5))
|
||||
.unwrap_or(true);
|
||||
if should_log {
|
||||
self.last_congestion_log = Some(now);
|
||||
debug!(next_hop = %next_hop, "Congestion detected, CE flag set on forwarded packet");
|
||||
}
|
||||
}
|
||||
|
||||
match self
|
||||
.send_encrypted_link_message_with_ce(&next_hop, &bytes, outgoing_ce)
|
||||
.await
|
||||
{
|
||||
Err(NodeError::MtuExceeded { mtu, .. }) => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::MtuExceeded, payload.len());
|
||||
self.send_mtu_exceeded_error(dest, datagram_ref.src_addr, mtu)
|
||||
.await;
|
||||
}
|
||||
Err(e) => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::SendError, payload.len());
|
||||
debug!(
|
||||
next_hop = %next_hop,
|
||||
dest = %dest,
|
||||
error = %e,
|
||||
"Failed to forward SessionDatagram"
|
||||
);
|
||||
}
|
||||
Ok(()) => {
|
||||
self.metrics().forwarding.record_forwarded(bytes.len());
|
||||
// Classify this transit forward by route class (partition
|
||||
// of forwarded_packets). Done here, at the data-plane
|
||||
// chokepoint, so the error-signal routing callers of
|
||||
// find_next_hop are excluded.
|
||||
let class = self.classify_forward(&dest, &next_hop);
|
||||
self.metrics().forwarding.record_route_class(class);
|
||||
if outgoing_ce {
|
||||
self.metrics().congestion.ce_forwarded.inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the next hop toward `dest` into its address plus the outgoing
|
||||
/// link's transport MTU. Returns `None` when there is no route.
|
||||
///
|
||||
/// The MTU defaults to `u16::MAX` (a no-op min-fold) when the peer's
|
||||
/// transport is not resolvable, matching the pre-refactor inline behavior
|
||||
/// where the MTU `if let` chain simply did not fire.
|
||||
fn resolve_next_hop(&mut self, dest: &NodeAddr) -> Option<NextHop> {
|
||||
let addr = *self.find_next_hop(dest)?.node_addr();
|
||||
let link_mtu = if let Some(peer) = self.peers.get(&addr)
|
||||
&& let Some(tid) = peer.transport_id()
|
||||
&& let Some(transport) = self.transports.get(&tid)
|
||||
{
|
||||
if let Some(addr) = peer.current_addr() {
|
||||
datagram.path_mtu = datagram.path_mtu.min(transport.link_mtu(addr));
|
||||
} else {
|
||||
datagram.path_mtu = datagram.path_mtu.min(transport.mtu());
|
||||
}
|
||||
}
|
||||
|
||||
// ECN CE relay: propagate incoming CE and detect local congestion
|
||||
let local_congestion = self.detect_congestion(&next_hop_addr);
|
||||
let outgoing_ce = incoming_ce || local_congestion;
|
||||
if local_congestion {
|
||||
self.metrics().congestion.congestion_detected.inc();
|
||||
let now = Instant::now();
|
||||
let should_log = self
|
||||
.last_congestion_log
|
||||
.map(|t| now.duration_since(t) >= Duration::from_secs(5))
|
||||
.unwrap_or(true);
|
||||
if should_log {
|
||||
self.last_congestion_log = Some(now);
|
||||
debug!(next_hop = %next_hop_addr, "Congestion detected, CE flag set on forwarded packet");
|
||||
}
|
||||
}
|
||||
|
||||
// Forward: re-encode (includes 0x00 type byte) and send
|
||||
let encoded = datagram.encode();
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message_with_ce(&next_hop_addr, &encoded, outgoing_ce)
|
||||
.await
|
||||
{
|
||||
match e {
|
||||
NodeError::MtuExceeded { mtu, .. } => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::MtuExceeded, payload.len());
|
||||
self.send_mtu_exceeded_error(&datagram, mtu).await;
|
||||
}
|
||||
_ => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::SendError, payload.len());
|
||||
debug!(
|
||||
next_hop = %next_hop_addr,
|
||||
dest = %datagram.dest_addr,
|
||||
error = %e,
|
||||
"Failed to forward SessionDatagram"
|
||||
);
|
||||
}
|
||||
match peer.current_addr() {
|
||||
Some(link_addr) => transport.link_mtu(link_addr),
|
||||
None => transport.mtu(),
|
||||
}
|
||||
} else {
|
||||
self.metrics().forwarding.record_forwarded(encoded.len());
|
||||
// Classify this transit forward by route class (partition of
|
||||
// forwarded_packets). Done here, at the data-plane chokepoint, so
|
||||
// the error-signal routing callers of find_next_hop are excluded.
|
||||
let class = self.classify_forward(&datagram.dest_addr, &next_hop_addr);
|
||||
self.metrics().forwarding.record_route_class(class);
|
||||
if outgoing_ce {
|
||||
self.metrics().congestion.ce_forwarded.inc();
|
||||
}
|
||||
}
|
||||
u16::MAX
|
||||
};
|
||||
Some(NextHop { addr, link_mtu })
|
||||
}
|
||||
|
||||
/// Attempt to warm the coordinate cache from session-layer payload headers.
|
||||
@@ -260,35 +301,41 @@ impl Node {
|
||||
/// If we can't route the error back to the source either, drop silently.
|
||||
/// No cascading errors.
|
||||
async fn send_routing_error(&mut self, original: &SessionDatagram) {
|
||||
// Rate limit: one error signal per destination per 100ms
|
||||
if !self
|
||||
.routing_error_rate_limiter
|
||||
.should_send(&original.dest_addr)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let my_addr = *self.node_addr();
|
||||
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
let default_ttl = self.config().node.session.default_ttl;
|
||||
|
||||
let error_payload =
|
||||
if let Some(coords) = self.coord_cache().get(&original.dest_addr, now_ms) {
|
||||
let coords = coords.clone();
|
||||
PathBroken::new(original.dest_addr, my_addr)
|
||||
.with_last_coords(coords)
|
||||
.encode()
|
||||
} else {
|
||||
CoordsRequired::new(original.dest_addr, my_addr).encode()
|
||||
// Pure decision: rate-limit gate + PathBroken/CoordsRequired choice +
|
||||
// error-PDU encode. Borrow the routing tables disjointly from
|
||||
// `&mut self.routing`, then release them before the reverse-hop lookup.
|
||||
let action = {
|
||||
let view = NodeRoutingView {
|
||||
coord_cache: &self.coord_cache,
|
||||
peers: &self.peers,
|
||||
tree_state: &self.tree_state,
|
||||
congested: false,
|
||||
};
|
||||
self.routing.synth_routing_error(
|
||||
&original.dest_addr,
|
||||
&original.src_addr,
|
||||
&my_addr,
|
||||
&view,
|
||||
now_ms,
|
||||
default_ttl,
|
||||
)
|
||||
};
|
||||
let RouteAction::SendError { toward, bytes } = match action {
|
||||
Some(action) => action,
|
||||
// Rate limited: drop silently. No cascading errors.
|
||||
None => return,
|
||||
};
|
||||
|
||||
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
|
||||
.with_ttl(self.config().node.session.default_ttl);
|
||||
|
||||
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
|
||||
// Resolve the reverse link hop only now, after the gate passed, so
|
||||
// `find_next_hop`'s coord-cache touch keeps its pre-refactor scope.
|
||||
let next_hop_addr = match self.find_next_hop(&toward) {
|
||||
Some(peer) => *peer.node_addr(),
|
||||
None => {
|
||||
debug!(
|
||||
@@ -300,9 +347,8 @@ impl Node {
|
||||
}
|
||||
};
|
||||
|
||||
let encoded = error_dg.encode();
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message(&next_hop_addr, &encoded)
|
||||
.send_encrypted_link_message(&next_hop_addr, &bytes)
|
||||
.await
|
||||
{
|
||||
debug!(
|
||||
@@ -324,37 +370,50 @@ impl Node {
|
||||
/// Called when `send_encrypted_link_message()` fails with
|
||||
/// `NodeError::MtuExceeded` during forwarding. The signal tells the
|
||||
/// source the bottleneck MTU so it can immediately reduce its path MTU.
|
||||
async fn send_mtu_exceeded_error(&mut self, original: &SessionDatagram, bottleneck_mtu: u16) {
|
||||
// Rate limit: reuse routing_error_rate_limiter keyed on dest_addr
|
||||
if !self
|
||||
.routing_error_rate_limiter
|
||||
.should_send(&original.dest_addr)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
///
|
||||
/// `dest` is the failed datagram's destination (rate-limit key); `toward`
|
||||
/// is its source, where the signal is routed back.
|
||||
async fn send_mtu_exceeded_error(
|
||||
&mut self,
|
||||
dest: NodeAddr,
|
||||
toward: NodeAddr,
|
||||
bottleneck_mtu: u16,
|
||||
) {
|
||||
let my_addr = *self.node_addr();
|
||||
let now_ms = Self::now_ms();
|
||||
let default_ttl = self.config().node.session.default_ttl;
|
||||
|
||||
let error_payload = MtuExceeded::new(original.dest_addr, my_addr, bottleneck_mtu).encode();
|
||||
// Pure decision: rate-limit gate + MtuExceeded PDU + encode.
|
||||
let action = self.routing.synth_mtu_exceeded(
|
||||
&dest,
|
||||
&toward,
|
||||
&my_addr,
|
||||
bottleneck_mtu,
|
||||
now_ms,
|
||||
default_ttl,
|
||||
);
|
||||
let RouteAction::SendError { toward, bytes } = match action {
|
||||
Some(action) => action,
|
||||
// Rate limited: drop silently. No cascading errors.
|
||||
None => return,
|
||||
};
|
||||
|
||||
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
|
||||
.with_ttl(self.config().node.session.default_ttl);
|
||||
|
||||
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
|
||||
// Resolve the reverse link hop only now, after the gate passed, so
|
||||
// `find_next_hop`'s coord-cache touch keeps its pre-refactor scope.
|
||||
let next_hop_addr = match self.find_next_hop(&toward) {
|
||||
Some(peer) => *peer.node_addr(),
|
||||
None => {
|
||||
debug!(
|
||||
src = %original.src_addr,
|
||||
dest = %original.dest_addr,
|
||||
src = %toward,
|
||||
dest = %dest,
|
||||
"Cannot route MtuExceeded signal back to source, dropping"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let encoded = error_dg.encode();
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message(&next_hop_addr, &encoded)
|
||||
.send_encrypted_link_message(&next_hop_addr, &bytes)
|
||||
.await
|
||||
{
|
||||
debug!(
|
||||
@@ -364,8 +423,8 @@ impl Node {
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
original_dest = %original.dest_addr,
|
||||
error_dest = %original.src_addr,
|
||||
original_dest = %dest,
|
||||
error_dest = %toward,
|
||||
bottleneck_mtu,
|
||||
"Sent MtuExceeded error signal"
|
||||
);
|
||||
|
||||
@@ -24,14 +24,14 @@ use crate::node::{Node, NodeError};
|
||||
use crate::noise::{
|
||||
HandshakeState, XK_HANDSHAKE_MSG1_SIZE, XK_HANDSHAKE_MSG2_SIZE, XK_HANDSHAKE_MSG3_SIZE,
|
||||
};
|
||||
use crate::proto::routing::{CoordsRequired, MtuExceeded, PathBroken};
|
||||
#[cfg(unix)]
|
||||
use crate::protocol::LinkMessageType;
|
||||
#[cfg(unix)]
|
||||
use crate::protocol::SESSION_DATAGRAM_HEADER_SIZE;
|
||||
use crate::protocol::{
|
||||
CoordsRequired, FspInnerFlags, MtuExceeded, PathBroken, PathMtuNotification, SessionAck,
|
||||
SessionDatagram, SessionMessageType, SessionMsg3, SessionReceiverReport, SessionSenderReport,
|
||||
SessionSetup,
|
||||
FspInnerFlags, PathMtuNotification, SessionAck, SessionDatagram, SessionMessageType,
|
||||
SessionMsg3, SessionReceiverReport, SessionSenderReport, SessionSetup,
|
||||
};
|
||||
use crate::protocol::{coords_wire_size, encode_coords};
|
||||
#[cfg(unix)]
|
||||
@@ -1125,7 +1125,7 @@ impl Node {
|
||||
// Send standalone CoordsWarmup immediately (rate-limited)
|
||||
if self
|
||||
.coords_response_rate_limiter
|
||||
.should_send(&msg.dest_addr)
|
||||
.should_send(&msg.dest_addr, Self::now_ms())
|
||||
{
|
||||
if let Some(entry) = self.sessions.get(&msg.dest_addr)
|
||||
&& entry.is_established()
|
||||
@@ -1186,7 +1186,7 @@ impl Node {
|
||||
// Send standalone CoordsWarmup immediately (rate-limited)
|
||||
if self
|
||||
.coords_response_rate_limiter
|
||||
.should_send(&msg.dest_addr)
|
||||
.should_send(&msg.dest_addr, Self::now_ms())
|
||||
{
|
||||
if let Some(entry) = self.sessions.get(&msg.dest_addr)
|
||||
&& entry.is_established()
|
||||
|
||||
+4
-37
@@ -90,43 +90,10 @@ pub struct ForwardingMetrics {
|
||||
}
|
||||
|
||||
/// Route class of a transit-forwarded packet, classified from tree
|
||||
/// coordinates at the forwarding decision point. The six variants
|
||||
/// partition `forwarded_packets` exactly.
|
||||
///
|
||||
/// Two variants are up-and-over forwards (destination not in the chosen
|
||||
/// peer's subtree); they differ in whether they depend on a child
|
||||
/// advertising cross-link reach *upward* to its parent:
|
||||
/// - `TreeDownCross`: the chosen peer is our tree descendant, but the
|
||||
/// destination is *not* in that child's subtree. The forward only fired
|
||||
/// because the child advertised cross-link reach upward to us, beyond its
|
||||
/// own subtree. If children advertised only their subtree upward, this
|
||||
/// forward would route up instead, so its count measures how much
|
||||
/// forwarding depends on the upward cross-link advertisement — the
|
||||
/// dive-to-tree-child cut-through.
|
||||
/// - `CrosslinkAscend`: the chosen peer is lateral (neither ancestor nor
|
||||
/// descendant) and the destination is not in its subtree. This is a node
|
||||
/// using its *own* cross-link, learned via the peer's split-horizon
|
||||
/// advertisement to its neighbors, so it does not depend on any upward
|
||||
/// advertisement. Tracked alongside `TreeDownCross` as the lateral
|
||||
/// up-and-over contrast.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RouteClass {
|
||||
/// Chosen peer is our ancestor (tree-up).
|
||||
TreeUp,
|
||||
/// Chosen peer is our descendant and dest is in its subtree (canonical
|
||||
/// tree-down).
|
||||
TreeDown,
|
||||
/// Chosen peer is our descendant but dest is *not* in its subtree: the
|
||||
/// dive-to-tree-child cut-through enabled by upward cross-link
|
||||
/// advertisement.
|
||||
TreeDownCross,
|
||||
/// Chosen peer is lateral and dest is in its subtree (subtree entry).
|
||||
CrosslinkDescend,
|
||||
/// Chosen peer is lateral and dest is not in its subtree (up-and-over).
|
||||
CrosslinkAscend,
|
||||
/// Chosen peer is the destination itself (degenerate direct hop).
|
||||
DirectPeer,
|
||||
}
|
||||
/// coordinates at the forwarding decision point. Defined by the sans-IO
|
||||
/// routing core and re-exported here for the forwarding-metrics counters
|
||||
/// ([`ForwardingMetrics::record_route_class`]).
|
||||
pub(crate) use crate::proto::routing::RouteClass;
|
||||
|
||||
impl ForwardingMetrics {
|
||||
/// Record a received packet of `bytes` payload (packets and bytes).
|
||||
|
||||
+99
-141
@@ -18,7 +18,6 @@ mod rate_limit;
|
||||
pub(crate) mod reject;
|
||||
mod reloadable;
|
||||
mod retry;
|
||||
mod routing_error_rate_limit;
|
||||
pub(crate) mod session;
|
||||
pub(crate) mod session_wire;
|
||||
pub(crate) mod stats;
|
||||
@@ -30,7 +29,6 @@ pub(crate) mod wire;
|
||||
|
||||
use self::rate_limit::HandshakeRateLimiter;
|
||||
use self::reloadable::Reloadable;
|
||||
use self::routing_error_rate_limit::RoutingErrorRateLimiter;
|
||||
|
||||
/// Half-range of the symmetric jitter applied to the per-session rekey timer.
|
||||
/// Each session draws an offset uniformly from `[-REKEY_JITTER_SECS,
|
||||
@@ -47,6 +45,7 @@ use crate::cache::CoordCache;
|
||||
use crate::node::session::SessionEntry;
|
||||
use crate::peer::{ActivePeer, PeerConnection};
|
||||
use crate::proto::discovery::{Discovery, DiscoveryBackoff, DiscoveryForwardRateLimiter};
|
||||
use crate::proto::routing::{self, Router, RoutingErrorRateLimiter};
|
||||
#[cfg(unix)]
|
||||
use crate::transport::ethernet::EthernetTransport;
|
||||
use crate::transport::nym::NymTransport;
|
||||
@@ -62,7 +61,7 @@ use crate::upper::hosts::HostMap;
|
||||
use crate::upper::icmp_rate_limit::IcmpRateLimiter;
|
||||
use crate::upper::tun::{TunError, TunOutboundRx, TunState, TunTx};
|
||||
use crate::utils::index::IndexAllocator;
|
||||
use crate::{Config, ConfigError, Identity, IdentityError, NodeAddr, PeerIdentity};
|
||||
use crate::{Config, ConfigError, Identity, IdentityError, NodeAddr, PeerIdentity, TreeCoordinate};
|
||||
use rand::Rng;
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::fmt;
|
||||
@@ -415,8 +414,8 @@ pub struct Node {
|
||||
msg1_rate_limiter: HandshakeRateLimiter,
|
||||
/// Rate limiter for ICMP Packet Too Big messages.
|
||||
icmp_rate_limiter: IcmpRateLimiter,
|
||||
/// Rate limiter for routing error signals (CoordsRequired / PathBroken).
|
||||
routing_error_rate_limiter: RoutingErrorRateLimiter,
|
||||
/// Routing-subsystem state (routing error-signal rate limiter).
|
||||
routing: Router,
|
||||
/// Rate limiter for source-side CoordsRequired/PathBroken responses.
|
||||
coords_response_rate_limiter: RoutingErrorRateLimiter,
|
||||
|
||||
@@ -658,9 +657,9 @@ impl Node {
|
||||
pending_outbound: HashMap::new(),
|
||||
msg1_rate_limiter,
|
||||
icmp_rate_limiter: IcmpRateLimiter::new(),
|
||||
routing_error_rate_limiter: RoutingErrorRateLimiter::new(),
|
||||
coords_response_rate_limiter: RoutingErrorRateLimiter::with_interval(
|
||||
std::time::Duration::from_millis(coords_response_interval_ms),
|
||||
routing: Router::new(),
|
||||
coords_response_rate_limiter: RoutingErrorRateLimiter::with_interval_ms(
|
||||
coords_response_interval_ms,
|
||||
),
|
||||
discovery: Discovery::new(
|
||||
DiscoveryBackoff::with_params(backoff_base_secs, backoff_max_secs),
|
||||
@@ -817,9 +816,9 @@ impl Node {
|
||||
pending_outbound: HashMap::new(),
|
||||
msg1_rate_limiter,
|
||||
icmp_rate_limiter: IcmpRateLimiter::new(),
|
||||
routing_error_rate_limiter: RoutingErrorRateLimiter::new(),
|
||||
coords_response_rate_limiter: RoutingErrorRateLimiter::with_interval(
|
||||
std::time::Duration::from_millis(coords_response_interval_ms),
|
||||
routing: Router::new(),
|
||||
coords_response_rate_limiter: RoutingErrorRateLimiter::with_interval_ms(
|
||||
coords_response_interval_ms,
|
||||
),
|
||||
discovery: Discovery::new(DiscoveryBackoff::new(), DiscoveryForwardRateLimiter::new()),
|
||||
pending_connects: Vec::new(),
|
||||
@@ -2607,11 +2606,27 @@ impl Node {
|
||||
|
||||
// 3. Bloom filter candidates — requires dest_coords for loop-free selection.
|
||||
// If no candidate is strictly closer, fall through to tree routing.
|
||||
let candidates: Vec<&ActivePeer> = self.destination_in_filters(dest_node_addr);
|
||||
// The sans-IO core assembles the candidate snapshot over the
|
||||
// `RoutingView` seam (enumerate peers, apply the bloom `may_reach`
|
||||
// filter, snapshot each), then picks the winner; the shell supplies
|
||||
// only the raw per-peer reads.
|
||||
let candidates = {
|
||||
let view = NodeRoutingView {
|
||||
coord_cache: &self.coord_cache,
|
||||
peers: &self.peers,
|
||||
tree_state: &self.tree_state,
|
||||
congested: false,
|
||||
};
|
||||
routing::routing_candidates(&view, dest_node_addr)
|
||||
};
|
||||
if !candidates.is_empty()
|
||||
&& let Some(peer) = self.select_best_candidate(&candidates, &dest_coords)
|
||||
&& let Some(next_hop) = routing::select_best_candidate(
|
||||
&candidates,
|
||||
&dest_coords,
|
||||
self.tree_state.my_coords(),
|
||||
)
|
||||
{
|
||||
return Some(peer);
|
||||
return self.peers.get(&next_hop);
|
||||
}
|
||||
|
||||
// 4. Greedy tree routing fallback
|
||||
@@ -2622,142 +2637,29 @@ impl Node {
|
||||
|
||||
/// Classify a transit forward by route class from tree coordinates.
|
||||
///
|
||||
/// Called at the transit chokepoint after `find_next_hop` returns a peer,
|
||||
/// so the six classes partition `forwarded_packets` exactly. The branch
|
||||
/// that `find_next_hop` took (bloom vs greedy-tree) is *not* the route
|
||||
/// class: a peer can be selected by either, so the cut-through splits
|
||||
/// (`TreeDownCross`, `CrosslinkAscend`) are decided here from coordinates,
|
||||
/// not from which branch fired.
|
||||
///
|
||||
/// Inputs: our coords (`tree_state.my_coords`), the chosen peer's coords
|
||||
/// (`tree_state.peer_coords`), and the destination coords (re-read from the
|
||||
/// coord cache, which `find_next_hop` just touched). Both the tree-down and
|
||||
/// cross-link branches split on whether the destination is in the chosen
|
||||
/// peer's subtree; when the dest coords are unavailable that test defaults
|
||||
/// to "not in subtree", i.e. the up-and-over variant (`TreeDownCross` for a
|
||||
/// descendant peer, `CrosslinkAscend` for a lateral one).
|
||||
/// Thin shell adapter over the pure [`routing::classify_forward`]: it
|
||||
/// pre-resolves the destination coordinates from the coord cache (the
|
||||
/// sole impurity — a read-only lookup, no LRU touch) and reads our own
|
||||
/// and the chosen peer's coordinates from tree state, then defers the
|
||||
/// six-way classification to the sans-IO routing core.
|
||||
pub(crate) fn classify_forward(
|
||||
&self,
|
||||
dest: &NodeAddr,
|
||||
chosen_peer: &NodeAddr,
|
||||
) -> metrics::RouteClass {
|
||||
// Degenerate: the next hop is the destination itself (Branch 2).
|
||||
if chosen_peer == dest {
|
||||
return metrics::RouteClass::DirectPeer;
|
||||
}
|
||||
|
||||
let my_addr = self.node_addr();
|
||||
let my_coords = self.tree_state.my_coords();
|
||||
|
||||
// Tree-up: the chosen peer is our ancestor.
|
||||
if my_coords.has_ancestor(chosen_peer) {
|
||||
return metrics::RouteClass::TreeUp;
|
||||
}
|
||||
|
||||
// Whether the destination is in the chosen peer's subtree. Both the
|
||||
// tree-down and cross-link splits below turn on this same test, so it
|
||||
// is computed once. On the live transit path the dest coords are
|
||||
// always present here: `find_next_hop` looks them up with an early
|
||||
// return, so a coord-cache miss yields no next hop to classify (the
|
||||
// caller signals `CoordsRequired` instead of forwarding). The miss
|
||||
// branch below is therefore defensive — reachable only by direct
|
||||
// unit-test calls — and defaults the test to "not in subtree", i.e.
|
||||
// the up-and-over variant of whichever branch fires (TreeDownCross for
|
||||
// a descendant peer, CrosslinkAscend for a lateral one), matching the
|
||||
// original cross-link default-to-ascend.
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
let dest_in_peer_subtree = self
|
||||
.coord_cache
|
||||
.get(dest, now_ms)
|
||||
.is_some_and(|dest_coords| dest_coords.has_ancestor(chosen_peer));
|
||||
|
||||
// Tree-down: the chosen peer is our descendant (we are its ancestor).
|
||||
// Split by subtree membership: a dest genuinely below the child is the
|
||||
// canonical tree-down; a dest *not* below it means we only forwarded
|
||||
// down because the child advertised cross-link reach upward, beyond its
|
||||
// own subtree — the dive-to-tree-child cut-through (TreeDownCross).
|
||||
if let Some(peer_coords) = self.tree_state.peer_coords(chosen_peer)
|
||||
&& peer_coords.has_ancestor(my_addr)
|
||||
{
|
||||
return if dest_in_peer_subtree {
|
||||
metrics::RouteClass::TreeDown
|
||||
} else {
|
||||
metrics::RouteClass::TreeDownCross
|
||||
};
|
||||
}
|
||||
|
||||
// Cross-link (lateral): split by whether the destination is in the
|
||||
// chosen peer's subtree. Descend = subtree entry; ascend = up-and-over
|
||||
// via the node's own cross-link (learned from the peer's split-horizon
|
||||
// advertisement, independent of any upward advertisement).
|
||||
if dest_in_peer_subtree {
|
||||
return metrics::RouteClass::CrosslinkDescend;
|
||||
}
|
||||
|
||||
metrics::RouteClass::CrosslinkAscend
|
||||
}
|
||||
|
||||
/// Select the best peer from a set of bloom filter candidates.
|
||||
///
|
||||
/// Uses distance from each candidate's tree coordinates to the destination
|
||||
/// as the primary metric (after link_cost). Only selects peers that are
|
||||
/// strictly closer to the destination than we are (self-distance check
|
||||
/// prevents routing loops).
|
||||
///
|
||||
/// Ordering: `(link_cost, distance_to_dest, node_addr)`.
|
||||
fn select_best_candidate<'a>(
|
||||
&'a self,
|
||||
candidates: &[&'a ActivePeer],
|
||||
dest_coords: &crate::tree::TreeCoordinate,
|
||||
) -> Option<&'a ActivePeer> {
|
||||
let my_distance = self.tree_state.my_coords().distance_to(dest_coords);
|
||||
|
||||
let mut best: Option<(&ActivePeer, f64, usize)> = None;
|
||||
|
||||
for &candidate in candidates {
|
||||
if !candidate.can_send() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let cost = candidate.link_cost();
|
||||
|
||||
let dist = self
|
||||
.tree_state
|
||||
.peer_coords(candidate.node_addr())
|
||||
.map(|pc| pc.distance_to(dest_coords))
|
||||
.unwrap_or(usize::MAX);
|
||||
|
||||
// Self-distance check: only consider peers strictly closer
|
||||
// to the destination than we are (prevents routing loops)
|
||||
if dist >= my_distance {
|
||||
continue;
|
||||
}
|
||||
|
||||
let dominated = match &best {
|
||||
None => true,
|
||||
Some((_, best_cost, best_dist)) => {
|
||||
cost < *best_cost
|
||||
|| (cost == *best_cost && dist < *best_dist)
|
||||
|| (cost == *best_cost
|
||||
&& dist == *best_dist
|
||||
&& candidate.node_addr() < best.as_ref().unwrap().0.node_addr())
|
||||
}
|
||||
};
|
||||
|
||||
if dominated {
|
||||
best = Some((candidate, cost, dist));
|
||||
}
|
||||
}
|
||||
|
||||
best.map(|(peer, _, _)| peer)
|
||||
}
|
||||
|
||||
/// Check if a destination is in any peer's bloom filter.
|
||||
pub fn destination_in_filters(&self, dest: &NodeAddr) -> Vec<&ActivePeer> {
|
||||
self.peers.values().filter(|p| p.may_reach(dest)).collect()
|
||||
let dest_coords = self.coord_cache.get(dest, now_ms).cloned();
|
||||
routing::classify_forward(
|
||||
dest,
|
||||
chosen_peer,
|
||||
self.node_addr(),
|
||||
self.tree_state.my_coords(),
|
||||
dest_coords.as_ref(),
|
||||
self.tree_state.peer_coords(chosen_peer),
|
||||
)
|
||||
}
|
||||
|
||||
/// Get the TUN packet sender channel.
|
||||
@@ -3002,6 +2904,62 @@ impl Node {
|
||||
}
|
||||
}
|
||||
|
||||
/// Shell-side [`routing::RoutingView`] seam over live `Node` state — the sole
|
||||
/// routing read adapter the shell retains. It hands the sans-IO routing core
|
||||
/// raw per-peer reads (enumeration plus `may_reach` / `can_send` / `link_cost`
|
||||
/// / `coords`) so the candidate assembly, selection, and error synthesis all
|
||||
/// live in `proto::routing::core`; no routing decision or assembly logic
|
||||
/// remains here.
|
||||
///
|
||||
/// Field-narrowed to `coord_cache` + `peers` + `tree_state` (never `&Node`
|
||||
/// whole) so it borrows disjointly from `&mut self.routing` on the
|
||||
/// forward/synth path, where the handler also holds the mutable `Router`.
|
||||
///
|
||||
/// Two call sites:
|
||||
/// - `find_next_hop` builds it to assemble bloom candidates via the `peer_*`
|
||||
/// reads; it never queries `is_congested`, so it leaves `congested` false.
|
||||
/// - `handle_session_datagram` builds it for `Router::route` / `synth_*`,
|
||||
/// which read `is_congested` (precomputed once for the resolved next hop)
|
||||
/// and `cached_coords`.
|
||||
pub(in crate::node) struct NodeRoutingView<'a> {
|
||||
pub(in crate::node) coord_cache: &'a CoordCache,
|
||||
pub(in crate::node) peers: &'a HashMap<NodeAddr, ActivePeer>,
|
||||
pub(in crate::node) tree_state: &'a TreeState,
|
||||
pub(in crate::node) congested: bool,
|
||||
}
|
||||
|
||||
impl routing::RoutingView for NodeRoutingView<'_> {
|
||||
fn is_congested(&self, _next_hop: &NodeAddr) -> bool {
|
||||
self.congested
|
||||
}
|
||||
|
||||
fn cached_coords(&self, dest: &NodeAddr, now_ms: u64) -> Option<TreeCoordinate> {
|
||||
self.coord_cache.get(dest, now_ms).cloned()
|
||||
}
|
||||
|
||||
fn peer_addrs(&self) -> Vec<NodeAddr> {
|
||||
self.peers.keys().copied().collect()
|
||||
}
|
||||
|
||||
fn peer_may_reach(&self, peer: &NodeAddr, dest: &NodeAddr) -> bool {
|
||||
self.peers.get(peer).is_some_and(|p| p.may_reach(dest))
|
||||
}
|
||||
|
||||
fn peer_can_send(&self, peer: &NodeAddr) -> bool {
|
||||
self.peers.get(peer).is_some_and(|p| p.can_send())
|
||||
}
|
||||
|
||||
fn peer_link_cost(&self, peer: &NodeAddr) -> f64 {
|
||||
self.peers
|
||||
.get(peer)
|
||||
.map_or(f64::INFINITY, |p| p.link_cost())
|
||||
}
|
||||
|
||||
fn peer_coords(&self, peer: &NodeAddr) -> Option<TreeCoordinate> {
|
||||
self.tree_state.peer_coords(peer).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
/// Project an MMP metrics block into the snapshot
|
||||
/// [`EntityMmp`](crate::control::snapshot::EntityMmp) shared by `show_peers`
|
||||
/// (link-layer, `path_mtu = None`) and `show_sessions` (session-layer,
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
//! Routing error signal rate limiting.
|
||||
//!
|
||||
//! Prevents routing error floods (CoordsRequired / PathBroken) by
|
||||
//! rate-limiting error signals per destination address at transit nodes.
|
||||
|
||||
use crate::NodeAddr;
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Rate limiter for routing error signals (CoordsRequired / PathBroken).
|
||||
///
|
||||
/// Tracks the last time a routing error was sent for each destination
|
||||
/// address and enforces a minimum interval to prevent floods.
|
||||
pub struct RoutingErrorRateLimiter {
|
||||
/// Maps destination NodeAddr to the last time we sent an error about it.
|
||||
last_sent: HashMap<NodeAddr, Instant>,
|
||||
/// Minimum interval between error signals for the same destination.
|
||||
min_interval: Duration,
|
||||
/// Maximum age of entries before cleanup.
|
||||
max_age: Duration,
|
||||
}
|
||||
|
||||
impl RoutingErrorRateLimiter {
|
||||
/// Create a new rate limiter.
|
||||
///
|
||||
/// Default: max 10 errors/sec per destination (100ms interval).
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
last_sent: HashMap::new(),
|
||||
min_interval: Duration::from_millis(100),
|
||||
max_age: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a rate limiter with a custom minimum interval.
|
||||
pub fn with_interval(min_interval: Duration) -> Self {
|
||||
Self {
|
||||
last_sent: HashMap::new(),
|
||||
min_interval,
|
||||
max_age: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if we should send a routing error for this destination.
|
||||
///
|
||||
/// Returns true if enough time has passed since the last error for
|
||||
/// this destination, or if this is the first error. Updates internal
|
||||
/// state when returning true.
|
||||
pub fn should_send(&mut self, dest_addr: &NodeAddr) -> bool {
|
||||
let now = Instant::now();
|
||||
|
||||
if let Some(&last) = self.last_sent.get(dest_addr)
|
||||
&& now.duration_since(last) < self.min_interval
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
self.last_sent.insert(*dest_addr, now);
|
||||
self.cleanup(now);
|
||||
true
|
||||
}
|
||||
|
||||
/// Remove entries older than max_age.
|
||||
fn cleanup(&mut self, now: Instant) {
|
||||
self.last_sent
|
||||
.retain(|_, &mut last| now.duration_since(last) < self.max_age);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn len(&self) -> usize {
|
||||
self.last_sent.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RoutingErrorRateLimiter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::thread;
|
||||
|
||||
fn addr(val: u8) -> NodeAddr {
|
||||
let mut bytes = [0u8; 16];
|
||||
bytes[0] = val;
|
||||
NodeAddr::from_bytes(bytes)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_first_send_allowed() {
|
||||
let mut limiter = RoutingErrorRateLimiter::new();
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rapid_sends_rate_limited() {
|
||||
let mut limiter = RoutingErrorRateLimiter::new();
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
assert!(!limiter.should_send(&addr(1)));
|
||||
assert!(!limiter.should_send(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_different_destinations_independent() {
|
||||
let mut limiter = RoutingErrorRateLimiter::new();
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
assert!(limiter.should_send(&addr(2)));
|
||||
assert!(!limiter.should_send(&addr(1)));
|
||||
assert!(!limiter.should_send(&addr(2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_send_allowed_after_interval() {
|
||||
let mut limiter = RoutingErrorRateLimiter::new();
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
|
||||
thread::sleep(Duration::from_millis(110));
|
||||
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_removes_old_entries() {
|
||||
let mut limiter = RoutingErrorRateLimiter::new();
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
assert!(limiter.should_send(&addr(2)));
|
||||
assert_eq!(limiter.len(), 2);
|
||||
|
||||
let future = Instant::now() + Duration::from_secs(11);
|
||||
limiter.cleanup(future);
|
||||
assert_eq!(limiter.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_preserves_recent_entries() {
|
||||
let mut limiter = RoutingErrorRateLimiter::new();
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
assert_eq!(limiter.len(), 1);
|
||||
|
||||
limiter.cleanup(Instant::now());
|
||||
assert_eq!(limiter.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_interval_custom_rate() {
|
||||
let mut limiter = RoutingErrorRateLimiter::with_interval(Duration::from_millis(500));
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
assert!(!limiter.should_send(&addr(1)));
|
||||
|
||||
// Still rate-limited after 200ms (would pass with default 100ms)
|
||||
thread::sleep(Duration::from_millis(200));
|
||||
assert!(!limiter.should_send(&addr(1)));
|
||||
|
||||
// Allowed after 500ms total
|
||||
thread::sleep(Duration::from_millis(350));
|
||||
assert!(limiter.should_send(&addr(1)));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user