Files
fips/src/node/handlers/forwarding.rs
T
Johnathan Corgan 274b09d4ff node: classify transit forwards by route class; regroup fipstop routing tab
Add six forwarding counters that partition transit-forwarded packets by their
tree relationship to the chosen next hop: tree-up (peer is our ancestor),
tree-down (peer is our descendant and the destination is within its subtree),
tree-down-cross (peer is our descendant but the destination is outside its
subtree), cross-link descend (lateral peer, destination within its subtree),
cross-link ascend (lateral peer, destination outside its subtree), and
direct-peer. The six classes sum to forwarded_packets, asserted by a unit
test. Classification is computed from tree coordinates at the transit
chokepoint, so the error-signal routing callers are excluded.

The two "outside the chosen peer's subtree" classes are both up-and-over
forwards but differ in what they depend on. Tree-down-cross is the
dive-to-tree-child cut-through: we forward down to our own child for a
destination not beneath it, which is only possible because the child
advertised cross-link reach upward to us, beyond its own subtree. Its count
measures how much forwarding depends on that upward advertisement, i.e. what
would change if cross-link advertisements were narrowed to subtree-entry only.
Cross-link ascend, by contrast, uses the node's own lateral cross-link learned
from a peer's split-horizon advertisement, so it does not depend on any upward
advertisement.

Surface the counters through the forwarding stats snapshot (control socket,
show_routing and show_status) and reorganize the fipstop routing tab so its
two columns separate own/endpoint traffic (received, delivered, originated)
from forwarded/transit traffic (the route-class breakdown and drop reasons),
with the tree-down-cross line visually flagged.
2026-06-20 14:56:54 +00:00

429 lines
17 KiB
Rust

//! SessionDatagram forwarding handler.
//!
//! Handles incoming SessionDatagram (0x00) link messages: decodes the
//! envelope, enforces hop limits, performs coordinate cache warming from
//! plaintext session-layer headers, routes to the next hop or delivers
//! locally, and generates error signals on routing failure.
use crate::NodeAddr;
use crate::node::reject::ForwardingReject;
use crate::node::session_wire::{
FSP_COMMON_PREFIX_SIZE, FSP_HEADER_SIZE, FSP_PHASE_ESTABLISHED, FSP_PHASE_MSG1, FSP_PHASE_MSG2,
FspCommonPrefix, parse_encrypted_coords,
};
use crate::node::{Node, NodeError};
use crate::protocol::{
CoordsRequired, MtuExceeded, PathBroken, SessionAck, SessionDatagram, SessionDatagramRef,
SessionSetup,
};
use std::time::{Duration, Instant};
use tracing::{debug, warn};
impl Node {
/// Handle an incoming SessionDatagram from a peer.
///
/// Called by `dispatch_link_message` for msg_type 0x00. The payload
/// has already had its msg_type byte stripped by dispatch.
pub(in crate::node) async fn handle_session_datagram(
&mut self,
_from: &NodeAddr,
payload: &[u8],
incoming_ce: bool,
) {
self.metrics().forwarding.record_received(payload.len());
let datagram_ref = match SessionDatagramRef::decode(payload) {
Ok(dg) => dg,
Err(e) => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::DecodeError, payload.len());
debug!(error = %e, "Malformed SessionDatagram");
return;
}
};
// TTL enforcement: decrement for forwarding and drop only if the
// received datagram was already exhausted.
if datagram_ref.ttl == 0 {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
debug!(
src = %datagram_ref.src_addr,
dest = %datagram_ref.dest_addr,
"SessionDatagram TTL exhausted, dropping"
);
return;
}
let forwarded_ttl = datagram_ref.ttl - 1;
// Coordinate cache warming from plaintext session-layer headers
self.try_warm_coord_cache_ref(&datagram_ref);
// Local delivery: dispatch to session layer handlers without
// materializing an owned SessionDatagram payload Vec.
if datagram_ref.dest_addr == *self.node_addr() {
self.metrics().forwarding.record_delivered(payload.len());
self.handle_session_payload(
&datagram_ref.src_addr,
datagram_ref.payload,
datagram_ref.path_mtu,
incoming_ce,
)
.await;
return;
}
let mut datagram = datagram_ref.into_owned();
datagram.ttl = forwarded_ttl;
// Find next hop toward destination
let next_hop_addr = match self.find_next_hop(&datagram.dest_addr) {
Some(peer) => *peer.node_addr(),
None => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::NoRoute, payload.len());
debug!(
src = %self.peer_display_name(&datagram.src_addr),
dest = %self.peer_display_name(&datagram.dest_addr),
bytes = payload.len(),
"Dropping transit SessionDatagram: no route to destination"
);
self.send_routing_error(&datagram).await;
return;
}
};
// Apply path_mtu min() from the outgoing link's transport MTU
if let Some(peer) = self.peers.get(&next_hop_addr)
&& let Some(tid) = peer.transport_id()
&& let Some(transport) = self.transports.get(&tid)
{
if let Some(addr) = peer.current_addr() {
datagram.path_mtu = datagram.path_mtu.min(transport.link_mtu(addr));
} else {
datagram.path_mtu = datagram.path_mtu.min(transport.mtu());
}
}
// ECN CE relay: propagate incoming CE and detect local congestion
let local_congestion = self.detect_congestion(&next_hop_addr);
let outgoing_ce = incoming_ce || local_congestion;
if local_congestion {
self.metrics().congestion.congestion_detected.inc();
let now = Instant::now();
let should_log = self
.last_congestion_log
.map(|t| now.duration_since(t) >= Duration::from_secs(5))
.unwrap_or(true);
if should_log {
self.last_congestion_log = Some(now);
debug!(next_hop = %next_hop_addr, "Congestion detected, CE flag set on forwarded packet");
}
}
// Forward: re-encode (includes 0x00 type byte) and send
let encoded = datagram.encode();
if let Err(e) = self
.send_encrypted_link_message_with_ce(&next_hop_addr, &encoded, outgoing_ce)
.await
{
match e {
NodeError::MtuExceeded { mtu, .. } => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::MtuExceeded, payload.len());
self.send_mtu_exceeded_error(&datagram, mtu).await;
}
_ => {
self.metrics()
.forwarding
.record_reject_bytes(ForwardingReject::SendError, payload.len());
debug!(
next_hop = %next_hop_addr,
dest = %datagram.dest_addr,
error = %e,
"Failed to forward SessionDatagram"
);
}
}
} else {
self.metrics().forwarding.record_forwarded(encoded.len());
// Classify this transit forward by route class (partition of
// forwarded_packets). Done here, at the data-plane chokepoint, so
// the error-signal routing callers of find_next_hop are excluded.
let class = self.classify_forward(&datagram.dest_addr, &next_hop_addr);
self.metrics().forwarding.record_route_class(class);
if outgoing_ce {
self.metrics().congestion.ce_forwarded.inc();
}
}
}
/// Attempt to warm the coordinate cache from session-layer payload headers.
///
/// Transit routers parse the 4-byte FSP common prefix to identify message
/// type, then extract plaintext coordinate fields from:
/// - SessionSetup (phase 0x1): src_coords + dest_coords
/// - SessionAck (phase 0x2): src_coords
/// - Encrypted with CP flag (phase 0x0): cleartext coords between header and ciphertext
///
/// Decode failures are logged and silently ignored — they don't block
/// forwarding.
fn try_warm_coord_cache_ref(&mut self, datagram: &SessionDatagramRef<'_>) {
let prefix = match FspCommonPrefix::parse(datagram.payload) {
Some(p) => p,
None => return,
};
let inner = &datagram.payload[FSP_COMMON_PREFIX_SIZE..];
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
match prefix.phase {
FSP_PHASE_MSG1 => match SessionSetup::decode(inner) {
Ok(setup) => {
self.coord_cache_mut()
.insert(datagram.src_addr, setup.src_coords, now_ms);
self.coord_cache_mut()
.insert(datagram.dest_addr, setup.dest_coords, now_ms);
debug!(
src = %datagram.src_addr,
dest = %datagram.dest_addr,
"Cached coords from SessionSetup"
);
}
Err(e) => {
debug!(error = %e, "Failed to decode SessionSetup for cache warming");
}
},
FSP_PHASE_MSG2 => match SessionAck::decode(inner) {
Ok(ack) => {
self.coord_cache_mut()
.insert(datagram.src_addr, ack.src_coords, now_ms);
self.coord_cache_mut()
.insert(datagram.dest_addr, ack.dest_coords, now_ms);
debug!(
src = %datagram.src_addr,
dest = %datagram.dest_addr,
"Cached coords from SessionAck"
);
}
Err(e) => {
debug!(error = %e, "Failed to decode SessionAck for cache warming");
}
},
FSP_PHASE_ESTABLISHED if prefix.has_coords() => {
// CP flag set: coords in cleartext between header and ciphertext.
// Parse coords from the cleartext section after the 12-byte header.
// inner starts after the 4-byte prefix, so we need 8 more bytes
// for the counter (header is 12 total = 4 prefix + 8 counter).
let coord_data = &datagram.payload[FSP_HEADER_SIZE..];
match parse_encrypted_coords(coord_data) {
Ok((src_coords, dest_coords, _bytes_consumed)) => {
if let Some(coords) = src_coords {
self.coord_cache_mut()
.insert(datagram.src_addr, coords, now_ms);
}
if let Some(coords) = dest_coords {
self.coord_cache_mut()
.insert(datagram.dest_addr, coords, now_ms);
}
debug!(
src = %datagram.src_addr,
dest = %datagram.dest_addr,
"Cached coords from encrypted message"
);
}
Err(e) => {
debug!(error = %e, "Failed to parse coords for cache warming");
}
}
}
_ => {
// Phase 0x0 without CP, error signals, unknown: no coords to cache
}
}
}
/// Generate and send a routing error signal back to the datagram's source.
///
/// If we have cached coords for the destination, send PathBroken (we know
/// where it is but can't reach it). Otherwise send CoordsRequired (we
/// don't know where it is).
///
/// If we can't route the error back to the source either, drop silently.
/// No cascading errors.
async fn send_routing_error(&mut self, original: &SessionDatagram) {
// Rate limit: one error signal per destination per 100ms
if !self
.routing_error_rate_limiter
.should_send(&original.dest_addr)
{
return;
}
let my_addr = *self.node_addr();
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let error_payload =
if let Some(coords) = self.coord_cache().get(&original.dest_addr, now_ms) {
let coords = coords.clone();
PathBroken::new(original.dest_addr, my_addr)
.with_last_coords(coords)
.encode()
} else {
CoordsRequired::new(original.dest_addr, my_addr).encode()
};
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
.with_ttl(self.config().node.session.default_ttl);
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
Some(peer) => *peer.node_addr(),
None => {
debug!(
src = %original.src_addr,
dest = %original.dest_addr,
"Cannot route error signal back to source, dropping"
);
return;
}
};
let encoded = error_dg.encode();
if let Err(e) = self
.send_encrypted_link_message(&next_hop_addr, &encoded)
.await
{
debug!(
next_hop = %next_hop_addr,
error = %e,
"Failed to send routing error signal"
);
} else {
debug!(
original_dest = %original.dest_addr,
error_dest = %original.src_addr,
"Sent routing error signal"
);
}
}
/// Generate and send an MtuExceeded error signal back to the datagram's source.
///
/// Called when `send_encrypted_link_message()` fails with
/// `NodeError::MtuExceeded` during forwarding. The signal tells the
/// source the bottleneck MTU so it can immediately reduce its path MTU.
async fn send_mtu_exceeded_error(&mut self, original: &SessionDatagram, bottleneck_mtu: u16) {
// Rate limit: reuse routing_error_rate_limiter keyed on dest_addr
if !self
.routing_error_rate_limiter
.should_send(&original.dest_addr)
{
return;
}
let my_addr = *self.node_addr();
let error_payload = MtuExceeded::new(original.dest_addr, my_addr, bottleneck_mtu).encode();
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
.with_ttl(self.config().node.session.default_ttl);
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
Some(peer) => *peer.node_addr(),
None => {
debug!(
src = %original.src_addr,
dest = %original.dest_addr,
"Cannot route MtuExceeded signal back to source, dropping"
);
return;
}
};
let encoded = error_dg.encode();
if let Err(e) = self
.send_encrypted_link_message(&next_hop_addr, &encoded)
.await
{
debug!(
next_hop = %next_hop_addr,
error = %e,
"Failed to send MtuExceeded error signal"
);
} else {
debug!(
original_dest = %original.dest_addr,
error_dest = %original.src_addr,
bottleneck_mtu,
"Sent MtuExceeded error signal"
);
}
}
/// Detect congestion for CE marking on forwarded datagrams.
///
/// Checks two signal sources:
/// 1. Outgoing link MMP metrics (loss rate, ETX) against configured thresholds
/// 2. Local transport congestion (kernel drops on any transport)
///
/// Returns `true` if any signal indicates congestion.
pub(in crate::node) fn detect_congestion(&self, next_hop: &NodeAddr) -> bool {
if !self.config().node.ecn.enabled {
return false;
}
// Outgoing link MMP metrics
if let Some(peer) = self.peers.get(next_hop)
&& let Some(mmp) = peer.mmp()
{
let metrics = &mmp.metrics;
if metrics.loss_rate() >= self.config().node.ecn.loss_threshold
|| metrics.etx >= self.config().node.ecn.etx_threshold
{
return true;
}
}
// Local transport congestion (kernel drops)
self.transport_drops.values().any(|s| s.dropping)
}
/// Sample transport congestion indicators.
///
/// Called from the tick handler (1s interval). For each transport,
/// queries the cumulative kernel drop counter and sets the `dropping`
/// flag if new drops occurred since the previous sample.
pub(in crate::node) fn sample_transport_congestion(&mut self) {
let mut new_drop_events = Vec::new();
for (&tid, transport) in &self.transports {
let congestion = transport.congestion();
let state = self.transport_drops.entry(tid).or_default();
if let Some(current) = congestion.recv_drops {
let new_drops = current > state.prev_drops;
if new_drops && !state.dropping {
new_drop_events.push(tid);
}
state.dropping = new_drops;
state.prev_drops = current;
}
}
for tid in new_drop_events {
self.metrics().congestion.kernel_drop_events.inc();
warn!(
transport_id = tid.as_u32(),
"Kernel recv drops first observed on transport"
);
}
}
}