mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 00:04:54 +00:00
Add six forwarding counters that partition transit-forwarded packets by their tree relationship to the chosen next hop: tree-up (peer is our ancestor), tree-down (peer is our descendant and the destination is within its subtree), tree-down-cross (peer is our descendant but the destination is outside its subtree), cross-link descend (lateral peer, destination within its subtree), cross-link ascend (lateral peer, destination outside its subtree), and direct-peer. The six classes sum to forwarded_packets, asserted by a unit test. Classification is computed from tree coordinates at the transit chokepoint, so the error-signal routing callers are excluded. The two "outside the chosen peer's subtree" classes are both up-and-over forwards but differ in what they depend on. Tree-down-cross is the dive-to-tree-child cut-through: we forward down to our own child for a destination not beneath it, which is only possible because the child advertised cross-link reach upward to us, beyond its own subtree. Its count measures how much forwarding depends on that upward advertisement, i.e. what would change if cross-link advertisements were narrowed to subtree-entry only. Cross-link ascend, by contrast, uses the node's own lateral cross-link learned from a peer's split-horizon advertisement, so it does not depend on any upward advertisement. Surface the counters through the forwarding stats snapshot (control socket, show_routing and show_status) and reorganize the fipstop routing tab so its two columns separate own/endpoint traffic (received, delivered, originated) from forwarded/transit traffic (the route-class breakdown and drop reasons), with the tree-down-cross line visually flagged.
429 lines
17 KiB
Rust
429 lines
17 KiB
Rust
//! SessionDatagram forwarding handler.
|
|
//!
|
|
//! Handles incoming SessionDatagram (0x00) link messages: decodes the
|
|
//! envelope, enforces hop limits, performs coordinate cache warming from
|
|
//! plaintext session-layer headers, routes to the next hop or delivers
|
|
//! locally, and generates error signals on routing failure.
|
|
|
|
use crate::NodeAddr;
|
|
use crate::node::reject::ForwardingReject;
|
|
use crate::node::session_wire::{
|
|
FSP_COMMON_PREFIX_SIZE, FSP_HEADER_SIZE, FSP_PHASE_ESTABLISHED, FSP_PHASE_MSG1, FSP_PHASE_MSG2,
|
|
FspCommonPrefix, parse_encrypted_coords,
|
|
};
|
|
use crate::node::{Node, NodeError};
|
|
use crate::protocol::{
|
|
CoordsRequired, MtuExceeded, PathBroken, SessionAck, SessionDatagram, SessionDatagramRef,
|
|
SessionSetup,
|
|
};
|
|
use std::time::{Duration, Instant};
|
|
use tracing::{debug, warn};
|
|
|
|
impl Node {
|
|
/// Handle an incoming SessionDatagram from a peer.
|
|
///
|
|
/// Called by `dispatch_link_message` for msg_type 0x00. The payload
|
|
/// has already had its msg_type byte stripped by dispatch.
|
|
pub(in crate::node) async fn handle_session_datagram(
|
|
&mut self,
|
|
_from: &NodeAddr,
|
|
payload: &[u8],
|
|
incoming_ce: bool,
|
|
) {
|
|
self.metrics().forwarding.record_received(payload.len());
|
|
|
|
let datagram_ref = match SessionDatagramRef::decode(payload) {
|
|
Ok(dg) => dg,
|
|
Err(e) => {
|
|
self.metrics()
|
|
.forwarding
|
|
.record_reject_bytes(ForwardingReject::DecodeError, payload.len());
|
|
debug!(error = %e, "Malformed SessionDatagram");
|
|
return;
|
|
}
|
|
};
|
|
|
|
// TTL enforcement: decrement for forwarding and drop only if the
|
|
// received datagram was already exhausted.
|
|
if datagram_ref.ttl == 0 {
|
|
self.metrics()
|
|
.forwarding
|
|
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
|
|
debug!(
|
|
src = %datagram_ref.src_addr,
|
|
dest = %datagram_ref.dest_addr,
|
|
"SessionDatagram TTL exhausted, dropping"
|
|
);
|
|
return;
|
|
}
|
|
let forwarded_ttl = datagram_ref.ttl - 1;
|
|
|
|
// Coordinate cache warming from plaintext session-layer headers
|
|
self.try_warm_coord_cache_ref(&datagram_ref);
|
|
|
|
// Local delivery: dispatch to session layer handlers without
|
|
// materializing an owned SessionDatagram payload Vec.
|
|
if datagram_ref.dest_addr == *self.node_addr() {
|
|
self.metrics().forwarding.record_delivered(payload.len());
|
|
self.handle_session_payload(
|
|
&datagram_ref.src_addr,
|
|
datagram_ref.payload,
|
|
datagram_ref.path_mtu,
|
|
incoming_ce,
|
|
)
|
|
.await;
|
|
return;
|
|
}
|
|
|
|
let mut datagram = datagram_ref.into_owned();
|
|
datagram.ttl = forwarded_ttl;
|
|
|
|
// Find next hop toward destination
|
|
let next_hop_addr = match self.find_next_hop(&datagram.dest_addr) {
|
|
Some(peer) => *peer.node_addr(),
|
|
None => {
|
|
self.metrics()
|
|
.forwarding
|
|
.record_reject_bytes(ForwardingReject::NoRoute, payload.len());
|
|
debug!(
|
|
src = %self.peer_display_name(&datagram.src_addr),
|
|
dest = %self.peer_display_name(&datagram.dest_addr),
|
|
bytes = payload.len(),
|
|
"Dropping transit SessionDatagram: no route to destination"
|
|
);
|
|
self.send_routing_error(&datagram).await;
|
|
return;
|
|
}
|
|
};
|
|
|
|
// Apply path_mtu min() from the outgoing link's transport MTU
|
|
if let Some(peer) = self.peers.get(&next_hop_addr)
|
|
&& let Some(tid) = peer.transport_id()
|
|
&& let Some(transport) = self.transports.get(&tid)
|
|
{
|
|
if let Some(addr) = peer.current_addr() {
|
|
datagram.path_mtu = datagram.path_mtu.min(transport.link_mtu(addr));
|
|
} else {
|
|
datagram.path_mtu = datagram.path_mtu.min(transport.mtu());
|
|
}
|
|
}
|
|
|
|
// ECN CE relay: propagate incoming CE and detect local congestion
|
|
let local_congestion = self.detect_congestion(&next_hop_addr);
|
|
let outgoing_ce = incoming_ce || local_congestion;
|
|
if local_congestion {
|
|
self.metrics().congestion.congestion_detected.inc();
|
|
let now = Instant::now();
|
|
let should_log = self
|
|
.last_congestion_log
|
|
.map(|t| now.duration_since(t) >= Duration::from_secs(5))
|
|
.unwrap_or(true);
|
|
if should_log {
|
|
self.last_congestion_log = Some(now);
|
|
debug!(next_hop = %next_hop_addr, "Congestion detected, CE flag set on forwarded packet");
|
|
}
|
|
}
|
|
|
|
// Forward: re-encode (includes 0x00 type byte) and send
|
|
let encoded = datagram.encode();
|
|
if let Err(e) = self
|
|
.send_encrypted_link_message_with_ce(&next_hop_addr, &encoded, outgoing_ce)
|
|
.await
|
|
{
|
|
match e {
|
|
NodeError::MtuExceeded { mtu, .. } => {
|
|
self.metrics()
|
|
.forwarding
|
|
.record_reject_bytes(ForwardingReject::MtuExceeded, payload.len());
|
|
self.send_mtu_exceeded_error(&datagram, mtu).await;
|
|
}
|
|
_ => {
|
|
self.metrics()
|
|
.forwarding
|
|
.record_reject_bytes(ForwardingReject::SendError, payload.len());
|
|
debug!(
|
|
next_hop = %next_hop_addr,
|
|
dest = %datagram.dest_addr,
|
|
error = %e,
|
|
"Failed to forward SessionDatagram"
|
|
);
|
|
}
|
|
}
|
|
} else {
|
|
self.metrics().forwarding.record_forwarded(encoded.len());
|
|
// Classify this transit forward by route class (partition of
|
|
// forwarded_packets). Done here, at the data-plane chokepoint, so
|
|
// the error-signal routing callers of find_next_hop are excluded.
|
|
let class = self.classify_forward(&datagram.dest_addr, &next_hop_addr);
|
|
self.metrics().forwarding.record_route_class(class);
|
|
if outgoing_ce {
|
|
self.metrics().congestion.ce_forwarded.inc();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Attempt to warm the coordinate cache from session-layer payload headers.
|
|
///
|
|
/// Transit routers parse the 4-byte FSP common prefix to identify message
|
|
/// type, then extract plaintext coordinate fields from:
|
|
/// - SessionSetup (phase 0x1): src_coords + dest_coords
|
|
/// - SessionAck (phase 0x2): src_coords
|
|
/// - Encrypted with CP flag (phase 0x0): cleartext coords between header and ciphertext
|
|
///
|
|
/// Decode failures are logged and silently ignored — they don't block
|
|
/// forwarding.
|
|
fn try_warm_coord_cache_ref(&mut self, datagram: &SessionDatagramRef<'_>) {
|
|
let prefix = match FspCommonPrefix::parse(datagram.payload) {
|
|
Some(p) => p,
|
|
None => return,
|
|
};
|
|
|
|
let inner = &datagram.payload[FSP_COMMON_PREFIX_SIZE..];
|
|
|
|
let now_ms = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map(|d| d.as_millis() as u64)
|
|
.unwrap_or(0);
|
|
|
|
match prefix.phase {
|
|
FSP_PHASE_MSG1 => match SessionSetup::decode(inner) {
|
|
Ok(setup) => {
|
|
self.coord_cache_mut()
|
|
.insert(datagram.src_addr, setup.src_coords, now_ms);
|
|
self.coord_cache_mut()
|
|
.insert(datagram.dest_addr, setup.dest_coords, now_ms);
|
|
debug!(
|
|
src = %datagram.src_addr,
|
|
dest = %datagram.dest_addr,
|
|
"Cached coords from SessionSetup"
|
|
);
|
|
}
|
|
Err(e) => {
|
|
debug!(error = %e, "Failed to decode SessionSetup for cache warming");
|
|
}
|
|
},
|
|
FSP_PHASE_MSG2 => match SessionAck::decode(inner) {
|
|
Ok(ack) => {
|
|
self.coord_cache_mut()
|
|
.insert(datagram.src_addr, ack.src_coords, now_ms);
|
|
self.coord_cache_mut()
|
|
.insert(datagram.dest_addr, ack.dest_coords, now_ms);
|
|
debug!(
|
|
src = %datagram.src_addr,
|
|
dest = %datagram.dest_addr,
|
|
"Cached coords from SessionAck"
|
|
);
|
|
}
|
|
Err(e) => {
|
|
debug!(error = %e, "Failed to decode SessionAck for cache warming");
|
|
}
|
|
},
|
|
FSP_PHASE_ESTABLISHED if prefix.has_coords() => {
|
|
// CP flag set: coords in cleartext between header and ciphertext.
|
|
// Parse coords from the cleartext section after the 12-byte header.
|
|
// inner starts after the 4-byte prefix, so we need 8 more bytes
|
|
// for the counter (header is 12 total = 4 prefix + 8 counter).
|
|
let coord_data = &datagram.payload[FSP_HEADER_SIZE..];
|
|
match parse_encrypted_coords(coord_data) {
|
|
Ok((src_coords, dest_coords, _bytes_consumed)) => {
|
|
if let Some(coords) = src_coords {
|
|
self.coord_cache_mut()
|
|
.insert(datagram.src_addr, coords, now_ms);
|
|
}
|
|
if let Some(coords) = dest_coords {
|
|
self.coord_cache_mut()
|
|
.insert(datagram.dest_addr, coords, now_ms);
|
|
}
|
|
debug!(
|
|
src = %datagram.src_addr,
|
|
dest = %datagram.dest_addr,
|
|
"Cached coords from encrypted message"
|
|
);
|
|
}
|
|
Err(e) => {
|
|
debug!(error = %e, "Failed to parse coords for cache warming");
|
|
}
|
|
}
|
|
}
|
|
_ => {
|
|
// Phase 0x0 without CP, error signals, unknown: no coords to cache
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Generate and send a routing error signal back to the datagram's source.
|
|
///
|
|
/// If we have cached coords for the destination, send PathBroken (we know
|
|
/// where it is but can't reach it). Otherwise send CoordsRequired (we
|
|
/// don't know where it is).
|
|
///
|
|
/// If we can't route the error back to the source either, drop silently.
|
|
/// No cascading errors.
|
|
async fn send_routing_error(&mut self, original: &SessionDatagram) {
|
|
// Rate limit: one error signal per destination per 100ms
|
|
if !self
|
|
.routing_error_rate_limiter
|
|
.should_send(&original.dest_addr)
|
|
{
|
|
return;
|
|
}
|
|
|
|
let my_addr = *self.node_addr();
|
|
|
|
let now_ms = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map(|d| d.as_millis() as u64)
|
|
.unwrap_or(0);
|
|
|
|
let error_payload =
|
|
if let Some(coords) = self.coord_cache().get(&original.dest_addr, now_ms) {
|
|
let coords = coords.clone();
|
|
PathBroken::new(original.dest_addr, my_addr)
|
|
.with_last_coords(coords)
|
|
.encode()
|
|
} else {
|
|
CoordsRequired::new(original.dest_addr, my_addr).encode()
|
|
};
|
|
|
|
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
|
|
.with_ttl(self.config().node.session.default_ttl);
|
|
|
|
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
|
|
Some(peer) => *peer.node_addr(),
|
|
None => {
|
|
debug!(
|
|
src = %original.src_addr,
|
|
dest = %original.dest_addr,
|
|
"Cannot route error signal back to source, dropping"
|
|
);
|
|
return;
|
|
}
|
|
};
|
|
|
|
let encoded = error_dg.encode();
|
|
if let Err(e) = self
|
|
.send_encrypted_link_message(&next_hop_addr, &encoded)
|
|
.await
|
|
{
|
|
debug!(
|
|
next_hop = %next_hop_addr,
|
|
error = %e,
|
|
"Failed to send routing error signal"
|
|
);
|
|
} else {
|
|
debug!(
|
|
original_dest = %original.dest_addr,
|
|
error_dest = %original.src_addr,
|
|
"Sent routing error signal"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Generate and send an MtuExceeded error signal back to the datagram's source.
|
|
///
|
|
/// Called when `send_encrypted_link_message()` fails with
|
|
/// `NodeError::MtuExceeded` during forwarding. The signal tells the
|
|
/// source the bottleneck MTU so it can immediately reduce its path MTU.
|
|
async fn send_mtu_exceeded_error(&mut self, original: &SessionDatagram, bottleneck_mtu: u16) {
|
|
// Rate limit: reuse routing_error_rate_limiter keyed on dest_addr
|
|
if !self
|
|
.routing_error_rate_limiter
|
|
.should_send(&original.dest_addr)
|
|
{
|
|
return;
|
|
}
|
|
|
|
let my_addr = *self.node_addr();
|
|
|
|
let error_payload = MtuExceeded::new(original.dest_addr, my_addr, bottleneck_mtu).encode();
|
|
|
|
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
|
|
.with_ttl(self.config().node.session.default_ttl);
|
|
|
|
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
|
|
Some(peer) => *peer.node_addr(),
|
|
None => {
|
|
debug!(
|
|
src = %original.src_addr,
|
|
dest = %original.dest_addr,
|
|
"Cannot route MtuExceeded signal back to source, dropping"
|
|
);
|
|
return;
|
|
}
|
|
};
|
|
|
|
let encoded = error_dg.encode();
|
|
if let Err(e) = self
|
|
.send_encrypted_link_message(&next_hop_addr, &encoded)
|
|
.await
|
|
{
|
|
debug!(
|
|
next_hop = %next_hop_addr,
|
|
error = %e,
|
|
"Failed to send MtuExceeded error signal"
|
|
);
|
|
} else {
|
|
debug!(
|
|
original_dest = %original.dest_addr,
|
|
error_dest = %original.src_addr,
|
|
bottleneck_mtu,
|
|
"Sent MtuExceeded error signal"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Detect congestion for CE marking on forwarded datagrams.
|
|
///
|
|
/// Checks two signal sources:
|
|
/// 1. Outgoing link MMP metrics (loss rate, ETX) against configured thresholds
|
|
/// 2. Local transport congestion (kernel drops on any transport)
|
|
///
|
|
/// Returns `true` if any signal indicates congestion.
|
|
pub(in crate::node) fn detect_congestion(&self, next_hop: &NodeAddr) -> bool {
|
|
if !self.config().node.ecn.enabled {
|
|
return false;
|
|
}
|
|
// Outgoing link MMP metrics
|
|
if let Some(peer) = self.peers.get(next_hop)
|
|
&& let Some(mmp) = peer.mmp()
|
|
{
|
|
let metrics = &mmp.metrics;
|
|
if metrics.loss_rate() >= self.config().node.ecn.loss_threshold
|
|
|| metrics.etx >= self.config().node.ecn.etx_threshold
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
// Local transport congestion (kernel drops)
|
|
self.transport_drops.values().any(|s| s.dropping)
|
|
}
|
|
|
|
/// Sample transport congestion indicators.
|
|
///
|
|
/// Called from the tick handler (1s interval). For each transport,
|
|
/// queries the cumulative kernel drop counter and sets the `dropping`
|
|
/// flag if new drops occurred since the previous sample.
|
|
pub(in crate::node) fn sample_transport_congestion(&mut self) {
|
|
let mut new_drop_events = Vec::new();
|
|
for (&tid, transport) in &self.transports {
|
|
let congestion = transport.congestion();
|
|
let state = self.transport_drops.entry(tid).or_default();
|
|
if let Some(current) = congestion.recv_drops {
|
|
let new_drops = current > state.prev_drops;
|
|
if new_drops && !state.dropping {
|
|
new_drop_events.push(tid);
|
|
}
|
|
state.dropping = new_drops;
|
|
state.prev_drops = current;
|
|
}
|
|
}
|
|
for tid in new_drop_events {
|
|
self.metrics().congestion.kernel_drop_events.inc();
|
|
warn!(
|
|
transport_id = tid.as_u32(),
|
|
"Kernel recv drops first observed on transport"
|
|
);
|
|
}
|
|
}
|
|
}
|