mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 08:14:42 +00:00
Tune overly aggressive discovery rate limiting
The default 30s post-failure backoff (300s cap, doubling per consecutive failure) was set to bound traffic from chatty apps looking up unreachable targets, but in practice it dominates cold-start mesh convergence: a single timed-out lookup during initial bloom-filter propagation suppresses any retry for 30s, and the existing reset triggers (parent change, new peer, first RTT, reconnection) don't fire on a stable post-handshake topology. The suppression window winds up dictating the protocol's effective time-to-converge instead of bounding repeat traffic. Replaces the single-lookup-with-internal-retry model (`timeout_secs`/`retry_interval_secs`/`max_attempts`) with a per-attempt timeout sequence in `node.discovery.attempt_timeouts_secs`, defaulting to `[1, 2, 4, 8]`. Each attempt sends a fresh LookupRequest with a new random request_id so successive attempts can take different forwarding paths as the bloom and tree state evolve. The destination is declared unreachable only after the sequence is exhausted (15s total at the default). Disables post-failure suppression by default (`backoff_base_secs`/ `backoff_max_secs` now `0`/`0`). The `DiscoveryBackoff` machinery stays in tree (inert at zero base/cap); operators with chatty apps generating repeat lookups against unreachable destinations can opt back in. `PendingLookup` field shape unchanged so the control-socket `show_routing` JSON (`pending_lookups[].attempt`/`initiated_ms`/ `last_sent_ms`) keeps the same schema for fipstop and external consumers; `last_sent_ms` now means "current-attempt start" under the new state machine.
This commit is contained in:
@@ -2,10 +2,12 @@
|
||||
//!
|
||||
//! Two complementary mechanisms:
|
||||
//!
|
||||
//! - **`DiscoveryBackoff`** (originator-side): Exponential backoff for failed
|
||||
//! lookups. After a lookup times out, suppresses re-initiation with
|
||||
//! increasing delays (30s → 60s → 300s cap). Reset on topology changes
|
||||
//! (parent change, new peer, first RTT, reconnection).
|
||||
//! - **`DiscoveryBackoff`** (originator-side, optional): Exponential
|
||||
//! suppression of fresh lookups after the per-attempt sequence in
|
||||
//! `node.discovery.attempt_timeouts_secs` has been exhausted.
|
||||
//! **Disabled by default** (base/cap = 0); the per-attempt sequence
|
||||
//! is the only retry pacing in the standard configuration. Reset on
|
||||
//! topology changes (parent change, new peer, first RTT, reconnection).
|
||||
//!
|
||||
//! - **`DiscoveryForwardRateLimiter`** (transit-side): Per-target minimum
|
||||
//! interval for forwarded requests. Defense-in-depth against misbehaving
|
||||
@@ -19,11 +21,11 @@ use std::time::{Duration, Instant};
|
||||
// Originator-side: Discovery Backoff
|
||||
// ============================================================================
|
||||
|
||||
/// Default base backoff after first lookup failure.
|
||||
const DEFAULT_BACKOFF_BASE_SECS: u64 = 30;
|
||||
/// Default base backoff after first lookup failure. `0` = disabled.
|
||||
const DEFAULT_BACKOFF_BASE_SECS: u64 = 0;
|
||||
|
||||
/// Default maximum backoff cap.
|
||||
const DEFAULT_BACKOFF_MAX_SECS: u64 = 300;
|
||||
/// Default maximum backoff cap. `0` = disabled.
|
||||
const DEFAULT_BACKOFF_MAX_SECS: u64 = 0;
|
||||
|
||||
/// Backoff multiplier per consecutive failure.
|
||||
const BACKOFF_MULTIPLIER: u64 = 2;
|
||||
@@ -49,7 +51,7 @@ struct BackoffEntry {
|
||||
}
|
||||
|
||||
impl DiscoveryBackoff {
|
||||
/// Create with default parameters (30s base, 300s cap).
|
||||
/// Create with default parameters (disabled — base/cap = 0).
|
||||
pub fn new() -> Self {
|
||||
Self::with_params(DEFAULT_BACKOFF_BASE_SECS, DEFAULT_BACKOFF_MAX_SECS)
|
||||
}
|
||||
@@ -245,7 +247,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_backoff_suppressed_after_failure() {
|
||||
let mut backoff = DiscoveryBackoff::new();
|
||||
// Backoff is opt-in; exercise the suppression path with explicit params.
|
||||
let mut backoff = DiscoveryBackoff::with_params(30, 300);
|
||||
backoff.record_failure(&addr(1));
|
||||
assert!(backoff.is_suppressed(&addr(1)));
|
||||
// Different target not affected
|
||||
@@ -254,7 +257,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_backoff_cleared_on_success() {
|
||||
let mut backoff = DiscoveryBackoff::new();
|
||||
let mut backoff = DiscoveryBackoff::with_params(30, 300);
|
||||
backoff.record_failure(&addr(1));
|
||||
assert!(backoff.is_suppressed(&addr(1)));
|
||||
|
||||
|
||||
@@ -423,31 +423,26 @@ impl Node {
|
||||
|
||||
/// Initiate a discovery lookup if one is not already pending for this target.
|
||||
///
|
||||
/// Checks: pending dedup, backoff, bloom filter pre-check. If all pass,
|
||||
/// initiates the lookup. If no tree peers have the target in their bloom
|
||||
/// filter, the lookup is skipped (bloom miss) and recorded as a failure
|
||||
/// for backoff purposes.
|
||||
/// Checks: pending dedup, post-failure backoff (off by default), bloom
|
||||
/// filter pre-check. If all pass, sends the first attempt's LookupRequest.
|
||||
/// Subsequent attempts (with fresh request_ids) are scheduled by
|
||||
/// [`Self::check_pending_lookups`] when each attempt's per-attempt timeout
|
||||
/// expires, using the sequence in `node.discovery.attempt_timeouts_secs`.
|
||||
pub(in crate::node) async fn maybe_initiate_lookup(&mut self, dest: &NodeAddr) {
|
||||
let now_ms = Self::now_ms();
|
||||
let lookup_timeout_ms = self.config.node.discovery.timeout_secs * 1000;
|
||||
|
||||
// Check pending lookup dedup (in-flight)
|
||||
if let Some(entry) = self.pending_lookups.get(dest) {
|
||||
let age_ms = now_ms.saturating_sub(entry.initiated_ms);
|
||||
let attempt = entry.attempt;
|
||||
if age_ms < lookup_timeout_ms {
|
||||
self.stats_mut().discovery.req_deduplicated += 1;
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
age_ms = age_ms,
|
||||
attempt = attempt,
|
||||
"Discovery lookup deduplicated, already pending"
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Dedup: any pending lookup means we are already trying.
|
||||
if self.pending_lookups.contains_key(dest) {
|
||||
self.stats_mut().discovery.req_deduplicated += 1;
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
"Discovery lookup deduplicated, already pending"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check backoff from previous failures
|
||||
// Optional post-failure suppression. Defaults are 0/0 (inert);
|
||||
// operators can opt in by setting `node.discovery.backoff_*_secs`.
|
||||
if self.discovery_backoff.is_suppressed(dest) {
|
||||
self.stats_mut().discovery.req_backoff_suppressed += 1;
|
||||
debug!(
|
||||
@@ -487,27 +482,33 @@ impl Node {
|
||||
}
|
||||
}
|
||||
|
||||
/// Check pending lookups for retry or timeout.
|
||||
/// Check pending lookups for next-attempt or final timeout.
|
||||
///
|
||||
/// Called periodically from the tick handler. For each pending lookup:
|
||||
/// - If retry interval elapsed and attempts remain: resend
|
||||
/// - If total timeout elapsed: fail, record backoff, send ICMP unreachable
|
||||
/// Called periodically from the tick handler. The lookup state machine
|
||||
/// runs through `node.discovery.attempt_timeouts_secs` (default
|
||||
/// `[1, 2, 4, 8]`): each entry is the deadline for one attempt. When the
|
||||
/// current attempt's deadline elapses:
|
||||
/// - If more entries remain: send the next attempt with a fresh
|
||||
/// `request_id`.
|
||||
/// - Otherwise: declare the destination unreachable, drop queued packets,
|
||||
/// and emit ICMPv6 destination-unreachable for each.
|
||||
pub(in crate::node) async fn check_pending_lookups(&mut self, now_ms: u64) {
|
||||
let timeout_ms = self.config.node.discovery.timeout_secs * 1000;
|
||||
let retry_ms = self.config.node.discovery.retry_interval_secs * 1000;
|
||||
let timeouts = self.config.node.discovery.attempt_timeouts_secs.clone();
|
||||
let max_attempts = timeouts.len() as u8;
|
||||
|
||||
// Collect targets needing action
|
||||
let mut to_retry: Vec<NodeAddr> = Vec::new();
|
||||
let mut to_timeout: Vec<NodeAddr> = Vec::new();
|
||||
|
||||
for (&target, entry) in &self.pending_lookups {
|
||||
let age = now_ms.saturating_sub(entry.initiated_ms);
|
||||
if age >= timeout_ms {
|
||||
to_timeout.push(target);
|
||||
} else if entry.attempt < self.config.node.discovery.max_attempts
|
||||
&& now_ms.saturating_sub(entry.last_sent_ms) >= retry_ms
|
||||
{
|
||||
to_retry.push(target);
|
||||
let attempt_idx = (entry.attempt as usize).saturating_sub(1);
|
||||
let attempt_timeout_ms = timeouts.get(attempt_idx).copied().unwrap_or(0) * 1000;
|
||||
if now_ms.saturating_sub(entry.last_sent_ms) >= attempt_timeout_ms {
|
||||
if entry.attempt >= max_attempts {
|
||||
to_timeout.push(target);
|
||||
} else {
|
||||
to_retry.push(target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -535,7 +536,7 @@ impl Node {
|
||||
self.stats_mut().discovery.resp_timed_out += 1;
|
||||
self.pending_lookups.remove(&addr);
|
||||
|
||||
// Record failure for backoff
|
||||
// Record failure for optional backoff
|
||||
self.discovery_backoff.record_failure(&addr);
|
||||
let failures = self.discovery_backoff.failure_count(&addr);
|
||||
|
||||
|
||||
+1
-1
@@ -260,7 +260,7 @@ struct PendingConnect {
|
||||
///
|
||||
/// The `addr_to_link` map enables dispatching incoming packets to the right
|
||||
/// connection before authentication completes.
|
||||
// Discovery lookup constants moved to config: node.discovery.timeout_secs, node.discovery.ttl
|
||||
// Discovery lookup constants moved to config: node.discovery.attempt_timeouts_secs, node.discovery.ttl
|
||||
pub struct Node {
|
||||
// === Identity ===
|
||||
/// This node's cryptographic identity.
|
||||
|
||||
Reference in New Issue
Block a user