Merge maint into master (admission-gate Msg2 silent-drop + integration suite)

This commit is contained in:
Johnathan Corgan
2026-05-26 20:45:50 +00:00
5 changed files with 508 additions and 0 deletions
+11
View File
@@ -145,6 +145,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`handshake.rs:1114` is unchanged. Introduces a shared
`Node::outbound_admission_check()` helper so the invariant is
grep-able and unit-testable.
- Inbound `handle_msg1` now silent-drops at `node.limits.max_peers`
saturation *before* building/sending Msg2, instead of replying with
Msg2 and then rejecting at `promote_connection`. Adds an early cap
check positioned after identity verification (so the
reconnect / cross-connection bypass for known peers still fires) and
before index allocation + Msg2 wire send. The late cap check inside
`promote_connection` is intentionally retained as
defense-in-depth. Wire savings observed in a 45 s ops tcpdump at
saturation: ~3.6 cap-denials/s × Msg2 (~104 B + AEAD compute) each.
Bigger win is cleaner peer-side semantics — no fake-completed
handshake whose subsequent data frames fail decryption on this side.
- Mesh-size estimator (`compute_mesh_size`) no longer double-counts the
parent's bloom cardinality during the transient cache window after a
local parent-switch. Symptom: `fipsctl show status` / fipstop displayed
+29
View File
@@ -207,6 +207,35 @@ impl Node {
possible_restart = true;
}
// Early cap check: at max_peers and this is a net-new identity?
// Bypass for known peers (reconnect / cross-connection) — admitting
// them doesn't grow peers.len(). This silent-drops the Msg1 before
// the Msg2 build/send and index allocation, avoiding wasted wire
// bytes and giving the peer cleaner semantics (no fake-completed
// handshake whose data frames subsequently fail decryption here).
// The late cap check inside promote_connection() is intentionally
// left in place as defense-in-depth.
if self.max_peers > 0 && self.peers.len() >= self.max_peers {
let is_known_active = self.peers.contains_key(&peer_node_addr);
let is_pending_outbound = self.connections.iter().any(|(_, conn)| {
conn.expected_identity()
.map(|id| *id.node_addr() == peer_node_addr)
.unwrap_or(false)
});
if !is_known_active && !is_pending_outbound {
debug!(
peer = %self.peer_display_name(&peer_node_addr),
max = self.max_peers,
"Silent-dropping Msg1 at max_peers cap (early gate; no Msg2 sent)"
);
// `link_id` was allocated above but `conn` is still a local
// (not yet inserted into self.connections / self.links /
// self.addr_to_link), so the local drop suffices.
self.msg1_rate_limiter.complete_handshake();
return;
}
}
// Epoch-based restart detection and duplicate msg1 handling.
//
// If we fell through from the addr_to_link check above with
+224
View File
@@ -1526,3 +1526,227 @@ fn nostr_discovery_outbound_admission_atomic_roundtrip() {
"after recovery store: traversal initiator/responder must see true"
);
}
/// Sender-side helper: build a wire-format Msg1 from a fresh peer
/// identity targeting `node_b`, *and* send it on the wire over `socket_a`
/// to `addr_b`. Returns the sender's NodeAddr so the test can assert on
/// identity-keyed maps.
///
/// Uses the same outbound-PeerConnection->Noise IK pattern as the
/// integration handshake tests, but inlined and unit-scoped.
async fn craft_and_send_msg1(
node_b: &Node,
sender_identity: &Identity,
socket_a: &tokio::net::UdpSocket,
addr_b: std::net::SocketAddr,
timestamp_ms: u64,
) -> NodeAddr {
use crate::node::wire::build_msg1;
use crate::utils::index::SessionIndex;
let peer_b_identity = PeerIdentity::from_pubkey_full(node_b.identity.pubkey_full());
let sender_pubkey_id = PeerIdentity::from_pubkey_full(sender_identity.pubkey_full());
let sender_node_addr = *sender_pubkey_id.node_addr();
let link_id = LinkId::new(0xDEAD_BEEF);
let mut conn = PeerConnection::outbound(link_id, peer_b_identity, timestamp_ms);
let sender_keypair = sender_identity.keypair();
let mut startup_epoch = [0u8; 8];
rand::Rng::fill_bytes(&mut rand::rng(), &mut startup_epoch);
let noise_msg1 = conn
.start_handshake(sender_keypair, startup_epoch, timestamp_ms)
.expect("start_handshake should produce noise msg1");
let sender_index = SessionIndex::new(0x5151);
let wire_msg1 = build_msg1(sender_index, &noise_msg1);
socket_a
.send_to(&wire_msg1, addr_b)
.await
.expect("sender_socket.send_to");
sender_node_addr
}
/// Helper: deliver a packet from `node`'s registered UDP transport to
/// `node.handle_msg1`. Returns Ok(()) on success or Err if the packet
/// was not received within `timeout`.
async fn pump_one_msg1_into_node(
node: &mut Node,
packet_rx: &mut crate::transport::PacketRx,
timeout_ms: u64,
) -> Result<(), &'static str> {
use tokio::time::{Duration, timeout};
let packet = timeout(Duration::from_millis(timeout_ms), packet_rx.recv())
.await
.map_err(|_| "timed out waiting for msg1 on packet_rx")?
.ok_or("packet channel closed")?;
node.handle_msg1(packet).await;
Ok(())
}
/// Verifies the early max_peers cap check in `handle_msg1` silent-drops
/// a Msg1 from a brand-new identity at saturation: no peer is admitted,
/// no Msg2 response goes back on the wire, and the msg1 rate-limiter
/// pending_count returns to baseline.
///
/// Wire-observable Msg2 absence is the load-bearing discriminator. With
/// the early cap gate removed (stash-verify), the late gate inside
/// `promote_connection` still rejects the new identity — but only
/// *after* `handle_msg1` has already built the Msg2 frame and
/// `transport.send(...wire_msg2)` has put it on the wire. The
/// post-call wire-side poll catches that Msg2 (FAIL pre-fix; the
/// silent timeout is the PASS post-fix).
#[tokio::test]
async fn handle_msg1_silent_drops_at_cap_for_new_peer() {
use crate::config::UdpConfig;
use tokio::time::{Duration, timeout};
let mut node = make_node();
node.set_max_peers(2);
inject_dummy_peers(&mut node, 2);
assert_eq!(node.peer_count(), 2, "precondition: at cap");
// === UDP transport setup for node_b (the unit under test) ===
let transport_id_b = TransportId::new(1);
let udp_config = UdpConfig {
bind_addr: Some("127.0.0.1:0".to_string()),
mtu: Some(1280),
..Default::default()
};
let (packet_tx_b, mut packet_rx_b) = packet_channel(64);
let mut transport_b = UdpTransport::new(transport_id_b, None, udp_config, packet_tx_b);
transport_b.start_async().await.unwrap();
let addr_b = transport_b.local_addr().unwrap();
node.transports
.insert(transport_id_b, TransportHandle::Udp(transport_b));
// === Sender-side socket ===
let socket_a = tokio::net::UdpSocket::bind("127.0.0.1:0")
.await
.expect("bind sender socket");
let before_peers = node.peer_count();
let before_pending = node.msg1_rate_limiter.pending_count();
// Fresh sender identity — never seen by `node`.
let sender = Identity::generate();
let sender_node_addr = craft_and_send_msg1(&node, &sender, &socket_a, addr_b, 1000).await;
// Sanity: new identity is not currently a peer.
assert!(
!node.peers.contains_key(&sender_node_addr),
"precondition: new sender not yet a peer"
);
// Pump the wire-arrived Msg1 into the node's handler.
pump_one_msg1_into_node(&mut node, &mut packet_rx_b, 1000)
.await
.expect("msg1 must reach packet_rx_b");
// Post-call state checks.
assert_eq!(
node.peer_count(),
before_peers,
"early cap gate must not adopt a new peer at saturation"
);
assert!(
!node.peers.contains_key(&sender_node_addr),
"new sender must not appear in peers map"
);
assert_eq!(
node.msg1_rate_limiter.pending_count(),
before_pending,
"rate limiter must rebalance: start_handshake() then \
complete_handshake() before silent-drop return"
);
// Wire-observable discriminator: with the early gate in place, no
// Msg2 should come back. With the gate removed, Msg2 IS sent
// before promote_connection rejects.
let mut buf = [0u8; 2048];
let recv = timeout(Duration::from_millis(300), socket_a.recv_from(&mut buf)).await;
let received_bytes = recv.ok().and_then(|inner| inner.ok()).map(|(n, _)| n);
assert!(
received_bytes.is_none(),
"Msg2 must NOT be sent in response when at max_peers cap; \
observed {received_bytes:?} wire bytes the fingerprint of \
the late-gate path replying with Msg2 before rejecting"
);
}
/// Verifies the bypass: at saturation, an inbound Msg1 from an
/// *existing* peer's identity is not silent-dropped by the early cap
/// check (the gate would otherwise wedge legitimate
/// reconnect/restart/rekey traffic against an at-cap node).
///
/// The cap-gate's `is_known_active = self.peers.contains_key(&peer_node_addr)`
/// branch admits this case; the downstream handling (restart-detect or
/// duplicate-msg1 resend) then runs per existing semantics. The
/// observable assertion here is the existing peer's continued
/// presence — the rate-limiter rebalance is the same in
/// bypass-admit and silent-drop, so this test isn't a discriminator
/// against the no-gate (stash) build; it's a regression check that the
/// gate doesn't accidentally evict known peers.
#[tokio::test]
async fn handle_msg1_admits_existing_peer_at_cap() {
use crate::config::UdpConfig;
let mut node = make_node();
node.set_max_peers(2);
inject_dummy_peers(&mut node, 1);
let existing_sender = Identity::generate();
let existing_pid = PeerIdentity::from_pubkey_full(existing_sender.pubkey_full());
let existing_node_addr = *existing_pid.node_addr();
let existing_link_id = LinkId::new(7777);
{
use crate::peer::ActivePeer;
let peer = ActivePeer::new(existing_pid, existing_link_id, 0);
node.peers.insert(existing_node_addr, peer);
}
assert_eq!(node.peer_count(), 2, "precondition: at cap");
let transport_id_b = TransportId::new(1);
let udp_config = UdpConfig {
bind_addr: Some("127.0.0.1:0".to_string()),
mtu: Some(1280),
..Default::default()
};
let (packet_tx_b, mut packet_rx_b) = packet_channel(64);
let mut transport_b = UdpTransport::new(transport_id_b, None, udp_config, packet_tx_b);
transport_b.start_async().await.unwrap();
let addr_b = transport_b.local_addr().unwrap();
node.transports
.insert(transport_id_b, TransportHandle::Udp(transport_b));
let socket_a = tokio::net::UdpSocket::bind("127.0.0.1:0")
.await
.expect("bind sender socket");
let before_pending = node.msg1_rate_limiter.pending_count();
let sender_node_addr =
craft_and_send_msg1(&node, &existing_sender, &socket_a, addr_b, 2000).await;
assert_eq!(
sender_node_addr, existing_node_addr,
"sanity: crafted msg1 carries the existing peer's NodeAddr"
);
pump_one_msg1_into_node(&mut node, &mut packet_rx_b, 1000)
.await
.expect("msg1 must reach packet_rx_b");
// Bypass must not evict the existing peer or grow peer count.
assert_eq!(node.peer_count(), 2, "peer count unchanged");
assert!(
node.peers.contains_key(&existing_node_addr),
"existing peer must still be present after bypass-admitted msg1"
);
assert_eq!(
node.msg1_rate_limiter.pending_count(),
before_pending,
"rate limiter must rebalance after the (bypass-admitted) handler returns"
);
}
+39
View File
@@ -57,6 +57,7 @@ ONLY_SUITE=""
# All integration suites matching ci.yml
STATIC_SUITES=(static-mesh static-chain)
REKEY_SUITES=(rekey rekey-accept-off rekey-outbound-only)
ADMISSION_SUITES=(admission-cap)
# Each entry: "display-name scenario [--flag value ...]"
CHAOS_SUITES=(
"smoke-10 smoke-10"
@@ -111,6 +112,9 @@ list_suites() {
echo " Rekey:"
for s in "${REKEY_SUITES[@]}"; do echo " $s"; done
echo ""
echo " Admission cap:"
for s in "${ADMISSION_SUITES[@]}"; do echo " $s"; done
echo ""
echo " Gateway:"
for s in "${GATEWAY_SUITES[@]}"; do echo " $s"; done
echo ""
@@ -320,6 +324,34 @@ run_rekey() {
record "rekey" $rc
}
# Run the admission-cap integration test
# Verifies the inbound max_peers early-gate silent-drops at scale by
# lowering node.max_peers on one mesh node and asserting via tcpdump
# that no Msg2 responses go to the sustained-retrying denied peers.
run_admission_cap() {
local compose="testing/static/docker-compose.yml"
local rc=0
info "[admission-cap] Generating configs"
bash testing/static/scripts/generate-configs.sh mesh || { record "admission-cap" 1; return; }
bash testing/static/scripts/admission-cap-test.sh inject-config || { record "admission-cap" 1; return; }
info "[admission-cap] Starting containers (mesh profile)"
docker compose -f "$compose" --profile mesh up -d || { record "admission-cap" 1; return; }
info "[admission-cap] Running admission-cap test"
if bash testing/static/scripts/admission-cap-test.sh; then
rc=0
else
rc=1
info "[admission-cap] Collecting failure logs"
docker compose -f "$compose" --profile mesh logs --no-color 2>&1 | tail -100
fi
docker compose -f "$compose" --profile mesh down --volumes --remove-orphans 2>/dev/null
record "admission-cap" $rc
}
# Run a chaos scenario
run_chaos() {
local name="$1"
@@ -568,6 +600,11 @@ run_integration() {
run_rekey_accept_off
run_rekey_outbound_only
# Admission cap (mesh profile, max_peers=1 on one node)
for _suite in "${ADMISSION_SUITES[@]}"; do
run_admission_cap
done
# Gateway
run_gateway
@@ -670,6 +707,8 @@ run_suite() {
run_rekey_accept_off ;;
rekey-outbound-only)
run_rekey_outbound_only ;;
admission-cap)
run_admission_cap ;;
gateway)
run_gateway ;;
acl-allowlist)
+205
View File
@@ -0,0 +1,205 @@
#!/bin/bash
# Integration test for the inbound max_peers admission gate.
#
# Verifies the silent-drop behavior of the early-gate in handle_msg1 at
# scale, using the mesh topology with one node's node.max_peers lowered
# to 1. This forces 2 of node-c's 3 configured peers (b, d, e) into a
# sustained denied state, and asserts via tcpdump that no Msg2 responses
# go back to those denied peers across a 60s capture window.
#
# Tested behavior:
# - Denied peers DO arrive at the cap'd node (inbound FMP-IK Msg1, 84 B)
# - Cap'd node sends NO Msg2 responses (104 B) to denied peers
# - Cap'd node maintains exactly max_peers active sessions
# - Admitted peer's session stays healthy throughout the window
#
# Usage:
# ./admission-cap-test.sh Run the test (containers must be up)
# ./admission-cap-test.sh inject-config Inject node.max_peers into generated configs
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CAP_NODE="${ADMISSION_CAP_NODE:-c}"
MAX_PEERS="${ADMISSION_MAX_PEERS:-1}"
CAPTURE_SECS="${ADMISSION_CAPTURE_SECS:-60}"
TOPOLOGY="mesh"
TOPO_FILE="$SCRIPT_DIR/../configs/topologies/$TOPOLOGY.yaml"
# ── inject-config subcommand ─────────────────────────────────────────
# Inject node.max_peers into generated configs. Called separately by CI
# before building Docker images.
if [ "${1:-}" = "inject-config" ]; then
echo "Injecting node.limits.max_peers: $MAX_PEERS into node-$CAP_NODE ($TOPOLOGY topology)..."
cfg="$SCRIPT_DIR/../generated-configs/$TOPOLOGY/node-$CAP_NODE.yaml"
if [ ! -f "$cfg" ]; then
echo " Error: $cfg not found (run generate-configs.sh $TOPOLOGY first)" >&2
exit 1
fi
# Insert under node.limits (the actual config path per src/config/node.rs).
# Three cases: limits.max_peers already present (update), limits: present
# without max_peers (append), or no limits block (insert full subtree).
if grep -qE "^ max_peers:" "$cfg"; then
sed -i -E "s/^ max_peers: *[0-9]+/ max_peers: $MAX_PEERS/" "$cfg"
elif grep -qE "^ limits:" "$cfg"; then
sed -i "/^ limits:/a\\
max_peers: $MAX_PEERS" "$cfg"
else
sed -i "/^node:/a\\
limits:\\
max_peers: $MAX_PEERS" "$cfg"
fi
echo " node-$CAP_NODE limits block:"
sed -n '/^ limits:/,/^ [a-z]/p' "$cfg" | head -5
exit 0
fi
stamp() { date '+%H:%M:%S'; }
info() { echo "[$(stamp)] $*"; }
fail() { echo "[$(stamp)] FAIL: $*"; exit 1; }
pass() { echo "[$(stamp)] PASS: $*"; }
# Extract docker_ip for a node from the topology file
node_ip() {
grep -A 5 "^ $1:" "$TOPO_FILE" \
| grep -m1 'docker_ip:' \
| sed 's/.*: *"*\([^"]*\)".*/\1/'
}
# Extract npub for a node from the topology file
node_npub() {
grep -A 5 "^ $1:" "$TOPO_FILE" \
| grep -m1 'npub:' \
| sed 's/.*: *"*\([^"]*\)".*/\1/'
}
# Extract configured peers list for a node from the topology file
node_peers() {
grep -A 5 "^ $1:" "$TOPO_FILE" \
| grep -m1 'peers:' \
| sed 's/.*\[\(.*\)\].*/\1/' \
| tr -d ' ' \
| tr ',' ' '
}
CAP_IP=$(node_ip "$CAP_NODE")
[ -n "$CAP_IP" ] || fail "could not resolve docker_ip for node-$CAP_NODE in $TOPO_FILE"
info "cap'd node: node-$CAP_NODE (ip $CAP_IP, max_peers=$MAX_PEERS)"
# ── Phase 1: wait for convergence ────────────────────────────────────
info "phase 1: wait for node-$CAP_NODE peer_count to reach $MAX_PEERS (90s timeout)"
deadline=$(($(date +%s) + 90))
pc=0
while [ "$(date +%s)" -lt "$deadline" ]; do
pc=$(docker exec fips-node-$CAP_NODE fipsctl show status 2>/dev/null \
| grep -m1 peer_count | sed 's/.*: *//' | tr -d ',' || echo 0)
[ "$pc" = "$MAX_PEERS" ] && break
sleep 2
done
[ "$pc" = "$MAX_PEERS" ] \
|| fail "node-$CAP_NODE peer_count=$pc after 90s, expected $MAX_PEERS"
info "node-$CAP_NODE converged: peer_count=$pc"
# Identify admitted vs denied peers among configured peers
ADMITTED_NPUBS=$(docker exec fips-node-$CAP_NODE fipsctl show peers 2>/dev/null \
| grep -oE 'npub1[a-z0-9]+' | sort -u || true)
DENIED=""
ADMITTED=""
for p in $(node_peers "$CAP_NODE"); do
npub=$(node_npub "$p")
if echo "$ADMITTED_NPUBS" | grep -q "$npub"; then
ADMITTED="$ADMITTED $p"
else
DENIED="$DENIED $p"
fi
done
ADMITTED=$(echo $ADMITTED | xargs)
DENIED=$(echo $DENIED | xargs)
info "admitted: ${ADMITTED:-<none>}"
info "denied (sustained-retry): ${DENIED:-<none>}"
[ -n "$DENIED" ] \
|| fail "no denied peers — test setup wrong (cap=$MAX_PEERS too high vs configured peers)"
# ── Phase 2: capture wire traffic for CAPTURE_SECS seconds ───────────
# Drives sustained load by restarting denied peer containers on a cadence
# during the capture window. Each restart resets the auto-reconnect
# exponential backoff (5s base / 300s cap), producing a fresh burst of
# Msg1s that exercises the silent-drop gate at meaningful rate. Without
# this loop the gate fires ~3-4 times per denied peer in a 60s window;
# with restarts every 15s we get ~30-50 firings across both denied peers.
info "phase 2: capture UDP/2121 on node-$CAP_NODE for ${CAPTURE_SECS}s, with denied-peer restart loop"
CAP_FILE=$(mktemp /tmp/admission-cap-pcap.XXXXXX.txt)
HELPER_IMAGE=$(docker inspect -f '{{.Config.Image}}' fips-node-$CAP_NODE 2>/dev/null)
[ -n "$HELPER_IMAGE" ] || fail "could not resolve helper image from fips-node-$CAP_NODE"
# Background: cycle denied peers to reset their backoff and drive load.
(
elapsed=0
while [ $elapsed -lt $((CAPTURE_SECS - 5)) ]; do
sleep 15
elapsed=$((elapsed + 15))
for n in $DENIED; do
docker restart "fips-node-$n" >/dev/null 2>&1 &
done
wait
info " [load-driver] restarted denied peers ($DENIED) at t+${elapsed}s"
done
) &
LOAD_PID=$!
# Foreground: tcpdump capture for CAPTURE_SECS
docker run --rm --net=container:fips-node-$CAP_NODE \
--cap-add NET_ADMIN --cap-add NET_RAW \
--entrypoint sh "$HELPER_IMAGE" \
-c "timeout $CAPTURE_SECS tcpdump -nn -i any 'udp port 2121' -l 2>&1 || true" \
> "$CAP_FILE" 2>&1
# Reap load-driver if it's still running (should be ~done)
wait $LOAD_PID 2>/dev/null || true
captured=$(wc -l < "$CAP_FILE")
info "captured $captured tcpdump lines → $CAP_FILE"
# ── Phase 3: per-denied-peer wire-level assertion ────────────────────
info "phase 3: per-denied-peer assertion (inbound Msg1 > 0, outbound Msg2 == 0)"
OVERALL=0
TOTAL_MSG1_IN=0
TOTAL_MSG2_OUT=0
for n in $DENIED; do
n_ip=$(node_ip "$n")
# Inbound: src=n_ip:* → dst=cap_ip:2121; FMP-IK Msg1 wire size = 84 B
msg1_in=$(grep -cE "IP $n_ip\.[0-9]+ > $CAP_IP\.2121: UDP, length 84" "$CAP_FILE" || true)
# Outbound: src=cap_ip:2121 → dst=n_ip:*; FMP-IK Msg2 wire size = 104 B
msg2_out=$(grep -cE "IP $CAP_IP\.2121 > $n_ip\.[0-9]+: UDP, length 104" "$CAP_FILE" || true)
info " node-$n ($n_ip): inbound Msg1 (len 84) = $msg1_in, outbound Msg2 (len 104) = $msg2_out"
TOTAL_MSG1_IN=$((TOTAL_MSG1_IN + msg1_in))
TOTAL_MSG2_OUT=$((TOTAL_MSG2_OUT + msg2_out))
if [ "$msg1_in" -eq 0 ]; then
info " FAIL: expected inbound Msg1 retries from denied peer (peer not sustained-retrying?)"
OVERALL=1
fi
if [ "$msg2_out" -gt 0 ]; then
info " FAIL: silent-drop gate leaked — expected 0 outbound Msg2, got $msg2_out"
OVERALL=1
fi
done
# ── Phase 4: cap'd node still at exactly max_peers ───────────────────
pc_final=$(docker exec fips-node-$CAP_NODE fipsctl show status 2>/dev/null \
| grep -m1 peer_count | sed 's/.*: *//' | tr -d ',' || echo 0)
info "node-$CAP_NODE final peer_count=$pc_final (expected $MAX_PEERS)"
[ "$pc_final" = "$MAX_PEERS" ] || OVERALL=1
if [ "$OVERALL" -eq 0 ]; then
pass "admission-cap: silent-drop gate verified at scale"
pass " denied peers: $(echo $DENIED | wc -w), capture: ${CAPTURE_SECS}s"
pass " total inbound Msg1 from denied: $TOTAL_MSG1_IN (sustained retries observed)"
pass " total outbound Msg2 to denied: $TOTAL_MSG2_OUT (silent-drop holds)"
rm -f "$CAP_FILE"
exit 0
else
info "--- tcpdump capture tail (last 50 lines) ---"
tail -50 "$CAP_FILE"
fail "admission-cap: see failures above (capture preserved at $CAP_FILE)"
fi