mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Merge branch 'maint' into master
This commit is contained in:
@@ -63,7 +63,7 @@ async fn make_test_node_ble(node_num: u8) -> TestNode {
|
||||
TestNode {
|
||||
node,
|
||||
transport_id,
|
||||
packet_rx,
|
||||
packet_rx: spanning_tree::bridge_to_unbounded(packet_rx),
|
||||
addr: ta,
|
||||
}
|
||||
}
|
||||
@@ -303,7 +303,7 @@ async fn test_ble_discovery() {
|
||||
let mut nodes = vec![TestNode {
|
||||
node,
|
||||
transport_id,
|
||||
packet_rx,
|
||||
packet_rx: spanning_tree::bridge_to_unbounded(packet_rx),
|
||||
addr: ta,
|
||||
}];
|
||||
cleanup_nodes(&mut nodes).await;
|
||||
|
||||
@@ -535,7 +535,6 @@ fn compute_mesh_size_skips_parent_under_stale_peer_declaration() {
|
||||
|
||||
/// 100-node random graph: bloom filter exchange at scale.
|
||||
#[tokio::test]
|
||||
#[ignore = "parallel-load flake class — re-enable when fixed (run solo with --ignored or --test-threads=1 in the meantime)"]
|
||||
async fn test_bloom_filter_convergence_100_nodes() {
|
||||
let _guard = lock_large_network_test().await;
|
||||
|
||||
|
||||
@@ -774,7 +774,6 @@ async fn test_apply_outgoing_link_mtu_to_response_unknown_peer_noop() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "parallel-load flake class — re-enable when fixed (run solo with --ignored or --test-threads=1 in the meantime)"]
|
||||
async fn test_response_path_mtu_three_node_chain() {
|
||||
// Topology: node0 — node1 — node2
|
||||
// Node0 initiates lookup for node2. The response travels node2→node1→node0.
|
||||
|
||||
@@ -104,7 +104,7 @@ async fn make_test_node_ethernet(interface: &str) -> TestNode {
|
||||
TestNode {
|
||||
node,
|
||||
transport_id,
|
||||
packet_rx,
|
||||
packet_rx: spanning_tree::bridge_to_unbounded(packet_rx),
|
||||
addr,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::*;
|
||||
use crate::PeerIdentity;
|
||||
use crate::transport::{LinkDirection, TransportAddr, packet_channel};
|
||||
use crate::transport::{LinkDirection, ReceivedPacket, TransportAddr, packet_channel};
|
||||
use crate::utils::index::SessionIndex;
|
||||
use std::time::Duration;
|
||||
|
||||
|
||||
@@ -670,7 +670,6 @@ fn simulate_forwarding(
|
||||
/// forwarding between every pair of nodes. Every packet must be delivered
|
||||
/// without loops.
|
||||
#[tokio::test]
|
||||
#[ignore = "parallel-load flake class — re-enable when fixed (run solo with --ignored or --test-threads=1 in the meantime)"]
|
||||
async fn test_routing_reachability_100_nodes() {
|
||||
let _guard = lock_large_network_test().await;
|
||||
|
||||
@@ -991,7 +990,6 @@ async fn test_routing_bloom_only_transit() {
|
||||
/// routing needs dest_coords at each hop for loop-free forwarding through
|
||||
/// non-adjacent nodes. Direct peer adjacency handles the last hop.
|
||||
#[tokio::test]
|
||||
#[ignore = "parallel-load flake class — re-enable when fixed (run solo with --ignored or --test-threads=1 in the meantime)"]
|
||||
async fn test_routing_source_only_coords_100_nodes() {
|
||||
let _guard = lock_large_network_test().await;
|
||||
|
||||
|
||||
@@ -571,7 +571,6 @@ async fn drain_to_quiescence(nodes: &mut [TestNode]) {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "parallel-load flake class — re-enable when fixed (run solo with --ignored or --test-threads=1 in the meantime)"]
|
||||
async fn test_session_100_nodes() {
|
||||
let _guard = lock_large_network_test().await;
|
||||
|
||||
@@ -1252,7 +1251,6 @@ async fn test_tun_outbound_3node_forwarded() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "parallel-load flake class — re-enable when fixed (run solo with --ignored or --test-threads=1 in the meantime)"]
|
||||
async fn test_tun_outbound_pending_queue_flush() {
|
||||
// Send multiple packets before session exists — all should be delivered
|
||||
let edges = vec![(0, 1)];
|
||||
|
||||
@@ -6,11 +6,51 @@
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::TreeAnnounce;
|
||||
use crate::transport::loopback::{LoopbackRegistry, LoopbackTransport, new_registry};
|
||||
use crate::tree::{CoordEntry, ParentDeclaration, TreeCoordinate};
|
||||
|
||||
static LARGE_NETWORK_TEST_LOCK: std::sync::LazyLock<tokio::sync::Mutex<()>> =
|
||||
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(()));
|
||||
|
||||
/// Process-wide shared loopback registry for node-level mesh tests.
|
||||
///
|
||||
/// All loopback test nodes register here so they can locate each other by
|
||||
/// address. Each node gets a unique synthetic address (`loopback:{n}`) from
|
||||
/// `LOOPBACK_ADDR_COUNTER`, so addresses never collide across concurrently
|
||||
/// running tests and stale entries from finished tests are harmless.
|
||||
static LOOPBACK_REGISTRY: std::sync::LazyLock<LoopbackRegistry> =
|
||||
std::sync::LazyLock::new(new_registry);
|
||||
|
||||
static LOOPBACK_ADDR_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
|
||||
/// Allocate the next globally-unique loopback address.
|
||||
fn next_loopback_addr() -> TransportAddr {
|
||||
let n = LOOPBACK_ADDR_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
TransportAddr::from_string(&format!("loopback:{}", n))
|
||||
}
|
||||
|
||||
/// Bridge a transport's bounded receive channel into the unbounded channel
|
||||
/// that `TestNode` holds.
|
||||
///
|
||||
/// Real transports (TCP, Ethernet, BLE) drain their kernel socket into a
|
||||
/// bounded `PacketRx` via a background receive task, so a bounded channel
|
||||
/// does not deadlock for them. `TestNode.packet_rx` is unbounded (required
|
||||
/// by the loopback path, which has no background reader); this spawns a
|
||||
/// forwarding task so non-loopback factories can still produce a `TestNode`.
|
||||
pub(super) fn bridge_to_unbounded(
|
||||
mut bounded_rx: PacketRx,
|
||||
) -> tokio::sync::mpsc::UnboundedReceiver<ReceivedPacket> {
|
||||
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
tokio::spawn(async move {
|
||||
while let Some(packet) = bounded_rx.recv().await {
|
||||
if tx.send(packet).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
rx
|
||||
}
|
||||
|
||||
pub(super) async fn lock_large_network_test() -> tokio::sync::MutexGuard<'static, ()> {
|
||||
LARGE_NETWORK_TEST_LOCK.lock().await
|
||||
}
|
||||
@@ -19,52 +59,42 @@ pub(super) async fn lock_large_network_test() -> tokio::sync::MutexGuard<'static
|
||||
pub(super) struct TestNode {
|
||||
pub(super) node: Node,
|
||||
pub(super) transport_id: TransportId,
|
||||
pub(super) packet_rx: PacketRx,
|
||||
pub(super) packet_rx: tokio::sync::mpsc::UnboundedReceiver<ReceivedPacket>,
|
||||
pub(super) addr: TransportAddr,
|
||||
}
|
||||
|
||||
/// Create a test node with a live UDP transport on localhost.
|
||||
/// Create a test node with an in-process loopback transport.
|
||||
pub(super) async fn make_test_node() -> TestNode {
|
||||
make_test_node_with_mtu(1280).await
|
||||
}
|
||||
|
||||
/// Create a test node with a specific transport MTU.
|
||||
///
|
||||
/// Uses the in-process loopback transport (not real UDP): packets are
|
||||
/// delivered directly to the destination node's unbounded receive channel
|
||||
/// via the shared registry. This avoids the kernel UDP receive-buffer
|
||||
/// overflow that drops handshake packets when many tests run in parallel
|
||||
/// under CPU contention. The `mtu` is enforced on send (MtuExceeded),
|
||||
/// mirroring UDP, so heterogeneous-MTU / PMTUD tests still exercise the
|
||||
/// forward-path bottleneck.
|
||||
pub(super) async fn make_test_node_with_mtu(mtu: u16) -> TestNode {
|
||||
use crate::config::UdpConfig;
|
||||
use crate::transport::udp::UdpTransport;
|
||||
|
||||
let mut node = make_node();
|
||||
let transport_id = TransportId::new(1);
|
||||
|
||||
// recv_buf_size and packet_channel are sized for large-network harness
|
||||
// tests (100-node burst patterns) under parallel-CPU load via
|
||||
// `cargo test --lib`. The daemon's 2 MB recv default is already
|
||||
// requested via UdpConfig; we ask for 8 MB so hosts with tuned
|
||||
// net.core.rmem_max get the larger budget (the kernel clamps to
|
||||
// rmem_max otherwise and the transport emits a warn). The
|
||||
// packet_channel(8192) is the actually-effective bump on hosts with
|
||||
// the typical 2 MB rmem_max — under parallel-test scheduler
|
||||
// contention the in-process channel between recv loop and the test's
|
||||
// packet_rx fills well before the kernel rcvbuf would.
|
||||
let udp_config = UdpConfig {
|
||||
bind_addr: Some("127.0.0.1:0".to_string()),
|
||||
mtu: Some(mtu),
|
||||
recv_buf_size: Some(8 * 1024 * 1024),
|
||||
..Default::default()
|
||||
};
|
||||
let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<ReceivedPacket>();
|
||||
let addr = next_loopback_addr();
|
||||
|
||||
let (packet_tx, packet_rx) = packet_channel(8192);
|
||||
let mut transport = UdpTransport::new(transport_id, None, udp_config, packet_tx);
|
||||
transport.start_async().await.unwrap();
|
||||
LOOPBACK_REGISTRY.lock().unwrap().insert(addr.clone(), tx);
|
||||
|
||||
let addr = TransportAddr::from_string(&transport.local_addr().unwrap().to_string());
|
||||
let loopback =
|
||||
LoopbackTransport::with_mtu(transport_id, addr.clone(), mtu, LOOPBACK_REGISTRY.clone());
|
||||
node.transports
|
||||
.insert(transport_id, TransportHandle::Udp(transport));
|
||||
.insert(transport_id, TransportHandle::Loopback(loopback));
|
||||
|
||||
TestNode {
|
||||
node,
|
||||
transport_id,
|
||||
packet_rx,
|
||||
packet_rx: rx,
|
||||
addr,
|
||||
}
|
||||
}
|
||||
@@ -240,9 +270,21 @@ pub(super) async fn process_available_packets(nodes: &mut [TestNode]) -> usize {
|
||||
COMMON_PREFIX_SIZE, CommonPrefix, FMP_VERSION, PHASE_ESTABLISHED, PHASE_MSG1, PHASE_MSG2,
|
||||
};
|
||||
|
||||
// Snapshot the number of packets queued at every node at the start of the
|
||||
// pass, before processing any node. Loopback delivery is synchronous, so a
|
||||
// packet sent during this pass would otherwise land in another node's
|
||||
// channel and be drained in the *same* pass. Real UDP defers such packets
|
||||
// to the next pass (socket round-trip + recv task), and several tests
|
||||
// depend on that one-hop-per-pass cadence. Bounding each node's drain to
|
||||
// its start-of-pass count preserves it regardless of iteration order.
|
||||
let queued: Vec<usize> = nodes.iter().map(|n| n.packet_rx.len()).collect();
|
||||
|
||||
let mut count = 0;
|
||||
for node in nodes.iter_mut() {
|
||||
while let Ok(packet) = node.packet_rx.try_recv() {
|
||||
for (node, &queued) in nodes.iter_mut().zip(queued.iter()) {
|
||||
for _ in 0..queued {
|
||||
let Ok(packet) = node.packet_rx.try_recv() else {
|
||||
break;
|
||||
};
|
||||
if packet.data.len() < COMMON_PREFIX_SIZE {
|
||||
continue;
|
||||
}
|
||||
@@ -756,7 +798,6 @@ pub(super) async fn cleanup_nodes(nodes: &mut [TestNode]) {
|
||||
/// Integration test: 100 nodes with random connectivity converge to a
|
||||
/// consistent spanning tree with the correct root.
|
||||
#[tokio::test]
|
||||
#[ignore = "parallel-load flake class — re-enable when fixed (run solo with --ignored or --test-threads=1 in the meantime)"]
|
||||
async fn test_spanning_tree_convergence_100_nodes() {
|
||||
let _guard = lock_large_network_test().await;
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ async fn make_test_node_tcp_with(config: Config) -> TestNode {
|
||||
TestNode {
|
||||
node,
|
||||
transport_id,
|
||||
packet_rx,
|
||||
packet_rx: spanning_tree::bridge_to_unbounded(packet_rx),
|
||||
addr,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
//! In-process loopback transport (test harness only).
|
||||
//!
|
||||
//! Delivers packets directly between nodes running in the same process via
|
||||
//! an unbounded in-process channel and a shared address-to-receiver
|
||||
//! registry, instead of going over real localhost UDP sockets. This is used
|
||||
//! by node-level multi-node tests to avoid the kernel UDP receive-buffer
|
||||
//! overflow that drops handshake packets when many tests run in parallel
|
||||
//! under CPU contention.
|
||||
//!
|
||||
//! An UNBOUNDED channel is used deliberately: the test harness drains
|
||||
//! packets sequentially (it fires the whole handshake burst before draining,
|
||||
//! with no background reader), so a bounded awaiting send would deadlock and
|
||||
//! a bounded try_send would drop. `UnboundedSender::send` is synchronous,
|
||||
//! never blocks, and never drops — it only errors if the receiver is gone —
|
||||
//! making delivery provably lossless and deadlock-free.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use tokio::sync::mpsc::UnboundedSender;
|
||||
|
||||
use super::{
|
||||
DiscoveredPeer, ReceivedPacket, Transport, TransportAddr, TransportError, TransportId,
|
||||
TransportState, TransportType,
|
||||
};
|
||||
|
||||
/// Shared registry mapping each loopback address to the receiver-side
|
||||
/// channel sender for the node listening on that address.
|
||||
///
|
||||
/// One registry instance is shared by all loopback transports in a given
|
||||
/// test run so they can locate each other by address.
|
||||
pub type LoopbackRegistry = Arc<Mutex<HashMap<TransportAddr, UnboundedSender<ReceivedPacket>>>>;
|
||||
|
||||
/// Create a fresh, empty loopback registry.
|
||||
pub fn new_registry() -> LoopbackRegistry {
|
||||
Arc::new(Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
/// Default loopback MTU, mirroring the UDP test path.
|
||||
const DEFAULT_LOOPBACK_MTU: u16 = 1280;
|
||||
|
||||
/// In-process loopback transport.
|
||||
pub struct LoopbackTransport {
|
||||
transport_id: TransportId,
|
||||
/// This transport's synthetic unique address (e.g. "loopback:7").
|
||||
my_addr: TransportAddr,
|
||||
/// Transport MTU. Enforced on send to mirror UDP's MtuExceeded behavior,
|
||||
/// so PMTUD/heterogeneous-MTU tests still exercise the forward-path
|
||||
/// bottleneck detection.
|
||||
mtu: u16,
|
||||
/// Shared address-to-receiver registry.
|
||||
registry: LoopbackRegistry,
|
||||
}
|
||||
|
||||
impl LoopbackTransport {
|
||||
/// Create a new loopback transport bound to `my_addr` with the default
|
||||
/// MTU, sharing `registry`.
|
||||
pub fn new(
|
||||
transport_id: TransportId,
|
||||
my_addr: TransportAddr,
|
||||
registry: LoopbackRegistry,
|
||||
) -> Self {
|
||||
Self::with_mtu(transport_id, my_addr, DEFAULT_LOOPBACK_MTU, registry)
|
||||
}
|
||||
|
||||
/// Create a new loopback transport with an explicit MTU.
|
||||
pub fn with_mtu(
|
||||
transport_id: TransportId,
|
||||
my_addr: TransportAddr,
|
||||
mtu: u16,
|
||||
registry: LoopbackRegistry,
|
||||
) -> Self {
|
||||
Self {
|
||||
transport_id,
|
||||
my_addr,
|
||||
mtu,
|
||||
registry,
|
||||
}
|
||||
}
|
||||
|
||||
/// This transport's synthetic loopback address.
|
||||
pub fn my_addr(&self) -> &TransportAddr {
|
||||
&self.my_addr
|
||||
}
|
||||
|
||||
/// Send data to a destination loopback address.
|
||||
///
|
||||
/// Looks up `dest_addr` in the shared registry and, if found, delivers a
|
||||
/// `ReceivedPacket` to its receiver. The packet's `remote_addr` is set to
|
||||
/// the sender's own address (`my_addr`), mirroring how UDP sets
|
||||
/// `remote_addr` from the datagram source, so the receiver's handlers
|
||||
/// learn the peer source.
|
||||
pub async fn send_async(
|
||||
&self,
|
||||
dest_addr: &TransportAddr,
|
||||
data: &[u8],
|
||||
) -> Result<usize, TransportError> {
|
||||
if data.len() > self.mtu as usize {
|
||||
return Err(TransportError::MtuExceeded {
|
||||
packet_size: data.len(),
|
||||
mtu: self.mtu,
|
||||
});
|
||||
}
|
||||
|
||||
let dest_tx = {
|
||||
let registry = self.registry.lock().map_err(|e| {
|
||||
TransportError::SendFailed(format!("registry lock poisoned: {}", e))
|
||||
})?;
|
||||
registry.get(dest_addr).cloned()
|
||||
};
|
||||
|
||||
match dest_tx {
|
||||
Some(tx) => {
|
||||
let packet =
|
||||
ReceivedPacket::new(self.transport_id, self.my_addr.clone(), data.to_vec());
|
||||
tx.send(packet).map_err(|_| {
|
||||
TransportError::SendFailed(format!("loopback receiver gone for {}", dest_addr))
|
||||
})?;
|
||||
Ok(data.len())
|
||||
}
|
||||
None => Err(TransportError::SendFailed(format!(
|
||||
"no loopback route to {}",
|
||||
dest_addr
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Asynchronous start (no-op; the transport is ready on construction).
|
||||
pub async fn start_async(&mut self) -> Result<(), TransportError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Asynchronous stop (no-op).
|
||||
pub async fn stop_async(&mut self) -> Result<(), TransportError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Transport for LoopbackTransport {
|
||||
fn transport_id(&self) -> TransportId {
|
||||
self.transport_id
|
||||
}
|
||||
|
||||
fn transport_type(&self) -> &TransportType {
|
||||
&TransportType::LOOPBACK
|
||||
}
|
||||
|
||||
fn state(&self) -> TransportState {
|
||||
TransportState::Up
|
||||
}
|
||||
|
||||
fn mtu(&self) -> u16 {
|
||||
self.mtu
|
||||
}
|
||||
|
||||
fn start(&mut self) -> Result<(), TransportError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stop(&mut self) -> Result<(), TransportError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn send(&self, _addr: &TransportAddr, _data: &[u8]) -> Result<(), TransportError> {
|
||||
// Synchronous send not supported — use send_async().
|
||||
Err(TransportError::NotSupported(
|
||||
"use send_async() for loopback transport".into(),
|
||||
))
|
||||
}
|
||||
|
||||
fn discover(&self) -> Result<Vec<DiscoveredPeer>, TransportError> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@
|
||||
//! underlying communication mechanisms (UDP, Ethernet, Tor, etc.) over
|
||||
//! which FIPS links are established.
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod loopback;
|
||||
pub mod tcp;
|
||||
pub mod tor;
|
||||
pub mod udp;
|
||||
@@ -18,6 +20,8 @@ pub mod ble;
|
||||
use ble::DefaultBleTransport;
|
||||
#[cfg(unix)]
|
||||
use ethernet::EthernetTransport;
|
||||
#[cfg(test)]
|
||||
use loopback::LoopbackTransport;
|
||||
use secp256k1::XOnlyPublicKey;
|
||||
use std::fmt;
|
||||
use std::net::SocketAddr;
|
||||
@@ -247,6 +251,14 @@ impl TransportType {
|
||||
reliable: true, // L2CAP SeqPacket guarantees delivery
|
||||
};
|
||||
|
||||
/// In-process loopback transport (test harness only).
|
||||
#[cfg(test)]
|
||||
pub const LOOPBACK: TransportType = TransportType {
|
||||
name: "loopback",
|
||||
connection_oriented: false,
|
||||
reliable: true, // in-process channel delivery is lossless
|
||||
};
|
||||
|
||||
/// Check if the transport is connectionless.
|
||||
pub fn is_connectionless(&self) -> bool {
|
||||
!self.connection_oriented
|
||||
@@ -870,6 +882,9 @@ pub enum TransportHandle {
|
||||
/// BLE L2CAP transport.
|
||||
#[cfg(target_os = "linux")]
|
||||
Ble(DefaultBleTransport),
|
||||
/// In-process loopback transport (test harness only).
|
||||
#[cfg(test)]
|
||||
Loopback(LoopbackTransport),
|
||||
}
|
||||
|
||||
impl TransportHandle {
|
||||
@@ -883,6 +898,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.start_async().await,
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.start_async().await,
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.start_async().await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -896,6 +913,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.stop_async().await,
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.stop_async().await,
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.stop_async().await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -909,6 +928,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.send_async(addr, data).await,
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.send_async(addr, data).await,
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.send_async(addr, data).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -922,6 +943,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.transport_id(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.transport_id(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.transport_id(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -935,6 +958,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.name(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.name(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -948,6 +973,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.transport_type(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.transport_type(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.transport_type(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -961,6 +988,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.state(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.state(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.state(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -974,6 +1003,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.mtu(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.mtu(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.mtu(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -990,6 +1021,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.link_mtu(addr),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.link_mtu(addr),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.link_mtu(addr),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1003,6 +1036,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(_) => None,
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(_) => None,
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1016,6 +1051,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(_) => None,
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(_) => None,
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1053,6 +1090,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.discover(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.discover(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.discover(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1066,6 +1105,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.auto_connect(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.auto_connect(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.auto_connect(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1079,6 +1120,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.accept_connections(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.accept_connections(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(t) => t.accept_connections(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1098,6 +1141,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.connect_async(addr).await,
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.connect_async(addr).await,
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(_) => Ok(()), // connectionless
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1115,6 +1160,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.connection_state_sync(addr),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.connection_state_sync(addr),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(_) => ConnectionState::Connected,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1131,6 +1178,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(t) => t.close_connection_async(addr).await,
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(t) => t.close_connection_async(addr).await,
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(_) => {} // connectionless no-op
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1153,6 +1202,8 @@ impl TransportHandle {
|
||||
TransportHandle::Tor(_) => TransportCongestion::default(),
|
||||
#[cfg(target_os = "linux")]
|
||||
TransportHandle::Ble(_) => TransportCongestion::default(),
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(_) => TransportCongestion::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1190,6 +1241,8 @@ impl TransportHandle {
|
||||
TransportHandle::Ble(t) => {
|
||||
serde_json::to_value(t.stats().snapshot()).unwrap_or_default()
|
||||
}
|
||||
#[cfg(test)]
|
||||
TransportHandle::Loopback(_) => serde_json::json!({}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
#!/bin/bash
|
||||
# Shared convergence wait helpers for FIPS integration tests.
|
||||
#
|
||||
# Source this file to get wait_for_links() and wait_for_peers().
|
||||
# Source this file to get wait_for_links(), wait_for_peers(), and
|
||||
# wait_until_connected().
|
||||
#
|
||||
# Usage:
|
||||
# source "$(dirname "$0")/../../lib/wait-converge.sh"
|
||||
# wait_for_links <container> <min_links> [timeout_secs]
|
||||
# wait_for_peers <container> <min_peers> [timeout_secs]
|
||||
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs]
|
||||
|
||||
# Wait until a container has at least min_links active links.
|
||||
# Returns 0 on success, 1 on timeout.
|
||||
@@ -49,3 +51,55 @@ wait_for_peers() {
|
||||
echo " $container: TIMEOUT waiting for $min_peers peer(s) after ${timeout}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Wait until a connectivity check reports every pair reachable, using a
|
||||
# progress-aware deadline instead of a fixed one.
|
||||
#
|
||||
# wait_until_connected <ping_fn> <max_secs> <stall_secs> [poll_secs]
|
||||
#
|
||||
# <ping_fn> is the name of a function that runs the suite's own
|
||||
# connectivity check and sets two globals each call:
|
||||
# PASSED number of reachable pairs this round
|
||||
# FAILED number of unreachable pairs this round
|
||||
#
|
||||
# The convergence signal is the suite's real pings (the same signal it
|
||||
# asserts on), not a structural proxy. Behaviour:
|
||||
# - converged: FAILED == 0 -> return 0.
|
||||
# - progressing: PASSED climbed past the best seen -> reset the stall
|
||||
# clock and keep waiting (slow-but-improving is not a failure, so it
|
||||
# does not false-time-out under CI load).
|
||||
# - stuck: PASSED has not improved for stall_secs -> return 1 (fail
|
||||
# fast rather than burn the whole budget on a genuinely wedged pair).
|
||||
# - hard cap: max_secs elapsed -> return 1 (never runs unbounded).
|
||||
#
|
||||
# Returns 0 once fully connected, 1 on stall or timeout.
|
||||
wait_until_connected() {
|
||||
local ping_fn="$1"
|
||||
local max_secs="$2"
|
||||
local stall_secs="$3"
|
||||
local poll_secs="${4:-1}"
|
||||
|
||||
local start_secs=$SECONDS
|
||||
local best=-1
|
||||
local last_progress=$SECONDS
|
||||
|
||||
while (( SECONDS - start_secs < max_secs )); do
|
||||
"$ping_fn"
|
||||
if (( FAILED == 0 )); then
|
||||
echo " converge: all $PASSED pair(s) reachable after $((SECONDS - start_secs))s"
|
||||
return 0
|
||||
fi
|
||||
if (( PASSED > best )); then
|
||||
best=$PASSED
|
||||
last_progress=$SECONDS
|
||||
echo " converge: $PASSED reachable, $FAILED pending (progressing) after $((SECONDS - start_secs))s"
|
||||
elif (( SECONDS - last_progress >= stall_secs )); then
|
||||
echo " converge: STUCK at $PASSED reachable / $FAILED pending — no progress for ${stall_secs}s (after $((SECONDS - start_secs))s)"
|
||||
return 1
|
||||
fi
|
||||
sleep "$poll_secs"
|
||||
done
|
||||
|
||||
echo " converge: TIMEOUT at $PASSED reachable / $FAILED pending after ${max_secs}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -14,6 +14,8 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SIDECAR_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
# shellcheck source=../../lib/wait-converge.sh
|
||||
source "$SCRIPT_DIR/../../lib/wait-converge.sh"
|
||||
|
||||
# Deterministic keys derived from: derive-keys.py sidecar-test node-{a,b,c}
|
||||
NODE_A_NSEC="9e688d0879fa9cd025fea0487ac23495080e3de626070fdb9b78dc1f619dd453"
|
||||
@@ -127,8 +129,17 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Allow a few more seconds for tree convergence and coordinate propagation
|
||||
sleep 3
|
||||
# Wait for end-to-end multi-hop connectivity (the same app-to-app pings
|
||||
# the test asserts on) with a progress-aware deadline, instead of a
|
||||
# blind fixed sleep that can fire before coordinates propagate across the
|
||||
# chain. The directed pings below remain the actual assertions.
|
||||
_sidecar_converged() {
|
||||
PASSED=0; FAILED=0
|
||||
if docker exec sidecar-b-app-1 ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
||||
if docker exec sidecar-c-app-1 ping6 -c1 -W2 "${NODE_A_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
||||
if docker exec sidecar-a-app-1 ping6 -c1 -W2 "${NODE_C_NPUB}.fips" >/dev/null 2>&1; then PASSED=$((PASSED+1)); else FAILED=$((FAILED+1)); fi
|
||||
}
|
||||
wait_until_connected _sidecar_converged "$CONVERGE_TIMEOUT" 10 || true
|
||||
|
||||
# ── Link verification ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -71,23 +71,6 @@ ping_all_quiet() {
|
||||
done
|
||||
}
|
||||
|
||||
# Wait until all ping pairs succeed or timeout.
|
||||
wait_for_full_connectivity() {
|
||||
local timeout="${1:-30}"
|
||||
local start_secs=$SECONDS
|
||||
|
||||
while (( SECONDS - start_secs < timeout )); do
|
||||
ping_all_quiet
|
||||
if [ "$FAILED" -eq 0 ]; then
|
||||
echo " All $PASSED pairs reachable after $((SECONDS - start_secs))s"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo " TIMEOUT: $PASSED passed, $FAILED failed after ${timeout}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "=== FIPS Ping Test ($PROFILE topology) ==="
|
||||
echo ""
|
||||
|
||||
@@ -108,8 +91,12 @@ elif [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
|
||||
wait_for_peers fips-node-d 3 20 || true
|
||||
wait_for_peers fips-node-e 3 20 || true
|
||||
fi
|
||||
# Wait for FSP-level connectivity (discovery + session establishment)
|
||||
wait_for_full_connectivity 30 || true
|
||||
# Wait for full pairwise connectivity, progress-aware: the actual pings
|
||||
# are the convergence signal, the deadline extends while more pairs come
|
||||
# up, and it only gives up if progress stalls — so it does not
|
||||
# false-time-out under load while routing is still converging. The
|
||||
# directed-pair ping assertions below remain the actual test.
|
||||
wait_until_connected ping_all_quiet 45 15 || true
|
||||
|
||||
# Reset counters for the actual test
|
||||
PASSED=0
|
||||
|
||||
@@ -149,8 +149,8 @@ trap 'echo ""; echo "Test interrupted"; exit 130' INT
|
||||
# BASELINE_CONVERGENCE_TIMEOUT must cover one full daemon
|
||||
# node.tree.reeval_interval_secs (default 60) plus a small margin
|
||||
# so any partition that only heals via the periodic TreeAnnounce
|
||||
# re-broadcast lands inside the convergence window. wait_for_full_baseline
|
||||
# early-exits on PASS, so successful reps are unaffected by the
|
||||
# re-broadcast lands inside the convergence window. The Phase-1 baseline
|
||||
# wait early-exits on PASS, so successful reps are unaffected by the
|
||||
# extra headroom.
|
||||
BASELINE_CONVERGENCE_TIMEOUT=65
|
||||
REKEY_SETTLE=12 # > DRAIN_WINDOW_SECS (10) so post-rekey samples are off the old session
|
||||
@@ -173,8 +173,8 @@ CONVERGENCE_PING_TIMEOUT=1
|
||||
# attempts drops the loss-math floor to ~(0.02)^MAX_PING_ATTEMPTS per
|
||||
# pair, making any residual failure attributable to a non-loss
|
||||
# mechanism rather than ICMP noise. Applied to Phase 1 (final strict
|
||||
# ping_all after wait_for_full_baseline converges) and Phase 3 / Phase
|
||||
# 5 (post-rekey strict asserts). The wait_for_full_baseline convergence
|
||||
# ping_all after the Phase-1 baseline wait converges) and Phase 3 / Phase
|
||||
# 5 (post-rekey strict asserts). The Phase-1 baseline wait's convergence
|
||||
# loop itself stays single-shot — its job is to detect when the mesh
|
||||
# first sees a fully clean 20-pair batch, and retries inside the loop
|
||||
# would conflate "transient ping loss" with "still converging."
|
||||
@@ -255,27 +255,11 @@ ping_all() {
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_full_baseline() {
|
||||
local timeout="${1:-30}"
|
||||
local start_secs=$SECONDS
|
||||
local best_passed=0
|
||||
local best_failed=20
|
||||
|
||||
while (( SECONDS - start_secs < timeout )); do
|
||||
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
|
||||
if [ "$PASSED" -gt "$best_passed" ]; then
|
||||
best_passed="$PASSED"
|
||||
best_failed="$FAILED"
|
||||
fi
|
||||
if [ "$FAILED" -eq 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
PASSED="$best_passed"
|
||||
FAILED="$best_failed"
|
||||
return 1
|
||||
# Connectivity probe for the progress-aware baseline wait: one full
|
||||
# all-pairs ping sweep, setting PASSED/FAILED (consumed by
|
||||
# wait_until_connected).
|
||||
_baseline_ping() {
|
||||
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
|
||||
}
|
||||
|
||||
phase_result() {
|
||||
@@ -370,13 +354,24 @@ echo "Config: rekey.after_secs=$REKEY_AFTER_SECS"
|
||||
echo ""
|
||||
|
||||
# ── Phase 1: Pre-rekey baseline ───────────────────────────────────────
|
||||
# Wait for full pre-rekey connectivity with a progress-aware deadline:
|
||||
# the all-pairs ping sweep is the convergence signal, the window extends
|
||||
# while more pairs come up, and it gives up only if progress stalls — so
|
||||
# it no longer false-times-out under concurrent CI load. The strict
|
||||
# ping_all below is the actual assertion, run only after convergence.
|
||||
echo "Phase 1: Pre-rekey connectivity (waiting for convergence)"
|
||||
wait_for_peers fips-node-a 2 "$BASELINE_CONVERGENCE_TIMEOUT" || true
|
||||
if wait_for_full_baseline "$BASELINE_CONVERGENCE_TIMEOUT"; then
|
||||
if wait_until_connected _baseline_ping "$BASELINE_CONVERGENCE_TIMEOUT" 20; then
|
||||
ping_all "" "$TIMEOUT" "$MAX_PING_ATTEMPTS"
|
||||
phase_result "Pre-rekey baseline (all 20 pairs)"
|
||||
if [ "$FAILED" -ne 0 ]; then
|
||||
echo ""
|
||||
dump_peer_connectivity
|
||||
echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ==="
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo " Best observed baseline before timeout: $PASSED/$((PASSED + FAILED)) passed"
|
||||
echo " Mesh did not reach a converged tree before timeout"
|
||||
ping_all quiet "$CONVERGENCE_PING_TIMEOUT"
|
||||
phase_result "Pre-rekey baseline (all 20 pairs)"
|
||||
echo ""
|
||||
dump_peer_connectivity
|
||||
|
||||
Reference in New Issue
Block a user