Add Ethernet transport with beacon discovery

Implement raw Ethernet transport using AF_PACKET SOCK_DGRAM on Linux
with EtherType 0x88B5 (IEEE experimental range) and 1-byte frame type
prefix (0x00=data, 0x01=beacon).

Transport implementation:
- EthernetConfig with interface, ethertype, MTU, buffer sizes, and
  four independent discovery knobs (discovery, announce, auto_connect,
  accept_connections)
- PacketSocket/AsyncPacketSocket wrappers with ioctl helpers for
  interface index, MAC address, and MTU queries
- EthernetTransport with Transport trait impl, async start/stop/send,
  receive loop dispatching data frames and discovery beacons
- Discovery beacons (34 bytes: type + version + x-only pubkey) with
  DiscoveryBuffer for peer accumulation and dedup
- Atomic statistics counters (frames, bytes, errors, beacons)
- Platform-gated with #[cfg(target_os = "linux")]

Transport-layer discovery integration:
- Promote auto_connect() and accept_connections() to Transport trait
  with default implementations and TransportHandle dispatch
- Extract initiate_connection() so both static peer config and
  discovery auto-connect share the same handshake initiation path
- Add poll_transport_discovery() to the tick handler to drain
  discovery buffers and auto-connect to discovered peers
- Enforce accept_connections() in handle_msg1() — transports with
  accept_connections=false silently drop inbound handshakes

Node integration:
- create_transports() handles Ethernet named instances
- resolve_ethernet_addr() parses "interface/mac" address format
- transport_mtu() generalized for multi-transport operation

Test harness:
- VethPair RAII struct for veth pair lifecycle management
- Three #[ignore] integration tests requiring root/CAP_NET_RAW:
  two-node handshake, data exchange, mixed transport coexistence
- Chaos harness: transport-aware topology model, VethManager for
  veth pairs between Docker containers, Ethernet-aware config gen,
  netem split (HTB+u32 for UDP, root netem for veth), transport-aware
  link flaps and node churn with veth re-setup
- Container entrypoint waits for configured Ethernet interfaces
  before starting FIPS (handles veth creation timing)
- New scenarios: ethernet-only (4-node ring), ethernet-mesh (6-node
  mixed UDP+Ethernet with netem and link flaps)

Documentation:
- fips-transport-layer.md: Ethernet section, beacon discovery, WiFi
  compatibility, updated discovery state, trait surface additions,
  implementation status table
- fips-configuration.md: Ethernet parameter table, named instances,
  peer address format, mixed UDP+Ethernet example, complete reference
- fips-wire-formats.md: Ethernet frame type prefix note
This commit is contained in:
Johnathan Corgan
2026-02-26 00:03:14 +00:00
parent 7260ad2878
commit d29da442ac
30 changed files with 2967 additions and 305 deletions
+4 -2
View File
@@ -27,8 +27,10 @@ RUN printf '%s\n' \
COPY fips fipsctl /usr/local/bin/
RUN chmod +x /usr/local/bin/fips /usr/local/bin/fipsctl
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/entrypoint.sh
# Static web page served via Python HTTP server
RUN printf 'Fuck IPs!\n' > /root/index.html
# Start dnsmasq, SSH server, iperf3, and HTTP server in background, then run FIPS
ENTRYPOINT ["/bin/bash", "-c", "dnsmasq && /usr/sbin/sshd && iperf3 -s -D && python3 -m http.server 8000 -d /root -b :: &>/dev/null & exec fips --config /etc/fips/fips.yaml"]
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+51
View File
@@ -0,0 +1,51 @@
#!/bin/bash
# Container entrypoint: start services and wait for Ethernet interfaces.
#
# If the FIPS config references Ethernet transports, wait for the
# interfaces to appear before starting the FIPS daemon. This handles
# the case where veth pairs are created from the host after the
# container starts.
set -e
# Start background services
dnsmasq
/usr/sbin/sshd
iperf3 -s -D
python3 -m http.server 8000 -d /root -b :: &>/dev/null &
CONFIG="/etc/fips/fips.yaml"
# Extract Ethernet interface names from the config file.
# Matches "interface: <name>" lines that appear under transports.ethernet.
# The sed strips the key prefix and any whitespace.
ETH_IFACES=""
if grep -q 'ethernet:' "$CONFIG" 2>/dev/null; then
ETH_IFACES=$(grep '^\s*interface:' "$CONFIG" \
| sed 's/.*interface:\s*//' \
| tr -d ' ' || true)
fi
if [ -n "$ETH_IFACES" ]; then
echo "Waiting for Ethernet interfaces: $ETH_IFACES"
DEADLINE=$((SECONDS + 30))
while [ $SECONDS -lt $DEADLINE ]; do
ALL_FOUND=true
for iface in $ETH_IFACES; do
if [ ! -e "/sys/class/net/$iface" ]; then
ALL_FOUND=false
break
fi
done
if $ALL_FOUND; then
echo "All Ethernet interfaces ready"
break
fi
sleep 0.2
done
if ! $ALL_FOUND; then
echo "WARNING: Timed out waiting for Ethernet interfaces"
fi
fi
exec fips --config "$CONFIG"
@@ -0,0 +1,67 @@
# Mixed transport mesh: 6 nodes, UDP + Ethernet edges
#
# Exercises both transports in a single mesh. UDP edges use static
# peer config; Ethernet edges use beacon discovery. Tests that the
# spanning tree converges across heterogeneous transports with netem
# and link flaps active.
#
# Topology:
#
# n01 ---udp--- n02 ---udp--- n03
# | | |
# eth eth udp
# | | |
# n04 ---eth--- n05 ---udp--- n06
#
# UDP edges: n01-n02, n02-n03, n03-n06, n05-n06
# Ethernet edges: n01-n04, n02-n05, n04-n05
scenario:
name: "ethernet-mesh"
seed: 42
duration_secs: 120
topology:
algorithm: explicit
num_nodes: 6
params:
adjacency:
- [n01, n02, udp]
- [n02, n03, udp]
- [n03, n06, udp]
- [n05, n06, udp]
- [n01, n04, ethernet]
- [n02, n05, ethernet]
- [n04, n05, ethernet]
netem:
enabled: true
default_policy:
delay_ms: [1, 10]
jitter_ms: [0, 2]
loss_pct: [0, 1]
mutation:
interval_secs: {min: 20, max: 40}
fraction: 0.3
policies:
normal:
delay_ms: [1, 10]
loss_pct: [0, 1]
degraded:
delay_ms: [30, 80]
jitter_ms: [5, 20]
loss_pct: [3, 8]
link_flaps:
enabled: true
interval_secs: {min: 20, max: 40}
max_down_links: 1
down_duration_secs: {min: 10, max: 20}
protect_connectivity: true
traffic:
enabled: false
logging:
rust_log: "info"
output_dir: "./sim-results"
@@ -0,0 +1,46 @@
# Ethernet-only: 4-node ring, all Ethernet transport
#
# All links use raw Ethernet (AF_PACKET) over veth pairs. Nodes
# discover each other via beacons (no static peer config). Tests
# that Ethernet transport and beacon discovery work end-to-end.
#
# Topology:
#
# n01 ---eth--- n02
# | |
# eth eth
# | |
# n04 ---eth--- n03
scenario:
name: "ethernet-only"
seed: 42
duration_secs: 90
topology:
algorithm: explicit
num_nodes: 4
default_transport: ethernet
params:
adjacency:
- [n01, n02]
- [n02, n03]
- [n03, n04]
- [n04, n01]
netem:
enabled: true
default_policy:
delay_ms: [1, 5]
jitter_ms: [0, 1]
loss_pct: [0, 0.5]
link_flaps:
enabled: false
traffic:
enabled: false
logging:
rust_log: "info"
output_dir: "./sim-results"
+1
View File
@@ -25,6 +25,7 @@ x-fips-common: &fips-common
context: ../..
cap_add:
- NET_ADMIN
- NET_RAW
devices:
- /dev/net/tun:/dev/net/tun
sysctls:
+56 -1
View File
@@ -54,6 +54,37 @@ def generate_peers_block(
return "\n".join(lines)
def _build_ethernet_config(iface: str) -> dict:
"""Build an Ethernet transport config dict for a single interface."""
return {
"interface": iface,
"discovery": True,
"announce": True,
"auto_connect": True,
"accept_connections": True,
"beacon_interval_secs": 10,
}
def _inject_ethernet_transports(parsed: dict, eth_ifaces: list[str]):
"""Inject Ethernet transport config into a parsed FIPS config.
For a single interface, uses the single-instance format.
For multiple interfaces, uses the named-instances format.
Pure-Ethernet nodes (no UDP peers) have their UDP transport removed.
"""
if not eth_ifaces:
return
transports = parsed.setdefault("transports", {})
if len(eth_ifaces) == 1:
transports["ethernet"] = _build_ethernet_config(eth_ifaces[0])
else:
transports["ethernet"] = {
iface: _build_ethernet_config(iface) for iface in eth_ifaces
}
def generate_node_config(
topology: SimTopology,
node_id: str,
@@ -72,7 +103,22 @@ def generate_node_config(
config = config.replace("{{NSEC}}", node.nsec)
config = config.replace("{{PEERS}}", peers_yaml)
if fips_overrides:
# Inject Ethernet transport config if this node has Ethernet edges
eth_ifaces = topology.ethernet_interfaces(node_id)
has_udp_peers = bool(outbound_peers) or _has_inbound_udp_peers(topology, node_id)
if eth_ifaces or not has_udp_peers:
parsed = yaml.safe_load(config)
if fips_overrides:
parsed = _deep_merge(parsed, fips_overrides)
if eth_ifaces:
_inject_ethernet_transports(parsed, eth_ifaces)
if not has_udp_peers and eth_ifaces:
# Pure-Ethernet node: remove UDP transport
transports = parsed.get("transports", {})
transports.pop("udp", None)
config = yaml.dump(parsed, default_flow_style=False, sort_keys=False)
elif fips_overrides:
parsed = yaml.safe_load(config)
merged = _deep_merge(parsed, fips_overrides)
config = yaml.dump(merged, default_flow_style=False, sort_keys=False)
@@ -80,6 +126,15 @@ def generate_node_config(
return config
def _has_inbound_udp_peers(topology: SimTopology, node_id: str) -> bool:
"""Check if any other node has a UDP outbound edge to this node."""
for peer_id in topology.nodes[node_id].peers:
edge = (min(node_id, peer_id), max(node_id, peer_id))
if topology.edge_transport.get(edge, "udp") == "udp":
return True
return False
def generate_npubs_env(topology: SimTopology) -> str:
"""Generate npubs.env content mapping NPUB_<ID>=<npub> for all nodes."""
lines = []
+34 -17
View File
@@ -15,7 +15,7 @@ from dataclasses import dataclass, field
from .docker_exec import docker_exec_quiet, is_container_running
from .scenario import LinkFlapsConfig
from .topology import SimTopology
from .topology import SimTopology, veth_interface_name
log = logging.getLogger(__name__)
@@ -138,7 +138,11 @@ class LinkManager:
log.info("Link UP: %s -- %s (was down %.0fs)", a, b, down_for)
def _set_loss(self, src_node: str, dst_node: str, netem_args: str) -> str | None:
"""Set netem args on the tc class for src->dst. Returns the previous netem args."""
"""Set netem args on the link for src->dst. Returns the previous netem args.
Transport-aware: UDP links use tc class on eth0, Ethernet links
use tc qdisc replace on the dedicated veth interface.
"""
if not self.netem_mgr:
return None
@@ -158,25 +162,38 @@ class LinkManager:
self.netem_mgr.down_nodes.add(src_node)
return None
dest_ip = self.topology.nodes[dst_node].docker_ip
transport = self.topology.transport_for_edge(src_node, dst_node)
states = self.netem_mgr.states.get(container, {})
link_state = states.get(dest_ip)
if link_state is None:
log.warning("No netem state for %s -> %s", src_node, dst_node)
return None
if transport == "ethernet":
# Ethernet: simple netem on veth
iface = veth_interface_name(src_node, dst_node)
veth_states = self.netem_mgr.veth_states.get(container, {})
veth_state = veth_states.get(iface)
if veth_state is None:
log.warning("No veth netem state for %s -> %s (%s)", src_node, dst_node, iface)
return None
# Save current params
prev_args = link_state.params.to_tc_args()
prev_args = veth_state.params.to_tc_args()
cmd = f"tc qdisc replace dev {iface} root netem {netem_args}"
docker_exec_quiet(container, cmd)
return prev_args
else:
# UDP: HTB class on eth0
dest_ip = self.topology.nodes[dst_node].docker_ip
# Apply new netem
cmd = (
f"tc qdisc replace dev {IFACE} parent {link_state.class_id} "
f"handle {link_state.netem_handle} netem {netem_args}"
)
docker_exec_quiet(container, cmd)
states = self.netem_mgr.states.get(container, {})
link_state = states.get(dest_ip)
if link_state is None:
log.warning("No netem state for %s -> %s", src_node, dst_node)
return None
return prev_args
prev_args = link_state.params.to_tc_args()
cmd = (
f"tc qdisc replace dev {IFACE} parent {link_state.class_id} "
f"handle {link_state.netem_handle} netem {netem_args}"
)
docker_exec_quiet(container, cmd)
return prev_args
def _would_disconnect(self, edge: tuple[str, str]) -> bool:
"""Check if removing this edge (plus currently-down edges) disconnects the graph."""
+184 -104
View File
@@ -18,7 +18,7 @@ from dataclasses import dataclass, field
from .docker_exec import docker_exec_quiet, is_container_running
from .scenario import BandwidthConfig, LinkPolicyOverride, NetemConfig, NetemPolicy
from .topology import SimTopology
from .topology import SimTopology, veth_interface_name
log = logging.getLogger(__name__)
@@ -67,6 +67,15 @@ class LinkNetemState:
rate_mbit: int = 0 # 0 = unlimited (1gbit default)
@dataclass
class VethNetemState:
"""Tracks the netem state for a veth interface (Ethernet link direction)."""
container: str
iface: str # e.g., "ve-n01-n02"
params: NetemParams = field(default_factory=NetemParams)
class NetemManager:
"""Manages per-link netem impairment across all containers."""
@@ -80,8 +89,10 @@ class NetemManager:
self.topology = topology
self.config = config
self.rng = rng
# Per-container, per-dest-ip netem state
# Per-container, per-dest-ip netem state (UDP links on eth0)
self.states: dict[str, dict[str, LinkNetemState]] = {}
# Per-container, per-veth netem state (Ethernet links)
self.veth_states: dict[str, dict[str, VethNetemState]] = {}
# Nodes currently down (updated by NodeManager) — skip tc ops on these
self.down_nodes: set[str] = set()
# Per-edge bandwidth: (node_a, node_b) -> rate in mbit
@@ -127,7 +138,11 @@ class NetemManager:
return self.config.default_policy
def setup_initial(self):
"""Set up HTB qdiscs and initial netem on all containers."""
"""Set up HTB qdiscs and initial netem on all containers.
UDP peers use HTB + u32 filters on eth0. Ethernet peers use a
simple root netem qdisc on their dedicated veth interface.
"""
if self._edge_rates:
log.info("Bandwidth pacing enabled (%d edges with rate limits)",
len(self._edge_rates) // 2)
@@ -135,69 +150,103 @@ class NetemManager:
for node_id in sorted(self.topology.nodes):
node = self.topology.nodes[node_id]
container = self.topology.container_name(node_id)
peer_ips = {}
# Split peers by transport type
udp_peers = {}
eth_peers = []
for peer_id in sorted(node.peers):
peer_ips[peer_id] = self.topology.nodes[peer_id].docker_ip
transport = self.topology.transport_for_edge(node_id, peer_id)
if transport == "ethernet":
eth_peers.append(peer_id)
else:
udp_peers[peer_id] = self.topology.nodes[peer_id].docker_ip
if not peer_ips:
continue
# Build all tc commands for this container
cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"]
cmds.append(
f"tc qdisc add dev {IFACE} root handle 1: htb default 99"
)
cmds.append(
f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit"
)
container_states = {}
for idx, (peer_id, dest_ip) in enumerate(peer_ips.items(), start=1):
class_id = f"1:{idx}"
netem_handle = f"{idx + 10}:"
# Sample initial params (per-edge override or default policy)
policy = self._policy_for_edge(node_id, peer_id)
params = self._sample_policy(policy)
rate = self._htb_rate(node_id, peer_id)
rate_mbit = self._edge_rates.get((node_id, peer_id), 0)
# --- UDP peers: HTB + u32 on eth0 ---
if udp_peers:
cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"]
cmds.append(
f"tc class add dev {IFACE} parent 1: classid {class_id} htb rate {rate}"
f"tc qdisc add dev {IFACE} root handle 1: htb default 99"
)
cmds.append(
f"tc qdisc add dev {IFACE} parent {class_id} "
f"handle {netem_handle} netem {params.to_tc_args()}"
)
cmds.append(
f"tc filter add dev {IFACE} parent 1: protocol ip "
f"prio {idx} u32 match ip dst {dest_ip}/32 flowid {class_id}"
f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit"
)
state = LinkNetemState(
container=container,
dest_ip=dest_ip,
class_id=class_id,
netem_handle=netem_handle,
params=params,
rate_mbit=rate_mbit,
)
container_states[dest_ip] = state
container_states = {}
# Execute all commands in one docker exec
full_cmd = " && ".join(cmds)
result = docker_exec_quiet(container, full_cmd, timeout=30)
if result is not None:
for idx, (peer_id, dest_ip) in enumerate(udp_peers.items(), start=1):
class_id = f"1:{idx}"
netem_handle = f"{idx + 10}:"
policy = self._policy_for_edge(node_id, peer_id)
params = self._sample_policy(policy)
rate = self._htb_rate(node_id, peer_id)
rate_mbit = self._edge_rates.get((node_id, peer_id), 0)
cmds.append(
f"tc class add dev {IFACE} parent 1: classid {class_id} htb rate {rate}"
)
cmds.append(
f"tc qdisc add dev {IFACE} parent {class_id} "
f"handle {netem_handle} netem {params.to_tc_args()}"
)
cmds.append(
f"tc filter add dev {IFACE} parent 1: protocol ip "
f"prio {idx} u32 match ip dst {dest_ip}/32 flowid {class_id}"
)
state = LinkNetemState(
container=container,
dest_ip=dest_ip,
class_id=class_id,
netem_handle=netem_handle,
params=params,
rate_mbit=rate_mbit,
)
container_states[dest_ip] = state
full_cmd = " && ".join(cmds)
result = docker_exec_quiet(container, full_cmd, timeout=30)
if result is not None:
log.info(
"Configured per-link netem on %s (%d UDP peers)",
container,
len(udp_peers),
)
else:
log.warning("Failed to configure netem on %s", container)
self.states[container] = container_states
# --- Ethernet peers: simple netem on veth ---
if eth_peers:
container_veth_states = {}
for peer_id in eth_peers:
iface = veth_interface_name(node_id, peer_id)
policy = self._policy_for_edge(node_id, peer_id)
params = self._sample_policy(policy)
cmd = (
f"tc qdisc del dev {iface} root 2>/dev/null || true && "
f"tc qdisc add dev {iface} root netem {params.to_tc_args()}"
)
result = docker_exec_quiet(container, cmd, timeout=10)
if result is not None:
log.debug("Veth netem on %s:%s -> %s", container, iface, params.to_tc_args())
else:
log.warning("Failed to configure veth netem on %s:%s", container, iface)
container_veth_states[iface] = VethNetemState(
container=container,
iface=iface,
params=params,
)
self.veth_states[container] = container_veth_states
log.info(
"Configured per-link netem on %s (%d peers)",
"Configured veth netem on %s (%d Ethernet peers)",
container,
len(peer_ips),
len(eth_peers),
)
else:
log.warning("Failed to configure netem on %s", container)
self.states[container] = container_states
def setup_node(self, node_id: str):
"""Re-apply HTB/netem/filters for a single node (after container restart).
@@ -206,45 +255,62 @@ class NetemManager:
class IDs and current netem params it had before going down.
"""
container = self.topology.container_name(node_id)
# Re-apply UDP netem (eth0 HTB + u32)
container_states = self.states.get(container)
if not container_states:
log.warning("No netem state for %s, skipping setup", container)
return
cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"]
cmds.append(
f"tc qdisc add dev {IFACE} root handle 1: htb default 99"
)
cmds.append(
f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit"
)
for dest_ip, state in container_states.items():
rate = f"{state.rate_mbit}mbit" if state.rate_mbit > 0 else "1gbit"
if container_states:
cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"]
cmds.append(
f"tc class add dev {IFACE} parent 1: classid {state.class_id} htb rate {rate}"
f"tc qdisc add dev {IFACE} root handle 1: htb default 99"
)
cmds.append(
f"tc qdisc add dev {IFACE} parent {state.class_id} "
f"handle {state.netem_handle} netem {state.params.to_tc_args()}"
)
# Extract the priority from class_id (e.g., "1:3" -> 3)
prio = state.class_id.split(":")[1]
cmds.append(
f"tc filter add dev {IFACE} parent 1: protocol ip "
f"prio {prio} u32 match ip dst {dest_ip}/32 flowid {state.class_id}"
f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit"
)
full_cmd = " && ".join(cmds)
result = docker_exec_quiet(container, full_cmd, timeout=30)
if result is not None:
for dest_ip, state in container_states.items():
rate = f"{state.rate_mbit}mbit" if state.rate_mbit > 0 else "1gbit"
cmds.append(
f"tc class add dev {IFACE} parent 1: classid {state.class_id} htb rate {rate}"
)
cmds.append(
f"tc qdisc add dev {IFACE} parent {state.class_id} "
f"handle {state.netem_handle} netem {state.params.to_tc_args()}"
)
prio = state.class_id.split(":")[1]
cmds.append(
f"tc filter add dev {IFACE} parent 1: protocol ip "
f"prio {prio} u32 match ip dst {dest_ip}/32 flowid {state.class_id}"
)
full_cmd = " && ".join(cmds)
result = docker_exec_quiet(container, full_cmd, timeout=30)
if result is not None:
log.info(
"Re-applied UDP netem on %s (%d peers)",
container,
len(container_states),
)
else:
log.warning("Failed to re-apply UDP netem on %s", container)
# Re-apply Ethernet veth netem
veth_states = self.veth_states.get(container)
if veth_states:
for iface, state in veth_states.items():
cmd = (
f"tc qdisc del dev {iface} root 2>/dev/null || true && "
f"tc qdisc add dev {iface} root netem {state.params.to_tc_args()}"
)
result = docker_exec_quiet(container, cmd, timeout=10)
if result is not None:
log.debug("Re-applied veth netem on %s:%s", container, iface)
else:
log.warning("Failed to re-apply veth netem on %s:%s", container, iface)
log.info(
"Re-applied netem on %s (%d peers)",
"Re-applied veth netem on %s (%d Ethernet peers)",
container,
len(container_states),
len(veth_states),
)
else:
log.warning("Failed to re-apply netem on %s", container)
def mutate(self):
"""Randomly mutate netem params on a fraction of links."""
@@ -280,6 +346,8 @@ class NetemManager:
def _update_link(self, node_a: str, node_b: str, params: NetemParams):
"""Update netem on both directions of a link."""
transport = self.topology.transport_for_edge(node_a, node_b)
for src, dst in [(node_a, node_b), (node_b, node_a)]:
if src in self.down_nodes:
continue
@@ -295,26 +363,38 @@ class NetemManager:
self.down_nodes.add(src)
continue
dest_ip = self.topology.nodes[dst].docker_ip
states = self.states.get(container, {})
state = states.get(dest_ip)
if state is None:
continue
cmd = (
f"tc qdisc replace dev {IFACE} parent {state.class_id} "
f"handle {state.netem_handle} netem {params.to_tc_args()}"
)
result = docker_exec_quiet(container, cmd)
if result is not None:
state.params = params
log.debug(
"Updated netem %s -> %s: %s",
src,
dst,
params.to_tc_args(),
if transport == "ethernet":
# Ethernet: simple netem replace on veth
iface = veth_interface_name(src, dst)
veth_states = self.veth_states.get(container, {})
state = veth_states.get(iface)
if state is None:
continue
cmd = f"tc qdisc replace dev {iface} root netem {params.to_tc_args()}"
result = docker_exec_quiet(container, cmd)
if result is not None:
state.params = params
log.debug("Updated veth netem %s:%s -> %s", src, iface, params.to_tc_args())
else:
# UDP: HTB class-based netem on eth0
dest_ip = self.topology.nodes[dst].docker_ip
states = self.states.get(container, {})
state = states.get(dest_ip)
if state is None:
continue
cmd = (
f"tc qdisc replace dev {IFACE} parent {state.class_id} "
f"handle {state.netem_handle} netem {params.to_tc_args()}"
)
result = docker_exec_quiet(container, cmd)
if result is not None:
state.params = params
log.debug(
"Updated netem %s -> %s: %s",
src,
dst,
params.to_tc_args(),
)
def _sample_policy(self, policy: NetemPolicy) -> NetemParams:
"""Sample concrete params from a policy's ranges."""
+9 -2
View File
@@ -42,11 +42,13 @@ class NodeManager:
rng: random.Random,
netem_mgr=None,
down_nodes: set[str] | None = None,
veth_mgr=None,
):
self.topology = topology
self.config = config
self.rng = rng
self.netem_mgr = netem_mgr
self.veth_mgr = veth_mgr
self.down_nodes = down_nodes or set()
self.node_states: dict[str, NodeState] = {
nid: NodeState(node_id=nid) for nid in topology.nodes
@@ -149,10 +151,15 @@ class NodeManager:
log.info("Node STARTED: %s (was down %.0fs)", node_id, down_for)
# Re-create veth pairs (container restart destroys netns)
if self.veth_mgr:
time.sleep(1)
self.veth_mgr.setup_node(node_id)
# Re-apply netem after a brief delay for the container to initialize
if self.netem_mgr:
# Small delay for container networking to be ready
time.sleep(1)
if not self.veth_mgr:
time.sleep(1)
self.netem_mgr.setup_node(node_id)
def _would_disconnect(self, node_id: str) -> bool:
+19 -1
View File
@@ -21,6 +21,7 @@ from .nodes import NodeManager
from .scenario import Scenario
from .topology import SimTopology, generate_topology
from .traffic import TrafficManager
from .veth import VethManager
log = logging.getLogger(__name__)
@@ -39,6 +40,7 @@ class SimRunner:
self._down_nodes: set[str] = set()
# Managers (initialized during setup)
self.veth_mgr: VethManager | None = None
self.netem_mgr: NetemManager | None = None
self.link_mgr: LinkManager | None = None
self.traffic_mgr: TrafficManager | None = None
@@ -125,7 +127,17 @@ class SimRunner:
log.info("Starting %d containers...", len(self.topology.nodes))
docker_compose(self.compose_file, ["up", "-d"])
# 6. Initialize managers
# 6. Set up veth pairs for Ethernet edges (before netem)
#
# The entrypoint script waits for configured Ethernet interfaces
# to appear before starting FIPS, so we just need to create the
# veth pairs promptly after containers are running.
if self.topology.has_ethernet():
self.veth_mgr = VethManager(self.topology)
log.info("Setting up Ethernet veth pairs...")
self.veth_mgr.setup_all()
# 7. Initialize managers
if s.netem.enabled:
bw = s.bandwidth if s.bandwidth.enabled else None
self.netem_mgr = NetemManager(self.topology, s.netem, self.rng, bandwidth=bw)
@@ -147,6 +159,7 @@ class SimRunner:
self.node_mgr = NodeManager(
self.topology, s.node_churn, self.rng,
netem_mgr=self.netem_mgr, down_nodes=self._down_nodes,
veth_mgr=self.veth_mgr,
)
def _warmup(self):
@@ -269,6 +282,11 @@ class SimRunner:
topology=self.topology,
)
# Clean up veth pairs
if self.veth_mgr:
log.info("Cleaning up veth pairs...")
self.veth_mgr.teardown_all()
# Stop containers
log.info("Stopping containers...")
docker_compose(
+22 -4
View File
@@ -22,6 +22,9 @@ class Range:
raise ValueError(f"{name}: min ({self.min}) must be >= 0")
VALID_TRANSPORTS = ("udp", "ethernet")
@dataclass
class TopologyConfig:
num_nodes: int = 10
@@ -30,6 +33,7 @@ class TopologyConfig:
ensure_connected: bool = True
subnet: str = "172.20.0.0/24"
ip_start: int = 10
default_transport: str = "udp"
@dataclass
@@ -186,6 +190,7 @@ def load_scenario(path: str) -> Scenario:
s.topology.ensure_connected = tc.get("ensure_connected", True)
s.topology.subnet = tc.get("subnet", "172.20.0.0/24")
s.topology.ip_start = int(tc.get("ip_start", 10))
s.topology.default_transport = tc.get("default_transport", "udp")
# Netem section
nc = raw.get("netem", {})
@@ -279,17 +284,30 @@ def _validate(s: Scenario):
raise ValueError("topology.num_nodes must be <= 250 (subnet limit)")
if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain", "explicit"):
raise ValueError(f"Unknown topology algorithm: {s.topology.algorithm}")
if s.topology.default_transport not in VALID_TRANSPORTS:
raise ValueError(
f"topology.default_transport: '{s.topology.default_transport}' "
f"not in {VALID_TRANSPORTS}"
)
if s.topology.algorithm == "explicit":
adj = s.topology.params.get("adjacency")
if not adj or not isinstance(adj, list):
raise ValueError("explicit topology requires params.adjacency list")
node_ids = set()
for i, pair in enumerate(adj):
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
for i, entry in enumerate(adj):
if not isinstance(entry, (list, tuple)) or len(entry) not in (2, 3):
raise ValueError(
f"explicit adjacency[{i}]: expected [nodeA, nodeB], got {pair}"
f"explicit adjacency[{i}]: expected [nodeA, nodeB] or "
f"[nodeA, nodeB, transport], got {entry}"
)
node_ids.update(str(p) for p in pair)
node_ids.update(str(p) for p in entry[:2])
if len(entry) == 3:
transport = str(entry[2])
if transport not in VALID_TRANSPORTS:
raise ValueError(
f"explicit adjacency[{i}]: transport '{transport}' "
f"not in {VALID_TRANSPORTS}"
)
if len(node_ids) != s.topology.num_nodes:
raise ValueError(
f"explicit adjacency references {len(node_ids)} nodes "
+87 -14
View File
@@ -18,12 +18,41 @@ class SimNode:
nsec: str # 64-char hex
npub: str # bech32 npub1...
peers: list[str] = field(default_factory=list)
# MAC addresses for Ethernet veth interfaces, keyed by peer_id
ethernet_macs: dict[str, str] = field(default_factory=dict)
@dataclass
class SimTopology:
nodes: dict[str, SimNode] = field(default_factory=dict)
edges: set[tuple[str, str]] = field(default_factory=set)
# Per-edge transport type; edges not in this dict default to "udp"
edge_transport: dict[tuple[str, str], str] = field(default_factory=dict)
def transport_for_edge(self, a: str, b: str) -> str:
"""Get the transport type for an edge (defaults to 'udp')."""
edge = _make_edge(a, b)
return self.edge_transport.get(edge, "udp")
def ethernet_edges(self) -> list[tuple[str, str]]:
"""Return all edges using Ethernet transport."""
return [e for e, t in self.edge_transport.items() if t == "ethernet"]
def has_ethernet(self) -> bool:
"""Check if any edges use Ethernet transport."""
return any(t == "ethernet" for t in self.edge_transport.values())
def ethernet_interfaces(self, node_id: str) -> list[str]:
"""Return the veth interface names for a node's Ethernet edges."""
ifaces = []
for (a, b), transport in self.edge_transport.items():
if transport != "ethernet":
continue
if a == node_id:
ifaces.append(veth_interface_name(a, b))
elif b == node_id:
ifaces.append(veth_interface_name(b, a))
return sorted(ifaces)
def is_connected(self) -> bool:
"""BFS connectivity check."""
@@ -61,20 +90,35 @@ class SimTopology:
return f"fips-node-{node_id}"
def directed_outbound(self) -> dict[str, list[str]]:
"""Assign each edge to exactly one node for outbound connection.
"""Assign each UDP edge to exactly one node for outbound connection.
Returns a mapping from node_id to the list of peers that node
should connect to (outbound only). Every edge appears in exactly
one direction, ensuring auto-reconnect is testable — if B goes
down, only A (the outbound owner) will attempt to reconnect.
Ethernet edges are excluded — they use beacon discovery instead
of static peer configuration.
Strategy: BFS spanning tree edges go parent→child. Non-tree
edges go from the lower node ID to the higher. This guarantees
every node is reachable via at least one inbound connection.
"""
# Only consider UDP edges for static peer config
udp_edges = {
e for e in self.edges
if self.edge_transport.get(e, "udp") == "udp"
}
outbound: dict[str, list[str]] = {nid: [] for nid in self.nodes}
# BFS spanning tree from first node
# Build UDP-only adjacency for BFS
udp_adj: dict[str, list[str]] = {nid: [] for nid in self.nodes}
for a, b in udp_edges:
udp_adj[a].append(b)
udp_adj[b].append(a)
# BFS spanning tree from first node (over UDP edges only)
root = min(self.nodes)
visited: set[str] = set()
tree_edges: set[tuple[str, str]] = set()
@@ -82,15 +126,15 @@ class SimTopology:
visited.add(root)
while queue:
node = queue.popleft()
for peer in self.nodes[node].peers:
for peer in udp_adj[node]:
if peer not in visited:
visited.add(peer)
queue.append(peer)
tree_edges.add((node, peer)) # parent → child
outbound[node].append(peer)
# Non-tree edges: lower ID → higher ID
for a, b in self.edges:
# Non-tree UDP edges: lower ID → higher ID
for a, b in udp_edges:
if (a, b) not in tree_edges and (b, a) not in tree_edges:
outbound[a].append(b) # a < b by _make_edge convention
@@ -134,7 +178,9 @@ def generate_topology(
adjacency = config.params.get("adjacency")
if not adjacency:
raise ValueError("explicit topology requires params.adjacency")
edges = _generate_explicit(adjacency)
edges, edge_transport = _generate_explicit(
adjacency, config.default_transport
)
# Validate all referenced nodes exist
for a, b in edges:
if a not in nodes:
@@ -144,12 +190,16 @@ def generate_topology(
else:
raise ValueError(f"Unknown algorithm: {config.algorithm}")
# For non-explicit topologies, all edges use the default transport
if config.algorithm != "explicit":
edge_transport = {e: config.default_transport for e in edges}
# Build peer lists from edges
for a, b in edges:
nodes[a].peers.append(b)
nodes[b].peers.append(a)
topo = SimTopology(nodes=nodes, edges=edges)
topo = SimTopology(nodes=nodes, edges=edges, edge_transport=edge_transport)
# Connectivity check with retry
if config.ensure_connected:
@@ -223,19 +273,42 @@ def _generate_erdos_renyi(
return edges
def _generate_explicit(adjacency: list) -> set[tuple[str, str]]:
def _generate_explicit(
adjacency: list, default_transport: str = "udp"
) -> tuple[set[tuple[str, str]], dict[tuple[str, str], str]]:
"""Build edges from an explicit adjacency list.
Each entry should be a 2-element list like ["n01", "n02"].
Each entry is a 2-element list ``[nodeA, nodeB]`` (uses default
transport) or a 3-element list ``[nodeA, nodeB, transport]``.
Returns ``(edges, edge_transport)`` where ``edge_transport`` maps
each edge to its transport type.
"""
edges = set()
for i, pair in enumerate(adjacency):
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
edge_transport: dict[tuple[str, str], str] = {}
for i, entry in enumerate(adjacency):
if not isinstance(entry, (list, tuple)) or len(entry) not in (2, 3):
raise ValueError(
f"explicit adjacency[{i}]: expected [nodeA, nodeB], got {pair}"
f"explicit adjacency[{i}]: expected [nodeA, nodeB] or "
f"[nodeA, nodeB, transport], got {entry}"
)
edges.add(_make_edge(str(pair[0]), str(pair[1])))
return edges
edge = _make_edge(str(entry[0]), str(entry[1]))
edges.add(edge)
transport = str(entry[2]) if len(entry) == 3 else default_transport
edge_transport[edge] = transport
return edges, edge_transport
def veth_interface_name(local: str, peer: str) -> str:
"""Generate the veth interface name inside a container.
Format: ``ve-{local}-{peer}`` (max 15 chars for IFNAMSIZ).
For typical node IDs like "n01", this yields "ve-n01-n02" (10 chars).
"""
name = f"ve-{local}-{peer}"
if len(name) > 15:
raise ValueError(f"veth interface name too long: {name!r} ({len(name)} > 15)")
return name
def _make_edge(a: str, b: str) -> tuple[str, str]:
+193
View File
@@ -0,0 +1,193 @@
"""Veth pair management for Ethernet transport edges.
Creates veth pairs between Docker containers for Ethernet-transport
edges. Each Ethernet edge gets a veth pair with one end moved into
each container's network namespace. Naming:
Host (temporary): vh{NN}{MM}a / vh{NN}{MM}b
Container: ve-{local}-{peer} (via veth_interface_name())
After creation, the container-side MAC addresses are queried and
stored in SimNode.ethernet_macs for use in config generation.
"""
from __future__ import annotations
import logging
import subprocess
from .docker_exec import docker_exec_quiet
from .topology import SimTopology, veth_interface_name
log = logging.getLogger(__name__)
class VethManager:
"""Manages veth pairs for Ethernet-transport edges."""
def __init__(self, topology: SimTopology):
self.topology = topology
# Track created host-side temp names for cleanup
self._host_pairs: list[tuple[str, str, str, str]] = []
# (node_a, node_b, host_name_a, host_name_b)
def setup_all(self):
"""Create veth pairs for all Ethernet edges.
For each Ethernet edge:
1. Get container PIDs
2. Create veth pair on host with temp names
3. Move ends into container network namespaces
4. Rename to final names and bring up
5. Query MACs and store in SimNode.ethernet_macs
"""
eth_edges = self.topology.ethernet_edges()
if not eth_edges:
return
log.info("Setting up %d Ethernet veth pairs...", len(eth_edges))
for a, b in eth_edges:
self._create_veth_pair(a, b)
log.info(
"Veth setup complete: %d pairs",
len(self._host_pairs),
)
def setup_node(self, node_id: str):
"""Re-create veth endpoints for a single node after container restart.
When a container restarts (node churn), its network namespace is
destroyed. We re-create the veth pairs for all Ethernet edges
involving this node.
"""
for a, b in self.topology.ethernet_edges():
if a != node_id and b != node_id:
continue
# Remove existing pair if any (host-side might still exist)
nn_a = a.replace("n", "")
nn_b = b.replace("n", "")
host_a = f"vh{nn_a}{nn_b}a"
_run_host(["ip", "link", "delete", host_a], check=False)
# Re-create
self._create_veth_pair(a, b)
def teardown_all(self):
"""Clean up all veth pairs."""
for _, _, host_a, _ in self._host_pairs:
_run_host(["ip", "link", "delete", host_a], check=False)
self._host_pairs.clear()
def _create_veth_pair(self, node_a: str, node_b: str):
"""Create a single veth pair between two containers."""
container_a = self.topology.container_name(node_a)
container_b = self.topology.container_name(node_b)
# Get container PIDs
pid_a = _get_container_pid(container_a)
pid_b = _get_container_pid(container_b)
if pid_a is None or pid_b is None:
log.warning(
"Cannot create veth %s--%s: container PID not found", node_a, node_b
)
return
# Generate names
nn_a = node_a.replace("n", "")
nn_b = node_b.replace("n", "")
host_a = f"vh{nn_a}{nn_b}a"
host_b = f"vh{nn_a}{nn_b}b"
final_a = veth_interface_name(node_a, node_b)
final_b = veth_interface_name(node_b, node_a)
# Clean up any stale pair
_run_host(["ip", "link", "delete", host_a], check=False)
# Create veth pair on host
ok = _run_host([
"ip", "link", "add", host_a, "type", "veth", "peer", "name", host_b,
])
if not ok:
log.warning("Failed to create veth pair %s/%s", host_a, host_b)
return
# Move into container namespaces
_run_host(["ip", "link", "set", host_a, "netns", str(pid_a)])
_run_host(["ip", "link", "set", host_b, "netns", str(pid_b)])
# Rename and bring up inside containers
docker_exec_quiet(
container_a,
f"ip link set {host_a} name {final_a} && ip link set {final_a} up",
timeout=10,
)
docker_exec_quiet(
container_b,
f"ip link set {host_b} name {final_b} && ip link set {final_b} up",
timeout=10,
)
# Query MAC addresses
mac_a = _get_mac_in_container(container_a, final_a)
mac_b = _get_mac_in_container(container_b, final_b)
if mac_a:
self.topology.nodes[node_a].ethernet_macs[node_b] = mac_a
if mac_b:
self.topology.nodes[node_b].ethernet_macs[node_a] = mac_b
self._host_pairs.append((node_a, node_b, host_a, host_b))
log.info(
"Veth %s(%s) -- %s(%s) MAC: %s / %s",
node_a, final_a, node_b, final_b,
mac_a or "?", mac_b or "?",
)
def _get_container_pid(container: str) -> int | None:
"""Get the PID of a running Docker container."""
try:
result = subprocess.run(
["docker", "inspect", "-f", "{{.State.Pid}}", container],
capture_output=True,
text=True,
timeout=10,
)
if result.returncode == 0:
pid = int(result.stdout.strip())
return pid if pid > 0 else None
except (subprocess.TimeoutExpired, ValueError):
pass
return None
def _get_mac_in_container(container: str, iface: str) -> str | None:
"""Query the MAC address of an interface inside a container."""
result = docker_exec_quiet(
container,
f"cat /sys/class/net/{iface}/address",
timeout=5,
)
if result is not None:
return result.strip()
return None
def _run_host(cmd: list[str], check: bool = True) -> bool:
"""Run a command on the host. Returns True on success."""
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=10,
)
if check and result.returncode != 0:
log.debug("Host cmd failed: %s -> %s", " ".join(cmd), result.stderr.strip())
return False
return result.returncode == 0
except subprocess.TimeoutExpired:
log.warning("Host cmd timed out: %s", " ".join(cmd))
return False