mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Add Ethernet transport with beacon discovery
Implement raw Ethernet transport using AF_PACKET SOCK_DGRAM on Linux with EtherType 0x88B5 (IEEE experimental range) and 1-byte frame type prefix (0x00=data, 0x01=beacon). Transport implementation: - EthernetConfig with interface, ethertype, MTU, buffer sizes, and four independent discovery knobs (discovery, announce, auto_connect, accept_connections) - PacketSocket/AsyncPacketSocket wrappers with ioctl helpers for interface index, MAC address, and MTU queries - EthernetTransport with Transport trait impl, async start/stop/send, receive loop dispatching data frames and discovery beacons - Discovery beacons (34 bytes: type + version + x-only pubkey) with DiscoveryBuffer for peer accumulation and dedup - Atomic statistics counters (frames, bytes, errors, beacons) - Platform-gated with #[cfg(target_os = "linux")] Transport-layer discovery integration: - Promote auto_connect() and accept_connections() to Transport trait with default implementations and TransportHandle dispatch - Extract initiate_connection() so both static peer config and discovery auto-connect share the same handshake initiation path - Add poll_transport_discovery() to the tick handler to drain discovery buffers and auto-connect to discovered peers - Enforce accept_connections() in handle_msg1() — transports with accept_connections=false silently drop inbound handshakes Node integration: - create_transports() handles Ethernet named instances - resolve_ethernet_addr() parses "interface/mac" address format - transport_mtu() generalized for multi-transport operation Test harness: - VethPair RAII struct for veth pair lifecycle management - Three #[ignore] integration tests requiring root/CAP_NET_RAW: two-node handshake, data exchange, mixed transport coexistence - Chaos harness: transport-aware topology model, VethManager for veth pairs between Docker containers, Ethernet-aware config gen, netem split (HTB+u32 for UDP, root netem for veth), transport-aware link flaps and node churn with veth re-setup - Container entrypoint waits for configured Ethernet interfaces before starting FIPS (handles veth creation timing) - New scenarios: ethernet-only (4-node ring), ethernet-mesh (6-node mixed UDP+Ethernet with netem and link flaps) Documentation: - fips-transport-layer.md: Ethernet section, beacon discovery, WiFi compatibility, updated discovery state, trait surface additions, implementation status table - fips-configuration.md: Ethernet parameter table, named instances, peer address format, mixed UDP+Ethernet example, complete reference - fips-wire-formats.md: Ethernet frame type prefix note
This commit is contained in:
@@ -27,8 +27,10 @@ RUN printf '%s\n' \
|
||||
COPY fips fipsctl /usr/local/bin/
|
||||
RUN chmod +x /usr/local/bin/fips /usr/local/bin/fipsctl
|
||||
|
||||
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
# Static web page served via Python HTTP server
|
||||
RUN printf 'Fuck IPs!\n' > /root/index.html
|
||||
|
||||
# Start dnsmasq, SSH server, iperf3, and HTTP server in background, then run FIPS
|
||||
ENTRYPOINT ["/bin/bash", "-c", "dnsmasq && /usr/sbin/sshd && iperf3 -s -D && python3 -m http.server 8000 -d /root -b :: &>/dev/null & exec fips --config /etc/fips/fips.yaml"]
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
# Container entrypoint: start services and wait for Ethernet interfaces.
|
||||
#
|
||||
# If the FIPS config references Ethernet transports, wait for the
|
||||
# interfaces to appear before starting the FIPS daemon. This handles
|
||||
# the case where veth pairs are created from the host after the
|
||||
# container starts.
|
||||
|
||||
set -e
|
||||
|
||||
# Start background services
|
||||
dnsmasq
|
||||
/usr/sbin/sshd
|
||||
iperf3 -s -D
|
||||
python3 -m http.server 8000 -d /root -b :: &>/dev/null &
|
||||
|
||||
CONFIG="/etc/fips/fips.yaml"
|
||||
|
||||
# Extract Ethernet interface names from the config file.
|
||||
# Matches "interface: <name>" lines that appear under transports.ethernet.
|
||||
# The sed strips the key prefix and any whitespace.
|
||||
ETH_IFACES=""
|
||||
if grep -q 'ethernet:' "$CONFIG" 2>/dev/null; then
|
||||
ETH_IFACES=$(grep '^\s*interface:' "$CONFIG" \
|
||||
| sed 's/.*interface:\s*//' \
|
||||
| tr -d ' ' || true)
|
||||
fi
|
||||
|
||||
if [ -n "$ETH_IFACES" ]; then
|
||||
echo "Waiting for Ethernet interfaces: $ETH_IFACES"
|
||||
DEADLINE=$((SECONDS + 30))
|
||||
while [ $SECONDS -lt $DEADLINE ]; do
|
||||
ALL_FOUND=true
|
||||
for iface in $ETH_IFACES; do
|
||||
if [ ! -e "/sys/class/net/$iface" ]; then
|
||||
ALL_FOUND=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
if $ALL_FOUND; then
|
||||
echo "All Ethernet interfaces ready"
|
||||
break
|
||||
fi
|
||||
sleep 0.2
|
||||
done
|
||||
if ! $ALL_FOUND; then
|
||||
echo "WARNING: Timed out waiting for Ethernet interfaces"
|
||||
fi
|
||||
fi
|
||||
|
||||
exec fips --config "$CONFIG"
|
||||
@@ -0,0 +1,67 @@
|
||||
# Mixed transport mesh: 6 nodes, UDP + Ethernet edges
|
||||
#
|
||||
# Exercises both transports in a single mesh. UDP edges use static
|
||||
# peer config; Ethernet edges use beacon discovery. Tests that the
|
||||
# spanning tree converges across heterogeneous transports with netem
|
||||
# and link flaps active.
|
||||
#
|
||||
# Topology:
|
||||
#
|
||||
# n01 ---udp--- n02 ---udp--- n03
|
||||
# | | |
|
||||
# eth eth udp
|
||||
# | | |
|
||||
# n04 ---eth--- n05 ---udp--- n06
|
||||
#
|
||||
# UDP edges: n01-n02, n02-n03, n03-n06, n05-n06
|
||||
# Ethernet edges: n01-n04, n02-n05, n04-n05
|
||||
|
||||
scenario:
|
||||
name: "ethernet-mesh"
|
||||
seed: 42
|
||||
duration_secs: 120
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 6
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02, udp]
|
||||
- [n02, n03, udp]
|
||||
- [n03, n06, udp]
|
||||
- [n05, n06, udp]
|
||||
- [n01, n04, ethernet]
|
||||
- [n02, n05, ethernet]
|
||||
- [n04, n05, ethernet]
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
delay_ms: [1, 10]
|
||||
jitter_ms: [0, 2]
|
||||
loss_pct: [0, 1]
|
||||
mutation:
|
||||
interval_secs: {min: 20, max: 40}
|
||||
fraction: 0.3
|
||||
policies:
|
||||
normal:
|
||||
delay_ms: [1, 10]
|
||||
loss_pct: [0, 1]
|
||||
degraded:
|
||||
delay_ms: [30, 80]
|
||||
jitter_ms: [5, 20]
|
||||
loss_pct: [3, 8]
|
||||
|
||||
link_flaps:
|
||||
enabled: true
|
||||
interval_secs: {min: 20, max: 40}
|
||||
max_down_links: 1
|
||||
down_duration_secs: {min: 10, max: 20}
|
||||
protect_connectivity: true
|
||||
|
||||
traffic:
|
||||
enabled: false
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,46 @@
|
||||
# Ethernet-only: 4-node ring, all Ethernet transport
|
||||
#
|
||||
# All links use raw Ethernet (AF_PACKET) over veth pairs. Nodes
|
||||
# discover each other via beacons (no static peer config). Tests
|
||||
# that Ethernet transport and beacon discovery work end-to-end.
|
||||
#
|
||||
# Topology:
|
||||
#
|
||||
# n01 ---eth--- n02
|
||||
# | |
|
||||
# eth eth
|
||||
# | |
|
||||
# n04 ---eth--- n03
|
||||
|
||||
scenario:
|
||||
name: "ethernet-only"
|
||||
seed: 42
|
||||
duration_secs: 90
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
default_transport: ethernet
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n02, n03]
|
||||
- [n03, n04]
|
||||
- [n04, n01]
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: false
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -25,6 +25,7 @@ x-fips-common: &fips-common
|
||||
context: ../..
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- NET_RAW
|
||||
devices:
|
||||
- /dev/net/tun:/dev/net/tun
|
||||
sysctls:
|
||||
|
||||
@@ -54,6 +54,37 @@ def generate_peers_block(
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _build_ethernet_config(iface: str) -> dict:
|
||||
"""Build an Ethernet transport config dict for a single interface."""
|
||||
return {
|
||||
"interface": iface,
|
||||
"discovery": True,
|
||||
"announce": True,
|
||||
"auto_connect": True,
|
||||
"accept_connections": True,
|
||||
"beacon_interval_secs": 10,
|
||||
}
|
||||
|
||||
|
||||
def _inject_ethernet_transports(parsed: dict, eth_ifaces: list[str]):
|
||||
"""Inject Ethernet transport config into a parsed FIPS config.
|
||||
|
||||
For a single interface, uses the single-instance format.
|
||||
For multiple interfaces, uses the named-instances format.
|
||||
Pure-Ethernet nodes (no UDP peers) have their UDP transport removed.
|
||||
"""
|
||||
if not eth_ifaces:
|
||||
return
|
||||
|
||||
transports = parsed.setdefault("transports", {})
|
||||
if len(eth_ifaces) == 1:
|
||||
transports["ethernet"] = _build_ethernet_config(eth_ifaces[0])
|
||||
else:
|
||||
transports["ethernet"] = {
|
||||
iface: _build_ethernet_config(iface) for iface in eth_ifaces
|
||||
}
|
||||
|
||||
|
||||
def generate_node_config(
|
||||
topology: SimTopology,
|
||||
node_id: str,
|
||||
@@ -72,7 +103,22 @@ def generate_node_config(
|
||||
config = config.replace("{{NSEC}}", node.nsec)
|
||||
config = config.replace("{{PEERS}}", peers_yaml)
|
||||
|
||||
if fips_overrides:
|
||||
# Inject Ethernet transport config if this node has Ethernet edges
|
||||
eth_ifaces = topology.ethernet_interfaces(node_id)
|
||||
has_udp_peers = bool(outbound_peers) or _has_inbound_udp_peers(topology, node_id)
|
||||
|
||||
if eth_ifaces or not has_udp_peers:
|
||||
parsed = yaml.safe_load(config)
|
||||
if fips_overrides:
|
||||
parsed = _deep_merge(parsed, fips_overrides)
|
||||
if eth_ifaces:
|
||||
_inject_ethernet_transports(parsed, eth_ifaces)
|
||||
if not has_udp_peers and eth_ifaces:
|
||||
# Pure-Ethernet node: remove UDP transport
|
||||
transports = parsed.get("transports", {})
|
||||
transports.pop("udp", None)
|
||||
config = yaml.dump(parsed, default_flow_style=False, sort_keys=False)
|
||||
elif fips_overrides:
|
||||
parsed = yaml.safe_load(config)
|
||||
merged = _deep_merge(parsed, fips_overrides)
|
||||
config = yaml.dump(merged, default_flow_style=False, sort_keys=False)
|
||||
@@ -80,6 +126,15 @@ def generate_node_config(
|
||||
return config
|
||||
|
||||
|
||||
def _has_inbound_udp_peers(topology: SimTopology, node_id: str) -> bool:
|
||||
"""Check if any other node has a UDP outbound edge to this node."""
|
||||
for peer_id in topology.nodes[node_id].peers:
|
||||
edge = (min(node_id, peer_id), max(node_id, peer_id))
|
||||
if topology.edge_transport.get(edge, "udp") == "udp":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def generate_npubs_env(topology: SimTopology) -> str:
|
||||
"""Generate npubs.env content mapping NPUB_<ID>=<npub> for all nodes."""
|
||||
lines = []
|
||||
|
||||
+34
-17
@@ -15,7 +15,7 @@ from dataclasses import dataclass, field
|
||||
|
||||
from .docker_exec import docker_exec_quiet, is_container_running
|
||||
from .scenario import LinkFlapsConfig
|
||||
from .topology import SimTopology
|
||||
from .topology import SimTopology, veth_interface_name
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -138,7 +138,11 @@ class LinkManager:
|
||||
log.info("Link UP: %s -- %s (was down %.0fs)", a, b, down_for)
|
||||
|
||||
def _set_loss(self, src_node: str, dst_node: str, netem_args: str) -> str | None:
|
||||
"""Set netem args on the tc class for src->dst. Returns the previous netem args."""
|
||||
"""Set netem args on the link for src->dst. Returns the previous netem args.
|
||||
|
||||
Transport-aware: UDP links use tc class on eth0, Ethernet links
|
||||
use tc qdisc replace on the dedicated veth interface.
|
||||
"""
|
||||
if not self.netem_mgr:
|
||||
return None
|
||||
|
||||
@@ -158,25 +162,38 @@ class LinkManager:
|
||||
self.netem_mgr.down_nodes.add(src_node)
|
||||
return None
|
||||
|
||||
dest_ip = self.topology.nodes[dst_node].docker_ip
|
||||
transport = self.topology.transport_for_edge(src_node, dst_node)
|
||||
|
||||
states = self.netem_mgr.states.get(container, {})
|
||||
link_state = states.get(dest_ip)
|
||||
if link_state is None:
|
||||
log.warning("No netem state for %s -> %s", src_node, dst_node)
|
||||
return None
|
||||
if transport == "ethernet":
|
||||
# Ethernet: simple netem on veth
|
||||
iface = veth_interface_name(src_node, dst_node)
|
||||
veth_states = self.netem_mgr.veth_states.get(container, {})
|
||||
veth_state = veth_states.get(iface)
|
||||
if veth_state is None:
|
||||
log.warning("No veth netem state for %s -> %s (%s)", src_node, dst_node, iface)
|
||||
return None
|
||||
|
||||
# Save current params
|
||||
prev_args = link_state.params.to_tc_args()
|
||||
prev_args = veth_state.params.to_tc_args()
|
||||
cmd = f"tc qdisc replace dev {iface} root netem {netem_args}"
|
||||
docker_exec_quiet(container, cmd)
|
||||
return prev_args
|
||||
else:
|
||||
# UDP: HTB class on eth0
|
||||
dest_ip = self.topology.nodes[dst_node].docker_ip
|
||||
|
||||
# Apply new netem
|
||||
cmd = (
|
||||
f"tc qdisc replace dev {IFACE} parent {link_state.class_id} "
|
||||
f"handle {link_state.netem_handle} netem {netem_args}"
|
||||
)
|
||||
docker_exec_quiet(container, cmd)
|
||||
states = self.netem_mgr.states.get(container, {})
|
||||
link_state = states.get(dest_ip)
|
||||
if link_state is None:
|
||||
log.warning("No netem state for %s -> %s", src_node, dst_node)
|
||||
return None
|
||||
|
||||
return prev_args
|
||||
prev_args = link_state.params.to_tc_args()
|
||||
cmd = (
|
||||
f"tc qdisc replace dev {IFACE} parent {link_state.class_id} "
|
||||
f"handle {link_state.netem_handle} netem {netem_args}"
|
||||
)
|
||||
docker_exec_quiet(container, cmd)
|
||||
return prev_args
|
||||
|
||||
def _would_disconnect(self, edge: tuple[str, str]) -> bool:
|
||||
"""Check if removing this edge (plus currently-down edges) disconnects the graph."""
|
||||
|
||||
+184
-104
@@ -18,7 +18,7 @@ from dataclasses import dataclass, field
|
||||
|
||||
from .docker_exec import docker_exec_quiet, is_container_running
|
||||
from .scenario import BandwidthConfig, LinkPolicyOverride, NetemConfig, NetemPolicy
|
||||
from .topology import SimTopology
|
||||
from .topology import SimTopology, veth_interface_name
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -67,6 +67,15 @@ class LinkNetemState:
|
||||
rate_mbit: int = 0 # 0 = unlimited (1gbit default)
|
||||
|
||||
|
||||
@dataclass
|
||||
class VethNetemState:
|
||||
"""Tracks the netem state for a veth interface (Ethernet link direction)."""
|
||||
|
||||
container: str
|
||||
iface: str # e.g., "ve-n01-n02"
|
||||
params: NetemParams = field(default_factory=NetemParams)
|
||||
|
||||
|
||||
class NetemManager:
|
||||
"""Manages per-link netem impairment across all containers."""
|
||||
|
||||
@@ -80,8 +89,10 @@ class NetemManager:
|
||||
self.topology = topology
|
||||
self.config = config
|
||||
self.rng = rng
|
||||
# Per-container, per-dest-ip netem state
|
||||
# Per-container, per-dest-ip netem state (UDP links on eth0)
|
||||
self.states: dict[str, dict[str, LinkNetemState]] = {}
|
||||
# Per-container, per-veth netem state (Ethernet links)
|
||||
self.veth_states: dict[str, dict[str, VethNetemState]] = {}
|
||||
# Nodes currently down (updated by NodeManager) — skip tc ops on these
|
||||
self.down_nodes: set[str] = set()
|
||||
# Per-edge bandwidth: (node_a, node_b) -> rate in mbit
|
||||
@@ -127,7 +138,11 @@ class NetemManager:
|
||||
return self.config.default_policy
|
||||
|
||||
def setup_initial(self):
|
||||
"""Set up HTB qdiscs and initial netem on all containers."""
|
||||
"""Set up HTB qdiscs and initial netem on all containers.
|
||||
|
||||
UDP peers use HTB + u32 filters on eth0. Ethernet peers use a
|
||||
simple root netem qdisc on their dedicated veth interface.
|
||||
"""
|
||||
if self._edge_rates:
|
||||
log.info("Bandwidth pacing enabled (%d edges with rate limits)",
|
||||
len(self._edge_rates) // 2)
|
||||
@@ -135,69 +150,103 @@ class NetemManager:
|
||||
for node_id in sorted(self.topology.nodes):
|
||||
node = self.topology.nodes[node_id]
|
||||
container = self.topology.container_name(node_id)
|
||||
peer_ips = {}
|
||||
|
||||
# Split peers by transport type
|
||||
udp_peers = {}
|
||||
eth_peers = []
|
||||
for peer_id in sorted(node.peers):
|
||||
peer_ips[peer_id] = self.topology.nodes[peer_id].docker_ip
|
||||
transport = self.topology.transport_for_edge(node_id, peer_id)
|
||||
if transport == "ethernet":
|
||||
eth_peers.append(peer_id)
|
||||
else:
|
||||
udp_peers[peer_id] = self.topology.nodes[peer_id].docker_ip
|
||||
|
||||
if not peer_ips:
|
||||
continue
|
||||
|
||||
# Build all tc commands for this container
|
||||
cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"]
|
||||
cmds.append(
|
||||
f"tc qdisc add dev {IFACE} root handle 1: htb default 99"
|
||||
)
|
||||
cmds.append(
|
||||
f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit"
|
||||
)
|
||||
|
||||
container_states = {}
|
||||
|
||||
for idx, (peer_id, dest_ip) in enumerate(peer_ips.items(), start=1):
|
||||
class_id = f"1:{idx}"
|
||||
netem_handle = f"{idx + 10}:"
|
||||
|
||||
# Sample initial params (per-edge override or default policy)
|
||||
policy = self._policy_for_edge(node_id, peer_id)
|
||||
params = self._sample_policy(policy)
|
||||
|
||||
rate = self._htb_rate(node_id, peer_id)
|
||||
rate_mbit = self._edge_rates.get((node_id, peer_id), 0)
|
||||
# --- UDP peers: HTB + u32 on eth0 ---
|
||||
if udp_peers:
|
||||
cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"]
|
||||
cmds.append(
|
||||
f"tc class add dev {IFACE} parent 1: classid {class_id} htb rate {rate}"
|
||||
f"tc qdisc add dev {IFACE} root handle 1: htb default 99"
|
||||
)
|
||||
cmds.append(
|
||||
f"tc qdisc add dev {IFACE} parent {class_id} "
|
||||
f"handle {netem_handle} netem {params.to_tc_args()}"
|
||||
)
|
||||
cmds.append(
|
||||
f"tc filter add dev {IFACE} parent 1: protocol ip "
|
||||
f"prio {idx} u32 match ip dst {dest_ip}/32 flowid {class_id}"
|
||||
f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit"
|
||||
)
|
||||
|
||||
state = LinkNetemState(
|
||||
container=container,
|
||||
dest_ip=dest_ip,
|
||||
class_id=class_id,
|
||||
netem_handle=netem_handle,
|
||||
params=params,
|
||||
rate_mbit=rate_mbit,
|
||||
)
|
||||
container_states[dest_ip] = state
|
||||
container_states = {}
|
||||
|
||||
# Execute all commands in one docker exec
|
||||
full_cmd = " && ".join(cmds)
|
||||
result = docker_exec_quiet(container, full_cmd, timeout=30)
|
||||
if result is not None:
|
||||
for idx, (peer_id, dest_ip) in enumerate(udp_peers.items(), start=1):
|
||||
class_id = f"1:{idx}"
|
||||
netem_handle = f"{idx + 10}:"
|
||||
|
||||
policy = self._policy_for_edge(node_id, peer_id)
|
||||
params = self._sample_policy(policy)
|
||||
|
||||
rate = self._htb_rate(node_id, peer_id)
|
||||
rate_mbit = self._edge_rates.get((node_id, peer_id), 0)
|
||||
cmds.append(
|
||||
f"tc class add dev {IFACE} parent 1: classid {class_id} htb rate {rate}"
|
||||
)
|
||||
cmds.append(
|
||||
f"tc qdisc add dev {IFACE} parent {class_id} "
|
||||
f"handle {netem_handle} netem {params.to_tc_args()}"
|
||||
)
|
||||
cmds.append(
|
||||
f"tc filter add dev {IFACE} parent 1: protocol ip "
|
||||
f"prio {idx} u32 match ip dst {dest_ip}/32 flowid {class_id}"
|
||||
)
|
||||
|
||||
state = LinkNetemState(
|
||||
container=container,
|
||||
dest_ip=dest_ip,
|
||||
class_id=class_id,
|
||||
netem_handle=netem_handle,
|
||||
params=params,
|
||||
rate_mbit=rate_mbit,
|
||||
)
|
||||
container_states[dest_ip] = state
|
||||
|
||||
full_cmd = " && ".join(cmds)
|
||||
result = docker_exec_quiet(container, full_cmd, timeout=30)
|
||||
if result is not None:
|
||||
log.info(
|
||||
"Configured per-link netem on %s (%d UDP peers)",
|
||||
container,
|
||||
len(udp_peers),
|
||||
)
|
||||
else:
|
||||
log.warning("Failed to configure netem on %s", container)
|
||||
|
||||
self.states[container] = container_states
|
||||
|
||||
# --- Ethernet peers: simple netem on veth ---
|
||||
if eth_peers:
|
||||
container_veth_states = {}
|
||||
for peer_id in eth_peers:
|
||||
iface = veth_interface_name(node_id, peer_id)
|
||||
policy = self._policy_for_edge(node_id, peer_id)
|
||||
params = self._sample_policy(policy)
|
||||
|
||||
cmd = (
|
||||
f"tc qdisc del dev {iface} root 2>/dev/null || true && "
|
||||
f"tc qdisc add dev {iface} root netem {params.to_tc_args()}"
|
||||
)
|
||||
result = docker_exec_quiet(container, cmd, timeout=10)
|
||||
if result is not None:
|
||||
log.debug("Veth netem on %s:%s -> %s", container, iface, params.to_tc_args())
|
||||
else:
|
||||
log.warning("Failed to configure veth netem on %s:%s", container, iface)
|
||||
|
||||
container_veth_states[iface] = VethNetemState(
|
||||
container=container,
|
||||
iface=iface,
|
||||
params=params,
|
||||
)
|
||||
|
||||
self.veth_states[container] = container_veth_states
|
||||
log.info(
|
||||
"Configured per-link netem on %s (%d peers)",
|
||||
"Configured veth netem on %s (%d Ethernet peers)",
|
||||
container,
|
||||
len(peer_ips),
|
||||
len(eth_peers),
|
||||
)
|
||||
else:
|
||||
log.warning("Failed to configure netem on %s", container)
|
||||
|
||||
self.states[container] = container_states
|
||||
|
||||
def setup_node(self, node_id: str):
|
||||
"""Re-apply HTB/netem/filters for a single node (after container restart).
|
||||
@@ -206,45 +255,62 @@ class NetemManager:
|
||||
class IDs and current netem params it had before going down.
|
||||
"""
|
||||
container = self.topology.container_name(node_id)
|
||||
|
||||
# Re-apply UDP netem (eth0 HTB + u32)
|
||||
container_states = self.states.get(container)
|
||||
if not container_states:
|
||||
log.warning("No netem state for %s, skipping setup", container)
|
||||
return
|
||||
|
||||
cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"]
|
||||
cmds.append(
|
||||
f"tc qdisc add dev {IFACE} root handle 1: htb default 99"
|
||||
)
|
||||
cmds.append(
|
||||
f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit"
|
||||
)
|
||||
|
||||
for dest_ip, state in container_states.items():
|
||||
rate = f"{state.rate_mbit}mbit" if state.rate_mbit > 0 else "1gbit"
|
||||
if container_states:
|
||||
cmds = [f"tc qdisc del dev {IFACE} root 2>/dev/null || true"]
|
||||
cmds.append(
|
||||
f"tc class add dev {IFACE} parent 1: classid {state.class_id} htb rate {rate}"
|
||||
f"tc qdisc add dev {IFACE} root handle 1: htb default 99"
|
||||
)
|
||||
cmds.append(
|
||||
f"tc qdisc add dev {IFACE} parent {state.class_id} "
|
||||
f"handle {state.netem_handle} netem {state.params.to_tc_args()}"
|
||||
)
|
||||
# Extract the priority from class_id (e.g., "1:3" -> 3)
|
||||
prio = state.class_id.split(":")[1]
|
||||
cmds.append(
|
||||
f"tc filter add dev {IFACE} parent 1: protocol ip "
|
||||
f"prio {prio} u32 match ip dst {dest_ip}/32 flowid {state.class_id}"
|
||||
f"tc class add dev {IFACE} parent 1: classid 1:99 htb rate 1gbit"
|
||||
)
|
||||
|
||||
full_cmd = " && ".join(cmds)
|
||||
result = docker_exec_quiet(container, full_cmd, timeout=30)
|
||||
if result is not None:
|
||||
for dest_ip, state in container_states.items():
|
||||
rate = f"{state.rate_mbit}mbit" if state.rate_mbit > 0 else "1gbit"
|
||||
cmds.append(
|
||||
f"tc class add dev {IFACE} parent 1: classid {state.class_id} htb rate {rate}"
|
||||
)
|
||||
cmds.append(
|
||||
f"tc qdisc add dev {IFACE} parent {state.class_id} "
|
||||
f"handle {state.netem_handle} netem {state.params.to_tc_args()}"
|
||||
)
|
||||
prio = state.class_id.split(":")[1]
|
||||
cmds.append(
|
||||
f"tc filter add dev {IFACE} parent 1: protocol ip "
|
||||
f"prio {prio} u32 match ip dst {dest_ip}/32 flowid {state.class_id}"
|
||||
)
|
||||
|
||||
full_cmd = " && ".join(cmds)
|
||||
result = docker_exec_quiet(container, full_cmd, timeout=30)
|
||||
if result is not None:
|
||||
log.info(
|
||||
"Re-applied UDP netem on %s (%d peers)",
|
||||
container,
|
||||
len(container_states),
|
||||
)
|
||||
else:
|
||||
log.warning("Failed to re-apply UDP netem on %s", container)
|
||||
|
||||
# Re-apply Ethernet veth netem
|
||||
veth_states = self.veth_states.get(container)
|
||||
if veth_states:
|
||||
for iface, state in veth_states.items():
|
||||
cmd = (
|
||||
f"tc qdisc del dev {iface} root 2>/dev/null || true && "
|
||||
f"tc qdisc add dev {iface} root netem {state.params.to_tc_args()}"
|
||||
)
|
||||
result = docker_exec_quiet(container, cmd, timeout=10)
|
||||
if result is not None:
|
||||
log.debug("Re-applied veth netem on %s:%s", container, iface)
|
||||
else:
|
||||
log.warning("Failed to re-apply veth netem on %s:%s", container, iface)
|
||||
log.info(
|
||||
"Re-applied netem on %s (%d peers)",
|
||||
"Re-applied veth netem on %s (%d Ethernet peers)",
|
||||
container,
|
||||
len(container_states),
|
||||
len(veth_states),
|
||||
)
|
||||
else:
|
||||
log.warning("Failed to re-apply netem on %s", container)
|
||||
|
||||
def mutate(self):
|
||||
"""Randomly mutate netem params on a fraction of links."""
|
||||
@@ -280,6 +346,8 @@ class NetemManager:
|
||||
|
||||
def _update_link(self, node_a: str, node_b: str, params: NetemParams):
|
||||
"""Update netem on both directions of a link."""
|
||||
transport = self.topology.transport_for_edge(node_a, node_b)
|
||||
|
||||
for src, dst in [(node_a, node_b), (node_b, node_a)]:
|
||||
if src in self.down_nodes:
|
||||
continue
|
||||
@@ -295,26 +363,38 @@ class NetemManager:
|
||||
self.down_nodes.add(src)
|
||||
continue
|
||||
|
||||
dest_ip = self.topology.nodes[dst].docker_ip
|
||||
|
||||
states = self.states.get(container, {})
|
||||
state = states.get(dest_ip)
|
||||
if state is None:
|
||||
continue
|
||||
|
||||
cmd = (
|
||||
f"tc qdisc replace dev {IFACE} parent {state.class_id} "
|
||||
f"handle {state.netem_handle} netem {params.to_tc_args()}"
|
||||
)
|
||||
result = docker_exec_quiet(container, cmd)
|
||||
if result is not None:
|
||||
state.params = params
|
||||
log.debug(
|
||||
"Updated netem %s -> %s: %s",
|
||||
src,
|
||||
dst,
|
||||
params.to_tc_args(),
|
||||
if transport == "ethernet":
|
||||
# Ethernet: simple netem replace on veth
|
||||
iface = veth_interface_name(src, dst)
|
||||
veth_states = self.veth_states.get(container, {})
|
||||
state = veth_states.get(iface)
|
||||
if state is None:
|
||||
continue
|
||||
cmd = f"tc qdisc replace dev {iface} root netem {params.to_tc_args()}"
|
||||
result = docker_exec_quiet(container, cmd)
|
||||
if result is not None:
|
||||
state.params = params
|
||||
log.debug("Updated veth netem %s:%s -> %s", src, iface, params.to_tc_args())
|
||||
else:
|
||||
# UDP: HTB class-based netem on eth0
|
||||
dest_ip = self.topology.nodes[dst].docker_ip
|
||||
states = self.states.get(container, {})
|
||||
state = states.get(dest_ip)
|
||||
if state is None:
|
||||
continue
|
||||
cmd = (
|
||||
f"tc qdisc replace dev {IFACE} parent {state.class_id} "
|
||||
f"handle {state.netem_handle} netem {params.to_tc_args()}"
|
||||
)
|
||||
result = docker_exec_quiet(container, cmd)
|
||||
if result is not None:
|
||||
state.params = params
|
||||
log.debug(
|
||||
"Updated netem %s -> %s: %s",
|
||||
src,
|
||||
dst,
|
||||
params.to_tc_args(),
|
||||
)
|
||||
|
||||
def _sample_policy(self, policy: NetemPolicy) -> NetemParams:
|
||||
"""Sample concrete params from a policy's ranges."""
|
||||
|
||||
@@ -42,11 +42,13 @@ class NodeManager:
|
||||
rng: random.Random,
|
||||
netem_mgr=None,
|
||||
down_nodes: set[str] | None = None,
|
||||
veth_mgr=None,
|
||||
):
|
||||
self.topology = topology
|
||||
self.config = config
|
||||
self.rng = rng
|
||||
self.netem_mgr = netem_mgr
|
||||
self.veth_mgr = veth_mgr
|
||||
self.down_nodes = down_nodes or set()
|
||||
self.node_states: dict[str, NodeState] = {
|
||||
nid: NodeState(node_id=nid) for nid in topology.nodes
|
||||
@@ -149,10 +151,15 @@ class NodeManager:
|
||||
|
||||
log.info("Node STARTED: %s (was down %.0fs)", node_id, down_for)
|
||||
|
||||
# Re-create veth pairs (container restart destroys netns)
|
||||
if self.veth_mgr:
|
||||
time.sleep(1)
|
||||
self.veth_mgr.setup_node(node_id)
|
||||
|
||||
# Re-apply netem after a brief delay for the container to initialize
|
||||
if self.netem_mgr:
|
||||
# Small delay for container networking to be ready
|
||||
time.sleep(1)
|
||||
if not self.veth_mgr:
|
||||
time.sleep(1)
|
||||
self.netem_mgr.setup_node(node_id)
|
||||
|
||||
def _would_disconnect(self, node_id: str) -> bool:
|
||||
|
||||
@@ -21,6 +21,7 @@ from .nodes import NodeManager
|
||||
from .scenario import Scenario
|
||||
from .topology import SimTopology, generate_topology
|
||||
from .traffic import TrafficManager
|
||||
from .veth import VethManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -39,6 +40,7 @@ class SimRunner:
|
||||
self._down_nodes: set[str] = set()
|
||||
|
||||
# Managers (initialized during setup)
|
||||
self.veth_mgr: VethManager | None = None
|
||||
self.netem_mgr: NetemManager | None = None
|
||||
self.link_mgr: LinkManager | None = None
|
||||
self.traffic_mgr: TrafficManager | None = None
|
||||
@@ -125,7 +127,17 @@ class SimRunner:
|
||||
log.info("Starting %d containers...", len(self.topology.nodes))
|
||||
docker_compose(self.compose_file, ["up", "-d"])
|
||||
|
||||
# 6. Initialize managers
|
||||
# 6. Set up veth pairs for Ethernet edges (before netem)
|
||||
#
|
||||
# The entrypoint script waits for configured Ethernet interfaces
|
||||
# to appear before starting FIPS, so we just need to create the
|
||||
# veth pairs promptly after containers are running.
|
||||
if self.topology.has_ethernet():
|
||||
self.veth_mgr = VethManager(self.topology)
|
||||
log.info("Setting up Ethernet veth pairs...")
|
||||
self.veth_mgr.setup_all()
|
||||
|
||||
# 7. Initialize managers
|
||||
if s.netem.enabled:
|
||||
bw = s.bandwidth if s.bandwidth.enabled else None
|
||||
self.netem_mgr = NetemManager(self.topology, s.netem, self.rng, bandwidth=bw)
|
||||
@@ -147,6 +159,7 @@ class SimRunner:
|
||||
self.node_mgr = NodeManager(
|
||||
self.topology, s.node_churn, self.rng,
|
||||
netem_mgr=self.netem_mgr, down_nodes=self._down_nodes,
|
||||
veth_mgr=self.veth_mgr,
|
||||
)
|
||||
|
||||
def _warmup(self):
|
||||
@@ -269,6 +282,11 @@ class SimRunner:
|
||||
topology=self.topology,
|
||||
)
|
||||
|
||||
# Clean up veth pairs
|
||||
if self.veth_mgr:
|
||||
log.info("Cleaning up veth pairs...")
|
||||
self.veth_mgr.teardown_all()
|
||||
|
||||
# Stop containers
|
||||
log.info("Stopping containers...")
|
||||
docker_compose(
|
||||
|
||||
@@ -22,6 +22,9 @@ class Range:
|
||||
raise ValueError(f"{name}: min ({self.min}) must be >= 0")
|
||||
|
||||
|
||||
VALID_TRANSPORTS = ("udp", "ethernet")
|
||||
|
||||
|
||||
@dataclass
|
||||
class TopologyConfig:
|
||||
num_nodes: int = 10
|
||||
@@ -30,6 +33,7 @@ class TopologyConfig:
|
||||
ensure_connected: bool = True
|
||||
subnet: str = "172.20.0.0/24"
|
||||
ip_start: int = 10
|
||||
default_transport: str = "udp"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -186,6 +190,7 @@ def load_scenario(path: str) -> Scenario:
|
||||
s.topology.ensure_connected = tc.get("ensure_connected", True)
|
||||
s.topology.subnet = tc.get("subnet", "172.20.0.0/24")
|
||||
s.topology.ip_start = int(tc.get("ip_start", 10))
|
||||
s.topology.default_transport = tc.get("default_transport", "udp")
|
||||
|
||||
# Netem section
|
||||
nc = raw.get("netem", {})
|
||||
@@ -279,17 +284,30 @@ def _validate(s: Scenario):
|
||||
raise ValueError("topology.num_nodes must be <= 250 (subnet limit)")
|
||||
if s.topology.algorithm not in ("random_geometric", "erdos_renyi", "chain", "explicit"):
|
||||
raise ValueError(f"Unknown topology algorithm: {s.topology.algorithm}")
|
||||
if s.topology.default_transport not in VALID_TRANSPORTS:
|
||||
raise ValueError(
|
||||
f"topology.default_transport: '{s.topology.default_transport}' "
|
||||
f"not in {VALID_TRANSPORTS}"
|
||||
)
|
||||
if s.topology.algorithm == "explicit":
|
||||
adj = s.topology.params.get("adjacency")
|
||||
if not adj or not isinstance(adj, list):
|
||||
raise ValueError("explicit topology requires params.adjacency list")
|
||||
node_ids = set()
|
||||
for i, pair in enumerate(adj):
|
||||
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
|
||||
for i, entry in enumerate(adj):
|
||||
if not isinstance(entry, (list, tuple)) or len(entry) not in (2, 3):
|
||||
raise ValueError(
|
||||
f"explicit adjacency[{i}]: expected [nodeA, nodeB], got {pair}"
|
||||
f"explicit adjacency[{i}]: expected [nodeA, nodeB] or "
|
||||
f"[nodeA, nodeB, transport], got {entry}"
|
||||
)
|
||||
node_ids.update(str(p) for p in pair)
|
||||
node_ids.update(str(p) for p in entry[:2])
|
||||
if len(entry) == 3:
|
||||
transport = str(entry[2])
|
||||
if transport not in VALID_TRANSPORTS:
|
||||
raise ValueError(
|
||||
f"explicit adjacency[{i}]: transport '{transport}' "
|
||||
f"not in {VALID_TRANSPORTS}"
|
||||
)
|
||||
if len(node_ids) != s.topology.num_nodes:
|
||||
raise ValueError(
|
||||
f"explicit adjacency references {len(node_ids)} nodes "
|
||||
|
||||
@@ -18,12 +18,41 @@ class SimNode:
|
||||
nsec: str # 64-char hex
|
||||
npub: str # bech32 npub1...
|
||||
peers: list[str] = field(default_factory=list)
|
||||
# MAC addresses for Ethernet veth interfaces, keyed by peer_id
|
||||
ethernet_macs: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimTopology:
|
||||
nodes: dict[str, SimNode] = field(default_factory=dict)
|
||||
edges: set[tuple[str, str]] = field(default_factory=set)
|
||||
# Per-edge transport type; edges not in this dict default to "udp"
|
||||
edge_transport: dict[tuple[str, str], str] = field(default_factory=dict)
|
||||
|
||||
def transport_for_edge(self, a: str, b: str) -> str:
|
||||
"""Get the transport type for an edge (defaults to 'udp')."""
|
||||
edge = _make_edge(a, b)
|
||||
return self.edge_transport.get(edge, "udp")
|
||||
|
||||
def ethernet_edges(self) -> list[tuple[str, str]]:
|
||||
"""Return all edges using Ethernet transport."""
|
||||
return [e for e, t in self.edge_transport.items() if t == "ethernet"]
|
||||
|
||||
def has_ethernet(self) -> bool:
|
||||
"""Check if any edges use Ethernet transport."""
|
||||
return any(t == "ethernet" for t in self.edge_transport.values())
|
||||
|
||||
def ethernet_interfaces(self, node_id: str) -> list[str]:
|
||||
"""Return the veth interface names for a node's Ethernet edges."""
|
||||
ifaces = []
|
||||
for (a, b), transport in self.edge_transport.items():
|
||||
if transport != "ethernet":
|
||||
continue
|
||||
if a == node_id:
|
||||
ifaces.append(veth_interface_name(a, b))
|
||||
elif b == node_id:
|
||||
ifaces.append(veth_interface_name(b, a))
|
||||
return sorted(ifaces)
|
||||
|
||||
def is_connected(self) -> bool:
|
||||
"""BFS connectivity check."""
|
||||
@@ -61,20 +90,35 @@ class SimTopology:
|
||||
return f"fips-node-{node_id}"
|
||||
|
||||
def directed_outbound(self) -> dict[str, list[str]]:
|
||||
"""Assign each edge to exactly one node for outbound connection.
|
||||
"""Assign each UDP edge to exactly one node for outbound connection.
|
||||
|
||||
Returns a mapping from node_id to the list of peers that node
|
||||
should connect to (outbound only). Every edge appears in exactly
|
||||
one direction, ensuring auto-reconnect is testable — if B goes
|
||||
down, only A (the outbound owner) will attempt to reconnect.
|
||||
|
||||
Ethernet edges are excluded — they use beacon discovery instead
|
||||
of static peer configuration.
|
||||
|
||||
Strategy: BFS spanning tree edges go parent→child. Non-tree
|
||||
edges go from the lower node ID to the higher. This guarantees
|
||||
every node is reachable via at least one inbound connection.
|
||||
"""
|
||||
# Only consider UDP edges for static peer config
|
||||
udp_edges = {
|
||||
e for e in self.edges
|
||||
if self.edge_transport.get(e, "udp") == "udp"
|
||||
}
|
||||
|
||||
outbound: dict[str, list[str]] = {nid: [] for nid in self.nodes}
|
||||
|
||||
# BFS spanning tree from first node
|
||||
# Build UDP-only adjacency for BFS
|
||||
udp_adj: dict[str, list[str]] = {nid: [] for nid in self.nodes}
|
||||
for a, b in udp_edges:
|
||||
udp_adj[a].append(b)
|
||||
udp_adj[b].append(a)
|
||||
|
||||
# BFS spanning tree from first node (over UDP edges only)
|
||||
root = min(self.nodes)
|
||||
visited: set[str] = set()
|
||||
tree_edges: set[tuple[str, str]] = set()
|
||||
@@ -82,15 +126,15 @@ class SimTopology:
|
||||
visited.add(root)
|
||||
while queue:
|
||||
node = queue.popleft()
|
||||
for peer in self.nodes[node].peers:
|
||||
for peer in udp_adj[node]:
|
||||
if peer not in visited:
|
||||
visited.add(peer)
|
||||
queue.append(peer)
|
||||
tree_edges.add((node, peer)) # parent → child
|
||||
outbound[node].append(peer)
|
||||
|
||||
# Non-tree edges: lower ID → higher ID
|
||||
for a, b in self.edges:
|
||||
# Non-tree UDP edges: lower ID → higher ID
|
||||
for a, b in udp_edges:
|
||||
if (a, b) not in tree_edges and (b, a) not in tree_edges:
|
||||
outbound[a].append(b) # a < b by _make_edge convention
|
||||
|
||||
@@ -134,7 +178,9 @@ def generate_topology(
|
||||
adjacency = config.params.get("adjacency")
|
||||
if not adjacency:
|
||||
raise ValueError("explicit topology requires params.adjacency")
|
||||
edges = _generate_explicit(adjacency)
|
||||
edges, edge_transport = _generate_explicit(
|
||||
adjacency, config.default_transport
|
||||
)
|
||||
# Validate all referenced nodes exist
|
||||
for a, b in edges:
|
||||
if a not in nodes:
|
||||
@@ -144,12 +190,16 @@ def generate_topology(
|
||||
else:
|
||||
raise ValueError(f"Unknown algorithm: {config.algorithm}")
|
||||
|
||||
# For non-explicit topologies, all edges use the default transport
|
||||
if config.algorithm != "explicit":
|
||||
edge_transport = {e: config.default_transport for e in edges}
|
||||
|
||||
# Build peer lists from edges
|
||||
for a, b in edges:
|
||||
nodes[a].peers.append(b)
|
||||
nodes[b].peers.append(a)
|
||||
|
||||
topo = SimTopology(nodes=nodes, edges=edges)
|
||||
topo = SimTopology(nodes=nodes, edges=edges, edge_transport=edge_transport)
|
||||
|
||||
# Connectivity check with retry
|
||||
if config.ensure_connected:
|
||||
@@ -223,19 +273,42 @@ def _generate_erdos_renyi(
|
||||
return edges
|
||||
|
||||
|
||||
def _generate_explicit(adjacency: list) -> set[tuple[str, str]]:
|
||||
def _generate_explicit(
|
||||
adjacency: list, default_transport: str = "udp"
|
||||
) -> tuple[set[tuple[str, str]], dict[tuple[str, str], str]]:
|
||||
"""Build edges from an explicit adjacency list.
|
||||
|
||||
Each entry should be a 2-element list like ["n01", "n02"].
|
||||
Each entry is a 2-element list ``[nodeA, nodeB]`` (uses default
|
||||
transport) or a 3-element list ``[nodeA, nodeB, transport]``.
|
||||
|
||||
Returns ``(edges, edge_transport)`` where ``edge_transport`` maps
|
||||
each edge to its transport type.
|
||||
"""
|
||||
edges = set()
|
||||
for i, pair in enumerate(adjacency):
|
||||
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
|
||||
edge_transport: dict[tuple[str, str], str] = {}
|
||||
for i, entry in enumerate(adjacency):
|
||||
if not isinstance(entry, (list, tuple)) or len(entry) not in (2, 3):
|
||||
raise ValueError(
|
||||
f"explicit adjacency[{i}]: expected [nodeA, nodeB], got {pair}"
|
||||
f"explicit adjacency[{i}]: expected [nodeA, nodeB] or "
|
||||
f"[nodeA, nodeB, transport], got {entry}"
|
||||
)
|
||||
edges.add(_make_edge(str(pair[0]), str(pair[1])))
|
||||
return edges
|
||||
edge = _make_edge(str(entry[0]), str(entry[1]))
|
||||
edges.add(edge)
|
||||
transport = str(entry[2]) if len(entry) == 3 else default_transport
|
||||
edge_transport[edge] = transport
|
||||
return edges, edge_transport
|
||||
|
||||
|
||||
def veth_interface_name(local: str, peer: str) -> str:
|
||||
"""Generate the veth interface name inside a container.
|
||||
|
||||
Format: ``ve-{local}-{peer}`` (max 15 chars for IFNAMSIZ).
|
||||
For typical node IDs like "n01", this yields "ve-n01-n02" (10 chars).
|
||||
"""
|
||||
name = f"ve-{local}-{peer}"
|
||||
if len(name) > 15:
|
||||
raise ValueError(f"veth interface name too long: {name!r} ({len(name)} > 15)")
|
||||
return name
|
||||
|
||||
|
||||
def _make_edge(a: str, b: str) -> tuple[str, str]:
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Veth pair management for Ethernet transport edges.
|
||||
|
||||
Creates veth pairs between Docker containers for Ethernet-transport
|
||||
edges. Each Ethernet edge gets a veth pair with one end moved into
|
||||
each container's network namespace. Naming:
|
||||
|
||||
Host (temporary): vh{NN}{MM}a / vh{NN}{MM}b
|
||||
Container: ve-{local}-{peer} (via veth_interface_name())
|
||||
|
||||
After creation, the container-side MAC addresses are queried and
|
||||
stored in SimNode.ethernet_macs for use in config generation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
from .docker_exec import docker_exec_quiet
|
||||
from .topology import SimTopology, veth_interface_name
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VethManager:
|
||||
"""Manages veth pairs for Ethernet-transport edges."""
|
||||
|
||||
def __init__(self, topology: SimTopology):
|
||||
self.topology = topology
|
||||
# Track created host-side temp names for cleanup
|
||||
self._host_pairs: list[tuple[str, str, str, str]] = []
|
||||
# (node_a, node_b, host_name_a, host_name_b)
|
||||
|
||||
def setup_all(self):
|
||||
"""Create veth pairs for all Ethernet edges.
|
||||
|
||||
For each Ethernet edge:
|
||||
1. Get container PIDs
|
||||
2. Create veth pair on host with temp names
|
||||
3. Move ends into container network namespaces
|
||||
4. Rename to final names and bring up
|
||||
5. Query MACs and store in SimNode.ethernet_macs
|
||||
"""
|
||||
eth_edges = self.topology.ethernet_edges()
|
||||
if not eth_edges:
|
||||
return
|
||||
|
||||
log.info("Setting up %d Ethernet veth pairs...", len(eth_edges))
|
||||
|
||||
for a, b in eth_edges:
|
||||
self._create_veth_pair(a, b)
|
||||
|
||||
log.info(
|
||||
"Veth setup complete: %d pairs",
|
||||
len(self._host_pairs),
|
||||
)
|
||||
|
||||
def setup_node(self, node_id: str):
|
||||
"""Re-create veth endpoints for a single node after container restart.
|
||||
|
||||
When a container restarts (node churn), its network namespace is
|
||||
destroyed. We re-create the veth pairs for all Ethernet edges
|
||||
involving this node.
|
||||
"""
|
||||
for a, b in self.topology.ethernet_edges():
|
||||
if a != node_id and b != node_id:
|
||||
continue
|
||||
# Remove existing pair if any (host-side might still exist)
|
||||
nn_a = a.replace("n", "")
|
||||
nn_b = b.replace("n", "")
|
||||
host_a = f"vh{nn_a}{nn_b}a"
|
||||
_run_host(["ip", "link", "delete", host_a], check=False)
|
||||
# Re-create
|
||||
self._create_veth_pair(a, b)
|
||||
|
||||
def teardown_all(self):
|
||||
"""Clean up all veth pairs."""
|
||||
for _, _, host_a, _ in self._host_pairs:
|
||||
_run_host(["ip", "link", "delete", host_a], check=False)
|
||||
self._host_pairs.clear()
|
||||
|
||||
def _create_veth_pair(self, node_a: str, node_b: str):
|
||||
"""Create a single veth pair between two containers."""
|
||||
container_a = self.topology.container_name(node_a)
|
||||
container_b = self.topology.container_name(node_b)
|
||||
|
||||
# Get container PIDs
|
||||
pid_a = _get_container_pid(container_a)
|
||||
pid_b = _get_container_pid(container_b)
|
||||
if pid_a is None or pid_b is None:
|
||||
log.warning(
|
||||
"Cannot create veth %s--%s: container PID not found", node_a, node_b
|
||||
)
|
||||
return
|
||||
|
||||
# Generate names
|
||||
nn_a = node_a.replace("n", "")
|
||||
nn_b = node_b.replace("n", "")
|
||||
host_a = f"vh{nn_a}{nn_b}a"
|
||||
host_b = f"vh{nn_a}{nn_b}b"
|
||||
final_a = veth_interface_name(node_a, node_b)
|
||||
final_b = veth_interface_name(node_b, node_a)
|
||||
|
||||
# Clean up any stale pair
|
||||
_run_host(["ip", "link", "delete", host_a], check=False)
|
||||
|
||||
# Create veth pair on host
|
||||
ok = _run_host([
|
||||
"ip", "link", "add", host_a, "type", "veth", "peer", "name", host_b,
|
||||
])
|
||||
if not ok:
|
||||
log.warning("Failed to create veth pair %s/%s", host_a, host_b)
|
||||
return
|
||||
|
||||
# Move into container namespaces
|
||||
_run_host(["ip", "link", "set", host_a, "netns", str(pid_a)])
|
||||
_run_host(["ip", "link", "set", host_b, "netns", str(pid_b)])
|
||||
|
||||
# Rename and bring up inside containers
|
||||
docker_exec_quiet(
|
||||
container_a,
|
||||
f"ip link set {host_a} name {final_a} && ip link set {final_a} up",
|
||||
timeout=10,
|
||||
)
|
||||
docker_exec_quiet(
|
||||
container_b,
|
||||
f"ip link set {host_b} name {final_b} && ip link set {final_b} up",
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
# Query MAC addresses
|
||||
mac_a = _get_mac_in_container(container_a, final_a)
|
||||
mac_b = _get_mac_in_container(container_b, final_b)
|
||||
|
||||
if mac_a:
|
||||
self.topology.nodes[node_a].ethernet_macs[node_b] = mac_a
|
||||
if mac_b:
|
||||
self.topology.nodes[node_b].ethernet_macs[node_a] = mac_b
|
||||
|
||||
self._host_pairs.append((node_a, node_b, host_a, host_b))
|
||||
|
||||
log.info(
|
||||
"Veth %s(%s) -- %s(%s) MAC: %s / %s",
|
||||
node_a, final_a, node_b, final_b,
|
||||
mac_a or "?", mac_b or "?",
|
||||
)
|
||||
|
||||
|
||||
def _get_container_pid(container: str) -> int | None:
|
||||
"""Get the PID of a running Docker container."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.State.Pid}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
pid = int(result.stdout.strip())
|
||||
return pid if pid > 0 else None
|
||||
except (subprocess.TimeoutExpired, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _get_mac_in_container(container: str, iface: str) -> str | None:
|
||||
"""Query the MAC address of an interface inside a container."""
|
||||
result = docker_exec_quiet(
|
||||
container,
|
||||
f"cat /sys/class/net/{iface}/address",
|
||||
timeout=5,
|
||||
)
|
||||
if result is not None:
|
||||
return result.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _run_host(cmd: list[str], check: bool = True) -> bool:
|
||||
"""Run a command on the host. Returns True on success."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
log.debug("Host cmd failed: %s -> %s", " ".join(cmd), result.stderr.strip())
|
||||
return False
|
||||
return result.returncode == 0
|
||||
except subprocess.TimeoutExpired:
|
||||
log.warning("Host cmd timed out: %s", " ".join(cmd))
|
||||
return False
|
||||
Reference in New Issue
Block a user