laantungir/fips
1
0
Fork 0
mirror of https://github.com/jmcorgan/fips.git synced 2026-07-30 19:46:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add connect/disconnect control commands and maelstrom chaos scenario

Browse Source 
Add runtime peer management to the FIPS daemon via control socket
commands, and a new chaos simulation scenario that exercises dynamic
topology mutation with ephemeral node identities.

Daemon (connect/disconnect commands):
- Extend control socket Request with optional params field
- Add commands.rs module for mutating command dispatch, separate from
  read-only queries
- Add api_connect() on Node: builds ephemeral PeerConfig (no auto-
  reconnect), pre-seeds identity cache, reuses initiate_peer_connection
- Add api_disconnect() on Node: calls remove_active_peer(), clears
  retry_pending to suppress reconnection
- Route non-show_* commands to async command dispatch in rx_loop

fipsctl CLI:
- Add Connect and Disconnect subcommands accepting npub or hostname
- Resolve hostnames from /etc/fips/hosts before sending to daemon
- Refactor socket I/O into reusable send_request helper

Chaos simulator (maelstrom scenario):
- Add PeerChurnManager: periodically disconnects a random active link
  and connects a random unconnected node pair via control socket
- Add send_command() to control.py using base64-encoded JSON payloads
  to avoid shell quoting issues in docker exec
- Add PeerChurnConfig to scenario with interval and ephemeral_fraction
- Ephemeral identity support: nodes configured without nsec generate
  fresh keypairs on restart; simulator queries show_status for new
  npub and updates its cache via on_node_restart callback
- Add maelstrom.yaml: all chaos dimensions (netem, link flaps, node
  churn, peer topology churn, traffic) with 50% ephemeral identity
This commit is contained in:
Johnathan Corgan
2026-03-20 05:44:31 +00:00
parent 8d51dbd268
commit 5053cf673d
13 changed files with 757 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
testing/chaos/scenarios/maelstrom.yaml
+76
View File
@@ -0,0 +1,76 @@
# Maelstrom: topology mutation + node churn + ephemeral identity
#
# Combines all chaos dimensions: netem impairments, link flaps, node
# churn, AND peer-level topology mutation (connect/disconnect).
# Half the nodes have ephemeral identities that change on restart.
#
# Default: 20 nodes. Override with --nodes flag.
scenario:
name: "maelstrom"
seed: 42
duration_secs: 600
topology:
num_nodes: 20
algorithm: erdos_renyi
params:
p: 0.3
ensure_connected: true
subnet: "172.20.0.0/16"
ip_start: 10
transport_mix:
udp: 0.8
tcp: 0.2
netem:
enabled: true
default_policy:
delay_ms: { min: 5, max: 50 }
jitter_ms: { min: 1, max: 10 }
loss_pct: { min: 0, max: 2 }
mutation:
interval_secs: { min: 20, max: 45 }
fraction: 0.3
policies:
normal:
delay_ms: [5, 20]
loss_pct: [0, 1]
degraded:
delay_ms: [50, 100]
jitter_ms: [10, 30]
loss_pct: [3, 8]
link_flaps:
enabled: true
interval_secs: { min: 30, max: 60 }
max_down_links: 3
down_duration_secs: { min: 10, max: 30 }
protect_connectivity: true
traffic:
enabled: true
max_concurrent: 5
interval_secs: { min: 5, max: 30 }
duration_secs: { min: 5, max: 60 }
parallel_streams: 4
node_churn:
enabled: true
interval_secs: { min: 60, max: 120 }
max_down_nodes: 3
down_duration_secs: { min: 30, max: 90 }
protect_connectivity: false
peer_churn:
enabled: true
interval_secs: { min: 8, max: 12 }
ephemeral_fraction: 0.5
bandwidth:
enabled: true
tiers_mbps: [1, 10, 100, 1000]
logging:
rust_log: "debug"
output_dir: "./sim-results"
testing/chaos/sim/config_gen.py
+17 -2
View File
@@ -107,8 +107,13 @@ def generate_node_config(
node_id: str,
outbound_peers: list[str],
fips_overrides: dict | None = None,
ephemeral: bool = False,
) -> str:
"""Generate a complete FIPS config YAML for one node."""
"""Generate a complete FIPS config YAML for one node.
If ephemeral is True, the nsec is omitted from the config so the
daemon generates a fresh keypair on each restart.
"""
template = _load_template()
node = topology.nodes[node_id]
peers_yaml = generate_peers_block(topology, node_id, outbound_peers)
@@ -120,6 +125,13 @@ def generate_node_config(
config = config.replace("{{NSEC}}", node.nsec)
config = config.replace("{{PEERS}}", peers_yaml)
# Ephemeral nodes: remove nsec so daemon generates fresh keys on restart
if ephemeral:
parsed = yaml.safe_load(config)
identity = parsed.get("node", {}).get("identity", {})
identity.pop("nsec", None)
config = yaml.dump(parsed, default_flow_style=False, sort_keys=False)
# Determine which transports this node participates in
eth_ifaces = topology.ethernet_interfaces(node_id)
has_tcp = bool(topology.tcp_peers(node_id))
@@ -168,14 +180,17 @@ def write_configs(
topology: SimTopology,
output_dir: str,
fips_overrides: dict | None = None,
ephemeral_nodes: set[str] | None = None,
):
"""Write all node configs and npubs.env to the output directory."""
os.makedirs(output_dir, exist_ok=True)
ephemeral_nodes = ephemeral_nodes or set()
outbound = topology.directed_outbound()
for node_id in topology.nodes:
config = generate_node_config(
topology, node_id, outbound[node_id], fips_overrides
topology, node_id, outbound[node_id], fips_overrides,
ephemeral=(node_id in ephemeral_nodes),
)
path = os.path.join(output_dir, f"{node_id}.yaml")
with open(path, "w") as f:
testing/chaos/sim/control.py
+46
View File
@@ -56,6 +56,52 @@ def query_node(container: str, command: str, timeout: int = 10) -> dict | None:
return response.get("data", {})
def send_command(
container: str, command: str, params: dict, timeout: int = 10
) -> dict | None:
"""Send a mutating command with params to a node's control socket.
Returns the response data dict, or None on failure.
Uses base64-encoded JSON to avoid shell quoting issues with
embedded quotes in the payload.
"""
import base64
payload = json.dumps({"command": command, "params": params})
b64 = base64.b64encode(payload.encode()).decode()
script = (
"import socket,json,sys,base64; "
"s=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM); "
f"s.connect('{CONTROL_SOCKET}'); "
f"s.sendall(base64.b64decode('{b64}')+b'\\n'); "
"s.shutdown(socket.SHUT_WR); "
"chunks=[]; "
"[chunks.append(d) for d in iter(lambda:s.recv(65536),b'')]; "
"print(b''.join(chunks).decode())"
)
stdout = docker_exec_quiet(container, f"python3 -c \"{script}\"", timeout=timeout)
if stdout is None:
return None
try:
response = json.loads(stdout.strip())
except json.JSONDecodeError as e:
log.warning("Invalid JSON from %s: %s", container, e)
return None
if response.get("status") != "ok":
msg = response.get("message", "unknown error")
log.debug("Command %s on %s: %s", command, container, msg)
return None
return response.get("data", {})
def query_status(container: str) -> dict | None:
"""Query a node's status (npub, ipv6, uptime, etc.)."""
return query_node(container, "show_status")
def query_tree(container: str) -> dict | None:
"""Query a node's spanning tree state."""
return query_node(container, "show_tree")
testing/chaos/sim/nodes.py
+6
View File
@@ -43,6 +43,7 @@ class NodeManager:
netem_mgr=None,
down_nodes: set[str] | None = None,
veth_mgr=None,
on_node_restart=None,
):
self.topology = topology
self.config = config
@@ -50,6 +51,7 @@ class NodeManager:
self.netem_mgr = netem_mgr
self.veth_mgr = veth_mgr
self.down_nodes = down_nodes or set()
self.on_node_restart = on_node_restart
self.node_states: dict[str, NodeState] = {
nid: NodeState(node_id=nid) for nid in topology.nodes
}
@@ -162,6 +164,10 @@ class NodeManager:
time.sleep(1)
self.netem_mgr.setup_node(node_id)
# Notify callback (e.g., refresh npub for ephemeral identity nodes)
if self.on_node_restart:
self.on_node_restart(node_id)
def _would_disconnect(self, node_id: str) -> bool:
"""Check if removing this node (plus currently-down nodes) disconnects the graph.
testing/chaos/sim/peer_churn.py
+178
View File
@@ -0,0 +1,178 @@
"""Peer-level topology churn via connect/disconnect commands.
Unlike NodeManager (which stops/starts containers), this uses the
fipsctl connect/disconnect API to dynamically add and remove individual
peer connections while nodes stay running. The topology graph evolves
over time: random links are disconnected and new random pairs connected.
Supports ephemeral identity nodes — half the nodes (configurable) get
new keypairs on each container restart, requiring the simulator to
track current npubs via show_status queries.
"""
from __future__ import annotations
import logging
import random
import time
from .control import query_status, send_command
from .scenario import PeerChurnConfig
from .topology import SimTopology
log = logging.getLogger(__name__)
class PeerChurnManager:
"""Manages peer-level topology churn using connect/disconnect commands."""
def __init__(
self,
topology: SimTopology,
config: PeerChurnConfig,
rng: random.Random,
down_nodes: set[str],
ephemeral_nodes: set[str] | None = None,
):
self.topology = topology
self.config = config
self.rng = rng
self.down_nodes = down_nodes
self.ephemeral_nodes = ephemeral_nodes or set()
# Current npub for each node (populated during init)
self.npub_cache: dict[str, str] = {}
# Currently active edges (start with topology edges)
self.active_edges: set[tuple[str, str]] = set()
for a, b in topology.edges:
self.active_edges.add(self._canonical(a, b))
# Edges disconnected by peer churn (not yet reconnected)
self.churned_count = 0
@staticmethod
def _canonical(a: str, b: str) -> tuple[str, str]:
"""Canonical edge ordering (sorted)."""
return (min(a, b), max(a, b))
@property
def churn_count(self) -> int:
return self.churned_count
def refresh_npub(self, node_id: str) -> str | None:
"""Query a node's current npub and update the cache.
Returns the npub or None if the query failed.
"""
container = self.topology.container_name(node_id)
status = query_status(container)
if status and "npub" in status:
npub = status["npub"]
self.npub_cache[node_id] = npub
return npub
return None
def refresh_all_npubs(self):
"""Populate npub cache for all nodes."""
for node_id in self.topology.nodes:
if node_id not in self.down_nodes:
self.refresh_npub(node_id)
log.info(
"Cached npubs for %d/%d nodes",
len(self.npub_cache),
len(self.topology.nodes),
)
def maybe_churn(self):
"""Disconnect a random active link, then connect a random new pair."""
# Skip if too many nodes are down
up_nodes = [n for n in self.topology.nodes if n not in self.down_nodes]
if len(up_nodes) < 3:
return
# Phase 1: Disconnect a random active link between up nodes
candidates = [
(a, b)
for a, b in self.active_edges
if a not in self.down_nodes and b not in self.down_nodes
]
if candidates:
edge = self.rng.choice(candidates)
if self._disconnect_edge(edge[0], edge[1]):
self.active_edges.discard(self._canonical(edge[0], edge[1]))
self.churned_count += 1
# Phase 2: Connect a random pair that isn't currently connected
non_edges = []
for i, a in enumerate(up_nodes):
for b in up_nodes[i + 1 :]:
if self._canonical(a, b) not in self.active_edges:
non_edges.append((a, b))
if non_edges:
a, b = self.rng.choice(non_edges)
if self._connect_edge(a, b):
self.active_edges.add(self._canonical(a, b))
def _disconnect_edge(self, a: str, b: str) -> bool:
"""Disconnect both sides of a link."""
npub_a = self.npub_cache.get(a)
npub_b = self.npub_cache.get(b)
if not npub_a or not npub_b:
log.debug("Missing npub for %s or %s, skipping disconnect", a, b)
return False
container_a = self.topology.container_name(a)
container_b = self.topology.container_name(b)
ok_a = send_command(container_a, "disconnect", {"npub": npub_b})
ok_b = send_command(container_b, "disconnect", {"npub": npub_a})
if ok_a is not None or ok_b is not None:
log.info("Peer DISCONNECT: %s -- %s", a, b)
return True
log.debug("Disconnect failed for %s -- %s", a, b)
return False
def _connect_edge(self, a: str, b: str) -> bool:
"""Connect both sides of a new link (mutual outbound)."""
npub_a = self.npub_cache.get(a)
npub_b = self.npub_cache.get(b)
if not npub_a or not npub_b:
log.debug("Missing npub for %s or %s, skipping connect", a, b)
return False
# Use UDP transport with the node's Docker IP
ip_a = self.topology.nodes[a].docker_ip
ip_b = self.topology.nodes[b].docker_ip
port = 2121 # Default UDP port
container_a = self.topology.container_name(a)
container_b = self.topology.container_name(b)
# Node A connects to B
ok_a = send_command(
container_a,
"connect",
{"npub": npub_b, "address": f"{ip_b}:{port}", "transport": "udp"},
)
# Node B connects to A
ok_b = send_command(
container_b,
"connect",
{"npub": npub_a, "address": f"{ip_a}:{port}", "transport": "udp"},
)
if ok_a is not None or ok_b is not None:
log.info("Peer CONNECT: %s -- %s (udp)", a, b)
return True
log.debug("Connect failed for %s -- %s", a, b)
return False
def restore_all(self):
"""No-op for teardown — peer connections are ephemeral."""
pass
testing/chaos/sim/runner.py
+62 -2
View File
@@ -19,6 +19,7 @@ from .links import LinkManager
from .logs import AnalysisResult, analyze_logs, collect_logs, write_sim_metadata
from .netem import NetemManager
from .nodes import NodeManager
from .peer_churn import PeerChurnManager
from .scenario import Scenario
from .topology import SimTopology, generate_topology
from .traffic import TrafficManager
@@ -46,6 +47,7 @@ class SimRunner:
self.link_mgr: LinkManager | None = None
self.traffic_mgr: TrafficManager | None = None
self.node_mgr: NodeManager | None = None
self.peer_churn_mgr: PeerChurnManager | None = None
def run(self) -> AnalysisResult | None:
"""Run the full simulation lifecycle."""
@@ -113,7 +115,23 @@ class SimRunner:
config_dir = os.path.normpath(
os.path.join(docker_network_dir, "generated-configs", "sim")
)
write_configs(self.topology, config_dir, self.scenario.fips_overrides)
# Select ephemeral identity nodes (if peer churn enabled)
self._ephemeral_nodes: set[str] = set()
if s.peer_churn.enabled and s.peer_churn.ephemeral_fraction > 0:
all_nodes = sorted(self.topology.nodes.keys())
count = int(len(all_nodes) * s.peer_churn.ephemeral_fraction)
self._ephemeral_nodes = set(self.rng.sample(all_nodes, count))
log.info(
"Ephemeral identity nodes (%d/%d): %s",
len(self._ephemeral_nodes),
len(all_nodes),
", ".join(sorted(self._ephemeral_nodes)),
)
write_configs(
self.topology, config_dir, self.scenario.fips_overrides,
ephemeral_nodes=self._ephemeral_nodes,
)
log.info("Wrote node configs to %s", config_dir)
# 3. Generate docker-compose.yml
@@ -167,6 +185,14 @@ class SimRunner:
self.topology, s.node_churn, self.rng,
netem_mgr=self.netem_mgr, down_nodes=self._down_nodes,
veth_mgr=self.veth_mgr,
on_node_restart=self._handle_node_restart,
)
if s.peer_churn.enabled:
self.peer_churn_mgr = PeerChurnManager(
self.topology, s.peer_churn, self.rng,
down_nodes=self._down_nodes,
ephemeral_nodes=self._ephemeral_nodes,
)
def _warmup(self):
@@ -177,6 +203,30 @@ class SimRunner:
self._sleep(wait)
self._take_snapshot("warmup")
# Populate npub cache after convergence (nodes must be running)
if self.peer_churn_mgr:
self.peer_churn_mgr.refresh_all_npubs()
def _handle_node_restart(self, node_id: str):
"""Called after a node container is restarted.
For ephemeral identity nodes, waits briefly for the daemon to
start, then queries its new npub and updates the peer churn
manager's cache.
"""
if not self.peer_churn_mgr:
return
if node_id not in self.peer_churn_mgr.ephemeral_nodes:
return
# Brief delay for daemon startup before querying control socket
time.sleep(2)
new_npub = self.peer_churn_mgr.refresh_npub(node_id)
if new_npub:
log.info("Ephemeral node %s new identity: %s...%s", node_id, new_npub[:12], new_npub[-6:])
else:
log.warning("Failed to refresh npub for ephemeral node %s", node_id)
def _simulation_loop(self):
"""Main event loop driving stochastic behavior."""
start = time.time()
@@ -189,6 +239,7 @@ class SimRunner:
next_flap = self._schedule_next(start, s.link_flaps.interval_secs) if self.link_mgr else float("inf")
next_traffic = self._schedule_next(start, s.traffic.interval_secs) if self.traffic_mgr else float("inf")
next_churn = self._schedule_next(start, s.node_churn.interval_secs) if self.node_mgr else float("inf")
next_peer_churn = self._schedule_next(start, s.peer_churn.interval_secs) if self.peer_churn_mgr else float("inf")
while not self._interrupted:
now = time.time()
@@ -222,17 +273,26 @@ class SimRunner:
next_churn = self._schedule_next(now, s.node_churn.interval_secs)
self.node_mgr.restore_expired()
# Peer churn (topology mutation)
if self.peer_churn_mgr:
if now >= next_peer_churn:
self.peer_churn_mgr.maybe_churn()
next_peer_churn = self._schedule_next(now, s.peer_churn.interval_secs)
# Status line
down_links = self.link_mgr.down_count if self.link_mgr else 0
down_nodes = self.node_mgr.down_count if self.node_mgr else 0
active = self.traffic_mgr.active_count if self.traffic_mgr else 0
peer_churns = self.peer_churn_mgr.churn_count if self.peer_churn_mgr else 0
status_extra = f" peer_churns={peer_churns}" if self.peer_churn_mgr else ""
print(
f"\r [{elapsed:.0f}s/{duration}s] "
f"nodes={len(self.topology.nodes)} "
f"edges={len(self.topology.edges)} "
f"links_down={down_links} "
f"nodes_down={down_nodes} "
f"traffic={active} ",
f"traffic={active}"
f"{status_extra} ",
end="",
flush=True,
)
testing/chaos/sim/scenario.py
+26
View File
@@ -106,6 +106,20 @@ class NodeChurnConfig:
protect_connectivity: bool = True
@dataclass
class PeerChurnConfig:
"""Peer-level topology churn via connect/disconnect commands.
When enabled, periodically disconnects a random active link and
connects a random unconnected node pair, causing the mesh topology
to evolve over time.
"""
enabled: bool = False
interval_secs: Range = field(default_factory=lambda: Range(8, 12))
ephemeral_fraction: float = 0.0
@dataclass
class BandwidthConfig:
"""Per-link bandwidth pacing via HTB rate limiting.
@@ -152,6 +166,7 @@ class Scenario:
link_flaps: LinkFlapsConfig = field(default_factory=LinkFlapsConfig)
traffic: TrafficConfig = field(default_factory=TrafficConfig)
node_churn: NodeChurnConfig = field(default_factory=NodeChurnConfig)
peer_churn: PeerChurnConfig = field(default_factory=PeerChurnConfig)
bandwidth: BandwidthConfig = field(default_factory=BandwidthConfig)
ingress: IngressConfig = field(default_factory=IngressConfig)
logging: LoggingConfig = field(default_factory=LoggingConfig)
@@ -279,6 +294,13 @@ def load_scenario(path: str) -> Scenario:
)
s.node_churn.protect_connectivity = nc2.get("protect_connectivity", True)
# Peer churn section
pc = raw.get("peer_churn", {})
s.peer_churn.enabled = pc.get("enabled", False)
if "interval_secs" in pc:
s.peer_churn.interval_secs = _parse_range(pc["interval_secs"], "peer_churn.interval_secs")
s.peer_churn.ephemeral_fraction = float(pc.get("ephemeral_fraction", 0.0))
# Bandwidth section
bw = raw.get("bandwidth", {})
s.bandwidth.enabled = bw.get("enabled", False)
@@ -399,6 +421,10 @@ def _validate(s: Scenario):
s.node_churn.down_duration_secs.validate("node_churn.down_duration_secs")
if s.node_churn.max_down_nodes >= s.topology.num_nodes:
raise ValueError("node_churn.max_down_nodes must be < topology.num_nodes")
if s.peer_churn.enabled:
s.peer_churn.interval_secs.validate("peer_churn.interval_secs")
if not 0.0 <= s.peer_churn.ephemeral_fraction <= 1.0:
raise ValueError("peer_churn.ephemeral_fraction must be between 0.0 and 1.0")
if s.bandwidth.enabled:
for tier in s.bandwidth.tiers_mbps:
if tier <= 0: