"""Ephemeral FIPS daemon lifecycle manager. Generates an isolated ``fips.yaml``, spawns ``fipsd --config `` as a subprocess **inside its own network namespace**, polls the control socket until the daemon is ready, and tears everything down cleanly (process + netns + temp dir). Why network namespaces: the FIPS daemon hardcodes a ``fd00::/8`` route on its TUN device (see ``fips/src/upper/tun.rs``). On a host that already runs a long-lived FIPS daemon (``fips0``), that route already exists, so a second daemon's TUN init fails with ``File exists (os error 17)``. Running each ephemeral daemon in its own netns gives it a fresh routing table where ``fd00::/8`` is free. The control socket is a Unix domain socket in ``/tmp`` (shared filesystem), so it remains reachable from the host. Each ephemeral daemon gets: - a unique temp config dir under ``/tmp/mesh-scan--/`` - a unique control socket path inside that dir - a unique network namespace (``fips-scan-``) — or a shared one for the self-test (so two daemons can peer over loopback) - unique UDP/TCP/DNS ports auto-allocated from the ephemeral range ``fipsd`` needs ``CAP_NET_ADMIN`` (for TUN + netns), so the spawn goes through ``sudo`` unless we already have the capability. """ from __future__ import annotations import logging import os import shutil import signal import socket import subprocess import tempfile import time from dataclasses import dataclass, field from pathlib import Path from control import ControlClient, ControlError log = logging.getLogger(__name__) FIPS_BINARY = os.environ.get("FIPS_BINARY", "fips") DEFAULT_NETNS_PREFIX = "fips-scan-" STARTUP_TIMEOUT = 15.0 STOP_TIMEOUT = 5.0 def _have_net_admin() -> bool: """True if this process can create a TUN device / netns without sudo.""" try: with open("/proc/self/status") as f: for line in f: if line.startswith("CapEff:"): eff = int(line.split()[1], 16) # CAP_NET_ADMIN is bit 12. return bool(eff & (1 << 12)) except OSError: pass return False def _free_udp_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: s.bind(("0.0.0.0", 0)) return s.getsockname()[1] def _free_tcp_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("0.0.0.0", 0)) return s.getsockname()[1] def _next_netns_name(index: int) -> str: return f"{DEFAULT_NETNS_PREFIX}{index}" # --------------------------------------------------------------------------- # Network namespace management # --------------------------------------------------------------------------- def _run(cmd: list[str], use_sudo: bool, timeout: float = 5.0) -> subprocess.CompletedProcess: full = (["sudo", "-n", *cmd] if use_sudo else cmd) return subprocess.run(full, capture_output=True, text=True, timeout=timeout) def _run_quiet(cmd: list[str], use_sudo: bool, timeout: float = 5.0) -> None: """Run a command, log failures at debug level only.""" try: r = _run(cmd, use_sudo, timeout) if r.returncode != 0 and r.stderr.strip(): log.debug("cmd failed: %s -> %s", " ".join(cmd), r.stderr.strip()) except Exception as e: log.debug("cmd exception: %s -> %s", " ".join(cmd), e) def _host_default_iface() -> str: """Return the host's default-route interface name, or 'eth0' fallback.""" try: r = subprocess.run(["ip", "route", "show", "default"], capture_output=True, text=True, timeout=3) for line in r.stdout.splitlines(): if "dev" in line: return line.split("dev")[1].split()[0] except Exception: pass return "eth0" # Subnet counter for veth pairs (10.250.N.0/30 — 2 usable hosts). _veth_counter = [0] def _next_veth_subnet() -> tuple[str, str, str]: """Return (host_addr, netns_addr, subnet) for the next veth pair.""" n = _veth_counter[0] _veth_counter[0] += 1 # 10.250.N.1 (host side) / 10.250.N.2 (netns side), /30. return f"10.250.{n}.1/30", f"10.250.{n}.2/30", f"10.250.{n}.0/30" def create_netns(name: str, use_sudo: bool | None = None) -> dict: """Create a network namespace with loopback + a veth pair for outbound NAT. The netns gets a veth link to the host with a 10.250.N.0/30 subnet and a default route via the host side. The host masquerades traffic from the netns out its default interface, so the ephemeral FIPS daemon inside can reach public seed nodes over UDP/TCP. Returns a dict with the veth interface names and subnet, for cleanup. """ if use_sudo is None: use_sudo = not _have_net_admin() # Linux interface names are limited to 15 chars. Use a short prefix # derived from the netns index instead of the full netns name. n = _veth_counter[0] - 1 # _next_veth_subnet already incremented info = {"veth_host": f"fsvh{n}", "veth_ns": f"fsvn{n}", "subnet": None} try: _run(["ip", "netns", "add", name], use_sudo) except subprocess.CalledProcessError as e: log.warning("ip netns add %s failed: %s", name, e.stderr.strip()) return info # Loopback up so ::1 binds work (DNS responder binds ::1). _run_quiet(["ip", "netns", "exec", name, "ip", "link", "set", "lo", "up"], use_sudo) # Create a veth pair: host end stays on the host, ns end moves into netns. host_addr, ns_addr, subnet = _next_veth_subnet() info["subnet"] = subnet _run_quiet(["ip", "link", "add", info["veth_host"], "type", "veth", "peer", "name", info["veth_ns"]], use_sudo) _run_quiet(["ip", "link", "set", info["veth_ns"], "netns", name], use_sudo) _run_quiet(["ip", "addr", "add", host_addr, "dev", info["veth_host"]], use_sudo) _run_quiet(["ip", "link", "set", info["veth_host"], "up"], use_sudo) _run_quiet(["ip", "netns", "exec", name, "ip", "addr", "add", ns_addr, "dev", info["veth_ns"]], use_sudo) _run_quiet(["ip", "netns", "exec", name, "ip", "link", "set", info["veth_ns"], "up"], use_sudo) # Default route in the netns via the host side of the veth. _run_quiet(["ip", "netns", "exec", name, "ip", "route", "add", "default", "via", host_addr.split("/")[0], "dev", info["veth_ns"]], use_sudo) # Enable IP forwarding + masquerade on the host for the netns subnet. _run_quiet(["sysctl", "-w", "net.ipv4.ip_forward=1"], use_sudo) out_iface = _host_default_iface() _run_quiet(["iptables", "-t", "nat", "-A", "POSTROUTING", "-s", subnet, "-o", out_iface, "-j", "MASQUERADE"], use_sudo) _run_quiet(["iptables", "-A", "FORWARD", "-i", info["veth_host"], "-o", out_iface, "-j", "ACCEPT"], use_sudo) _run_quiet(["iptables", "-A", "FORWARD", "-i", out_iface, "-o", info["veth_host"], "-j", "ACCEPT"], use_sudo) log.info("netns %s: veth %s<->%s subnet %s nat via %s", name, info["veth_host"], info["veth_ns"], subnet, out_iface) return info def delete_netns(name: str, use_sudo: bool | None = None, info: dict | None = None) -> None: """Delete a network namespace and its veth / iptables rules.""" if use_sudo is None: use_sudo = not _have_net_admin() # Deleting the netns also destroys the ns-end of the veth; the host-end # is removed below. iptables rules referencing the veth are cleaned up # best-effort. if info: subnet = info.get("subnet") out_iface = _host_default_iface() if subnet: _run_quiet(["iptables", "-t", "nat", "-D", "POSTROUTING", "-s", subnet, "-o", out_iface, "-j", "MASQUERADE"], use_sudo) _run_quiet(["iptables", "-D", "FORWARD", "-i", info.get("veth_host", ""), "-o", out_iface, "-j", "ACCEPT"], use_sudo) _run_quiet(["iptables", "-D", "FORWARD", "-i", out_iface, "-o", info.get("veth_host", ""), "-j", "ACCEPT"], use_sudo) _run_quiet(["ip", "link", "del", info.get("veth_host", "")], use_sudo) try: _run(["ip", "netns", "del", name], use_sudo) except Exception as e: log.debug("ip netns del %s: %s", name, e) def exec_in_netns(name: str, cmd: list[str], use_sudo: bool | None = None, **kwargs) -> subprocess.CompletedProcess: """Run a command inside a network namespace.""" if use_sudo is None: use_sudo = not _have_net_admin() full = (["sudo", "-n"] if use_sudo else []) + ["ip", "netns", "exec", name] + cmd return subprocess.run(full, **kwargs) @dataclass class EphemeralDaemon: """A running ephemeral fipsd process + its resources.""" config_dir: Path config_path: Path socket_path: str netns: str owns_netns: bool udp_port: int tcp_port: int dns_port: int process: subprocess.Popen used_sudo: bool netns_info: dict | None = None client: ControlClient = field(init=False) def __post_init__(self) -> None: self.client = ControlClient(self.socket_path) # -- lifecycle --------------------------------------------------------- def wait_ready(self, timeout: float = STARTUP_TIMEOUT) -> None: """Poll the control socket until ``show_status`` responds.""" deadline = time.time() + timeout last_err = "" while time.time() < deadline: if self.process.poll() is not None: raise RuntimeError(f"fipsd exited early (code={self.process.returncode})") try: status = self.client.show_status() if status.get("state") in ("running", "initialized", "ready"): if status.get("tun_state") == "active": return # TUN not yet active; keep waiting briefly. except ControlError as e: last_err = str(e) time.sleep(0.3) raise RuntimeError(f"fipsd did not become ready in {timeout}s: {last_err}") def stop(self) -> None: """Disconnect peers, SIGTERM, then SIGKILL if needed; clean up netns + dir.""" try: self.client.disconnect_all_peers() except Exception as e: log.debug("disconnect-all failed: %s", e) self._terminate() if self.owns_netns: delete_netns(self.netns, self.used_sudo, info=self.netns_info) self._cleanup_dir() def _terminate(self) -> None: if self.process.poll() is not None: return try: self.process.send_signal(signal.SIGTERM) except Exception: pass try: self.process.wait(timeout=STOP_TIMEOUT) except subprocess.TimeoutExpired: log.warning("fipsd in netns %s did not exit on SIGTERM, SIGKILL", self.netns) try: self.process.kill() self.process.wait(timeout=2) except Exception: pass def _cleanup_dir(self) -> None: try: shutil.rmtree(self.config_dir, ignore_errors=True) except Exception: pass # -- reachability helper ---------------------------------------------- def run_in_netns(self, cmd: list[str], timeout: float = 30.0, **kwargs) -> subprocess.CompletedProcess: """Run a command (e.g. ping6) inside this daemon's netns. Defaults to capturing stdout/stderr as text with a 30s hard timeout (callers can override). The timeout prevents hung probes from blocking the scan indefinitely. """ kwargs.setdefault("capture_output", True) kwargs.setdefault("text", True) kwargs.setdefault("timeout", timeout) return exec_in_netns(self.netns, cmd, self.used_sudo, **kwargs) # -- context manager --------------------------------------------------- def __enter__(self) -> "EphemeralDaemon": return self def __exit__(self, *exc) -> None: self.stop() # --------------------------------------------------------------------------- # Spawn # --------------------------------------------------------------------------- def _build_config( socket_path: str, udp_port: int, tcp_port: int, dns_port: int, ) -> str: # Inside a netns we can use the canonical "fips0" TUN name — no collision. # Mirrors the structure in sovereign_browser/src/net_services.c:349 # and fips/src/config/{node,transport,upper/config}.rs. return ( "node:\n" " identity:\n" " persistent: false\n" f" control:\n" f" socket_path: \"{socket_path}\"\n" " log_level: warn\n" # Enable Nostr discovery so the ephemeral node learns about other # mesh nodes from the advert relays (not just its direct peer). # Without this, the identity cache stays at 1 (only the seed) and # find_next_hop returns None for every non-peer target. # policy: open lets it consider adverts for non-configured peers. # advertise: false — the scanner node is ephemeral, don't publish. " discovery:\n" " nostr:\n" " enabled: true\n" " advertise: false\n" " policy: open\n" " advert_relays:\n" " - \"wss://relay.damus.io\"\n" " - \"wss://nos.lol\"\n" " - \"wss://offchain.pub\"\n" "tun:\n" " enabled: true\n" " name: \"fips0\"\n" " mtu: 1280\n" "dns:\n" " enabled: true\n" " bind_addr: \"::1\"\n" f" port: {dns_port}\n" "transports:\n" " udp:\n" f" bind_addr: \"0.0.0.0:{udp_port}\"\n" " tcp:\n" f" bind_addr: \"0.0.0.0:{tcp_port}\"\n" "peers: []\n" ) def spawn_daemon(tag: str = "scan", index: int = 0, netns: str | None = None) -> EphemeralDaemon: """Create the config dir + netns, write fips.yaml, spawn fipsd, return handle. If ``netns`` is given, the daemon is spawned in that (shared) namespace and does not own it (the caller is responsible for deleting it). This is used by the self-test to put two daemons in one netns so they can peer over loopback. The daemon is *not* waited on for readiness — call ``wait_ready()``. """ use_sudo = not _have_net_admin() config_dir = Path(tempfile.mkdtemp(prefix=f"mesh-scan-{tag}-")) socket_path = str(config_dir / "control.sock") udp_port = _free_udp_port() tcp_port = _free_tcp_port() dns_port = _free_udp_port() owns_netns = netns is None netns_info = None if owns_netns: netns = _next_netns_name(index) netns_info = create_netns(netns, use_sudo) else: # Shared netns (self-test): ensure it exists with NAT. The caller # is responsible for creating/deleting it, but if they passed a # name without creating it, create_netns is idempotent enough. pass config = _build_config(socket_path, udp_port, tcp_port, dns_port) config_path = config_dir / "fips.yaml" config_path.write_text(config) os.chmod(config_path, 0o600) # Spawn fips inside the netns. fips_argv = [FIPS_BINARY, "--config", str(config_path)] if use_sudo: spawn_argv = ["sudo", "-n", "ip", "netns", "exec", netns, *fips_argv] else: spawn_argv = ["ip", "netns", "exec", netns, *fips_argv] log.info("spawning fipsd: netns=%s udp=%d tcp=%d dns=%d sudo=%s", netns, udp_port, tcp_port, dns_port, use_sudo) proc = subprocess.Popen( spawn_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True, ) return EphemeralDaemon( config_dir=config_dir, config_path=config_path, socket_path=socket_path, netns=netns, owns_netns=owns_netns, netns_info=netns_info, udp_port=udp_port, tcp_port=tcp_port, dns_port=dns_port, process=proc, used_sudo=use_sudo, ) # --------------------------------------------------------------------------- # Stale-namespace cleanup (--cleanup-stale) # --------------------------------------------------------------------------- def cleanup_stale_netns(prefix: str = DEFAULT_NETNS_PREFIX) -> int: """Remove any network namespaces whose name starts with ``prefix``.""" use_sudo = not _have_net_admin() try: out = _run(["ip", "netns", "list"], use_sudo).stdout except Exception as e: log.warning("could not list netns: %s", e) return 0 count = 0 for line in out.splitlines(): name = line.split()[0] if line.strip() else "" if name.startswith(prefix): try: _run(["ip", "netns", "del", name], use_sudo) log.info("removed stale netns %s", name) count += 1 except Exception as e: log.warning("failed to remove stale netns %s: %s", name, e) return count # Back-compat alias used by mesh_scan.py's --cleanup-stale flag. cleanup_stale_tun_devices = cleanup_stale_netns # Extend ControlClient with a helper to disconnect all peers (used on stop). def _disconnect_all_peers(self: ControlClient) -> None: try: peers = self.show_peers().get("peers", []) or [] except ControlError: return for p in peers: npub = p.get("npub") if npub: try: self.disconnect(npub) except ControlError: pass ControlClient.disconnect_all_peers = _disconnect_all_peers # type: ignore[attr-defined]