diff --git a/.gitignore b/.gitignore index d0dee68..ce9b98e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ agents/ deploy/ vps.env -reference/ +/reference/ dist/ *.ipk diff --git a/Cargo.toml b/Cargo.toml index ca24947..0c9a543 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,7 @@ assets = [ ["packaging/debian/fips.tmpfiles", "/usr/lib/tmpfiles.d/fips.conf", "644"], ["target/release/fips-gateway", "/usr/bin/", "755"], ["packaging/debian/fips-gateway.service", "/lib/systemd/system/fips-gateway.service", "644"], - ["docs/fips-security.md", "/usr/share/doc/fips/fips-security.md", "644"], + ["docs/design/fips-security.md", "/usr/share/doc/fips/fips-security.md", "644"], ] conf-files = ["/etc/fips/fips.yaml", "/etc/fips/hosts", "/etc/fips/fips.nft"] diff --git a/README.md b/README.md index 8c7ed7f..915ca07 100644 --- a/README.md +++ b/README.md @@ -403,7 +403,7 @@ Ethernet, Tor, and Bluetooth (BLE) with a small live mesh of deployed nodes. NAT traversal — peers publish endpoint adverts on public Nostr relays, exchange candidates via NIP-59 gift-wrapped offers/answers, and establish direct paths through NATs using STUN-assisted - punching (behind the `nostr-discovery` cargo feature) + punching ### Near-term priorities diff --git a/docs/README.md b/docs/README.md index 89d2c64..6a5652d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,5 +1,54 @@ # FIPS Documentation -| Directory | Description | -|-----------|-------------| -| [design/](design/) | Protocol design specifications and analysis | +FIPS (Free Internetworking Peering System) is a self-organizing +encrypted mesh network built on Nostr identities, capable of +operating over arbitrary transports — local networks, the public +internet, Tor, Bluetooth, or point-to-point links — without central +infrastructure. + +With FIPS, your machine becomes a node in the mesh with a +self-generated cryptographic identity. There are two ways to +deploy it. + +**As an overlay** on top of existing IP networks, FIPS lets +your node reach any other FIPS node wherever it sits — behind a NAT, on a +different ISP, on a phone over cellular, on a laptop with only +Bluetooth in range, or behind a Tor onion. The mesh forwards +IPv6 traffic transparently and end-to-end encrypted, with no +central VPN concentrator or coordinating server. + +**From the ground up** over raw Ethernet, WiFi, or Bluetooth, +FIPS provides a complete permissionless network +without any pre-existing IP infrastructure, ISP, or DNS. Any +node that joins the link gets routable IPv6 addresses, peer +discovery, and a path to every other node automatically. + +Either way, existing networking software runs over it unchanged: +SSH, HTTP servers, file transfer, anything IPv6-native works the +same way it would on a local network. + +New to FIPS? Start with the [Getting Started](getting-started.md) +guide. + +## Documentation Sections + +### [Tutorials](tutorials/) + +If you are starting from scratch and want a guided path to a +working mesh, go here. + +### [How-To Guides](how-to/) + +If you have a specific task in mind — enabling a feature, +deploying a component, diagnosing a problem — go here. + +### [Reference](reference/) + +If you need to look up wire formats, configuration keys, command +flags, or counter inventories, go here. + +### [Design](design/) + +If you want to understand how the mesh self-organizes, why FIPS +makes the choices it does, or how the pieces fit together, go +here. diff --git a/docs/design/README.md b/docs/design/README.md index b9e2d8a..015372c 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -1,52 +1,65 @@ -# FIPS Design Documents +# FIPS Design -Protocol design specifications for the Federated Interoperable Peering -System — a self-organizing encrypted mesh network built on Nostr identities. +Architectural and protocol-level explanations for FIPS — the *why* +and the *how* behind the wire and the system. For wire formats and +configuration keys, see [reference/](../reference/). For task +recipes, see [how-to/](../how-to/). For end-to-end lessons, see +[tutorials/](../tutorials/). ## Reading Order -Start with the introduction, then follow the protocol stack from bottom to -top. After the stack, the mesh operation document explains how all the -pieces work together. Supporting references provide deeper dives into -specific topics. +Start with [fips-concepts.md](fips-concepts.md) for the +novice-friendly framing of what FIPS is and why, then move to +[fips-architecture.md](fips-architecture.md) for the protocol stack, +identity model, and two-layer encryption walkthrough. From there, +follow the protocol stack from bottom to top. After the stack, +[fips-mesh-operation.md](fips-mesh-operation.md) explains how the +pieces work together at runtime. Cross-cutting and supporting +documents cover specific subsystems in detail. + +### Foundations + +| Document | Description | +| -------- | ----------- | +| [fips-concepts.md](fips-concepts.md) | What FIPS is, why it exists, mental model | +| [fips-architecture.md](fips-architecture.md) | Protocol stack, identity, two-layer encryption | +| [fips-prior-work.md](fips-prior-work.md) | Designs and protocols FIPS builds on | ### Protocol Stack | Document | Description | | -------- | ----------- | -| [fips-intro.md](fips-intro.md) | Protocol introduction: goals, architecture, layer model | | [fips-transport-layer.md](fips-transport-layer.md) | Transport layer: datagram delivery over arbitrary media | | [fips-mesh-layer.md](fips-mesh-layer.md) | FIPS Mesh Protocol (FMP): peer authentication, link encryption, forwarding | | [fips-session-layer.md](fips-session-layer.md) | FIPS Session Protocol (FSP): end-to-end encryption, sessions | | [fips-ipv6-adapter.md](fips-ipv6-adapter.md) | IPv6 adaptation: TUN interface, DNS, MTU enforcement | +### Cross-Cutting + +| Document | Description | +| -------- | ----------- | +| [fips-mmp.md](fips-mmp.md) | Metrics Measurement Protocol (link + session) | +| [fips-mtu.md](fips-mtu.md) | Path MTU model, encapsulation overhead, PMTUD | +| [fips-security.md](fips-security.md) | `fips0` interface threat model and default-deny baseline | + ### Mesh Behavior | Document | Description | | -------- | ----------- | | [fips-mesh-operation.md](fips-mesh-operation.md) | How the mesh operates: routing, discovery, error recovery | -| [fips-wire-formats.md](fips-wire-formats.md) | Wire format reference for all message types | -| [fips-nostr-discovery.md](fips-nostr-discovery.md) | Optional Nostr-mediated peer discovery and UDP NAT hole-punch (behind `nostr-discovery` feature) | +| [fips-nostr-discovery.md](fips-nostr-discovery.md) | Optional Nostr-mediated peer discovery and UDP NAT hole-punch | +| [port-advertisement-and-nat-traversal.md](port-advertisement-and-nat-traversal.md) | Nostr-signaled port advertisement and UDP NAT-traversal protocol; generic, with FIPS as an example implementation | -### Supporting References +### Deeper Dives | Document | Description | | -------- | ----------- | | [fips-spanning-tree.md](fips-spanning-tree.md) | Spanning tree algorithms: root discovery, parent selection, coordinates | -| [fips-bloom-filters.md](fips-bloom-filters.md) | Bloom filter math: FPR analysis, size classes, split-horizon | - -### Implementation - -| Document | Description | -| -------- | ----------- | -| [fips-configuration.md](fips-configuration.md) | YAML configuration reference | - -### Supplemental - -| Document | Description | -| -------- | ----------- | +| [fips-bloom-filters.md](fips-bloom-filters.md) | Bloom filter properties: FPR analysis, size classes, split-horizon | | [spanning-tree-dynamics.md](spanning-tree-dynamics.md) | Spanning tree walkthroughs: convergence scenarios, worked examples | -## Document Relationships +### Adjacent Components -![Document relationships](document-relationships.svg) +| Document | Description | +| -------- | ----------- | +| [fips-gateway.md](fips-gateway.md) | `fips-gateway` service: outbound (LAN-to-mesh) DNS-proxy + virtual-IP NAT and inbound (mesh-to-LAN) port-forwarding, sharing one nftables table | diff --git a/docs/design/diagrams/fips-coordinate-discovery.svg b/docs/design/diagrams/fips-coordinate-discovery.svg index 6feee57..34e1778 100644 --- a/docs/design/diagrams/fips-coordinate-discovery.svg +++ b/docs/design/diagrams/fips-coordinate-discovery.svg @@ -82,7 +82,7 @@ LookupRequest - guided by bloom filters at each hop + guided by bloom filters at each hop; transits do not cache @@ -99,23 +99,19 @@ LookupResponse + coords - - - cache D + + + + LookupResponse + coords - - - - LookupResponse + coords + + + + LookupResponse + coords - - - cache D - - - - - LookupResponse + coords + + + cache D @@ -125,32 +121,34 @@ Phase 3 - Routing + Data flow - + - Data + Data + coords - - cached coords + + + cache D - - - - Data + + + + Data + coords - - cached coords + + + cache D - - - Data + + + Data + coords - cached coords enable efficient forwarding — no re-discovery needed + transits cache coords from in-flight data; subsequent traffic forwards without re-discovery - Each transit node caches coordinates from the LookupResponse return path + LookupResponse caches coords at the originator only; transit caches warm during the subsequent data flow diff --git a/docs/design/diagrams/fips-identity-derivation.svg b/docs/design/diagrams/fips-identity-derivation.svg index 05ac257..ddb6ad6 100644 --- a/docs/design/diagrams/fips-identity-derivation.svg +++ b/docs/design/diagrams/fips-identity-derivation.svg @@ -71,7 +71,7 @@ - one-way hash + SHA-256, truncate to 16 bytes @@ -95,7 +95,7 @@ - add fd00::/8 prefix + 0xfd + node_addr[0..15] diff --git a/docs/design/diagrams/fips-protocol-stack.svg b/docs/design/diagrams/fips-protocol-stack.svg index ad20d4b..906892e 100644 --- a/docs/design/diagrams/fips-protocol-stack.svg +++ b/docs/design/diagrams/fips-protocol-stack.svg @@ -19,7 +19,7 @@ Application Layer Interface Native FIPS API — for FIPS-aware applications - IPv6 Shim — for traditional IP application backward compatibility + IPv6 adapter — for traditional IP application backward compatibility diff --git a/docs/design/diagrams/fips-routing-decision.svg b/docs/design/diagrams/fips-routing-decision.svg index ca094fa..b3629f7 100644 --- a/docs/design/diagrams/fips-routing-decision.svg +++ b/docs/design/diagrams/fips-routing-decision.svg @@ -76,54 +76,54 @@ No - + 3 - Bloom filter - hit? + Coords + known? - + - Yes - - Rank candidates by - tree distance and - link performance + No + + No route → error signal - - - - Forward to 'best' - - + - No + Yes - + 4 - Coords - known? + Bloom filter + hit? - + Yes - - Greedy tree forward + + Rank candidates by + tree distance and + link performance + + + + + Forward to 'best' No - + 5 - - No route → error signal + + Greedy tree forward @@ -149,5 +149,5 @@ Control flow direction - Each hop evaluates destinations in priority order 1–4, falling through on miss + Each hop checks 1–4 in priority order, falling through to greedy tree (5) when bloom yields no candidate; missing coords is the only error path diff --git a/docs/design/document-relationships.svg b/docs/design/document-relationships.svg deleted file mode 100644 index dcc0860..0000000 --- a/docs/design/document-relationships.svg +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - - - Document Relationships - - - - fips-intro.md - - - - - - - - fips-transport-layer.md - - - - fips-mesh-operation.md - - - - - - - fips-mesh-layer.md - - - - - - - - - - fips-session-layer.md - - - - - - - fips-ipv6-adapter.md - - - - - - - - - - fips-spanning-tree.md - - - - - - - fips-bloom-filters.md - - - - - - - fips-wire-formats.md - (referenced by all layer docs) - - - - fips-configuration.md - (standalone reference) - - - - spanning-tree-dynamics.md - (companion to fips-spanning-tree) - - - - Protocol stack - - Mesh behavior - - Supporting references - - Wire formats - - Implementation - diff --git a/docs/design/fips-architecture.md b/docs/design/fips-architecture.md new file mode 100644 index 0000000..7d909b0 --- /dev/null +++ b/docs/design/fips-architecture.md @@ -0,0 +1,264 @@ +# FIPS Architecture + +The protocol architecture, identity system, and two-layer encryption +model. For the higher-level "what is FIPS and why" framing, see +[fips-concepts.md](fips-concepts.md). For prior art and academic +citations, see [fips-prior-work.md](fips-prior-work.md). + +## Protocol Architecture + +FIPS is organized in three protocol layers, each with distinct +responsibilities and clean service boundaries. No layer depends on +the specifics of the layers above or below it — transport plugins +know nothing about sessions, the routing layer knows nothing about +application addressing, and applications know nothing about which +physical media carry their traffic. This separation means new +transports, protocol features, and application interfaces can be +added independently. + +![Protocol Stack](diagrams/fips-protocol-stack.svg) + +### Mapping to Traditional Networking + +Readers familiar with the OSI model or TCP/IP networking may find it +helpful to see how FIPS concepts relate to traditional layers: + +![OSI Mapping](diagrams/fips-osi-mapping.svg) + +Note that FMP spans what would traditionally be separate link and +network layers. This is intentional — in a self-organizing mesh, the +same layer that authenticates peers also makes routing decisions, +because routing depends on authenticated peer state (spanning tree +positions, bloom filters). + +### Layer Responsibilities + +**Transport layer**: Delivers datagrams between endpoints over a +specific medium. Each transport type (UDP socket, Ethernet interface, +radio modem) implements the same abstract interface: send and receive +datagrams, report MTU. The transport layer knows nothing about FIPS +identities, routing, or encryption. It provides raw datagram delivery +to FMP above. + +See [fips-transport-layer.md](fips-transport-layer.md) for the +transport layer specification. + +**FIPS Mesh Protocol (FMP)**: Manages peer connections, authenticates +peers via Noise IK handshakes, and encrypts all traffic on each link. +FMP is where the mesh organizes itself — nodes exchange spanning tree +announcements and bloom filters with their direct peers, and FMP +makes forwarding decisions for transit traffic. FMP provides +authenticated, encrypted forwarding to FSP above. + +See [fips-mesh-layer.md](fips-mesh-layer.md) for the FMP specification +and [fips-mesh-operation.md](fips-mesh-operation.md) for how FMP's +routing and self-organization work in practice. + +**FIPS Session Protocol (FSP)**: Provides end-to-end authenticated +encryption between any two nodes, regardless of how many intermediate +hops separate them. FSP manages session lifecycle (setup, data +transfer, teardown), caches destination coordinates for efficient +routing, and handles the warmup strategy that keeps transit node +caches populated. Session dispatch uses index-based routing inspired +by [WireGuard](https://www.wireguard.com/), enabling O(1) packet +demultiplexing. FSP provides a datagram service to applications above. + +See [fips-session-layer.md](fips-session-layer.md) for the FSP +specification. + +**IPv6 adaptation layer**: Sits above FSP as a service on port 256, +adapting the FIPS datagram service for unmodified IPv6 applications. +Provides DNS resolution (npub → fd00::/8 address), identity cache +management, IPv6 header compression, MTU enforcement, and a TUN +interface. This is the primary way existing applications use the FIPS +mesh. + +See [fips-ipv6-adapter.md](fips-ipv6-adapter.md) for the IPv6 adapter. + +### Node Architecture + +Application services sit at the top of the stack, dispatched by FSP +port number: the IPv6 TUN adapter (port 256) maps npubs to `fd00::/8` +addresses with header compression so unmodified IP applications can +use the network transparently, while the native datagram API +addresses destinations directly by npub. + +![Node Architecture](diagrams/fips-node-architecture.svg) + +The mesh routes application traffic across heterogeneous transports +transparently. A packet may traverse WiFi, Ethernet, UDP/IP, and Tor +links on its way from source to destination — the application never +needs to know which transports are involved. Each hop is independently +encrypted at the link layer, while a single end-to-end session +protects the payload across the entire path. + +![Architecture Overview](diagrams/fips-architecture-overview.svg) + +![Mesh Topology](diagrams/fips-mesh-topology.svg) + +## Identity System + +FIPS uses [Nostr](https://github.com/nostr-protocol/nips) keypairs +(secp256k1) as node identities. The public key identifies the node; +the private key signs protocol messages and establishes encrypted +sessions. + +The public key (or its bech32-encoded npub form) is the primary means +for application-layer software to identify communication endpoints. +Internally, the protocol derives a `node_addr` (a 16-byte SHA-256 hash +of the pubkey) used as the routing identifier in packet headers, and +an IPv6 address derived from the node_addr for the TUN adapter. +Applications use the pubkey or npub; the routing layer uses node_addr; +unmodified IPv6 applications use the derived `fd00::/8` address. All +three are deterministically derived from the same keypair. + +### FIPS Identity Handling + +![Identity Derivation](diagrams/fips-identity-derivation.svg) + +The pubkey is the node's cryptographic identity, used in Noise +handshakes for both link encryption (IK) and session encryption (XK). +It is never exposed beyond the endpoints of an encrypted channel. The node_addr, a one-way +SHA-256 hash truncated to 16 bytes, serves as the routing identifier +in packet headers and bloom filters. Intermediate routers see only +node_addrs — they can forward traffic without learning the Nostr +identities of the endpoints. An observer can verify "does this +node_addr belong to pubkey X?" if they already know the pubkey, but +cannot enumerate communicating identities by inspecting traffic. The +IPv6 address prepends `fd` to the first 15 bytes of the node_addr, +providing a ULA overlay address for unmodified IP applications via the +TUN interface. + +Below the FIPS identity layer, each transport uses its own native +addressing — IP:port or hostname:port addresses, MAC addresses, +.onion identifiers. These **link addresses** are opaque to everything +above FMP and discarded once link authentication completes. + +### Identity Verification + +The Noise Protocol Framework mutually authenticates both peer-to-peer +link connections (at FMP) and end-to-end session traffic (at FSP), +proving each party controls the private key for their claimed +identity. + +See [fips-mesh-layer.md](fips-mesh-layer.md) for peer authentication +and [fips-session-layer.md](fips-session-layer.md) for end-to-end +session establishment. + +Key rotation changes the node's identity — a new keypair produces a +new node_addr and IPv6 address, requiring all sessions to be +re-established. Migration mechanisms that allow a node to announce a +successor key are a future consideration. + +## Two-Layer Encryption + +FIPS uses independent encryption at two protocol layers: + +| Layer | Scope | Pattern | Purpose | +| ----- | ----- | ------- | ------- | +| **FMP (Mesh)** | Hop-by-hop | Noise IK | Encrypt all traffic on each peer link | +| **FSP (Session)** | End-to-end | Noise XK | Encrypt application payload between endpoints | + +### Link Layer (Hop-by-Hop) + +When two nodes establish a direct connection, they perform a [Noise +IK](https://noiseprotocol.org/) handshake. This authenticates both +parties and establishes symmetric keys for encrypting all traffic on +that link. Every packet between direct peers is encrypted — gossip +messages, routing queries, and forwarded session datagrams alike. + +The IK pattern is used because outbound connections know the peer's +npub from configuration, while inbound connections learn the +initiator's identity from the first handshake message. + +### Session Layer (End-to-End) + +FIPS establishes end-to-end encrypted sessions between any two +communicating nodes using Noise XK, regardless of how many hops +separate them. The initiator knows the destination's npub (required +for XK's pre-message); the responder learns the initiator's identity +from the third handshake message. Unlike the link-layer IK pattern +where the initiator's identity is revealed in msg1, XK delays +identity disclosure until msg3, providing stronger initiator identity +protection for traffic traversing untrusted intermediate nodes. + +A packet from A to D through intermediate nodes B and C: + +1. A encrypts payload with A↔D session key (FSP) +2. A wraps in SessionDatagram, encrypts with A↔B link key (FMP), + sends to B +3. B decrypts link layer, reads destination node_addr, re-encrypts + with B↔C link key, forwards to C +4. C decrypts link layer, re-encrypts with C↔D link key, forwards + to D +5. D decrypts link layer, then decrypts session layer to get payload + +Intermediate nodes route based on destination node_addr but cannot +read session-layer payloads. Each hop strips one link encryption and +applies the next — the session-layer ciphertext passes through +untouched. + +Both layers always apply, even between adjacent peers — a packet to a +direct neighbor is still encrypted twice. This uniform model means no +special cases for local vs remote destinations, and topology changes +(a direct peer becomes reachable only through intermediaries) don't +affect existing sessions. + +See [fips-mesh-layer.md](fips-mesh-layer.md) for link encryption and +[fips-session-layer.md](fips-session-layer.md) for session encryption. + +## Routing and Mesh Operation + +Forwarding decisions are local. Each node combines spanning-tree +coordinates with peer bloom filters to choose a next hop, falling back +to greedy tree routing when bloom filters have not converged. Discovery +warms transit node caches with destination coordinates, and three +explicit error signals (CoordsRequired, PathBroken, MtuExceeded) drive +recovery when forwarding fails. The full routing decision process, +discovery protocol, and error-recovery integration view live in +[fips-mesh-operation.md](fips-mesh-operation.md). + +## Transport Abstraction + +FIPS treats the communication medium as a pluggable component. UDP, +TCP, raw Ethernet, Tor, and BLE all implement the same small datagram +interface (send, receive, report MTU) and feed peers into a single FMP +routing layer; radio and serial transports are in the planned set. +Multi-transport nodes bridge between networks transparently. The +transport-layer specification — including per-transport categories, +the trait surface, the connection model, and implementation status — +is in [fips-transport-layer.md](fips-transport-layer.md). + +## Security + +FIPS defends against four adversary classes (transport observers, +active transport attackers, intermediate routers, and adversarial +mesh nodes) through layered controls: hop-by-hop FMP link encryption, +end-to-end FSP session encryption with stronger initiator identity +protection, signed and replay-protected gossip, and rate-limited +handshake processing. The threat-model details and per-layer +mitigations are in [fips-mesh-layer.md](fips-mesh-layer.md), and the +operator-facing controls (default-deny baseline, peer ACLs, +filesystem permissions, cryptographic primitives) are consolidated in +[fips-security.md](fips-security.md) and +[../reference/security.md](../reference/security.md). + +## MTU as a Cross-Cutting Concern + +MTU is not owned by any single layer. The transport layer reports +per-link MTU, FMP carries `path_mtu` in SessionDatagram and +LookupResponse to track the minimum along a path, FSP echoes the +observed forward-path MTU back to the source, and the IPv6 adapter +enforces the resulting effective MTU at the TUN with ICMP Packet Too +Big and TCP MSS clamping. The unified design — encapsulation overhead +budget, proactive PMTUD, reactive MtuExceeded, and per-destination +storage — is in [fips-mtu.md](fips-mtu.md). + +## Approaches Considered but Rejected + +One design alternative evaluated and ruled out during the architecture +pass was onion routing, rejected because it requires the sender to +know the full path upfront (incompatible with self-organizing +routing) and prevents per-hop error feedback (incompatible with +CoordsRequired/PathBroken recovery). The canonical mention lives in +[fips-mesh-operation.md](fips-mesh-operation.md#privacy-considerations). diff --git a/docs/design/fips-bloom-filters.md b/docs/design/fips-bloom-filters.md index fda6565..1e71356 100644 --- a/docs/design/fips-bloom-filters.md +++ b/docs/design/fips-bloom-filters.md @@ -153,6 +153,15 @@ this node, and this node thinks it can reach the same destination through Q. Split-horizon is computed per-peer: the outbound filter for peer Q merges all tree peer inbound filters except Q's. +### Filter Propagation Diagram + +![Bloom filter propagation on a spanning tree](diagrams/fips-bloom-propagation.svg) + +The outbound filter for peer Q merges this node's identity with tree +peer inbound filters except Q's (split-horizon exclusion). Upward +filters (child → parent) contain the child's subtree, while downward +filters (parent → child) contain the complement. + ### Directional Asymmetry Because merge is restricted to tree peers, outgoing filters exhibit @@ -202,9 +211,10 @@ Filter updates are event-driven, not periodic: ### Rate Limiting -Updates are rate-limited at 500ms minimum interval per peer to prevent -storms during topology changes. Multiple pending changes within the -cooldown period are coalesced into a single announcement. +Updates are rate-limited at a 500ms minimum interval per peer +(`node.bloom.update_debounce_ms`) to prevent storms during topology +changes. Multiple pending changes within the cooldown period are +coalesced into a single announcement. ### Propagation Scope @@ -249,21 +259,14 @@ Where `filter_bits = 8 × (512 << size_class)` — 8,192 for v1. ## Wire Format -FilterAnnounce messages are carried inside encrypted link-layer frames: - -| Offset | Field | Size | Description | -| ------ | ----- | ---- | ----------- | -| 0 | msg_type | 1 byte | 0x20 | -| 1 | sequence | 8 bytes LE | Monotonic counter for freshness | -| 9 | hash_count | 1 byte | Number of hash functions (5 in v1) | -| 10 | size_class | 1 byte | Filter size: `512 << size_class` bytes | -| 11 | filter_bits | 1,024 bytes | Bloom filter bit array (v1) | - -**v1 total**: 1,035 bytes payload, 1,064 bytes with link encryption -overhead. - -See [fips-wire-formats.md](fips-wire-formats.md) for the complete wire -format reference. +The FilterAnnounce byte layout (`msg_type 0x20`, sequence, hash_count, +size_class, filter_bits) lives in +[../reference/wire-formats.md](../reference/wire-formats.md). The +v1 plaintext payload is 1,035 bytes (11-byte header + 1,024-byte +filter); link encryption adds 36 bytes of FMP framing (16-byte outer +header + 4-byte inner timestamp + 16-byte AEAD tag), bringing the +on-the-wire size to roughly 1,071 bytes before the underlying +transport's per-packet overhead. ## Scale and Size Classes @@ -320,9 +323,8 @@ The envisioned approach is that hub nodes near the root — which carry the largest downward filters — would use larger size classes, while leaf nodes and resource-constrained nodes continue with smaller filters. A node receiving a filter larger than its own size class folds it down locally. -The mechanism by which heterogeneous filter sizes propagate through the -tree is a future design direction not specified in v1. See -[IDEA-0043](../../ideas/IDEA-0043-heterogeneous-filter-propagation.md). +The mechanism by which heterogeneous filter sizes propagate through +the tree is a future design direction not specified in v1. ### Folding @@ -335,6 +337,33 @@ The hash function design supports folding: membership tests at a smaller size use `hash(item, i) % smaller_bit_count`, which maps to the same bit positions that folding produces. +## Mesh Size Estimation + +Each filter's saturation can be inverted into an estimated entry count +via the standard formula `n ≈ -(m/k) · ln(1 − X/m)`, where `m` is the +filter size in bits, `k` is the hash count, and `X` is the population +count. Combining the parent's inbound filter with the children's +inbound filters gives an estimate of the whole network: parent + each +child's subtree are disjoint by construction, and adding 1 for the +node itself yields the total. The result is cached on the node and +exposed through the control socket and `fipstop` dashboard. + +The estimator refuses to produce a value when any contributing filter +is above the antipoison FPR cap (`node.bloom.max_inbound_fpr`, +default `0.05`); a partial aggregate would silently underestimate. +Consumers handle the resulting `None` by displaying an "unknown" +state rather than a misleading number. + +## Antipoison: Inbound FPR Cap + +Inbound `FilterAnnounce` payloads are checked against +`node.bloom.max_inbound_fpr` (default `0.05`). Filters whose +estimated false positive rate exceeds the cap are dropped silently +(no NACK on the wire) — they would otherwise inflate downstream +candidate evaluation cost without contributing useful discrimination. +The cap also gates filters from feeding into mesh size estimation, +as described above. + ## Implementation Status | Feature | Status | @@ -349,6 +378,8 @@ positions that folding produces. | 500ms rate limiting | **Implemented** | | FilterAnnounce gossip (all peers) | **Implemented** | | Filter cardinality logging | **Implemented** | +| Mesh size estimation (parent + children + 1) | **Implemented** | +| Inbound FPR cap (antipoison) | **Implemented** | | Size class negotiation | Future direction | | Folding support | Future direction | | Adaptive filter sizing | Future direction | @@ -357,6 +388,7 @@ positions that folding produces. - [fips-mesh-operation.md](fips-mesh-operation.md) — How bloom filters fit into routing -- [fips-wire-formats.md](fips-wire-formats.md) — FilterAnnounce wire format +- [../reference/wire-formats.md](../reference/wire-formats.md) — + FilterAnnounce wire format - [fips-spanning-tree.md](fips-spanning-tree.md) — The coordinate system that bloom filter candidates are ranked by diff --git a/docs/design/fips-concepts.md b/docs/design/fips-concepts.md new file mode 100644 index 0000000..cb6ab02 --- /dev/null +++ b/docs/design/fips-concepts.md @@ -0,0 +1,123 @@ +# FIPS Concepts + +A novice-friendly introduction to what FIPS is, why it exists, and the +mental model behind a self-organizing mesh. For the protocol stack, +identity system, and encryption walkthrough, see +[fips-architecture.md](fips-architecture.md). For prior art and +academic citations, see [fips-prior-work.md](fips-prior-work.md). + +## What is FIPS? + +FIPS is a self-organizing mesh network that can operate natively over a +variety of physical and logical media, such as local area networks, +Bluetooth, serial links, or the existing internet as an overlay. The +long-term goal is infrastructure that can function alongside or +ultimately replace dependence on the Internet itself. Systems running +FIPS establish peer connections, authenticate each other, and route +traffic for each other without any central authority or global topology +knowledge, and allow end-to-end encrypted sessions between any two +nodes regardless of how many hops separate them. + +Nodes in the mesh route traffic for each other using Nostr identities +(npubs) as network addresses. Applications can access the mesh through +a native FIPS datagram service, or through an IPv6 adaptation layer +that presents each node as an IPv6 endpoint for compatibility with +existing IP-based applications. + +## Why FIPS? + +**Self-sovereign identity**: FIPS nodes generate their own addresses, +node IDs, and security credentials without coordination with any +central authority. These identities can be long-term fixed or may be +ephemeral, changed at any time. These identities are not visible to +the FIPS network itself — they are used only at the application layer +and for end-to-end session encryption. + +**Infrastructure independence**: The internet depends on centralized +infrastructure — ISPs, backbone providers, DNS, certificate +authorities. FIPS works over any transport that can carry packets: a +serial connection, onion-routed connections through Tor, local area +networking, radio links between remote sites, or the existing internet +as an overlay. When the internet is unavailable, unreliable, or +untrusted, the mesh still works. + +**Privacy by design**: FIPS provides secure, authenticated, and +encrypted communication between any two nodes in the mesh, independent +of the mix of transports used along the routed path between them. +Furthermore, the mesh itself is designed to minimize metadata exposure +— intermediate nodes route packets without learning the identities of +the endpoints. + +**Zero configuration**: Nodes discover each other and build routing +automatically. Connect to one peer and you can reach the entire mesh. +The network self-heals around failures and adapts to changing topology. + +## A Self-Organizing Mesh + +Traditional networks are built top-down. A central authority assigns +addresses, configures routing tables, provisions hardware, and manages +the topology. If the authority disappears or the infrastructure fails, +the network fails with it. Nodes cannot reach each other without +infrastructure mediating the connection. + +FIPS inverts this model. There is no central authority, no address +assignment service, no routing table pushed from above. Each node +generates its own identity from a cryptographic keypair. Each node +independently decides which peers to connect to and which transports +to use. From these local decisions alone, the network self-organizes: + +- A **spanning tree** forms through distributed parent selection, + giving every node a coordinate in the network without any node + knowing the full topology +- **Bloom filters** propagate through gossip, so each node learns + which peers can reach which destinations — again without global + knowledge +- **Routing decisions** are made locally at each hop, using only the + node's immediate peers and cached coordinate information + +Each peer link and end-to-end session actively measures RTT, loss, +jitter, and goodput through a lightweight in-band Metrics Measurement +Protocol (MMP), providing operator visibility and a foundation for +quality-aware routing. + +The result is a network that builds itself from the bottom up, heals +around failures automatically, and scales without central coordination. +Adding a node is as simple as connecting to one existing peer — the +network integrates the new node through its normal mesh protocols. + +## Specific Design Goals + +- **Nostr-native identity and cryptography** — Use Nostr keypairs as + node identities and leverage secp256k1, Schnorr signatures, and + SHA-256 +- **Transport agnostic** — Support overlay, shared medium, and + point-to-point transports transparently +- **Self-organizing** — Automatic topology discovery and route + optimization +- **Privacy preserving** — Minimize metadata leakage across untrusted + links +- **Resilient** — Self-healing with graceful degradation + +Non-goals include: + +- **Reliable delivery** — FIPS provides a best-effort datagram + service; retransmission and ordering are left to applications or + higher-layer protocols +- **Anonymity** — Direct peers learn each other's identity; FIPS + minimizes metadata exposure but is not an anonymity network like Tor +- **Congestion control** — FIPS measures link quality but does not + implement flow control or congestion avoidance at the mesh layer + +## Where to Read Next + +- [fips-architecture.md](fips-architecture.md) — protocol stack, + identity system, two-layer encryption, MTU as a cross-cutting + concern +- [fips-spanning-tree.md](fips-spanning-tree.md) — how the tree forms + and reconverges +- [fips-bloom-filters.md](fips-bloom-filters.md) — how reachability + information propagates +- [fips-mesh-operation.md](fips-mesh-operation.md) — how the pieces + work together at runtime +- [fips-prior-work.md](fips-prior-work.md) — designs and protocols + FIPS builds on diff --git a/docs/design/fips-gateway.md b/docs/design/fips-gateway.md index 84e754c..b514f95 100644 --- a/docs/design/fips-gateway.md +++ b/docs/design/fips-gateway.md @@ -1,398 +1,536 @@ -# FIPS Outbound LAN Gateway +# FIPS Gateway -`fips-gateway` is a sidecar binary that runs alongside the FIPS daemon, enabling -unmodified LAN hosts to reach mesh destinations. It provides DNS resolution of -`.fips` names to virtual IPv6 addresses from a managed pool and configures -kernel nftables NAT rules for traffic forwarding through the fips0 TUN -interface. LAN clients need no FIPS software — any device that can resolve DNS -and send IPv6 packets can use the gateway. +The FIPS gateway lets unmodified IPv6 hosts on a LAN exchange traffic +with the mesh without running any FIPS software themselves. It is a +niche feature — most operators will never enable it. The gateway +runs most conveniently on a system that is already providing network +services (DHCP, DNS, RA) to a LAN segment, since hosts on that +segment already get IP assignment and a default route from that box. +The canonical example is an OpenWrt-based WiFi access point: every +client that associates with the AP already has the AP as default +router and DNS server, which is exactly the placement the gateway +needs. The OpenWrt ipk ships with the `gateway:` block of +`/etc/fips/fips.yaml` pre-populated and the integration glue +(dnsmasq forwarding, RA route for the virtual pool, global-scope +IPv6 prefix on `br-lan`) automated by the init script — +[`packaging/openwrt-ipk/files/etc/init.d/fips-gateway`](https://github.com/jmcorgan/fips/blob/master/packaging/openwrt-ipk/files/etc/init.d/fips-gateway). +The operator only needs to enable and start the service. Running the +gateway on a non-OpenWrt LAN-edge host (a Linux router/server, for +example) is technically possible but requires manual integration: +distributing a route to the virtual-IP pool, wiring DNS forwarding so +LAN clients send `.fips` queries to the gateway, configuring sysctls +and capabilities. That path is supported but tedious; it is the +secondary path. + +The feature has two halves that share common machinery and have +their own unique parts. + +The **outbound half** carries traffic from LAN to mesh. A non-FIPS +LAN workstation resolves `.fips` (or a `.fips` host alias) via +the gateway's DNS proxy, which returns a virtual IPv6 address from a +managed pool. The kernel routes the LAN packet to that virtual IP via +a route to the pool CIDR (RA-advertised, statically distributed, or +on-link via the default route). The gateway runs nftables NAT so the +packet appears on the mesh as if it had originated from the gateway's +own FIPS identity: prerouting DNAT rewrites the destination from the +virtual IP to the real `fd00::/8` mesh address, and postrouting +masquerade rewrites the source from the LAN host's address to the +gateway's `fips0` address. Return traffic follows the conntrack +reverse path back to the originating LAN host, with postrouting SNAT +restoring the virtual IP as source so the client sees a response from +the address it connected to. + +The **inbound half** carries traffic from mesh to LAN. A +configuration entry in `gateway.port_forwards[]` exposes a LAN +service (`host:port`) on a port of the gateway's mesh-side `fips0` +address. Mesh peers reach it as `.fips:`. +A prerouting DNAT rule keyed on `(iif=fips0, l4proto, dport)` +rewrites the destination to the LAN target; a LAN-side masquerade in +postrouting rewrites the mesh peer's source so the LAN target sees a +reachable LAN address and conntrack steers replies back through the +gateway. This is the inverse of port-forwarding on a conventional NAT +router. + +The two halves are independent and can be configured separately. +Inbound port-forwards work without any outbound configuration (just +a port-forward list and the table); outbound works without any +inbound forwards. They share the same nftables table, the same +binary, the same control socket, and the same atomic-rebuild +strategy. That shared machinery is what makes them halves of one +feature rather than two separate features. ## Architecture -The gateway is a separate binary (`fips-gateway`), not part of the FIPS daemon. -It connects to the daemon indirectly: `.fips` DNS queries are forwarded to the -daemon's built-in resolver (localhost:5354), which resolves names to mesh -addresses and primes its identity cache as a side effect. The gateway then -allocates a virtual IP, installs NAT rules, and returns the virtual IP to the -LAN client. +### The `fips-gateway` Service + +The gateway is a separate binary, [`fips-gateway`](https://github.com/jmcorgan/fips/blob/master/src/bin/fips-gateway.rs), +not part of the FIPS daemon. It reads the same `/etc/fips/fips.yaml` +the daemon reads (via `--config`, or the standard search path), but +acts on the `gateway.*` block. It needs `CAP_NET_ADMIN` to install +nftables rules, manage proxy NDP entries, and add the pool route. +The CLI is documented in +[../reference/cli-fips-gateway.md](../reference/cli-fips-gateway.md). + +The gateway connects to the daemon indirectly. The outbound half +forwards `.fips` DNS queries to the daemon's built-in resolver +(default `[::1]:5354`); the daemon resolves the name to a mesh +address and primes its identity cache as a side effect. The inbound +half does not require any daemon plumbing at all — packets that +arrive on `fips0` after the daemon's TUN injection path are matched +by the nftables rules on `fips0` ingress. There is no shared memory, +no IPC channel, and no startup ordering coupling beyond "the daemon's +DNS responder must be reachable before the gateway starts serving +LAN queries", which the gateway enforces with a bounded reachability +probe at startup. + +### nftables Table Layout + +All gateway rules live in a single nftables table, `inet +fips_gateway`, with two chains: + +- `prerouting` — `type nat hook prerouting priority dstnat (-100)`, + for both LAN→mesh DNAT (per virtual-IP mapping) and mesh→LAN DNAT + (per port-forward). +- `postrouting` — `type nat hook postrouting priority srcnat (100)`, + for both the always-on `oifname fips0` masquerade, the per-mapping + return-path SNAT, and (when any port-forward is configured) the + LAN-side masquerade for inbound traffic. + +The table is rebuilt atomically on every change. The rebuild +sequence — delete the existing table (ignore `ENOENT` on first +call), then create a new table with chains and the full rule set in +a single netlink batch — avoids reliance on kernel rule-handle +tracking, which the rustables crate does not expose. The table stays +small (one always-on masquerade plus two rules per active outbound +mapping plus one rule per inbound forward, with one extra masquerade +when any forward is present), so rebuilds are cheap. + +### Control Socket + +`fips-gateway` exposes a Unix-domain control socket at +`/run/fips/gateway.sock` (`root:fips`, mode `0770`) with two +commands: `show_gateway` and `show_mappings`. The protocol is the +same line-delimited JSON used by the daemon's control socket. The +shapes are documented in the +[Gateway command catalog](../reference/control-socket.md#gateway-command-catalog). +There is no `fipsctl gateway` subcommand; clients (including +`fipstop`'s gateway view) talk to the socket directly. + +### Diagram ```text - LAN Client - | - DNS query (.fips) - | - v - +---------------------+ - | DNS Proxy | Listens on [::]:53 - | (dns.rs) | - +---------------------+ - | | - .fips query non-.fips → REFUSED - | - v - FIPS Daemon Resolver - (localhost:5354) - | - mesh address (fd00::/8) - | - v - +---------------------+ - | Virtual IP Pool | Allocates from pool CIDR - | (pool.rs) | - +---------------------+ - | - pool event (new/removed mapping) - | - v - +----------+----------+ - | | - v v -+---------------+ +-----------------+ -| NAT Manager | | Network Setup | -| (nat.rs) | | (net.rs) | -| DNAT/SNAT/ | | Proxy NDP, | -| masquerade | | pool route | -+---------------+ +-----------------+ - | | - v v - nftables rules ip -6 neigh proxy - (inet fips_gateway) ip -6 route local + LAN clients + │ + DNS query (.fips) │ IPv6 packet + for outbound │ to virtual IP + │ or mesh peer + ▼ + ┌───────────────────────────────────┐ + │ fips-gateway │ + │ │ + │ ┌──────────────┐ ┌───────────┐ │ + │ │ DNS proxy │ │ Virtual │ │ + │ │ ([::]:53) │─▶│ IP pool │ │ + │ │ .fips only │ │ (state │ │ + │ └──────┬───────┘ │ machine) │ │ + │ │ └─────┬─────┘ │ + │ │ │ │ + │ forward to │ pool │ + │ daemon resolver │ events │ + │ ([::1]:5354) ▼ │ + │ │ ┌───────────┐ │ + │ │ │ NAT │ │ + │ │ │ manager │ │ + │ │ │ (rebuild │ │ + │ │ │ inet │ │ + │ │ │ fips_ │ │ + │ │ │ gateway) │ │ + │ │ └─────┬─────┘ │ + │ │ │ │ + │ │ ┌─────▼─────┐ │ + │ │ │ net │ │ + │ │ │ setup │ │ + │ │ │ (proxy │ │ + │ │ │ NDP, lo │ │ + │ │ │ route) │ │ + │ │ └───────────┘ │ + │ │ │ + │ │ control socket │ + │ │ /run/fips/ │ + │ │ gateway.sock │ + └─────────┼─────────────────────────┘ + │ + ▼ + FIPS daemon resolver + ([::1]:5354) + │ + ▼ + fips0 TUN interface + │ + ▼ + the mesh ``` -### Data Flow +The DNS proxy and the virtual IP pool are exclusive to the outbound +half. The NAT manager and the kernel-side machinery (nftables table, +`fips0` and LAN interfaces, conntrack) are shared. The inbound half +contributes per-port-forward rules to the same table without +involving the DNS proxy or the pool. -1. LAN client queries `hostname.fips` via DNS -2. Gateway forwards to daemon resolver (localhost:5354) -3. Daemon resolves name to mesh address (fd00::/8), primes identity cache -4. Gateway allocates virtual IP from pool, creates DNAT/SNAT rules and proxy NDP - entry -5. Gateway returns AAAA record with virtual IP to client -6. Client sends traffic to virtual IP -7. Kernel DNAT rewrites destination to mesh address, masquerade rewrites source - to gateway's fips0 address -8. Traffic flows through fips0 into the mesh -9. Return traffic follows the reverse path via conntrack +## The Outbound Half (LAN → Mesh) -## NAT Pipeline +### DNS Resolution Flow -The gateway manages a dedicated nftables table (`inet fips_gateway`) containing -two chains with rules that translate between virtual IPs and mesh addresses. +1. A LAN client sends a DNS query to the gateway's listener (default + `[::]:53`, configurable via `gateway.dns.listen`). +2. If the question is not for a `.fips` domain, the gateway replies + `REFUSED`. The proxy is intentionally narrow — it does not resolve + public DNS, and the LAN's primary resolver should hold port 53 on + the gateway host (the OpenWrt init script wires this up by binding + the gateway listener to a non-conflicting port and configuring + dnsmasq to forward `.fips` queries there). +3. The gateway forwards the query to the daemon resolver + (`gateway.dns.upstream`, default `[::1]:5354`). The daemon must + match: an IPv6 socket bound to `[::1]` does not accept v4-mapped + traffic, so a `127.0.0.1:5354` upstream cannot reach a daemon + bound on `[::1]:5354`. +4. If the daemon is unreachable or times out (5 s), the gateway + replies `SERVFAIL`. If the daemon returns `NXDOMAIN` or a + non-`AAAA` answer, the gateway forwards the response unchanged. +5. The gateway extracts the AAAA (`fd00::/8`) record from the + daemon's response. This resolution primes the daemon's identity + cache as a side effect — a prerequisite for `fips0` routing, + because the daemon needs the cache entry to map the mesh address + back to a `NodeAddr` for forwarding. +6. The gateway allocates a virtual IP from the pool for that mesh + address (idempotent: an existing mapping is reused and its TTL + refreshed). +7. If a new mapping was created, the pool emits `MappingCreated`, + which the main loop turns into `add_mapping` calls on the NAT + manager and `add_proxy_ndp` on the network setup. +8. The gateway returns an `AAAA` response containing the virtual IP, + with the configured TTL (default 60 s). -### Prerouting DNAT +### Virtual IP Pool -A per-mapping rule in the `prerouting` chain (priority dstnat / -100) rewrites -the destination address from the virtual IP to the corresponding fd00::/8 mesh -address: +The pool allocates IPv6 addresses from a configured CIDR (default +`fd01::/112`). Each address maps to one mesh destination, keyed by +`NodeAddr` rather than by hostname — different `.fips` aliases for +the same node share a virtual IP. Address 0 (the network-equivalent) +is reserved; the rest are allocatable. The pool is capped at 2^16 +addresses regardless of prefix length, to bound memory. + +The pool tracks state per address: ```text -match: ip6 daddr == +Allocated ──→ Active ──→ Draining ──→ Free + │ ▲ + └──────────────────────────────────┘ + (TTL expired, no sessions) +``` + +| State | Meaning | +| ----- | ------- | +| Allocated | DNS query created the mapping; no NAT sessions yet. | +| Active | Conntrack reports at least one session for this virtual IP. | +| Draining | TTL has expired; sessions may still be in progress, or grace period is running after sessions ended. | +| Free | Reclaimed and available for new allocations. | + +Transitions: + +- **Allocated → Active**: conntrack sessions count goes above zero. +- **Allocated → Free**: TTL expires before any session is ever + observed. +- **Active → Draining**: TTL expires (sessions may or may not still + be present). +- **Draining → Free**: session count is zero and the grace period + has elapsed since draining began. + +Timing: + +- **TTL** (`gateway.dns.ttl`, default 60 s) is both the DNS TTL + returned to the client and the mapping's idle lifetime. Repeated + DNS queries for the same destination refresh the + `last_referenced` timestamp. +- **Grace period** (`gateway.pool_grace_period`, default 60 s) is + the dwell time after the last session ends before the address is + recycled. It prevents immediate reuse from confusing hosts with + cached DNS responses. +- **Tick interval**: the pool re-evaluates state every 10 s. + +Active session counts come from `/proc/net/nf_conntrack`: an entry +counts as a session if its original destination is the virtual IP. + +If the pool is exhausted, new DNS queries return `SERVFAIL`. +Existing mappings are never evicted prematurely — the correctness of +in-flight sessions takes precedence over fresh allocations. + +### NAT Pipeline (Outbound) + +Three rule classes in `inet fips_gateway` together implement the +LAN→mesh path: + +**Prerouting DNAT (per mapping)** rewrites the destination from the +virtual IP to the corresponding mesh address: + +```text +match: nfproto ipv6 && ip6 daddr == action: dnat to ``` -After DNAT, the kernel routes the packet through fips0 via the standard routing -table. +After DNAT, the kernel routes the packet through `fips0` via the +standard routing table. -### Postrouting Masquerade - -A single masquerade rule in the `postrouting` chain (priority srcnat / 100) -rewrites the source address of all traffic exiting via fips0 to the gateway's -own fips0 address: +**Postrouting masquerade (`oifname fips0`)** rewrites the source of +all traffic exiting via `fips0` to the gateway's own `fips0` address: ```text -match: oifname == "fips0" +match: oifname == "fips0" action: masquerade ``` -This is critical. Without masquerade, LAN client source addresses (e.g., -`fd01::5` from the virtual pool) would appear as the source on the mesh. These -addresses are meaningless to mesh peers, so return traffic would be black-holed. -Masquerade ensures all mesh traffic appears to originate from the gateway's own -FIPS identity. +This rule is critical. Without it, LAN client source addresses (for +example `fd02::20` from the LAN's RA-advertised prefix, or virtual +addresses from another forwarding domain) would appear as the source +on the mesh. Those addresses are meaningless to mesh nodes, so +return traffic would be black-holed. Masquerade ensures all mesh +traffic appears to originate from the gateway's own FIPS identity. -### Postrouting SNAT - -A per-mapping rule in the `postrouting` chain rewrites the source address of -return traffic from the mesh address back to the virtual IP: +**Postrouting SNAT (per mapping)** rewrites the source of return +traffic from the mesh address back to the virtual IP: ```text -match: ip6 saddr == +match: nfproto ipv6 && ip6 saddr == action: snat to ``` -This ensures LAN hosts see responses from the virtual IP they connected to, -not from the raw fd00::/8 mesh address. +Without it, the LAN client would see replies from the raw +`fd00::/8` mesh address rather than from the virtual IP it had +originally connected to, breaking application-layer assumptions about +the destination address. -### Atomic Table Rebuild +### Network Requirements (Outbound) -The entire nftables table is rebuilt atomically on every mapping change. The -rebuild sequence is: delete the existing table (ignore ENOENT on first call), -then create a new table with all chains, the masquerade rule, and all -per-mapping DNAT/SNAT rules in a single netlink batch. +The gateway host needs IPv6 forwarding enabled +(`net.ipv6.conf.all.forwarding=1`), proxy NDP enabled on the LAN +interface, `CAP_NET_ADMIN` for `fips-gateway`, and a `local + dev lo` route so the kernel accepts packets to the pool +as locally owned and runs them through the NAT chains. LAN clients +need a route to the pool via the gateway and DNS resolution that +forwards `.fips` queries there. On OpenWrt the init script handles +all of this; on other Linux hosts the operator handles it manually. +Full setup is documented in +[../how-to/deploy-gateway.md](../how-to/deploy-gateway.md). -This approach avoids relying on kernel rule handle tracking, which the rustables -crate does not expose. The table is small — one masquerade rule plus two rules -per active mapping — so rebuilding is cheap. +## The Inbound Half (Mesh → LAN) -## Virtual IP Pool Lifecycle +### Configuration Shape -The pool allocates IPv6 addresses from a configured CIDR range (e.g., -`fd01::/112`). Each address maps to one FIPS mesh destination (keyed by -NodeAddr, not hostname). Address 0 (network equivalent) is reserved; the -remaining addresses are available for allocation. +Inbound port-forwards live in `gateway.port_forwards[]`. Each entry +is a triple: -### State Machine +| Field | Type | Notes | +| ----- | ---- | ----- | +| `listen_port` | `u16` | Port on the gateway's `fips0` address. Must be non-zero. | +| `proto` | `tcp` \| `udp` | Match protocol. | +| `target` | `[ipv6]:port` | LAN destination. IPv4 targets are rejected at parse time by `SocketAddrV6`. | + +Validation runs at startup and on every config reload: +`(listen_port, proto)` must be unique across the list, and zero +listen ports are rejected. Forwards are independent of outbound +configuration: a gateway with no `pool` consumers can still expose +inbound services (the pool route and DNS proxy still run, since they +are part of the same binary, but they sit idle). + +### NAT Pipeline (Inbound) + +For each port-forward, a single prerouting DNAT rule matches +mesh-originated traffic landing on the gateway's `fips0` address +and rewrites it to the LAN target: ```text -ALLOCATED ──→ ACTIVE ──→ DRAINING ──→ FREE - │ ↑ - └───────────────────────────────────┘ - (TTL expired, no sessions) +match: iifname == "fips0" && nfproto ipv6 + && l4proto == && th dport == +action: dnat to : ``` -| State | Description | -| ----- | ----------- | -| Allocated | DNS query created the mapping. No NAT sessions yet. | -| Active | Conntrack reports at least one active session. | -| Draining | TTL expired but sessions remain, or sessions ended and grace period is running. | -| Free | Reclaimed. Virtual IP returned to the available pool. | +The match clause is deliberately narrow: -### Transitions +- **`iifname == "fips0"`** restricts the rule to traffic that + arrived from the mesh. LAN-side ingress is never subject to + inbound forwarding. +- **`nfproto ipv6`** is enforced both here and at config-load time + (`SocketAddrV6` rejects IPv4 targets); FIPS is IPv6-only end to + end. +- **`l4proto + dport`** narrows the match to one + `(listen_port, proto)` pair per rule. Unique-tuple validation + ensures no two rules contend for the same packet. -- **Allocated to Active**: Conntrack reports sessions > 0. -- **Allocated to Free**: TTL expired with no sessions ever created. -- **Active to Draining**: TTL expired (sessions may or may not remain). -- **Draining to Free**: Sessions drop to zero and the grace period elapses. +When *any* port-forward is configured, a single LAN-side masquerade +is added to postrouting: -### Timing +```text +match: iifname == "fips0" && oifname == + && nfproto ipv6 +action: masquerade +``` -- **TTL**: Default 60 seconds (matches DNS TTL). Repeated DNS queries for the - same destination reset the `last_referenced` timestamp. -- **Grace period**: Default 60 seconds after draining begins with zero sessions. - Prevents immediate reuse that could confuse hosts with cached DNS responses. -- **Tick interval**: The pool evaluates state transitions every 10 seconds. +Without this rule, the LAN target would attempt to reply directly to +the mesh peer's `fd00::/8` source address, which is not reachable on +the LAN. Masquerade rewrites the source to the gateway's LAN-side +address so the target sees a reachable peer and conntrack routes +the reply back through the gateway. -### Conntrack Integration +This LAN-side masquerade is independent of the `oifname fips0` +masquerade in the outbound pipeline; the two have disjoint match +clauses (different `iifname`/`oifname` combinations) and coexist +without interaction when both directions are active. -The pool queries `/proc/net/nf_conntrack` to count active sessions per virtual -IP. A session is counted if any conntrack entry's original destination matches -the virtual IP address. +### Independence From Outbound -### Pool Exhaustion +The inbound half does not require: -If no addresses are available, new DNS queries return SERVFAIL. Existing -mappings are never evicted prematurely — correctness of active sessions takes -priority over new allocations. The pool is capped at 2^16 addresses regardless -of CIDR prefix length to prevent excessive memory allocation. +- A virtual-IP pool. Mesh peers connect directly to the gateway's + own `fips0` address, which the FIPS daemon already owns. +- DNS resolution. Mesh peers reach the gateway as + `.fips:` using their own resolver (or a + numeric mesh address); the gateway's DNS proxy is not in the path. +- A daemon-side identity cache for the LAN target. The target is a + LAN-side IPv6 address, not a mesh address; no `fd00::/8` lookup + happens for it. -## DNS Resolution Flow +A gateway configured with port-forwards but with no LAN clients ever +issuing `.fips` DNS queries will have an empty pool and zero +outbound mappings, but its inbound forwards work normally. The +inverse is also true: a gateway that serves only outbound LAN→mesh +traffic has zero entries in the port-forwards list and no LAN-side +masquerade. -1. Gateway listens on configured address (default `[::]:53`). -2. Client sends DNS query. -3. If the query is not for a `.fips` domain, return `REFUSED`. -4. Forward the query to the daemon resolver at `127.0.0.1:5354` (configurable). -5. If the daemon is unreachable or times out (5 seconds), return `SERVFAIL`. -6. If the daemon returns NXDOMAIN or an error, forward the response as-is. -7. Extract the AAAA record (fd00::/8 mesh address) from the daemon's response. -8. Allocate a virtual IP from the pool for this destination (idempotent — if a - mapping already exists, reuse it and refresh the TTL). -9. If a new mapping was created, emit a `MappingCreated` event to install NAT - rules and proxy NDP entry. -10. Build and return an AAAA response containing the virtual IP with the - configured TTL. +## Atomic Table Rebuild (Common) -The daemon's resolver populates its identity cache as a side effect of -resolution. This is required for fips0 routing to work — without the cache -entry, the daemon cannot map the fd00::/8 address back to a NodeAddr for mesh -routing. +Both halves contribute rules to the same `inet fips_gateway` table, +and that table is rebuilt as one unit on every state change — +mapping added, mapping removed, port-forwards updated. The rebuild +sequence is: -## Network Requirements +1. Delete the existing table in its own batch (ignore `ENOENT`). +2. In a fresh batch: add the table; add the `prerouting` and + `postrouting` chains; add the always-on `oifname fips0` + masquerade; add per-mapping DNAT/SNAT rules for every active + pool entry; add per-port-forward DNAT rules; add the LAN-side + masquerade if any port-forwards exist. +3. Send the batch as a single netlink transaction. -### Gateway Host - -The following must be true on the machine running `fips-gateway`: - -- **FIPS daemon running** with TUN enabled (fips0 interface must exist) and DNS - resolver on port 5354 -- **IPv6 forwarding enabled**: `sysctl -w net.ipv6.conf.all.forwarding=1` -- **Proxy NDP enabled**: `sysctl -w net.ipv6.conf.all.proxy_ndp=1` -- **CAP_NET_ADMIN**: Required for nftables table management and proxy NDP - manipulation (run as root or set the capability) -- **Pool route**: The gateway adds `local dev lo` at startup, which - tells the kernel to accept packets destined for pool addresses as - locally-owned, enabling NAT processing. This route is cleaned up on shutdown. - -### LAN Clients - -LAN clients need no FIPS software. They require: - -- **Route to virtual IP pool**: `ip -6 route add via - `. This can be pushed via DHCP, configured on the LAN - router, or set per-host. -- **DNS resolution**: Either configure the LAN's main DNS server to forward - `.fips` queries to the gateway, or point individual hosts at the gateway for - DNS (noting that non-`.fips` queries will get `REFUSED`). +The rustables crate does not expose rule-handle tracking, so +incremental update of individual rules is not available. Atomic +rebuild was chosen for simplicity and correctness: it eliminates an +entire class of partial-update inconsistency bugs at the cost of +repeating the (cheap) rule construction on every change. The total +rule count is bounded by the pool capacity (2 per mapping, capped +at 2^16) and the port-forward count, both of which are small in +practice. ## Configuration Reference -All configuration lives under the `gateway` key in `fips.yaml`: +The full `gateway.*` block — pool CIDR, LAN interface, DNS +listen/upstream/TTL, pool grace period, conntrack timeouts, and +inbound port-forwards — is documented in the +[Gateway section](../reference/configuration.md#gateway-gateway) +of the configuration reference. The same block governs both halves; +fields specific to one half (`pool`, `dns.*` for outbound; +`port_forwards[]` for inbound) are simply unused when the other +half is not in play. -```yaml -gateway: - enabled: true - pool: "fd01::/112" - lan_interface: "enp3s0" - dns: - listen: "[::]:53" - upstream: "127.0.0.1:5354" - ttl: 60 - pool_grace_period: 60 - conntrack: - tcp_established: 432000 - udp_timeout: 30 - udp_assured: 180 - icmp_timeout: 30 -``` +## Operations and Troubleshooting -| Field | Type | Default | Description | -| ----- | ---- | ------- | ----------- | -| `enabled` | bool | `false` | Enable the gateway. Must be `true` for `fips-gateway` to start. | -| `pool` | string (CIDR) | required | Virtual IP pool range (e.g., `fd01::/112`). | -| `lan_interface` | string | required | LAN-facing interface for proxy NDP entries. | -| `dns.listen` | string | `[::]:53` | Address and port for the gateway DNS listener. | -| `dns.upstream` | string | `127.0.0.1:5354` | FIPS daemon DNS resolver address. | -| `dns.ttl` | u32 | `60` | DNS response TTL in seconds. Also governs mapping TTL. | -| `pool_grace_period` | u64 | `60` | Seconds after last session before a mapping is reclaimed. | -| `conntrack.tcp_established` | u64 | `432000` | TCP established timeout (seconds). 5 days. | -| `conntrack.udp_timeout` | u64 | `30` | UDP unreplied timeout (seconds). | -| `conntrack.udp_assured` | u64 | `180` | UDP bidirectional (assured) timeout (seconds). | -| `conntrack.icmp_timeout` | u64 | `30` | ICMP timeout (seconds). | - -## Troubleshooting - -### "No gateway section in configuration" - -The `fips-gateway` binary loads the same config file as the daemon. If it -cannot find a `gateway:` section, use the `--config` flag to point at the -correct file: - -```bash -fips-gateway --config /etc/fips/fips.yaml -``` - -### DNS Queries Fail - -Verify the daemon resolver is running and reachable: - -```bash -dig @127.0.0.1 -p 5354 hostname.fips AAAA -``` - -If this fails, the daemon is not running or its DNS resolver is not enabled. -Check that the daemon config has `dns.enabled: true` (enabled by default). - -### Ping Works But TCP Does Not - -This usually means the masquerade rule is missing or misconfigured. Inspect the -nftables table: - -```bash -nft list table inet fips_gateway -``` - -Verify the postrouting chain contains a masquerade rule matching `oifname -"fips0"`. Without masquerade, the mesh peer sees a source address it cannot -route replies to. - -### Connection Timeout - -Check that IPv6 forwarding is enabled: - -```bash -sysctl net.ipv6.conf.all.forwarding -``` - -Verify the pool route exists: - -```bash -ip -6 route show table local | grep -``` - -If the route is missing, the kernel does not recognize pool addresses as local -and drops the packets before NAT can process them. - -### Virtual IP Unreachable From Client - -Verify the client has a route to the pool via the gateway: - -```bash -ip -6 route get -``` - -On the gateway, verify proxy NDP entries exist for allocated virtual IPs: - -```bash -ip -6 neigh show proxy -``` - -If proxy NDP entries are missing, the gateway cannot answer Neighbor Solicitation -requests for virtual IPs on the LAN, so clients cannot resolve the link-layer -address. - -### Port 53 Conflict - -If another DNS server (systemd-resolved, dnsmasq) is using port 53, the -gateway cannot bind. Options: - -```bash -# Check what is using port 53 -ss -tulnp | grep :53 - -# Use an alternate listen address -dns: - listen: "192.168.1.1:5353" -``` - -Then configure LAN clients to query the alternate port, or run a forwarding -stub on port 53 that delegates `.fips` queries to the gateway. +- [../tutorials/deploy-fips-gateway.md](../tutorials/deploy-fips-gateway.md) + — end-to-end walkthrough on OpenWrt. +- [../how-to/deploy-gateway.md](../how-to/deploy-gateway.md) — + recipe for non-OpenWrt Linux hosts and inbound-port-forwarding + configuration. +- [../how-to/troubleshoot-gateway.md](../how-to/troubleshoot-gateway.md) + — diagnostic recipes (DNS failures, ping working but TCP not, + conntrack inspection, pool exhaustion, port-53 conflicts, + port-forward verification). +- [../reference/cli-fips-gateway.md](../reference/cli-fips-gateway.md) + — command-line interface. +- [../reference/control-socket.md](../reference/control-socket.md#gateway-command-catalog) + — `show_gateway` and `show_mappings` commands. ## Security Considerations -- **LAN trust boundary**: The gateway DNS listener is accessible to any host on - the LAN. Any host that can reach the DNS port and route to the virtual IP pool - can access mesh destinations through the gateway. Access restriction must be - enforced at the network level (firewall rules on the LAN interface). +### Outbound -- **Identity masking**: All LAN traffic appears on the mesh under the gateway's - own FIPS identity. Mesh peers cannot determine which LAN host originated a - connection. This provides privacy for LAN hosts but means the gateway's - reputation covers all its clients. +- **LAN trust boundary.** The DNS listener and the virtual-IP pool + are reachable by every host on the LAN. Any LAN host that can + resolve `.fips` and route to the pool CIDR can reach mesh + destinations. There is no per-client authentication; access + restriction is a network-level concern, enforced with firewall + rules on the LAN interface or on the gateway host itself. +- **Identity masking.** All outbound LAN traffic appears on the + mesh under the gateway's own FIPS identity. Mesh nodes cannot + determine which LAN host originated a connection. This provides + privacy for LAN hosts but means the gateway's reputation covers + all of its clients — and that abusive behavior from one LAN host + is attributed to the gateway, not to the host. +- **Plaintext between client and gateway.** Traffic between the LAN + client and the gateway is unencrypted at the IP layer. FIPS + encryption (FSP) protects the segment between the gateway and the + destination mesh node; application-layer encryption (TLS, SSH, + Noise) is the only thing that provides true end-to-end protection + through the gateway. +- **Pool addresses are ephemeral.** Virtual IPs are allocated + dynamically and recycled. They are not authenticated and not + bound to client identity — a LAN host connecting to a virtual IP + is trusting the gateway's recent DNS response. +- **DNS upstream trust.** The outbound half's correctness depends + on the FIPS daemon's resolver returning honest `fd00::/8` + answers; a compromised daemon could redirect LAN clients to + arbitrary mesh nodes. -- **Plaintext at the gateway**: Traffic between LAN hosts and the gateway is - unencrypted at the IP layer. FIPS encryption (FSP) protects traffic between - the gateway and the destination mesh peer. Application-layer encryption (TLS, - SSH) provides end-to-end protection through the gateway. +### Inbound -- **Pool addresses are ephemeral**: Virtual IPs are allocated dynamically and - recycled. They are not authenticated or bound to client identity. A LAN host - connecting to a virtual IP is trusting the gateway's DNS response. +- **Port exposure.** Each entry in `port_forwards[]` exposes the + matched `(listen_port, proto)` on the gateway's mesh-side + address to every reachable mesh peer. Inbound port-forwards are + not gated by any peer ACL beyond what FMP normally enforces; + treat them with the same care as a public-internet port forward. +- **Mesh peer trust.** The LAN target sees connections that have + been masqueraded to the gateway's LAN address. The target cannot + distinguish one mesh peer from another, and there is no + authenticated peer identity available to the LAN target — any + application-layer authentication or rate-limiting must run on + the target itself. +- **Return-path masquerade exposes the gateway's LAN address.** + The LAN-side masquerade rewrites the mesh peer's source to the + gateway's LAN address. A malicious or buggy LAN target can use + this to send unsolicited traffic back at the gateway, or to + probe other LAN hosts via the gateway's network position; LAN + segmentation (VLANs, host firewalls) is the right control. -- **No client identity verification**: The gateway does not authenticate LAN - clients. Any host that can send packets is served. +### Common -## Future Work - -- **IPv4 pool support**: NAT46 translation via TAYGA or Jool, allowing LAN - hosts to use IPv4 virtual addresses while the mesh remains IPv6. -- **Inbound gateway**: Exposing LAN services to mesh peers (mesh to LAN - direction), requiring port-forwarding or reverse-proxy configuration. -- **fipstop Gateway tab**: Monitoring integration showing pool utilization, - active mappings, and NAT session counts. -- **Gateway control socket**: Status queries via `fipsctl gateway status` and - `fipsctl gateway mappings` for operational visibility. +- **No client identity verification.** The gateway authenticates + neither LAN clients nor mesh peers beyond what the underlying + layers already do — `fips0` ingress carries an FSP-authenticated + payload, the LAN side is whoever the LAN admits. ## References -- [fips-ipv6-adapter.md](fips-ipv6-adapter.md) — IPv6 adapter and TUN interface - design -- [fips-configuration.md](fips-configuration.md) — Configuration reference -- [fips-intro.md](fips-intro.md) — Protocol overview and architecture +- [fips-ipv6-adapter.md](fips-ipv6-adapter.md) — IPv6 adapter and + TUN interface design. +- [fips-architecture.md](fips-architecture.md) — protocol layer + architecture. +- [fips-concepts.md](fips-concepts.md) — protocol overview. +- [../reference/configuration.md](../reference/configuration.md) — + configuration reference. +- [../reference/cli-fips-gateway.md](../reference/cli-fips-gateway.md) + — `fips-gateway` CLI. +- [../reference/control-socket.md](../reference/control-socket.md) — + control-socket protocol and command catalog. +- [../how-to/deploy-gateway.md](../how-to/deploy-gateway.md) — + gateway host and LAN client setup. +- [../how-to/troubleshoot-gateway.md](../how-to/troubleshoot-gateway.md) + — diagnostic recipes. +- [../tutorials/deploy-fips-gateway.md](../tutorials/deploy-fips-gateway.md) + — OpenWrt walkthrough. diff --git a/docs/design/fips-intro.md b/docs/design/fips-intro.md deleted file mode 100644 index ead1774..0000000 --- a/docs/design/fips-intro.md +++ /dev/null @@ -1,822 +0,0 @@ -# FIPS: Free Internetworking Peering System - -## What is FIPS? - -FIPS is a self-organizing mesh network that can operate natively over a -variety of physical and logical media, such as local area networks, -Bluetooth, serial links, or the existing internet as an overlay. The -long-term goal is infrastructure that can function alongside or ultimately -replace dependence on the Internet itself. Systems running FIPS establish -peer connections, authenticate each other, and route traffic for each other -without any central authority or global topology knowledge, and allow -end-to-end encrypted sessions between any two nodes regardless of how many -hops separate them. - -Nodes in the mesh route traffic for each other using Nostr identities -(npubs) as network addresses. Applications can access the mesh through a -native FIPS datagram service, or through an IPv6 adaptation layer that -presents each node as an IPv6 endpoint for compatibility with existing -IP-based applications. - -## Why FIPS? - -**Self-sovereign identity**: FIPS nodes generate their own addresses, node -IDs, and security credentials without coordination with any central -authority. These identities can be long-term fixed or may be ephemeral, -changed at any time. These identities are not visible to the FIPS network -itself — they are used only at the application layer and for end-to-end -session encryption. - -**Infrastructure independence**: The internet depends on centralized -infrastructure — ISPs, backbone providers, DNS, certificate authorities. -FIPS works over any transport that can carry packets: a serial connection, -onion-routed connections through Tor, local area networking, radio links -between remote sites, or the existing internet as an overlay. When the -internet is unavailable, unreliable, or untrusted, the mesh still works. - -**Privacy by design**: FIPS provides secure, authenticated, and encrypted -communication between any two nodes in the mesh, independent of the mix of -transports used along the routed path between them. Furthermore, the mesh -itself is designed to minimize metadata exposure — intermediate nodes route -packets without learning the identities of the endpoints. - -**Zero configuration**: Nodes discover each other and build routing -automatically. Connect to one peer and you can reach the entire mesh. The -network self-heals around failures and adapts to changing topology. - -## A Self-Organizing Mesh - -Traditional networks are built top-down. A central authority assigns -addresses, configures routing tables, provisions hardware, and manages the -topology. If the authority disappears or the infrastructure fails, the -network fails with it. Nodes cannot reach each other without infrastructure -mediating the connection. - -FIPS inverts this model. There is no central authority, no address -assignment service, no routing table pushed from above. Each node generates -its own identity from a cryptographic keypair. Each node independently -decides which peers to connect to and which transports to use. From these -local decisions alone, the network self-organizes: - -- A **spanning tree** forms through distributed parent selection, giving - every node a coordinate in the network without any node knowing the full - topology -- **Bloom filters** propagate through gossip, so each node learns which - peers can reach which destinations — again without global knowledge -- **Routing decisions** are made locally at each hop, using only the node's - immediate peers and cached coordinate information - -Each peer link and end-to-end session actively measures RTT, loss, jitter, -and goodput through a lightweight in-band Metrics Measurement Protocol -(MMP), providing operator visibility and a foundation for quality-aware -routing. - -The result is a network that builds itself from the bottom up, heals around -failures automatically, and scales without central coordination. Adding a -node is as simple as connecting to one existing peer — the network -integrates the new node through its normal mesh protocols. - -## Specific Design Goals - -- **Nostr-native identity and cryptography** — Use Nostr keypairs as node - identities and leverage secp256k1, Schnorr signatures, and SHA-256 -- **Transport agnostic** — Support overlay, shared medium, and - point-to-point transports transparently -- **Self-organizing** — Automatic topology discovery and route optimization -- **Privacy preserving** — Minimize metadata leakage across untrusted links -- **Resilient** — Self-healing with graceful degradation - -Non-goals include: - -- **Reliable delivery** — FIPS provides a best-effort datagram service; - retransmission and ordering are left to applications or higher-layer - protocols -- **Anonymity** — Direct peers learn each other's identity; FIPS minimizes - metadata exposure but is not an anonymity network like Tor -- **Congestion control** — FIPS measures link quality but does not implement - flow control or congestion avoidance at the mesh layer - ---- - -## Protocol Architecture - -FIPS is organized in three protocol layers, each with distinct -responsibilities and clean service boundaries. No layer depends on the -specifics of the layers above or below it — transport plugins know nothing -about sessions, the routing layer knows nothing about application addressing, -and applications know nothing about which physical media carry their traffic. -This separation means new transports, protocol features, and application -interfaces can be added independently. - -![Protocol Stack](diagrams/fips-protocol-stack.svg) - -### Mapping to Traditional Networking - -Readers familiar with the OSI model or TCP/IP networking may find it helpful -to see how FIPS concepts relate to traditional layers: - -![OSI Mapping](diagrams/fips-osi-mapping.svg) - -Note that FMP spans what would traditionally be separate link and network -layers. This is intentional — in a self-organizing mesh, the same layer that -authenticates peers also makes routing decisions, because routing depends on -authenticated peer state (spanning tree positions, bloom filters). - -### Layer Responsibilities - -**Transport layer**: Delivers datagrams between endpoints over a specific -medium. Each transport type (UDP socket, Ethernet interface, radio modem) -implements the same abstract interface: send and receive datagrams, report -MTU. The transport layer knows nothing about FIPS identities, routing, or -encryption. It provides raw datagram delivery to FMP above. - -See [fips-transport-layer.md](fips-transport-layer.md) for the transport layer -specification. - -**FIPS Mesh Protocol (FMP)**: Manages peer connections, authenticates peers -via Noise IK handshakes, and encrypts all traffic on each link. FMP is where -the mesh organizes itself — nodes exchange spanning tree announcements and -bloom filters with their direct peers, and FMP makes forwarding decisions -for transit traffic. FMP provides authenticated, encrypted forwarding to FSP -above. - -See [fips-mesh-layer.md](fips-mesh-layer.md) for the FMP specification and -[fips-mesh-operation.md](fips-mesh-operation.md) for how FMP's routing and -self-organization work in practice. - -**FIPS Session Protocol (FSP)**: Provides end-to-end authenticated -encryption between any two nodes, regardless of how many intermediate hops -separate them. FSP manages session lifecycle (setup, data transfer, -teardown), caches destination coordinates for efficient routing, and handles -the warmup strategy that keeps transit node caches populated. Session -dispatch uses index-based routing inspired by -[WireGuard](https://www.wireguard.com/), enabling O(1) packet -demultiplexing. FSP provides a datagram service to applications above. - -See [fips-session-layer.md](fips-session-layer.md) for the FSP specification. - -**IPv6 adaptation layer**: Sits above FSP as a service on port 256, adapting -the FIPS datagram service for unmodified IPv6 applications. Provides DNS -resolution (npub → fd00::/8 address), identity cache management, IPv6 header -compression, MTU enforcement, and a TUN interface. This is the primary way -existing applications use the FIPS mesh. - -See [fips-ipv6-adapter.md](fips-ipv6-adapter.md) for the IPv6 adapter. - -### Node Architecture - -Application services sit at the top of the stack, dispatched by FSP port -number: the IPv6 TUN adapter (port 256) maps npubs to `fd00::/8` addresses -with header compression so unmodified IP applications can use the network -transparently, while the native datagram API addresses destinations directly -by npub. - -![Node Architecture](diagrams/fips-node-architecture.svg) - -The mesh routes application traffic across heterogeneous transports -transparently. A packet may traverse WiFi, Ethernet, UDP/IP, and Tor links -on its way from source to destination — the application never needs to know -which transports are involved. Each hop is independently encrypted at the -link layer, while a single end-to-end session protects the payload across -the entire path. - -![Architecture Overview](diagrams/fips-architecture-overview.svg) - ---- - -## Identity System - -FIPS uses [Nostr](https://github.com/nostr-protocol/nips) keypairs -(secp256k1) as node identities. The public key identifies the node; the -private key signs protocol messages and establishes encrypted sessions. - -The public key (or its bech32-encoded npub form) is the primary means for -application-layer software to identify communication endpoints. Internally, -the protocol derives a `node_addr` (a 16-byte SHA-256 hash of the pubkey) -used as the routing identifier in packet headers, and an IPv6 address derived -from the node_addr for the TUN adapter. Applications use the pubkey or npub; -the routing layer uses node_addr; unmodified IPv6 applications use the -derived `fd00::/8` address. All three are deterministically derived from the -same keypair. - -### FIPS Identity Handling - -![Identity Derivation](diagrams/fips-identity-derivation.svg) - -The pubkey is the node's cryptographic identity, used in Noise IK handshakes -for both link and session encryption. It is never exposed beyond the -endpoints of an encrypted channel. The node_addr, a one-way SHA-256 hash -truncated to 16 bytes, serves as the routing identifier in packet headers -and bloom filters. Intermediate routers see only node_addrs — they can -forward traffic without learning the Nostr identities of the endpoints. An -observer can verify "does this node_addr belong to pubkey X?" if they already -know the pubkey, but cannot enumerate communicating identities by inspecting -traffic. The IPv6 -address prepends `fd` to the first 15 bytes of the node_addr, providing a -ULA overlay address for unmodified IP applications via the TUN interface. - -Below the FIPS identity layer, each transport uses its own native addressing -— IP:port or hostname:port addresses, MAC addresses, .onion identifiers. These **link -addresses** are opaque to everything above FMP and discarded once link -authentication completes. - -### Identity Verification - -The Noise Protocol Framework mutually authenticates both peer-to-peer link -connections (at FMP) and end-to-end session traffic (at FSP), proving each -party controls the private key for their claimed identity. - -See [fips-mesh-layer.md](fips-mesh-layer.md) for peer authentication and -[fips-session-layer.md](fips-session-layer.md) for end-to-end session -establishment. - -Key rotation changes the node's identity — a new keypair produces a new -node_addr and IPv6 address, requiring all sessions to be re-established. -Migration mechanisms that allow a node to announce a successor key are a -future consideration. - ---- - -## Two-Layer Encryption - -FIPS uses independent encryption at two protocol layers: - -| Layer | Scope | Pattern | Purpose | -| ----- | ----- | ------- | ------- | -| **FMP (Mesh)** | Hop-by-hop | Noise IK | Encrypt all traffic on each peer link | -| **FSP (Session)** | End-to-end | Noise XK | Encrypt application payload between endpoints | - -### Link Layer (Hop-by-Hop) - -When two nodes establish a direct connection, they perform a [Noise -IK](https://noiseprotocol.org/) handshake. This authenticates both parties -and establishes symmetric keys for encrypting all traffic on that link. -Every packet between direct peers is encrypted — gossip messages, routing -queries, and forwarded session datagrams alike. - -The IK pattern is used because outbound connections know the peer's npub -from configuration, while inbound connections learn the initiator's identity -from the first handshake message. - -### Session Layer (End-to-End) - -FIPS establishes end-to-end encrypted sessions between any two communicating -nodes using Noise XK, regardless of how many hops separate them. The -initiator knows the destination's npub (required for XK's pre-message); -the responder learns the initiator's identity from the third handshake -message. Unlike the link-layer IK pattern where the initiator's identity -is revealed in msg1, XK delays identity disclosure until msg3, providing -stronger initiator identity protection for traffic traversing untrusted -intermediate nodes. - -A packet from A to D through intermediate nodes B and C: - -1. A encrypts payload with A↔D session key (FSP) -2. A wraps in SessionDatagram, encrypts with A↔B link key (FMP), sends to B -3. B decrypts link layer, reads destination node_addr, re-encrypts with B↔C - link key, forwards to C -4. C decrypts link layer, re-encrypts with C↔D link key, forwards to D -5. D decrypts link layer, then decrypts session layer to get payload - -Intermediate nodes route based on destination node_addr but cannot read -session-layer payloads. Each hop strips one link encryption and applies the -next — the session-layer ciphertext passes through untouched. - -Both layers always apply, even between adjacent peers — a packet to a direct -neighbor is still encrypted twice. This uniform model means no special cases -for local vs remote destinations, and topology changes (a direct peer -becomes reachable only through intermediaries) don't affect existing -sessions. - -See [fips-mesh-layer.md](fips-mesh-layer.md) for link encryption and -[fips-session-layer.md](fips-session-layer.md) for session encryption. - ---- - -## Routing and Mesh Operation - -Each node makes forwarding decisions using only local information — its -immediate peers, their bloom filters, and cached coordinates — rather than -centrally distributed routing tables or global topology knowledge. Two -complementary mechanisms provide the information each node needs. - -### Spanning Tree: The Coordinate System - -![Mesh Topology](diagrams/fips-mesh-topology.svg) - -Nodes self-organize into a spanning tree through gossip — each node -exchanges announcements with its direct peers and independently selects a -parent. Because every node applies the same rule (prefer the root with the -smallest node_addr), the network converges on a single agreed-upon root -without any voting or coordination. This is the same principle behind the -[Spanning Tree -Protocol](https://en.wikipedia.org/wiki/Spanning_Tree_Protocol) used in -Ethernet bridging since the 1980s: purely local decisions that converge to -consistent global state. The resulting tree gives every node a -**coordinate** — its path from itself to the root. Using tree coordinates -for routing is adapted from -[Yggdrasil](https://yggdrasil-network.github.io/)'s -[Ironwood](https://github.com/Arceliar/ironwood) routing library. - -These coordinates enable distance calculations between any two nodes: the -distance is the number of hops from each node to their lowest common -ancestor in the tree. This provides a metric for routing decisions without -any node needing to know the full network topology. - -The tree maintains itself through gossip — nodes exchange TreeAnnounce -messages with their peers, propagating parent selections and ancestry -chains. Changes cascade through the tree proportional to depth, not network -size. If the network partitions, each segment converges to its own new root -through the same process and reconverges automatically when segments rejoin. - -See [fips-spanning-tree.md](fips-spanning-tree.md) for the tree algorithms -and [spanning-tree-dynamics.md](spanning-tree-dynamics.md) for detailed -convergence walkthroughs. - -### Bloom Filters: Candidate Selection - -The spanning tree provides a coordinate system for distance-based routing, -but on its own each node would only know about its immediate neighbors. -Bloom filters complement the tree by distributing reachability knowledge -across the entire mesh — each node learns which destinations are reachable -through which peers, without any node needing a complete view of the -network. - -Each node's peer-advertised [bloom -filter](https://en.wikipedia.org/wiki/Bloom_filter) is a compact, fixed-size -data structure that answers one question: "can this peer possibly reach -destination D?" The answer is either "no" (definitive) or "maybe" -(probabilistic — false positives are possible). Because the filter size is -constant regardless of how many destinations it represents, bloom filters -scale efficiently as the network grows. This is candidate selection for -routing — bloom filters narrow the set of peers worth considering, and the -actual forwarding decision ranks those candidates by tree distance and link -quality. - -Filters propagate transitively through tree edges, with each node computing -outbound filters by merging the filters received from its tree peers (parent -and children) using a -[split-horizon](https://en.wikipedia.org/wiki/Split_horizon_route_advertisement) -technique borrowed from distance-vector routing. All peers — including -non-tree mesh shortcuts — receive FilterAnnounce messages, but only tree -peers' filters are merged into outgoing computation. This prevents filter -saturation where mesh shortcuts would cause every filter to converge toward -the full network. - -See [fips-bloom-filters.md](fips-bloom-filters.md) for filter parameters and -mathematical properties. - -![Bloom filter propagation on a spanning tree](diagrams/fips-bloom-propagation.svg) - -The outbound filter for peer Q merges this node's identity with tree peer -inbound filters except Q's (split-horizon exclusion). This creates -directional asymmetry: upward filters (child → parent) contain the child's -subtree, while downward filters (parent → child) contain the complement. -Mesh peers receive filters but their inbound filters are not merged -transitively — they provide single-hop shortcut visibility only. - -A node with multiple peers receives genuinely different filters from each. -In the diagram, R receives {B, D, E} from B and {C, F} from C — two disjoint -subtrees. When R needs to reach F, only C's filter matches. This is where -bloom filters provide real candidate selection: a node with several peers -can narrow the forwarding choice before consulting tree coordinates. Leaf -nodes like D have only one peer, so their single inbound filter is -necessarily near-complete (everything except themselves) and offers no -selection — but leaf nodes have no choice to make anyway. - -Bloom filter sizing (bit count and hash functions) requires further analysis -based on actual deployment scenarios. The FMP wire format is versioned to -accommodate future parameter changes as operational experience accumulates. - -### Routing Decisions - -At each hop, FMP makes a local forwarding decision using the following -priority chain: - -1. **Local delivery** — the destination is this node -2. **Direct peer** — the destination is an authenticated neighbor -3. **Bloom-guided candidate selection** — bloom filters identify peers that - can reach the destination; tree coordinates rank them by distance and - link quality -4. **[Greedy routing](https://en.wikipedia.org/wiki/Greedy_embedding)** — - fallback when bloom filters haven't converged; forward to the peer that - minimizes tree distance to the destination -5. **No route** — destination unreachable; send error signal to source - -All multi-hop routing depends on knowing the destination's tree coordinates. -These are cached at each node after being learned through discovery -(LookupRequest/LookupResponse) or session establishment (SessionSetup). The -coordinate cache is the critical piece that enables efficient forwarding. - -![Per-hop routing decision flowchart](diagrams/fips-routing-decision.svg) - -### Coordinate Caching and Discovery - -When a node first needs to reach an unknown destination, it sends a -LookupRequest that propagates through the network guided by bloom filters -and loop prevention. The destination responds with its coordinates, which -the source and intermediate nodes along the return path cache. Subsequent -traffic routes efficiently using the cached coordinates. - -Session establishment (SessionSetup) also carries coordinates, warming -transit node caches along the path so that data packets can be forwarded -without individual discovery at each hop. - -![Coordinate discovery and cache warming sequence](diagrams/fips-coordinate-discovery.svg) - -### Error Recovery - -When routing fails — because cached coordinates are stale, a path has -broken, or a packet exceeds a link's MTU — transit nodes signal the source: - -- **CoordsRequired**: A transit node lacks the destination's coordinates. - The source re-initiates discovery and resets its coordinate warmup - strategy. -- **PathBroken**: Greedy routing reached a dead end. The source re-discovers - the destination's current coordinates. -- **MtuExceeded**: A transit node cannot forward a packet because it exceeds - the next-hop link MTU. The source adjusts its path MTU estimate. - -All three signals trigger active recovery, and are rate-limited to prevent -storms during topology changes. - -See [fips-mesh-operation.md](fips-mesh-operation.md) for the complete -routing and mesh behavior description. - -### Metrics Measurement Protocol (MMP) - -Each peer link runs an instance of the Metrics Measurement Protocol, which -measures link quality through in-band report exchange. MMP computes smoothed -round-trip time (SRTT), packet loss rate, interarrival jitter, goodput, and -one-way delay trend — all derived from counter and timestamp fields already -present in the FMP wire format, with no additional probing traffic required. - -MMP operates in three modes. **Full** mode exchanges both SenderReports and -ReceiverReports to compute all metrics including RTT. **Lightweight** mode -exchanges only ReceiverReports, providing loss and jitter but not RTT — useful -for constrained links. **Minimal** mode disables reports entirely, relying -only on spin bit and congestion echo flags in the frame header. - -Reports are sent at RTT-adaptive intervals (clamped to 100 ms–2 s), so -high-latency links don't generate excessive measurement traffic while -low-latency links converge quickly. Each metric carries both short-term and -long-term exponentially weighted moving averages, enabling detection of -quality changes against a stable baseline. - -MMP serves dual roles: operator visibility and cost-based parent selection. -Periodic log lines report per-link RTT, loss, jitter, and goodput. MMP -computes an Expected Transmission Count (ETX) from bidirectional delivery -ratios, which feeds into cost-based parent selection where each node -evaluates `effective_depth = depth + link_cost` using -`link_cost = etx * (1.0 + srtt_ms / 100.0)`. ETX is not yet used in -`find_next_hop()` candidate ranking for data forwarding. - -See [fips-mesh-layer.md](fips-mesh-layer.md) for MMP operating modes, report -scheduling, and the spin bit design. - ---- - -## Transport Abstraction - -FIPS treats the communication medium as a pluggable component. Every transport -— whether a UDP socket, an Ethernet interface, a Tor circuit, or a radio modem -— implements the same simple interface: send a datagram to an address, receive -datagrams, and report the link MTU. The rest of the protocol stack sees no -difference between them. - -A **transport** is a driver for a particular medium. A **link** is a peer -connection established over a transport. Transport addresses (IP:port, MAC -address, .onion) are opaque to all layers above FMP — they exist only to -deliver datagrams and are discarded once FMP has authenticated the peer via -the Noise IK handshake. From that point on, the peer is identified solely by -its cryptographic identity. - -Transports fall into three categories based on their connectivity model: - -| Category | Examples | Characteristics | -| -------- | -------- | --------------- | -| Overlay | UDP/IP, Tor | Tunnels FIPS over existing networks | -| Shared medium | Ethernet, WiFi, Bluetooth, Radio | Local broadcast, peer discovery | -| Point-to-point | Serial, dialup | Fixed connections, no discovery | - -These categories differ in addressing, MTU, reliability, and whether they -support local discovery, but FMP handles all of them uniformly. A node -running multiple transports simultaneously bridges between those networks -automatically — peers from all transports feed into a single spanning tree, -and the router selects the best path regardless of which medium carries it. -If one transport fails, traffic reroutes through alternatives without -application involvement. - -Some transports support an optional discovery capability — the ability to -broadcast and listen for announcements indicating the availability of FIPS -endpoints on the local medium. Shared media like Ethernet, WiFi, Bluetooth, -and radio are natural fits for this, as they can reach nearby devices without -prior configuration. When discovery is available, nodes can automatically -find and peer with other FIPS nodes on the same medium. Transports that -lack discovery (such as configured UDP endpoints) simply skip this step and -connect directly to configured addresses. For internet-reachable nodes, -endpoint discovery via signed Nostr events allows nodes to publish and -consume transport addresses through public relays — available behind the -`nostr-discovery` cargo feature. - -NAT traversal for internet-connected nodes is supported via STUN-assisted -UDP hole punching, with offer/answer signaling carried over Nostr gift-wrap -events. Once a direct UDP path is established, the punched socket is handed -into the standard FIPS transport/session stack. Nodes that cannot establish -a direct path remain reachable through port forwarding, a publicly addressed -peer, or relay through other mesh nodes. The Nostr-mediated discovery and -NAT traversal paths are gated by the `nostr-discovery` cargo feature and -configured under `node.discovery.nostr.*`. - -> **Implementation status**: UDP/IP, TCP/IP, Ethernet, Tor -> (SOCKS5 outbound + directory-mode inbound via onion service), -> and Bluetooth (BLE L2CAP CoC) transports are implemented. -> All others are future directions. - -See [fips-transport-layer.md](fips-transport-layer.md) for the full transport -layer specification. - ---- - -## Security - -FIPS is designed around four classes of adversary, each addressed by a -different layer of the protocol. - -### Transport Observers - -A passive observer on the underlying transport — someone monitoring a WiFi -network, tapping an Ethernet segment, or inspecting UDP traffic — sees only -encrypted packets. The FMP link-layer Noise IK session encrypts all traffic -between direct peers, including routing gossip and forwarded session -datagrams. The observer can infer timing, packet sizes, and which transport -endpoints are exchanging traffic, but cannot read content or determine -FIPS-level node identities from the encrypted packets. Traffic analysis — -correlating timing and volume patterns across multiple vantage points to -infer communication relationships — is not defended against (see -[Specific Design Goals](#specific-design-goals)). - -### Active Attackers on the Transport - -An adversary who can inject, modify, drop, or replay packets on the -transport is also defeated by the FMP link-layer Noise IK session. Mutual -authentication prevents impersonation, AEAD encryption detects tampering, -and counter-based nonces with a sliding replay window reject replayed -packets. - -### Other FIPS Nodes (Intermediate Routers) - -The most important adversary class is the operators of other nodes in the -mesh — the peers that forward your traffic. FIPS treats every intermediate -router as potentially adversarial. The FSP session layer establishes a -completely independent Noise XK session between the communicating endpoints, -so intermediate nodes cannot read application payloads even though they -decrypt and re-encrypt the link-layer envelope at each hop. - -Routing headers expose only the destination's node_addr — an opaque -SHA-256 hash of the actual public key. Intermediate routers can forward -traffic without learning which Nostr identities are communicating. An -observer can verify "does this node_addr belong to pubkey X?" if they -already know the pubkey, but cannot enumerate communicating identities by -inspecting routed traffic. - -| Entity | Can See | -| ------ | ------- | -| Transport observer | Encrypted packets, timing, packet sizes | -| Direct peer | Your npub, traffic volume, timing | -| Intermediate router | Source and destination node_addrs, packet size | -| Destination | Your npub, payload content | - -### Adversarial Nodes Disrupting the Mesh - -Beyond passive observation, a malicious node could attempt to disrupt -routing by injecting false spanning tree announcements, advertising bogus -bloom filters, or claiming invalid tree positions. FMP mitigates these -through signed TreeAnnounce messages verified by direct peers, transitive -ancestry chain validation, replay protection via sequence numbers, and -discretionary peering — node operators choose who to peer with, so an -attacker with many identities still needs real nodes to accept their -connections. Handshake rate limiting further constrains how fast an attacker -can establish new links. In fully open networks with automatic peer -discovery, Sybil resistance relies primarily on rate limiting; discretionary -peering provides stronger resistance in curated deployments where operators -vet their peers. An attacker who controls all of a target node's direct -peers can completely control its view of the network (an eclipse attack); -diverse peering across independent operators and transports is the primary -mitigation. - ---- - -## Prior Work - -FIPS builds on proven designs rather than inventing new cryptography or routing -algorithms. Nearly every major design decision has deployed precedent. - -### Spanning Tree Self-Organization - -The idea that distributed nodes can build a spanning tree through purely local -decisions — each node selecting a parent based on announcements from its -neighbors — dates to the -[IEEE 802.1D Spanning Tree Protocol](https://en.wikipedia.org/wiki/Spanning_Tree_Protocol) -(STP, 1985). STP demonstrated that a network-wide tree emerges from a simple -deterministic rule (lowest bridge ID wins root election) applied independently -at each node. FIPS uses the same principle — lowest node address determines the -root — adapted from an Ethernet bridging context to a general-purpose overlay -mesh. - -### Tree Coordinate Routing - -The spanning tree coordinates, bloom filter candidate selection, and greedy -routing algorithms are adapted from -[Yggdrasil v0.5](https://yggdrasil-network.github.io/2023/10/22/upcoming-v05-release.html) -and its [Ironwood](https://github.com/Arceliar/ironwood) routing library. -Yggdrasil's key insight was using the tree path from root to node as a -routable coordinate, enabling greedy forwarding without global routing tables. -FIPS adapts these algorithms for multi-transport operation, Nostr identity -integration, and constrained MTU environments. - -The theoretical foundation for greedy routing on tree embeddings draws on -[Kleinberg's work](https://www.cs.cornell.edu/home/kleinber/swn.pdf) on -navigable small-world networks, which showed that greedy forwarding succeeds -in O(log² n) steps when the network has hierarchical structure. Thorup-Zwick -compact routing schemes separately demonstrated that sublinear routing state -is achievable with bounded stretch, motivating the use of tree coordinates -rather than full routing tables. - -### Split-Horizon Bloom Filter Propagation - -FIPS distributes reachability information using bloom filters computed with a -split-horizon rule: when advertising to a peer, exclude that peer's own -contributions. This technique is borrowed from distance-vector routing -protocols — [RIP](https://en.wikipedia.org/wiki/Routing_Information_Protocol) -(1988) and [Babel](https://www.irif.fr/~jch/software/babel/) use split-horizon -to prevent routing loops by not advertising a route back to the neighbor it was -learned from. FIPS applies the same principle to probabilistic set -advertisements rather than distance-vector tables. - -### Cryptographic Identity as Network Address - -FIPS nodes are identified by their Nostr public keys (secp256k1). The network -address *is* the cryptographic identity — there is no separate address -assignment or registration step. -[CJDNS](https://github.com/cjdelisle/cjdns) pioneered this approach in -overlay meshes, deriving IPv6 addresses from the double-SHA-512 of each node's -public key. Tor [.onion addresses](https://spec.torproject.org/rend-spec-v3) -and the IETF -[Host Identity Protocol](https://en.wikipedia.org/wiki/Host_Identity_Protocol) -(HIP) follow the same principle. FIPS uses Nostr's existing key infrastructure -rather than introducing a new identity scheme. - -### Dual-Layer Encryption - -FIPS encrypts traffic twice: FMP provides hop-by-hop link encryption -(protecting against transport-layer observers), while FSP provides independent -end-to-end session encryption (protecting against intermediate FIPS nodes). -This layered approach mirrors [Tor](https://www.torproject.org/), where each -relay peels one layer of encryption (hop-by-hop) while the innermost layer -protects end-to-end payload. [I2P](https://geti2p.net/) uses a similar -garlic routing scheme with tunnel-layer and end-to-end encryption. Unlike Tor -and I2P, FIPS does not provide anonymity — its dual encryption protects -confidentiality and integrity rather than hiding traffic patterns. - -### Noise Protocol Framework - -FIPS uses the [Noise Protocol Framework](https://noiseprotocol.org/) at both -protocol layers, with different handshake patterns chosen for each layer's -threat model. FMP link encryption uses **Noise IK**, providing mutual -authentication with a single round trip where the initiator knows the -responder's static key in advance. -[WireGuard](https://www.wireguard.com/) uses the same IK base pattern -(extended with a pre-shared key as IKpsk2) for VPN tunnels. FSP session -encryption uses **Noise XK**, the same pattern used by the -[Lightning Network](https://github.com/lightning/bolts/blob/master/08-transport.md), -where the initiator's static key is transmitted in a third message rather -than the first. XK provides stronger initiator identity hiding at the cost -of an additional round trip — a worthwhile tradeoff for session-layer traffic -that traverses untrusted intermediate nodes. At the link layer, where both -peers are configured and directly connected, IK's single round trip is -preferred. - -### Index-Based Session Dispatch - -FIPS uses locally-assigned 32-bit session indices to demultiplex incoming -packets to the correct cryptographic session in O(1) time, without parsing -source addresses or performing expensive lookups. This directly follows -[WireGuard's](https://www.wireguard.com/papers/wireguard.pdf) receiver index -approach, where each peer assigns a random index during handshake and the -remote side includes it in every packet header. - -### Transport-Agnostic Overlay Mesh - -FIPS is designed to operate over any datagram-capable transport — UDP, raw -Ethernet, Bluetooth, radio, serial — through a uniform transport abstraction. -Several mesh overlays have demonstrated transport-agnostic design: -[CJDNS](https://github.com/cjdelisle/cjdns) runs over UDP and Ethernet, -[Yggdrasil](https://yggdrasil-network.github.io/) supports TCP and TLS -transports, and [Tor](https://www.torproject.org/) can use pluggable -transports to tunnel through various media. FIPS extends this pattern to -shared-medium transports (radio, BLE) with per-transport MTU and discovery -capabilities. - -### Metrics Measurement Protocol - -MMP's design assembles well-established measurement techniques into a unified -per-link protocol. The SenderReport/ReceiverReport exchange structure follows -[RTCP](https://www.rfc-editor.org/rfc/rfc3550) (RFC 3550), which uses the -same report pairing for media stream quality monitoring in RTP sessions. MMP's -jitter computation uses the RTCP interarrival jitter algorithm directly. - -The smoothed RTT estimator uses the Jacobson/Karels algorithm -([RFC 6298](https://www.rfc-editor.org/rfc/rfc6298)), the same SRTT -computation used in TCP for retransmission timeout calculation since 1988. -MMP derives RTT from timestamp-echo in ReceiverReports with dwell-time -compensation, rather than from packet round-trips. - -The spin bit in the FMP frame header follows the -[QUIC](https://www.rfc-editor.org/rfc/rfc9000) spin bit -([RFC 9312](https://www.rfc-editor.org/rfc/rfc9312)) — a single bit that -alternates each round trip, enabling passive latency measurement. FIPS -implements the spin bit state machine but relies on timestamp-echo for SRTT, -as irregular mesh traffic makes spin bit RTT unreliable. - -The Expected Transmission Count (ETX) metric, computed from bidirectional -delivery ratios, was introduced by -[De Couto et al. (2003)](https://pdos.csail.mit.edu/papers/grid:mobicom03/paper.pdf) -for wireless mesh routing and is used in protocols including -[OLSR](https://en.wikipedia.org/wiki/Optimized_Link_State_Routing_Protocol) -and [Babel](https://www.irif.fr/~jch/software/babel/). FIPS computes ETX -per-link from MMP loss measurements for future use in candidate ranking. - -The CE (Congestion Experienced) echo flag provides hop-by-hop -[ECN](https://en.wikipedia.org/wiki/Explicit_Congestion_Notification) -signaling, following the TCP/IP ECN echo pattern (RFC 3168). Transit nodes -detect congestion via MMP loss/ETX metrics or kernel buffer drops and set -the CE flag on forwarded frames; destination nodes mark ECN-capable IPv6 -packets accordingly. - -### Cryptographic Primitives - -FIPS reuses [Nostr's](https://github.com/nostr-protocol/nips) cryptographic -stack — secp256k1 for identity keys, Schnorr signatures for authentication, -SHA-256 for hashing, and ChaCha20-Poly1305 for authenticated encryption. This -is the same primitive set used across Bitcoin, Nostr, and a growing ecosystem -of self-sovereign identity systems. No novel cryptography is introduced. - ---- - -## Further Reading - -### Protocol Layers - -| Document | Description | -| -------- | ----------- | -| [fips-transport-layer.md](fips-transport-layer.md) | Transport layer: abstraction, types, services provided to FMP | -| [fips-mesh-layer.md](fips-mesh-layer.md) | FMP: peer authentication, link encryption, forwarding | -| [fips-session-layer.md](fips-session-layer.md) | FSP: end-to-end encryption, session lifecycle | -| [fips-ipv6-adapter.md](fips-ipv6-adapter.md) | IPv6 adaptation: DNS, TUN interface, MTU enforcement | - -### Mesh Behavior and Wire Formats - -| Document | Description | -| -------- | ----------- | -| [fips-mesh-operation.md](fips-mesh-operation.md) | How the mesh operates: routing, discovery, error recovery | -| [fips-wire-formats.md](fips-wire-formats.md) | Complete wire format reference for all protocol layers | - -### Supporting References - -| Document | Description | -| -------- | ----------- | -| [fips-spanning-tree.md](fips-spanning-tree.md) | Spanning tree algorithms and data structures | -| [fips-bloom-filters.md](fips-bloom-filters.md) | Bloom filter parameters, math, and computation | -| [spanning-tree-dynamics.md](spanning-tree-dynamics.md) | Scenario walkthroughs: convergence, partitions, recovery | - -### Implementation - -| Document | Description | -| -------- | ----------- | -| [fips-configuration.md](fips-configuration.md) | YAML configuration reference | - -### External References - -- [IEEE 802.1D Spanning Tree Protocol](https://en.wikipedia.org/wiki/Spanning_Tree_Protocol) -- [Yggdrasil Network](https://yggdrasil-network.github.io/) -- [Yggdrasil v0.5 Release Notes](https://yggdrasil-network.github.io/2023/10/22/upcoming-v05-release.html) -- [Ironwood Routing Library](https://github.com/Arceliar/ironwood) -- [Kleinberg — The Small-World Phenomenon](https://www.cs.cornell.edu/home/kleinber/swn.pdf) -- [CJDNS](https://github.com/cjdelisle/cjdns) -- [Tor Project](https://www.torproject.org/) -- [I2P](https://geti2p.net/) -- [Host Identity Protocol (HIP)](https://en.wikipedia.org/wiki/Host_Identity_Protocol) -- [Babel Routing Protocol](https://www.irif.fr/~jch/software/babel/) -- [Noise Protocol Framework](https://noiseprotocol.org/) -- [WireGuard](https://www.wireguard.com/) -- [WireGuard Whitepaper](https://www.wireguard.com/papers/wireguard.pdf) -- [Lightning Network BOLT #8 — Transport](https://github.com/lightning/bolts/blob/master/08-transport.md) -- [QUIC (RFC 9000)](https://www.rfc-editor.org/rfc/rfc9000) -- [QUIC Spin Bit (RFC 9312)](https://www.rfc-editor.org/rfc/rfc9312) -- [RTCP (RFC 3550)](https://www.rfc-editor.org/rfc/rfc3550) -- [TCP SRTT / RTO (RFC 6298)](https://www.rfc-editor.org/rfc/rfc6298) -- [ECN (RFC 3168)](https://www.rfc-editor.org/rfc/rfc3168) -- [ETX — De Couto et al. 2003](https://pdos.csail.mit.edu/papers/grid:mobicom03/paper.pdf) -- [OLSR](https://en.wikipedia.org/wiki/Optimized_Link_State_Routing_Protocol) -- [Nostr Protocol](https://github.com/nostr-protocol/nips) diff --git a/docs/design/fips-ipv6-adapter.md b/docs/design/fips-ipv6-adapter.md index 6c6b49e..383559a 100644 --- a/docs/design/fips-ipv6-adapter.md +++ b/docs/design/fips-ipv6-adapter.md @@ -69,6 +69,24 @@ Known cache population mechanisms: - **Inbound traffic**: Authenticated sessions from other nodes populate the cache with their identity information +### Mesh-Interface Query Filter + +The DNS responder is intended for local applications resolving `.fips` +names; queries arriving over the mesh interface itself are dropped. The +daemon records the index of the TUN interface at startup and compares +it against the arrival interface of each incoming UDP DNS query. When +they match — meaning the query came from another mesh node, not from a +local socket — the responder discards the query without replying. + +The check is implemented in +[`is_mesh_interface_query`](../../src/upper/dns.rs) and prevents two +classes of misbehaviour: a peer asking the daemon to resolve `.fips` +names on its behalf (which would let one node use another as an +identity-cache priming proxy), and accidental query loops where a +misconfigured resolver forwards `.fips` queries back into the mesh. +Local applications binding to the host's loopback or non-mesh +interfaces are unaffected. + ## IPv6 Address Derivation FIPS addresses use the IPv6 Unique Local Address (ULA) prefix `fd00::/8`: @@ -125,41 +143,24 @@ entry hasn't been evicted by memory pressure. ## MTU Enforcement -FIPS does not provide fragmentation or reassembly at the session or mesh -protocol layers — every datagram must fit in a single transport-layer packet. -Some transports may perform fragmentation and reassembly internally (e.g., BLE -L2CAP) and can advertise a larger virtual MTU than the physical medium -supports, but this is transparent to FIPS. The mesh layer provides two -facilities to manage MTU across heterogeneous paths: route discovery can -constrain results to paths that support a required minimum MTU, and transit -nodes that cannot forward an oversized datagram send an MtuExceeded error -signal back to the source. The adapter must ensure that IPv6 packets from -applications fit within the FIPS encapsulation budget after all layers of -wrapping. +The adapter sits at the boundary between the host's IPv6 stack and the +FIPS encapsulation budget. Its job is to keep IPv6 packets small +enough that they fit through the FIPS protocol envelope on every link +along the path. The cross-cutting MTU model — proactive +SessionDatagram `path_mtu` annotation, reactive MtuExceeded signals, +end-to-end PathMtuNotification echo, and per-destination MTU storage +— is documented in [fips-mtu.md](fips-mtu.md). What the adapter +contributes is the IPv6-specific overhead accounting and the TUN-side +enforcement integration. -### Encapsulation Overhead +### IPv6-Specific Overhead -| Layer | Overhead | Purpose | -| ----- | -------- | ------- | -| Link encryption | 37 bytes | 16-byte outer header + 5-byte inner header (timestamp + msg_type) + 16-byte AEAD tag | -| SessionDatagram body | 35 bytes | ttl + path_mtu + src_addr + dest_addr (msg_type counted in inner header) | -| FSP header | 12 bytes | 4-byte prefix + 8-byte counter (used as AEAD AAD) | -| FSP inner header | 6 bytes | 4-byte timestamp + 1-byte msg_type + 1-byte inner_flags (inside AEAD) | -| Session AEAD tag | 16 bytes | ChaCha20-Poly1305 tag on session-encrypted payload | -| **Protocol envelope** | **106 bytes** | `FIPS_OVERHEAD` constant | -| Port header | 4 bytes | src_port + dst_port (DataPacket service dispatch) | -| IPv6 compression | −33 bytes | 40-byte IPv6 header → 7-byte format + residual | -| **IPv6 data path total** | **77 bytes** | `FIPS_IPV6_OVERHEAD` constant | - -Coordinate piggybacking (CP flag) adds variable overhead: `2 + entries × 16` -per coordinate, with both src and dst coords sent. The send path skips the -CP flag if adding coords would exceed the transport MTU. - -The `FIPS_OVERHEAD` constant (106 bytes) represents the base protocol -envelope overhead (link encryption + routing + session encryption). For IPv6 -traffic, FSP port multiplexing adds 4 bytes (port header) while IPv6 header -compression saves 33 bytes (40-byte header → 7-byte format + residual), -yielding a net `FIPS_IPV6_OVERHEAD` of 77 bytes. +For IPv6 traffic, FSP port multiplexing adds 4 bytes (port header) +while IPv6 header compression saves 33 bytes (40-byte header → +7-byte format + residual), yielding a net `FIPS_IPV6_OVERHEAD` of +77 bytes on top of the base `FIPS_OVERHEAD` (106 bytes) protocol +envelope. The full encapsulation breakdown lives in +[fips-mtu.md](fips-mtu.md#encapsulation-overhead). ### Effective IPv6 MTU @@ -183,48 +184,51 @@ transport path MTU for the IPv6 adapter is therefore: 1280 + 77 = 1357 bytes ``` -Transports with smaller MTUs (radio at ~250 bytes, serial at 256 bytes) cannot -support the IPv6 adapter without some form of internal fragmentation and -reassembly. Otherwise, applications on those transports must use the native -FIPS datagram API. +Transports with smaller MTUs (radio at ~250 bytes, serial at 256 +bytes) cannot support the IPv6 adapter without some form of internal +fragmentation and reassembly. Otherwise, applications on those +transports must use the native FIPS datagram API. -### ICMP Packet Too Big +### TUN-Side ICMP Packet Too Big -When an outbound packet at the TUN exceeds the effective IPv6 MTU, the adapter -generates an ICMPv6 Packet Too Big message and delivers it back to the -application via the TUN. This triggers the kernel's Path MTU Discovery (PMTUD) -mechanism, which adjusts TCP segment sizes for subsequent transmissions. +When an outbound packet at the TUN exceeds the effective IPv6 MTU, +the adapter generates an ICMPv6 Packet Too Big message and delivers +it back to the application via the TUN. This triggers the kernel's +Path MTU Discovery mechanism, which adjusts TCP segment sizes for +subsequent transmissions. -ICMP Packet Too Big generation is rate-limited per source address (100ms -interval) to prevent storms from applications sending many oversized packets. +ICMP Packet Too Big generation is rate-limited per source address +(100ms interval) to prevent storms from applications sending many +oversized packets. The ICMP response is delivered locally back through +the TUN; no network traversal is needed, so delivery is reliable. -The ICMP response is delivered locally (back through the TUN to the kernel) — -no network traversal is needed, so delivery is reliable. +### TUN-Side TCP MSS Clamping -### TCP MSS Clamping - -The adapter intercepts TCP SYN and SYN-ACK packets at the TUN interface and -clamps the Maximum Segment Size (MSS) option: +The adapter intercepts TCP SYN and SYN-ACK packets at the TUN +interface and clamps the Maximum Segment Size (MSS) option: ```text clamped_mss = effective_ipv6_mtu - 40 (IPv6 header) - 20 (TCP header) ``` -This prevents TCP connections from negotiating segment sizes that would exceed -the FIPS path MTU. Clamping is applied in two places: +Clamping is applied in two places: - **TUN reader** (outbound): Clamps MSS on outbound SYN packets - **TUN writer** (inbound): Clamps MSS on inbound SYN-ACK packets -Together, these ensure both directions of a TCP connection use appropriately -sized segments from the start, avoiding the initial oversized packet loss -that would occur with ICMP Packet Too Big alone. +Together, these ensure both directions of a TCP connection use +appropriately sized segments from the start, avoiding the initial +oversized packet loss that would occur with ICMP Packet Too Big +alone. The conditional clamp (per-flow lookup with cold-flow +fallback) and the rationale for `max_mss` semantics are in +[fips-mtu.md](fips-mtu.md#tcp-mss-clamping). ### ICMP Rate Limiting -ICMPv6 error generation is rate-limited per source address using a token bucket -(100ms interval). This matches the standard ICMP rate limiting approach and -prevents amplification when an application sends a burst of oversized packets. +ICMPv6 error generation is rate-limited per source address using a +token bucket (100ms interval). This matches the standard ICMP rate +limiting approach and prevents amplification when an application sends +a burst of oversized packets. ## TUN Interface @@ -287,20 +291,16 @@ path. ### Configuration -```yaml -tun: - enabled: true - name: fips0 - mtu: 1280 -``` +The TUN block (`tun.*`) is documented in +[../reference/configuration.md](../reference/configuration.md). ### Privileges -TUN device creation requires `CAP_NET_ADMIN`. Options: - -- Run as root -- Set capability: `sudo setcap cap_net_admin+ep ./target/debug/fips` -- Pre-created persistent TUN device +TUN device creation requires `CAP_NET_ADMIN`. The shipped Debian +systemd unit runs the daemon as `root` by default; for the +alternative — running under a dedicated unprivileged service +account with the capability granted on the binary — see +[../how-to/run-as-unprivileged-user.md](../how-to/run-as-unprivileged-user.md). ## Implementation Status @@ -314,6 +314,7 @@ TUN device creation requires `CAP_NET_ADMIN`. Options: | ICMP rate limiting (per-source) | **Implemented** | | TCP MSS clamping (SYN + SYN-ACK) | **Implemented** | | DNS service (.fips domain) | **Implemented** | +| DNS responder mesh-interface filter | **Implemented** | | Port-based service multiplexing (port 256) | **Implemented** | | IPv6 header compression (format 0x00) | **Implemented** | | Per-destination route MTU (netlink) | Planned | @@ -324,38 +325,27 @@ TUN device creation requires `CAP_NET_ADMIN`. Options: ## Design Considerations -### Path MTU Discovery +### Path MTU Discovery and No-Fragmentation Policy -Two complementary mechanisms support full PMTUD: - -1. **Proactive**: The `path_mtu` field (2 bytes) in the SessionDatagram envelope - is implemented at the FMP level. The source sets it to its outbound link MTU - minus overhead; each transit node applies - `min(current, own_outbound_mtu - overhead)`. The destination receives the - forward-path minimum. PathMtuNotification is handled at the session layer; - the destination sends the observed forward-path MTU back to the source, - which applies it with decrease-immediate / increase-requires-3-consecutive - hysteresis. - -2. **Reactive**: When a transit node cannot forward a packet (MTU exceeded), it - sends an error signal back to the source. This handles the in-flight gap - between a path MTU decrease and the source learning via the echo. - -Both are needed: proactive handles steady state; reactive handles the transient -window when oversized packets hit a new bottleneck before the source adapts. - -### No Fragmentation - -FIPS remains a pure datagram service with no fragmentation at transit nodes. -Session-layer encryption is end-to-end — the AEAD tag authenticates the entire -plaintext. Fragmenting encrypted datagrams would require either exposing -plaintext structure to transit nodes (unacceptable) or reassembly before -decryption (opens attack surface). +Path MTU Discovery (proactive `path_mtu` annotation, reactive +MtuExceeded, end-to-end PathMtuNotification) and the no-fragmentation +policy that drives the design both live in the unified MTU treatment +at [fips-mtu.md](fips-mtu.md). The adapter is a consumer of that +model — its job is to enforce the resulting effective IPv6 MTU at the +TUN with ICMP Packet Too Big and TCP MSS clamping. ## References -- [fips-intro.md](fips-intro.md) — Protocol overview and architecture +- [fips-concepts.md](fips-concepts.md) — Protocol overview +- [fips-architecture.md](fips-architecture.md) — Layer architecture and + identity model - [fips-session-layer.md](fips-session-layer.md) — FSP (below the adapter) -- [fips-wire-formats.md](fips-wire-formats.md) — FSP and SessionDatagram wire - formats -- [fips-configuration.md](fips-configuration.md) — TUN configuration parameters +- [fips-mtu.md](fips-mtu.md) — Unified path MTU model (proactive, + reactive, hysteresis, no-fragmentation) +- [../reference/wire-formats.md](../reference/wire-formats.md) — FSP and + SessionDatagram wire formats +- [../reference/configuration.md](../reference/configuration.md) — TUN + configuration parameters +- [../how-to/run-as-unprivileged-user.md](../how-to/run-as-unprivileged-user.md) + — privilege options for the daemon, including the unprivileged + service-account path diff --git a/docs/design/fips-mesh-layer.md b/docs/design/fips-mesh-layer.md index c5a0b8b..9de389a 100644 --- a/docs/design/fips-mesh-layer.md +++ b/docs/design/fips-mesh-layer.md @@ -215,8 +215,8 @@ The plaintext inside the encrypted frame begins with a 5-byte inner header (4-byte session-relative timestamp followed by a message type byte), then the message-specific payload. -See [fips-wire-formats.md](fips-wire-formats.md) for the complete wire format -specification. +See [../reference/wire-formats.md](../reference/wire-formats.md) for +the complete wire format specification. ### What Encryption Provides @@ -296,7 +296,7 @@ Roaming addresses *mid-session* NAT rebinding. Establishing the initial UDP path through NAT is a separate concern, addressed by the optional Nostr-mediated overlay discovery and STUN-assisted hole punching feature (see [fips-transport-layer.md](fips-transport-layer.md) and -[fips-configuration.md](fips-configuration.md)). +[../reference/configuration.md](../reference/configuration.md)). ## Replay Protection @@ -341,7 +341,9 @@ Additional protections: memory usage - **Handshake timeout**: Stale pending handshakes are cleaned up after a configurable timeout -- **Allowlist/blocklist**: Optional peer filtering before handshake processing +- **Peer ACL**: Optional allowlist / denylist filtering of peer npubs + before handshake processing (loaded from `/etc/fips/peers.allow` and + `/etc/fips/peers.deny`, mtime-watched and reloaded automatically) ## Disconnect @@ -361,6 +363,72 @@ links. On node shutdown, Disconnect is sent to all active peers before transports are stopped. +## Rekey + +FMP periodically negotiates a fresh Noise session over each established link +to bound forward-secrecy exposure: limiting AEAD nonce reuse risk, bounding +the volume of ciphertext recoverable from a stolen long-term static key, and +rotating the session indices that addressed packets carry on the wire. + +A rekey is initiated when either threshold is reached on the link's current +session: `node.rekey.after_secs` (default 120) elapsed since the link came +up or last rekeyed, or `node.rekey.after_messages` (default 65536) frames +sent. Either side can be the initiator independently. Rekey is on by +default and can be disabled via `node.rekey.enabled: false` (the +configuration tree is documented in +[../reference/configuration.md](../reference/configuration.md)). + +### Mechanism + +A rekey reuses the Noise IK pattern of the initial handshake, but the two +messages travel over the existing link as ordinary encrypted FMP frames +rather than as plaintext bootstrap packets. The initiator builds a fresh +`HandshakeState`, generates msg1, and sends it through the current session; +the responder consumes msg1, builds msg2, and replies. After both sides +have exchanged messages and finalised the new keys, traffic transitions +from the old session to the new one. + +Cutover is signalled in-band by the **K-bit** in the FMP flags byte. Each +side starts emitting frames under the new session with K set; on receipt +of the first K-marked frame the peer accepts the cutover and follows +suit. A new pair of session indices is allocated as part of the new +session, replacing the old indices on subsequent frames (see +[Index Properties](#index-properties)). + +### Drain Window + +To absorb in-flight reordering across the cutover, the old session is not +discarded immediately. Each peer retains it in a `previous_session` slot +on the active-peer state for `DRAIN_WINDOW_SECS = 10` seconds (a +compile-time constant in `src/node/handlers/rekey.rs`). During the +window, decrypt attempts fall back to `previous_session` when the new +session rejects a frame, so a packet sent under the old keys that +arrives a few hundred milliseconds late still decrypts. After the +window expires, the old session is dropped. + +### Dual-Initiation Race + +On high-latency links, both sides' rekey timers can fire close enough +together that each peer's msg1 crosses the other in flight. Without +arbitration, each side would act as both initiator and responder, end +up with two different Noise sessions, and lose connectivity at cutover. +FMP arbitrates with a deterministic tie-breaker: the peer with the +**numerically smaller `NodeAddr`** wins the role of initiator and +discards any inbound msg1 it sees during the race; the larger-`NodeAddr` +peer abandons its own initiation and processes the inbound msg1 as +responder. The same tie-breaker is applied to cross-connection races +during initial handshake. + +### Operator Visibility + +Successful cutover is reported at INFO level on the K-bit observation; +intermediate steps (handshake start, msg1/msg2 exchange, drain-window +fallback decrypts) log at DEBUG/TRACE. Failures (handshake error, +drain-window expiry without cutover) log at WARN. + +The end-to-end rekey at the session layer follows a parallel design; +see [fips-session-layer.md](fips-session-layer.md). + ## Liveness Detection FMP detects link liveness through a combination of explicit heartbeats and @@ -368,8 +436,8 @@ traffic observation. ### Heartbeat -A Heartbeat message (0x51) is sent to each active peer every -`node.heartbeat_interval_secs` (default 10s). The heartbeat is a minimal +A Heartbeat message (0x51) is sent to each active peer at a configurable +interval (`node.heartbeat_interval_secs`). The heartbeat is a minimal encrypted frame with no payload beyond the standard inner header (timestamp + message type). Any successfully decrypted frame — data, gossip, MMP report, or heartbeat — resets the peer's last-receive timestamp tracked by the MMP @@ -377,8 +445,8 @@ receiver. ### Dead Timeout -When no traffic (of any kind) is received from a peer for -`node.link_dead_timeout_secs` (default 30s), the peer is declared dead and +When no traffic (of any kind) is received from a peer for the +configured `node.link_dead_timeout_secs` window, the peer is declared dead and removed via `remove_active_peer()`. This triggers the full teardown cascade: spanning tree parent reselection (if the dead peer was the parent), TreeAnnounce propagation, coordinate cache flush, and bloom filter recompute. @@ -386,156 +454,74 @@ TreeAnnounce propagation, coordinate cache flush, and bloom filter recompute. If the dead peer is eligible for auto-reconnect (see [Auto-Reconnect] (#auto-reconnect)), reconnection is scheduled immediately after removal. -The heartbeat is independent of MMP — it is needed because idle links in -Lightweight MMP mode have no guaranteed periodic traffic (gossip is -event-driven, and MMP reports require at least one side running Full mode). +The heartbeat is independent of MMP. Gossip is event-driven, Lightweight +produces receiver reports only when traffic arrives, and Minimal emits no +reports at all — so on a fully idle link no MMP-mode combination +guarantees periodic activity. The heartbeat is the always-on liveness +signal. ## Link Message Types -FMP defines eight message types carried inside encrypted frames: +FMP defines several encrypted message types carried inside the +established-frame envelope. They group naturally by purpose: -| Type | Name | Purpose | -| ---- | ---- | ------- | -| 0x10 | TreeAnnounce | Spanning tree state announcements between peers | -| 0x20 | FilterAnnounce | Bloom filter reachability updates | -| 0x30 | LookupRequest | Coordinate discovery — flood toward destination | -| 0x31 | LookupResponse | Coordinate discovery — response with coordinates | -| 0x00 | SessionDatagram | Encapsulated session-layer payload for forwarding | -| 0x01 | SenderReport | MMP sender-side metrics report | -| 0x02 | ReceiverReport | MMP receiver-side metrics report | -| 0x50 | Disconnect | Orderly link teardown with reason code | -| 0x51 | Heartbeat | Link liveness probe | +- **Routing gossip**: TreeAnnounce carries spanning-tree announcements + between direct peers; FilterAnnounce carries bloom-filter + reachability updates between direct peers. Both are peer-to-peer + (not forwarded). +- **Discovery**: LookupRequest is forwarded through tree peers under + bloom-filter guidance to find a destination's coordinates; + LookupResponse routes back to the requester via reverse-path lookup + in `recent_requests`. +- **Forwarded payload**: SessionDatagram carries a session-layer + payload hop-by-hop toward the destination. +- **Metrics**: SenderReport and ReceiverReport carry the link-layer + MMP report stream peer-to-peer. +- **Liveness and lifecycle**: Heartbeat is a minimal frame sent + peer-to-peer to keep the link alive; Disconnect carries an orderly + teardown reason code peer-to-peer. -Additionally, handshake messages (phase 0x1 msg1, phase 0x2 msg2) are sent -unencrypted before the link session is established. +Handshake messages (phase 0x1 msg1, phase 0x2 msg2) travel before +encryption is established and are identified by the FMP common-prefix +`phase` field rather than a `msg_type` byte. -TreeAnnounce and FilterAnnounce are exchanged between direct peers only — they -are not forwarded. LookupRequest and LookupResponse are forwarded through the -mesh (flooded with deduplication). SessionDatagram is forwarded hop-by-hop -toward the destination. Disconnect is peer-to-peer. - -See [fips-mesh-operation.md](fips-mesh-operation.md) for how these messages -work together to build and maintain the mesh, and -[fips-wire-formats.md](fips-wire-formats.md) for byte-level message layouts. +See [../reference/wire-formats.md](../reference/wire-formats.md) for +byte-level message layouts and the canonical FMP message type +catalog, and [fips-mesh-operation.md](fips-mesh-operation.md) for how +these messages work together to build and maintain the mesh. ## Metrics Measurement Protocol (MMP) -Each active peer link runs an instance of the Metrics Measurement Protocol, -providing per-link quality metrics to the operator and to the spanning tree -layer for cost-based parent selection. +MMP runs on every active link to provide per-link quality metrics +(SRTT, loss, jitter, goodput, OWD trend, ETX) to the operator and to +the spanning tree layer for cost-based parent selection. Reports are +exchanged peer-to-peer between direct neighbors at RTT-adaptive +intervals clamped to `[1s, 5s]`, with a 200 ms cold-start floor for +the first five SRTT samples. -### Metrics Tracked +The CE (Congestion Experienced) bit in the FMP flags byte carries +hop-by-hop ECN signaling: transit nodes detect congestion on outgoing +links (via MMP loss/ETX or `SO_RXQ_OVFL` kernel drops) and set CE on +forwarded packets, which the destination then mirrors to the IPv6 +Traffic Class for ECN-capable flows. -MMP computes the following metrics from the per-frame counter and timestamp -fields in the FMP wire format: - -- **SRTT** — Smoothed round-trip time (Jacobson/RFC 6298, α=1/8). Derived - from timestamp-echo in ReceiverReports with dwell-time compensation. -- **Loss rate** — Bidirectional loss inferred from counter gaps. Tracked as - both instantaneous (per-interval) and long-term EWMA. -- **Jitter** — Interarrival jitter (RFC 3550 algorithm) in microseconds. -- **Goodput** — Bytes per second of payload data (excludes MMP reports). -- **OWD trend** — One-way delay trend (µs/s, signed). Indicates congestion - buildup before loss occurs. -- **ETX** — Expected Transmission Count, computed from bidirectional delivery - ratios. Used in cost-based parent selection via - `link_cost = etx * (1.0 + srtt_ms / 100.0)`; not yet used in - `find_next_hop()` candidate ranking. -- **Dual EWMA trends** — Short-term (α=1/4) and long-term (α=1/32) trend - indicators for both RTT and loss, enabling change detection. - -### Operating Modes - -MMP supports three modes, configured via `node.mmp.mode`: - -| Mode | Reports Exchanged | Metrics Available | -| ---- | ----------------- | ----------------- | -| **Full** (default) | SenderReport + ReceiverReport | All metrics including RTT, loss, jitter, goodput, OWD trend | -| **Lightweight** | ReceiverReport only | Loss (from counter gaps), jitter, OWD trend. No RTT. | -| **Minimal** | None | Spin bit and CE echo flags only. No computed metrics. | - -### Report Scheduling - -Reports are sent at RTT-adaptive intervals, clamped to [100ms, 2s]. A -cold-start interval of 500ms is used before SRTT converges. The interval -formula is `clamp(2 × SRTT, 100ms, 2000ms)`. - -### Spin Bit and RTT - -The SP (spin bit) flag in the FMP inner header follows the QUIC spin bit -pattern: reflected on receive, toggled on send when the reflected value -matches the last sent value. The spin bit state machine runs for TX -reflection, but **RTT samples from the spin bit are discarded**. In a mesh -protocol where frames are sent irregularly (tree announces, bloom filters, -MMP reports on different timers), inter-frame processing delays inflate spin -bit RTT measurements unpredictably. Timestamp-echo from ReceiverReports -(with dwell-time compensation) is the sole SRTT source. - -### ECN Congestion Signaling - -The CE (Congestion Experienced) flag (bit 1 in the FMP flags byte) provides -hop-by-hop congestion signaling through the mesh. Transit nodes detect -congestion on outgoing links and set CE on forwarded packets; once set, the -flag stays set for all subsequent hops to the destination. - -**Congestion detection** (`detect_congestion()`) triggers on any of: - -- Outgoing link MMP loss rate ≥ `node.ecn.loss_threshold` (default 5%) -- Outgoing link MMP ETX ≥ `node.ecn.etx_threshold` (default 3.0) -- Kernel receive buffer drops detected on any local transport (via - `SO_RXQ_OVFL` on UDP) - -**CE relay**: The forwarding path computes `outgoing_ce = incoming_ce || -local_congestion`. The `send_encrypted_link_message_with_ce()` method ORs -`FLAG_CE` into the FMP header flags when ce is true. The original -`send_encrypted_link_message()` delegates with `ce_flag=false`, leaving the -20+ existing call sites unchanged. - -**IPv6 ECN-CE marking**: When a CE-flagged DataPacket arrives at its final -destination, the IPv6 Traffic Class ECN bits are marked CE (0b11) before -TUN delivery — but only for ECN-capable packets (ECT(0) or ECT(1)). Not-ECT -packets are never marked per RFC 3168. The host TCP stack then echoes ECE in -ACKs, triggering sender cwnd reduction through standard congestion control. - -**Session-layer tracking**: The `ecn_ce_count` field in MMP ReceiverReports -tracks CE-flagged packets received per link, providing end-to-end visibility -into congestion propagation. - -**Monitoring**: `CongestionStats` tracks four counters — `ce_forwarded`, -`ce_received`, `congestion_detected`, and `kernel_drop_events` — exposed via -`fipsctl show routing` (congestion block) and `fipstop` (routing tab). -Rate-limited warn logging (5s interval) alerts on congestion detection events. - -See `node.ecn.*` in -[fips-configuration.md](fips-configuration.md#ecn-signaling-nodeecn) for -tuning parameters. - -### Operator Logging - -MMP emits periodic link metrics at info level (configurable via -`node.mmp.log_interval_secs`, default 30s): - -```text -MMP link metrics peer=node-b rtt=2.3ms loss=0.2% jitter=0.1ms goodput=76.0MB/s tx_pkts=1234 rx_pkts=5678 -``` - -Teardown logs include final SRTT, loss rate, jitter, ETX, goodput, and -cumulative tx/rx packet and byte counts. +For the full MMP design — operating modes, report scheduling, spin +bit interaction, ECN, and the algorithmic details shared with +session-layer MMP — see [fips-mmp.md](fips-mmp.md). For the +SenderReport and ReceiverReport byte layouts, see +[../reference/wire-formats.md](../reference/wire-formats.md). +Configuration knobs live under `node.mmp.*` and `node.ecn.*` in +[../reference/configuration.md](../reference/configuration.md). ## Security Properties ### Threat Resistance -| Threat | Mitigation | -| ------ | ---------- | -| Connection exhaustion | Token bucket rate limit + connection count limit | -| CPU exhaustion (msg1 flood) | Rate limit before crypto operations | -| Replay attacks | Counter-based nonces with sliding window | -| State confusion | Strict handshake state machine validation | -| Spoofed encrypted packets | Index lookup + AEAD verification | -| Spoofed msg2 | Index lookup + Noise ephemeral key binding | -| Address spoofing | Cryptographic authority, not address-based | -| Session correlation | Index rotation on rekey | +The link-layer threat-resistance matrix (connection exhaustion, CPU +exhaustion, replay, state confusion, spoofing variants, address +spoofing, session correlation) is consolidated in +[../reference/security.md](../reference/security.md) along with the +session-layer matrix and operator-facing controls. ### Unauthenticated Attack Surface @@ -580,13 +566,17 @@ an attacker sends invalid packets to elicit responses. | Metrics Measurement Protocol (MMP) | **Implemented** | | ECN congestion signaling (CE relay, IPv6 marking) | **Implemented** | | Rekey with index rotation | **Implemented** | -| Allowlist/blocklist | Planned | +| Peer ACL (allowlist / denylist) | **Implemented** | ## References -- [fips-intro.md](fips-intro.md) — Protocol overview and architecture +- [fips-concepts.md](fips-concepts.md) — Protocol overview +- [fips-architecture.md](fips-architecture.md) — Layer architecture and + identity model - [fips-transport-layer.md](fips-transport-layer.md) — Transport layer (below FMP) - [fips-session-layer.md](fips-session-layer.md) — FSP (above FMP) +- [fips-mmp.md](fips-mmp.md) — Metrics Measurement Protocol (link + session) - [fips-mesh-operation.md](fips-mesh-operation.md) — How FMP's routing and self-organization work in practice -- [fips-wire-formats.md](fips-wire-formats.md) — Byte-level wire format reference +- [../reference/wire-formats.md](../reference/wire-formats.md) — Byte-level + wire format reference diff --git a/docs/design/fips-mesh-operation.md b/docs/design/fips-mesh-operation.md index ebefc35..8fe8c20 100644 --- a/docs/design/fips-mesh-operation.md +++ b/docs/design/fips-mesh-operation.md @@ -31,183 +31,51 @@ and self-healing. ## Spanning Tree Formation and Maintenance -### What the Spanning Tree Provides +For routing purposes, the spanning tree provides each node with a +coordinate (its ancestry path from itself to the root) plus a way to +compute distance between any two nodes (hops to their lowest common +ancestor). The strictly-decreasing distance invariant gives greedy +forwarding its loop-freedom. -The spanning tree gives each node a **coordinate**: its ancestry path from -itself to the root, expressed as a sequence of node_addrs. These coordinates -enable: +The tree forms through distributed parent selection — root is the +smallest node_addr (no election), and each node picks the peer with +the lowest `effective_depth = depth + link_cost`. Cost-aware parent +selection lets the tree trade hop count for link quality once MMP has +accumulated SRTT and ETX metrics. Hysteresis (20% improvement +required to switch) and hold-down (suppress non-mandatory +re-evaluation after a switch) keep the tree stable under metric +noise. Partitions self-resolve — each segment converges to its own +root and reconverges to the smallest reachable root when segments +rejoin. -- **Distance calculation**: The tree distance between two nodes is the number - of hops from each to their lowest common ancestor (LCA). This provides a - routing metric without any node knowing the full topology. -- **Greedy routing**: At each hop, forward to the peer that minimizes tree - distance to the destination. The strictly-decreasing distance invariant - guarantees loop-free forwarding. +Liveness is detected via FMP heartbeats; dead-peer removal triggers +tree reconvergence and bloom filter recomputation for the affected +subtree. The heartbeat and dead-timeout mechanism lives at the link +layer; see [fips-mesh-layer.md](fips-mesh-layer.md#liveness-detection). -### How the Tree Forms - -Nodes self-organize into a spanning tree through distributed parent selection: - -1. **Root discovery**: The node with the smallest node_addr becomes the root. - No election protocol — this is a consequence of each node independently - preferring lower-addressed roots. -2. **Parent selection**: Each node selects a single parent from among its - direct peers based on which offers the lowest effective depth (tree depth - weighted by local link cost). -3. **Coordinate computation**: Once a node has a parent, its coordinate is - computed from its ancestry path. - -### How the Tree Maintains Itself - -Nodes exchange **TreeAnnounce** messages with their direct peers (not -forwarded — peer-to-peer only). Each TreeAnnounce carries the sender's -current ancestry chain and a sequence number. - -Changes cascade through the tree: - -- A node that changes its parent recomputes its coordinates and announces to - all peers -- Each receiving peer evaluates whether the change affects its own parent - selection -- Only nodes that actually change their coordinates (root or depth changed) - propagate further - -TreeAnnounce propagation is rate-limited at 500ms minimum interval per peer. -A tree of depth D reconverges in roughly D×0.5s to D×1.0s. - -### How the Tree Adapts to Link Quality - -The initial tree forms based on hop count alone — all links default to a -cost of 1.0 before measurements are available. As the Metrics Measurement -Protocol (MMP) accumulates bidirectional delivery ratios and round-trip -time estimates, each node computes a per-link cost: - -```text -link_cost = ETX × (1.0 + SRTT_ms / 100.0) -``` - -ETX (Expected Transmission Count) captures loss — a perfect link has -ETX = 1.0, while 10% loss in each direction yields ETX ≈ 1.23. The SRTT -term weights latency so that a low-loss but high-latency link (e.g., a -satellite hop) costs more than a low-loss, low-latency link. - -Parent selection uses **effective depth** rather than raw hop count: - -```text -effective_depth = peer.depth + link_cost_to_peer -``` - -This allows a node to trade a shorter but lossy path for a longer but -higher-quality one. A node two hops from the root over clean links -(effective depth ≈ 3.0) is preferred over a node one hop away over a -degraded link (effective depth ≈ 4.5). - -Parent reselection is triggered by three paths: - -1. **TreeAnnounce**: When a peer announces a new tree position, the node - re-evaluates using current link costs -2. **Periodic re-evaluation**: Every 60s (configurable), the node - re-evaluates its parent choice using the latest MMP metrics, catching - gradual link degradation that doesn't trigger TreeAnnounce -3. **Parent loss**: When the current parent is removed, the node - immediately selects the best alternative - -To prevent oscillation from metric noise, parent switches are subject to -**hysteresis**: a candidate must offer an effective depth at least 20% -better than the current parent to trigger a switch. A **hold-down period** -(default 30s) suppresses non-mandatory re-evaluation after a switch, -allowing MMP metrics to stabilize on the new link before reconsidering. - -### Flap Dampening - -Unstable links that repeatedly connect and disconnect can cause cascading -tree reconvergence. The spanning tree uses flap dampening with hysteresis -and hold-down periods to suppress rapid parent oscillation. Links that flap -above a configurable threshold are temporarily penalized, preventing them -from being selected as parent until the link stabilizes. - -### Link Liveness - -Each node sends a dedicated **Heartbeat** message (0x51, 1 byte, no -payload) to every peer at a fixed interval (default 10s). Any -authenticated encrypted frame — heartbeat, MMP report, TreeAnnounce, -data packet — resets the peer's liveness timer. On an idle link with no -application data or topology changes, the heartbeat is the only traffic -that keeps the link alive. - -Peers that are silent for a configurable dead timeout (default 30s) are -considered dead and removed from the peer table. With the default 10s -heartbeat interval, a peer must miss three consecutive heartbeats before -removal. This triggers tree reconvergence and bloom filter recomputation -for the affected subtree. - -### Partition Handling - -If the network partitions, each segment independently rediscovers its own -root (the smallest node_addr in the segment) and reconverges. When segments -rejoin, nodes discover the globally-smallest root through TreeAnnounce -exchange and reconverge to a single tree. - -See [fips-spanning-tree.md](fips-spanning-tree.md) for algorithm details -and [spanning-tree-dynamics.md](spanning-tree-dynamics.md) for convergence -walkthroughs. +For the parent-selection algorithm, hold-down/hysteresis details, and +the convergence walkthroughs, see +[fips-spanning-tree.md](fips-spanning-tree.md) and +[spanning-tree-dynamics.md](spanning-tree-dynamics.md). ## Bloom Filter Gossip and Propagation -### What Bloom Filters Provide +For routing purposes, each node maintains a bloom filter per peer +that answers "can peer P possibly reach destination D?" — either "no" +(definitive) or "maybe" (probabilistic). Because filters propagate +along tree edges with split-horizon exclusion, a bloom hit on a tree +peer reliably indicates which subtree contains the destination, and +tree-coordinate distance ranks competing matches. -Each node maintains a bloom filter per peer, answering: "can peer P possibly -reach destination D?" The answer is either "no" (definitive) or "maybe" -(probabilistic — false positives are possible). +FilterAnnounce updates are event-driven (peer changes, tree +restructuring, local identity changes) and rate-limited to prevent +storms. False positives at large scale never cause loops — the +self-distance check at each hop guarantees forward progress, and +mismatched bloom matches fall through to greedy tree routing. -Because filters propagate along tree edges with split-horizon exclusion, -they encode directional reachability: a bloom hit on a tree peer reliably -indicates which subtree contains the destination. When multiple peers match, -tree coordinate distance ranks them. - -### How Filters Propagate - -Nodes exchange **FilterAnnounce** messages with all direct peers. Each -FilterAnnounce replaces the previous filter for that peer — there is no -incremental update. - -Filter computation uses **tree-only merge with split-horizon exclusion**: -the outbound filter for peer Q is computed by merging the local node's own -identity, its leaf-only dependents (if any), and the inbound filters from -tree peers (parent and children) *except* Q. Filters from non-tree mesh -peers are stored locally for routing queries but are not merged into -outgoing filters. This prevents saturation where mesh shortcuts cause -filters to converge toward the full network. - -The restriction creates **directional asymmetry**: upward filters -(child → parent) contain the child's subtree, while downward filters -(parent → child) contain the complement. Together they cover the entire -network. - -Filters propagate transitively through tree edges. At steady state, every -reachable destination appears in at least one tree peer's filter. - -### Update Triggers - -Filter updates are event-driven, not periodic: - -- Peer connects or disconnects -- A peer's incoming filter changes (triggers recomputation for other peers) -- Tree relationship changes (new parent, new child, parent switch) -- Local state changes (new identity, leaf-only dependent changes) - -Updates are rate-limited at 500ms to prevent storms during topology changes. - -### Scale Properties - -At moderate network sizes, bloom filters are highly accurate. At larger -scales (~1M nodes), hub nodes with many peers may see elevated false positive -rates (7–15% for nodes with 20+ peers). False positives may cause a packet -to be forwarded toward the wrong subtree, but the self-distance check at -each hop prevents loops and the packet falls through to greedy tree routing. - -See [fips-bloom-filters.md](fips-bloom-filters.md) for filter parameters, -FPR calculations, and size class folding. +For the filter computation, split-horizon merge rules, FPR analysis, +size classes, and folding, see +[fips-bloom-filters.md](fips-bloom-filters.md). ## Routing Decision Process @@ -222,34 +90,33 @@ priority chain. This is the core routing algorithm. 2. **Direct peer** — The destination is an authenticated neighbor. Forward directly. No coordinates or bloom filters needed. -3. **Bloom-guided routing** — One or more peers' bloom filters contain the +3. **Coordinate cache check** — Multi-hop forwarding requires the + destination's tree coordinates to be in the local cache. On miss, + `find_next_hop()` returns None immediately — bloom filters are never + consulted — and the source receives a CoordsRequired error signal. + +4. **Bloom-guided routing** — One or more peers' bloom filters contain the destination. Select the best peer by composite key: - `(link_cost, tree_distance, node_addr)`. This requires the destination's - tree coordinates to be in the local coordinate cache. + `(link_cost, tree_distance, node_addr)`. -4. **Greedy tree routing** — Fallback when bloom filters haven't converged - for this destination. Forward to the peer that minimizes tree distance. - Also requires destination coordinates. +5. **Greedy tree routing** — Fall-through when bloom yields no candidate. + Forward to the peer that minimizes tree distance. If the tree has no + next hop closer to the destination, the source receives a PathBroken + error signal. -5. **No route** — Destination unreachable. Generate an error signal - (CoordsRequired or PathBroken) back to the source. +### Convergence Requirements -### The Coordinate Requirement - -All multi-hop routing (steps 3–4) requires the destination's tree coordinates -to be in the local coordinate cache. Without coordinates, `find_next_hop()` -returns None immediately — bloom filters are never even consulted. - -This creates two simultaneous convergence requirements for multi-hop routing: +Multi-hop routing depends on two propagation processes that must run +to convergence simultaneously: 1. **Bloom convergence**: Filters must propagate so peers advertise reachability 2. **Coordinate availability**: Destination coordinates must be cached at every transit node on the path -Both must be satisfied simultaneously. Bloom convergence without coordinates -causes a coordinate cache miss. Coordinates without bloom convergence falls -through to greedy tree routing (functional but suboptimal). +Bloom convergence without coordinates trips step 3 (coord-cache miss → +CoordsRequired). Coordinates without bloom convergence falls through to +greedy tree routing — functional but suboptimal. ### Candidate Ranking @@ -270,6 +137,10 @@ A peer with a bloom filter hit but no entry in the peer ancestry table (missing TreeAnnounce) defaults to maximum distance and is effectively invisible to routing. +### Routing Decision Flowchart + +![Per-hop routing decision flowchart](diagrams/fips-routing-decision.svg) + ### Loop Prevention The routing decision enforces strict progress: a packet is only forwarded @@ -284,48 +155,12 @@ PathBroken error. ## Coordinate Caching -The coordinate cache maps `NodeAddr → TreeCoordinate` and is the critical -data structure for multi-hop routing. Without it, forwarding decisions cannot -be made. - -### Unified Cache - -The coordinate cache is a single unified cache. All sources — SessionSetup -transit, CP-flagged data packets, LookupResponse — write to the same cache. - -### Population Sources - -| Source | When | What | -| ------ | ---- | ---- | -| SessionSetup transit | Session establishment | Both src and dest coordinates | -| SessionAck transit | Session establishment | Both src and dest coordinates | -| CP-flagged data packet | Warmup or recovery | Both src and dest coordinates (cleartext) | -| LookupResponse | Discovery | Target's coordinates | - -### Eviction - -- **TTL-based**: Entries expire after 300s (configurable) -- **Refresh on use**: Active routing refreshes the TTL, keeping hot entries - alive -- **LRU**: When full, least recently used entries are evicted first -- **Flush on parent change**: When the local node's tree parent changes, the - entire cache is flushed. Parent changes mean the node's own coordinates - have changed, making relative distance calculations with cached coordinates - potentially invalid. Flushing is preferred over stale routing: the cost of - re-discovery is lower than routing packets to dead ends. - -### Cache and Session Timer Ordering - -Timer values are ordered so that idle sessions tear down before transit -caches expire: - -| Timer | Default | Purpose | -| ----- | ------- | ------- | -| Session idle | 90s | Session teardown | -| Coordinate cache TTL | 300s | Coordinate expiration | - -When traffic stops, the session tears down at 90s. When traffic resumes, a -fresh SessionSetup re-warms transit caches (still within their 300s TTL). +The coordinate cache maps `NodeAddr → TreeCoordinate` and is the +critical data structure for multi-hop routing. The session layer owns +this cache (its eviction policy, TTL/refresh semantics, parent-change +flush, and timer ordering with session idle timeout); see +[fips-session-layer.md](fips-session-layer.md#coordinate-cache) for +the canonical treatment. ## Discovery Protocol @@ -375,28 +210,17 @@ where a request might arrive via both tree and fallback paths. Single-path forwarding is more fragile than flooding — if any transit node on the path has a stale bloom filter or loses a link, the request fails. -To compensate, the originator retries: +To compensate, each discovery is a sequence of attempts with growing +per-attempt timeouts. The default sequence is `[1s, 2s, 4s, 8s]` +(configurable via `node.discovery.attempt_timeouts_secs`); the destination +is declared unreachable only after the full sequence is exhausted (15s +total at default). -- **T=0**: Initial lookup sent -- **T=5s**: Retry if no response (configurable via `retry_interval_secs`) -- **T=10s**: Timeout, fail (configurable via `timeout_secs`) - -The default `max_attempts` is 2 (initial + one retry). Each retry generates -a fresh `request_id` and re-evaluates bloom filter matches, so it can take -a different path if the tree has restructured. - -### Per-Attempt Timeouts - -Each discovery is a sequence of attempts with growing per-attempt timeouts. -Default sequence is `[1s, 2s, 4s, 8s]` (configurable via -`node.discovery.attempt_timeouts_secs`). When the current attempt's deadline -elapses without a `LookupResponse`, the originator sends another -`LookupRequest` with a **fresh `request_id`** and the next entry in the -sequence as its deadline. Fresh request_ids let each attempt take a -different forwarding path as the bloom and tree state evolve, which is -particularly useful during cold-start convergence. The destination is -declared unreachable only after the full sequence is exhausted (15s total -with the default). +When the current attempt's deadline elapses without a `LookupResponse`, +the originator sends another `LookupRequest` with a **fresh `request_id`** +and the next entry in the sequence as its deadline. Fresh `request_id`s +let each attempt take a different forwarding path as the bloom and tree +state evolve, which is particularly useful during cold-start convergence. ### Originator Backoff (optional, off by default) @@ -462,6 +286,10 @@ verification at the source confirms the target holds the claimed position. The `path_mtu` field is excluded from the proof because it is a transit annotation modified at each hop. +### Coordinate Discovery Sequence + +![Coordinate discovery and cache warming sequence](diagrams/fips-coordinate-discovery.svg) + ### Discovery Outcome On receiving a verified LookupResponse, the source caches the target's @@ -473,50 +301,19 @@ If discovery times out (no response after all retry attempts), queued packets receive ICMPv6 Destination Unreachable and the target enters backoff. -## SessionSetup Self-Bootstrapping +## Coordinate Cache Warming -SessionSetup is the mechanism that warms transit node coordinate caches -along a path, enabling subsequent data packets to route efficiently. - -### How It Works - -SessionSetup carries plaintext coordinates (outside the Noise handshake -payload, visible to transit nodes): - -- **src_coords**: Source's current tree coordinates -- **dest_coords**: Destination's tree coordinates (learned from discovery) - -As the SessionSetup transits each intermediate node: - -1. The transit node extracts both coordinate sets -2. Caches `src_addr → src_coords` and `dest_addr → dest_coords` in its - coordinate cache -3. Forwards the message using the cached destination coordinates - -SessionAck returns along the reverse path, carrying both the responder's -and initiator's coordinates and warming caches in the other direction. This -ensures return-path transit nodes can route even when the reverse path -diverges from the forward path (e.g., after tree reconvergence). - -### Result - -After the handshake completes, the entire forward and reverse paths have -cached coordinates for both endpoints. Subsequent data packets use minimal -headers (no coordinates) and route efficiently through the warmed caches. - -## Hybrid Coordinate Warmup (CP + CoordsWarmup) - -The CP flag in the FSP common prefix and the standalone CoordsWarmup message -(0x14) together provide a hybrid cache-warming mechanism that complements -SessionSetup. See [fips-session-layer.md](fips-session-layer.md) for the -full warmup strategy. - -Transit nodes parse the CP flag from the FSP header and extract source and -destination coordinates from the cleartext section between the header and -ciphertext — no decryption needed. This is the same caching operation -performed for SessionSetup coordinates. CoordsWarmup messages use the same -CP-flag format and are handled identically by transit nodes via the existing -`try_warm_coord_cache()` path. +SessionSetup carries plaintext source and destination coordinates, +which transit nodes cache as the message travels — warming the +forward path. SessionAck carries them back along the reverse path, +warming return-path caches. Steady-state data packets piggyback +coordinates via the FSP CP flag during the warmup window, falling +back to standalone CoordsWarmup messages when piggybacking would +exceed the transport MTU. See +[fips-session-layer.md](fips-session-layer.md#hybrid-coordinate-warmup-strategy) +for the canonical hybrid-warmup design (SessionSetup +self-bootstrapping plus CP-flag piggyback plus standalone +CoordsWarmup). ## Error Recovery @@ -716,21 +513,10 @@ routing decisions but retains its own end-to-end encryption and identity. ## Packet Type Summary -| Message | Typical Size | When | Forwarded? | -| ------- | ------------ | ---- | ---------- | -| TreeAnnounce | Variable (depth-dependent) | Topology changes | No (peer-to-peer) | -| FilterAnnounce | ~1 KB | Topology changes | No (peer-to-peer) | -| LookupRequest | ~300 bytes | First contact, recovery | Yes (bloom-guided tree) | -| LookupResponse | ~400 bytes | Response to discovery | Yes (greedy routed) | -| SessionDatagram + SessionSetup | ~232–402 bytes | Session establishment | Yes (routed) | -| SessionDatagram + SessionAck | ~170 bytes | Session confirmation | Yes (routed) | -| SessionDatagram + Data (minimal) | 77 bytes + IPv6 payload | Bulk IPv6 traffic (compressed) | Yes (routed) | -| SessionDatagram + Data (with CP) | 77 + coords + IPv6 payload | Warmup/recovery (compressed) | Yes (routed) | -| SessionDatagram + CoordsRequired | 70 bytes | Cache miss error | Yes (routed) | -| SessionDatagram + PathBroken | 70+ bytes | Dead-end error | Yes (routed) | -| Disconnect | 2 bytes | Link teardown | No (peer-to-peer) | - -See [fips-wire-formats.md](fips-wire-formats.md) for byte-level layouts. +For typical sizes, forwarding category, and the byte-level layouts +of each FMP and FSP message type, see +[../reference/wire-formats.md](../reference/wire-formats.md). The +canonical Packet Type Summary table lives there. ## Privacy Considerations @@ -784,11 +570,14 @@ recovery). ## References -- [fips-intro.md](fips-intro.md) — Protocol overview +- [fips-concepts.md](fips-concepts.md) — Protocol overview +- [fips-architecture.md](fips-architecture.md) — Layer architecture and + identity model - [fips-mesh-layer.md](fips-mesh-layer.md) — FMP specification - [fips-spanning-tree.md](fips-spanning-tree.md) — Tree algorithms and data structures - [fips-bloom-filters.md](fips-bloom-filters.md) — Filter parameters and math -- [fips-wire-formats.md](fips-wire-formats.md) — Wire format reference +- [../reference/wire-formats.md](../reference/wire-formats.md) — Wire + format reference - [spanning-tree-dynamics.md](spanning-tree-dynamics.md) — Convergence walkthroughs diff --git a/docs/design/fips-mmp.md b/docs/design/fips-mmp.md new file mode 100644 index 0000000..562b0a0 --- /dev/null +++ b/docs/design/fips-mmp.md @@ -0,0 +1,218 @@ +# Metrics Measurement Protocol (MMP) + +The Metrics Measurement Protocol provides per-link and per-session +quality metrics — SRTT, loss, jitter, goodput, ETX, and one-way delay +trend — using only counter and timestamp fields already present in +the FMP and FSP wire formats. No additional probing traffic is +required. The same algorithms and report message format are used at +both layers; only the routing scope and configuration namespace +differ. + +This document is the canonical home for the MMP design. For the +link-layer instance's role inside FMP, see +[fips-mesh-layer.md](fips-mesh-layer.md). For the session-layer +instance's role inside FSP, see +[fips-session-layer.md](fips-session-layer.md). For the byte-level +SenderReport and ReceiverReport layouts, see +[../reference/wire-formats.md](../reference/wire-formats.md). + +## Two Layers, One Protocol + +MMP runs at two layers: + +- **Link-layer MMP**: One instance per active FMP peer link. Reports + are exchanged peer-to-peer between direct neighbors and measure the + quality of that single hop. +- **Session-layer MMP**: One instance per established FSP session. + Reports are encrypted end-to-end and forwarded through every transit + link, measuring end-to-end quality independent of hop count. + +The algorithms (SRTT estimation, jitter computation, loss inference, +ETX) are identical at both layers. The differences are configuration +namespace, report intervals, and routing scope. See +[Layer Differences](#layer-differences) below. + +## Metrics Tracked + +MMP computes the following metrics from the per-frame counter and +timestamp fields: + +- **SRTT** — Smoothed round-trip time (Jacobson/RFC 6298, α=1/8). + Derived from timestamp-echo in ReceiverReports with dwell-time + compensation. +- **Loss rate** — Bidirectional loss inferred from counter gaps. + Tracked as both instantaneous (per-interval) and long-term EWMA. +- **Jitter** — Interarrival jitter (RFC 3550 algorithm) in + microseconds. +- **Goodput** — Bytes per second of payload data (excludes MMP + reports). +- **OWD trend** — One-way delay trend (µs/s, signed). Indicates + congestion buildup before loss occurs. +- **ETX** — Expected Transmission Count, computed from bidirectional + delivery ratios. Used in cost-based parent selection via + `link_cost = etx * (1.0 + srtt_ms / 100.0)`, and in bloom-filter + candidate ranking inside `find_next_hop()` (the same `link_cost` + is the primary key when choosing among bloom-filter peers, with + tree distance as the tie-breaker). +- **Dual EWMA trends** — Short-term (α=1/4) and long-term (α=1/32) + trend indicators for both RTT and loss, enabling change detection. + +Session-layer MMP additionally tracks the observed forward-path MTU; +see [fips-mtu.md](fips-mtu.md) for the end-to-end path-MTU mechanism. + +## Operating Modes + +MMP supports three modes: + +| Mode | Reports Exchanged | Metrics Available | +| ---- | ----------------- | ----------------- | +| **Full** (default) | SenderReport + ReceiverReport | All metrics including RTT, loss, jitter, goodput, OWD trend | +| **Lightweight** | ReceiverReport only | Loss (from counter gaps), jitter, OWD trend. No RTT. | +| **Minimal** | None | Spin bit and CE echo flags only. No computed metrics. | + +The mode is configured per layer (`node.mmp.mode` and +`node.session_mmp.mode`). + +## Report Scheduling + +Reports are sent at RTT-adaptive intervals computed as +`clamp(2 × SRTT, low, high)`. A cold-start interval is used until SRTT +has converged. + +| Layer | Adaptive bounds | Cold-start | +| ----- | --------------- | ---------- | +| Link | `[1s, 5s]` | 200 ms (first 5 samples) | +| Session | `[500ms, 10s]` | 1 s | + +The session-layer bounds are higher because session reports are +encrypted and forwarded through every transit link, so bandwidth cost +is proportional to path length. + +## Spin Bit and RTT + +The SP (spin bit) flag in the FMP inner header follows the QUIC spin +bit pattern: reflected on receive, toggled on send when the reflected +value matches the last sent value. The spin bit state machine runs +for TX reflection, but **RTT samples from the spin bit are +discarded**. In a mesh protocol where frames are sent irregularly +(tree announces, bloom filters, MMP reports on different timers), +inter-frame processing delays inflate spin bit RTT measurements +unpredictably. Timestamp-echo from ReceiverReports (with dwell-time +compensation) is the sole SRTT source. + +The spin bit lives in the link-layer FMP inner header, so this +mechanism applies to link-layer MMP only. Session-layer MMP carries +its spin bit in the FSP encrypted inner header but uses it the same +way: reflected for diagnostic visibility, not used for SRTT. + +## ECN Congestion Signaling + +The CE (Congestion Experienced) flag (bit 1 in the FMP flags byte) +provides hop-by-hop congestion signaling through the mesh. Transit +nodes detect congestion on outgoing links and set CE on forwarded +packets; once set, the flag stays set for all subsequent hops to the +destination. + +**Congestion detection** triggers on any of: + +- Outgoing link MMP loss rate ≥ `node.ecn.loss_threshold` (default 5%) +- Outgoing link MMP ETX ≥ `node.ecn.etx_threshold` (default 3.0) +- Kernel receive buffer drops detected on any local transport (via + `SO_RXQ_OVFL` on UDP) + +**CE relay**: The forwarding path computes +`outgoing_ce = incoming_ce || local_congestion`. Once CE is set on a +packet, it remains set for the rest of the forward path. + +**IPv6 ECN-CE marking**: When a CE-flagged DataPacket arrives at its +final destination, the IPv6 Traffic Class ECN bits are marked CE +(0b11) before TUN delivery — but only for ECN-capable packets (ECT(0) +or ECT(1)). Not-ECT packets are never marked per RFC 3168. The host +TCP stack then echoes ECE in ACKs, triggering sender cwnd reduction +through standard congestion control. + +**Session-layer tracking**: The `ecn_ce_count` field in MMP +ReceiverReports tracks CE-flagged packets received per link, providing +end-to-end visibility into congestion propagation. + +ECN signaling is a link-layer mechanism. Session-layer MMP only +observes the CE counter as part of the report stream; CE marking is +not generated end-to-end. Tuning parameters live under `node.ecn.*` +in [../reference/configuration.md](../reference/configuration.md). + +## Send Failure Backoff (Session Layer Only) + +When a session MMP report cannot be delivered (destination unreachable, +no route), the sender applies exponential backoff to the probe +interval — a standard distributed-systems pattern for transient +failure handling: + +- Each consecutive failure doubles the interval: 2x, 4x, 8x, 16x, 32x +- Backoff caps at 32x the base interval (5 consecutive failures) +- A successful send resets to the normal SRTT-based interval +- Debug logging is suppressed after 3 consecutive failures; a summary + is logged when the destination becomes reachable again + +This prevents wasted CPU and log noise when a session's remote +endpoint has departed the network but the local session has not yet +timed out. Link-layer MMP has no equivalent — link-layer reports are +peer-to-peer over an authenticated link, so delivery failure is +indistinguishable from link death and the link-liveness mechanism +takes over. + +## Layer Differences + +| Aspect | Link layer | Session layer | +| ------ | ---------- | ------------- | +| Routing scope | Peer-to-peer (one hop) | End-to-end (forwarded through every hop) | +| Configuration namespace | `node.mmp.*` | `node.session_mmp.*` | +| Report bounds | `[1s, 5s]` | `[500ms, 10s]` | +| Cold-start interval | 200 ms (first 5 samples) | 1 s | +| Bandwidth cost | One link | Proportional to path length | +| Send-failure backoff | Not applicable | Yes | +| Path-MTU echo | Not applicable | PathMtuNotification (see [fips-mtu.md](fips-mtu.md)) | +| Idle-timeout interaction | None | Reports do **not** reset session idle timer | + +## Idle Timeout Interaction (Session Layer Only) + +MMP reports (SenderReport, ReceiverReport) and PathMtuNotification do +**not** reset the session idle timer. Only application data +(DataPacket, type 0x10) resets `last_activity`. This ensures sessions +with no application traffic tear down after +`node.session.idle_timeout_secs` (default 90s), while MMP continues +providing measurement data up to the teardown moment. + +## Operator Logging + +Both layers emit periodic metrics at info level. The interval is +`node.mmp.log_interval_secs` for link-layer (default 30s) and +`node.session_mmp.log_interval_secs` for session-layer (default 30s). + +Link-layer: + +```text +MMP link metrics peer=node-b rtt=2.3ms loss=0.2% jitter=0.1ms goodput=76.0MB/s tx_pkts=1234 rx_pkts=5678 +``` + +Session-layer: + +```text +MMP session metrics session=npub1tdwa...84le rtt=4.3ms loss=0.6% jitter=0.2ms goodput=71.3MB/s mtu=1472 tx_pkts=1234 rx_pkts=5678 +``` + +Teardown logs include final SRTT, loss rate, jitter, ETX, goodput, +and cumulative tx/rx packet and byte counts. + +## See also + +- [fips-mesh-layer.md](fips-mesh-layer.md) — link-layer MMP integration + inside FMP +- [fips-session-layer.md](fips-session-layer.md) — session-layer MMP + integration inside FSP +- [fips-mtu.md](fips-mtu.md) — PathMtuNotification, the session-only + end-to-end path-MTU echo +- [../reference/wire-formats.md](../reference/wire-formats.md) — + SenderReport (0x01 / 0x11) and ReceiverReport (0x02 / 0x12) byte + layouts +- [../reference/configuration.md](../reference/configuration.md) — + full `node.mmp.*`, `node.session_mmp.*`, and `node.ecn.*` knob tables diff --git a/docs/design/fips-mtu.md b/docs/design/fips-mtu.md new file mode 100644 index 0000000..ea88aab --- /dev/null +++ b/docs/design/fips-mtu.md @@ -0,0 +1,316 @@ +# FIPS Path MTU and Encapsulation Overhead + +MTU is a cross-cutting concern in FIPS. No single layer owns it: the +transport reports per-link MTU, FMP propagates `path_mtu` along +forward and reverse paths, FSP echoes the observed path MTU end-to-end +back to the source, and the IPv6 adapter enforces the resulting +effective MTU at the TUN interface. This document is the canonical +home for the unified MTU model. + +For operator-facing diagnostic recipes (interpreting `MtuExceeded` +counters, tuning IPv6 application MSS, troubleshooting cold-flow +oversize), see the relevant how-to under `docs/how-to/`. + +## The MTU Problem in FIPS + +A FIPS path can traverse heterogeneous link types — UDP/IP (1280 +default, IPv6 minimum), Ethernet (interface MTU − 3, typically 1497), +BLE (negotiated ATT_MTU per link), Tor stream (1400 default), radio +(51–222) — within a single end-to-end session. +The minimum MTU along the path determines the largest datagram a +session can deliver. Several properties make this harder than in +classic IP networks: + +- **No fragmentation.** FIPS does not fragment at transit nodes (see + [No fragmentation policy](#no-fragmentation-policy)). A datagram + that exceeds the next-hop link MTU is dropped, and the source is + signaled. +- **Forward/reverse path asymmetry.** After tree reconvergence the + return path may diverge from the forward path, so the bottleneck + on each direction can differ. +- **First-flow race.** The very first SessionDatagram races + destination discovery — the source has not yet learned the path MTU + but must pick a payload size for the queued packet. +- **Variable per-link MTU.** Some transports (BLE, TCP via + `TCP_MAXSEG`) report different MTUs for different links rather than + a single transport-wide value. + +The unified MTU model below combines proactive and reactive +mechanisms to converge on a working effective MTU within the first +few packets of a session, then maintain it across topology changes. + +## Encapsulation Overhead + +The byte budget for a FIPS-encapsulated packet: + +| Layer | Overhead | Purpose | +| ----- | -------- | ------- | +| Link encryption | 37 bytes | 16-byte outer header + 5-byte inner header (timestamp + msg_type) + 16-byte AEAD tag | +| SessionDatagram body | 35 bytes | ttl + path_mtu + src_addr + dest_addr (msg_type counted in inner header) | +| FSP header | 12 bytes | 4-byte prefix + 8-byte counter (used as AEAD AAD) | +| FSP inner header | 6 bytes | 4-byte timestamp + 1-byte msg_type + 1-byte inner_flags (inside AEAD) | +| Session AEAD tag | 16 bytes | ChaCha20-Poly1305 tag on session-encrypted payload | +| **Protocol envelope** | **106 bytes** | `FIPS_OVERHEAD` constant — the base payload budget for any service | + +`FIPS_OVERHEAD = 106` is the constant the rest of the system reasons +about. Coordinate piggybacking via the CP flag adds variable extra +overhead — `2 + entries × 16` bytes per coordinate, with both source +and destination coordinates carried — and the send path skips the CP +flag if adding coords would exceed the transport MTU. + +Service-specific overheads layer on top of `FIPS_OVERHEAD`: + +| Service | Overhead | Note | +| ------- | -------- | ---- | +| DataPacket port header | +4 bytes | Always present for port-multiplexed services | +| IPv6 compression | −33 bytes | 40-byte IPv6 header → 7-byte format + residual | +| **IPv6 effective overhead** | **77 bytes** | `FIPS_IPV6_OVERHEAD` constant | + +See [fips-ipv6-adapter.md](fips-ipv6-adapter.md) for the IPv6 +compression scheme that lets the adapter reach `FIPS_IPV6_OVERHEAD`. + +## Per-Link MTU Reporting + +Each transport implements two MTU methods on its trait: + +- `mtu() -> u16` — Transport-wide default MTU. +- `link_mtu(addr: &TransportAddr) -> u16` — Per-link MTU for a + specific remote address. The default implementation falls back to + `mtu()`, so transports with uniform MTU (UDP, raw Ethernet) need + not override it. + +FMP uses `link_mtu()` when it needs to reason about a specific +outbound link — typically for `path_mtu` annotation in +SessionDatagram and LookupResponse. Per-transport defaults: + +| Transport | Default MTU | Per-link MTU source | +| --------- | ----------- | ------------------- | +| UDP | 1280 (IPv6 minimum) | uniform (`mtu()` fallback) | +| Ethernet | interface MTU − 3 (typically 1497) | uniform | +| TCP | 1400 | derived from `TCP_MAXSEG` per connection | +| Tor | 1400 | uniform | +| BLE | 2048 default; negotiated ATT_MTU per link | per-link (overrides `mtu()`) | + +For TCP, the per-connection `TCP_MAXSEG` query lets FMP discover the +actual MSS the kernel negotiated for each connection, rather than +assuming a single value across all TCP peers. + +## Proactive PMTUD: SessionDatagram path_mtu + +Every SessionDatagram and LookupResponse carries a 2-byte `path_mtu` +field. The source initializes it to its outbound link MTU; each +transit node applies `min(current, link_mtu(next_hop))` before +forwarding. The destination receives the forward-path minimum. + +For SessionDatagram, the receiver of the forward-path minimum is the +session-layer destination, which then echoes the value back to the +source via PathMtuNotification (see +[End-to-end echo](#end-to-end-echo-pathmtunotification)). + +For LookupResponse, the receiver is the original requester, and the +annotation is reverse-path-only: the LookupResponse path is the +return path of the lookup, so the annotated `path_mtu` reflects what +the requester can use to reach the discovered destination over the +discovered path. + +Because the field is initialized by the source and mins as it travels, +it converges to the bottleneck without any additional probing. The +first SessionDatagram on a fresh session may carry an over-estimate +(the source has not yet been told a smaller min), which is what makes +the reactive MtuExceeded path necessary. + +## Reactive PMTUD: MtuExceeded + +When a transit node receives a SessionDatagram whose total wire size +exceeds the next-hop `link_mtu`, it cannot forward without +fragmentation. Instead: + +1. The transit node generates a SessionDatagram addressed back to the + source carrying an `MtuExceeded` payload (msg_type 0x22). The + payload identifies the destination, the reporting router, and the + bottleneck MTU. +2. The error is routed via `find_next_hop(src_addr)`. If the source + is also unreachable, the error is dropped silently (no cascading + errors). +3. The original oversized packet is dropped. + +The source's FSP layer applies the reported bottleneck immediately — +unlike the increase case (see hysteresis below), decrease is always +take-the-lower-value because the original packet has already been +dropped. The source can then reduce payload sizes on subsequent +SessionDatagrams. + +MtuExceeded is the reactive complement to the proactive `path_mtu` +field. The proactive field tracks the minimum along the forward path +under steady-state convergence; MtuExceeded handles the in-flight gap +when an oversized packet hits a new bottleneck (forward path shifted, +peer's outbound MTU dropped, BLE renegotiated) before the source has +adapted. + +Error generation is rate-limited at 100ms per destination at the +transit node to prevent storms during topology changes. + +## End-to-End Echo: PathMtuNotification + +PathMtuNotification (msg_type 0x13, session-layer) provides +end-to-end path MTU feedback, adapting RFC 1191 Path MTU Discovery +for overlay networks — the transit-node `min()` propagation replaces +ICMP Packet Too Big. + +Mechanism: + +1. The source sets `path_mtu` in each SessionDatagram envelope to its + outbound link MTU. +2. Each transit node applies `min(current, transport.link_mtu(addr))` + before forwarding. +3. The destination receives the forward-path minimum and sends a + PathMtuNotification (2-byte body: `u16 LE path_mtu`) back to the + source. +4. The source applies the notification with hysteresis: + - **Decrease**: immediate (take lower value). + - **Increase**: requires 3 consecutive higher-value notifications + spanning at least 2 × notification interval. +5. Notifications are sent on first measurement, on any decrease, and + periodically at `max(10s, 5 × SRTT)`. + +The hysteresis on increase prevents oscillation when the path MTU +fluctuates around a boundary; the immediate decrease prevents +delivering oversized packets after a path has narrowed. + +PathMtuNotification is wrapped in a session-layer encrypted message +and travels back to the source via the session's normal forwarding +path. It is part of the session-layer MMP report stream's traffic +budget and (along with SenderReport and ReceiverReport) does not +reset the session idle timer. + +## Per-Destination MTU Storage + +Two storage locations track per-destination MTU, serving different +consumers: + +- **Session-canonical** (`MmpSessionState.path_mtu`, type + `PathMtuState`). Holds the running end-to-end path MTU for an + established FSP session. Updated by both `PathMtuNotification` + (proactive, end-to-end echo) and reactive `MtuExceeded` from + transit routers. Read by the session layer when constructing + outbound `SessionDatagram` envelopes. + +- **TCP-clamp mirror** (`path_mtu_lookup`, a + `HashMap` on the Node). Read by the + TUN-side TCP MSS clamp (`per_flow_max_mss` in + `src/upper/tun.rs`) at first-SYN time so outbound TCP flows + are clamped to the per-destination MTU rather than a generic + ceiling. Written from four sites, all using tighter-only + semantics — the clamp is never loosened: + - Discovery's `LookupResponse` handler — reverse-path + annotated value carried back by the discovery target. + - `seed_path_mtu_for_link_peer` when a peer is promoted to + an active link, seeding with the new link's `link_mtu` + so traffic to that peer immediately uses the per-link + value rather than a generic default. + - The reactive `MtuExceeded` handler, mirroring the + bottleneck reported by a transit router. + - The proactive `PathMtuNotification` handler, mirroring + the new effective end-to-end value so a fresh TCP flow + benefits immediately from PMTU knowledge the session has + already acquired. + +All four writers apply the same tighter-only rule, so the mirror +converges to the smallest MTU any signal has reported for that +destination and a subsequent looser observation cannot widen it. + +## TCP MSS Clamping + +The IPv6 adapter intercepts TCP SYN and SYN-ACK packets at the TUN +interface and clamps the Maximum Segment Size (MSS) option to: + +```text +clamped_mss = effective_ipv6_mtu - 40 (IPv6 header) - 20 (TCP header) +``` + +Clamping is applied in two places: + +- **TUN reader** (outbound): clamps MSS on outbound SYN packets +- **TUN writer** (inbound): clamps MSS on inbound SYN-ACK packets + +Together these ensure both directions of a TCP connection use +appropriately-sized segments from the start, avoiding the initial +oversized-packet loss that would occur if the adapter relied on ICMP +Packet Too Big alone. + +Clamping is **conditional**: when `per_flow_max_mss` already has an +entry for the flow, that entry is used; otherwise the clamp falls +back to a ceiling derived from the most pessimistic effective IPv6 +MTU the adapter knows about (1143 with the typical 1280 transport +floor). The fallback handles cold-flow first-SYN traffic — the very +first SYN of a flow may arrive before the MMP path-MTU echo and any +per-flow lookup has been populated, so the conservative ceiling +prevents the SYN-ACK chain from negotiating a too-large MSS that +would later drop. + +The adapter integrates with the MTU subsystem rather than owning it. +The "why we clamp and what `max_mss` means" lives here in the MTU +design; the "how the clamp is implemented at the TUN" lives in the +[IPv6 adapter](fips-ipv6-adapter.md#tcp-mss-clamping) doc. + +## ICMP Packet Too Big + +When an outbound packet at the TUN exceeds the effective IPv6 MTU, +the adapter generates an ICMPv6 Packet Too Big message and delivers +it back to the application via the TUN. This triggers the kernel's +Path MTU Discovery mechanism for non-TCP traffic and for any TCP flow +where MSS clamping was insufficient. + +ICMPv6 Packet Too Big generation is rate-limited per source address +(100ms interval) to prevent storms from applications sending many +oversized packets. The ICMP response is delivered locally back +through the TUN; no network traversal is needed, so delivery is +reliable. + +## No Fragmentation Policy + +FIPS does not perform fragmentation at transit nodes: + +- **Why no transit fragmentation.** Session-layer encryption is + end-to-end — the AEAD tag authenticates the entire plaintext. + Fragmenting an encrypted SessionDatagram would require either + exposing plaintext structure to transit nodes (unacceptable) or + reassembling before decryption (opens an attack surface — a transit + node could replay or withhold fragments to influence reassembly). +- **Why no source-side fragmentation.** The source doesn't need + fragmentation because the proactive `path_mtu` field plus the + reactive MtuExceeded signal converge on a working size within the + first few packets. Applications that need oversized payloads run + TCP over the IPv6 adapter, which has its own segmentation under + MSS clamping. + +Some transports may perform fragmentation and reassembly internally +(e.g., BLE L2CAP) and can advertise a larger virtual MTU than the +physical medium supports — this is transparent to FIPS. + +## Operational Considerations + +Diagnosing MTU-related symptoms (handshakes succeed but bulk +transfers stall, ssh hangs after `Welcome` banner, sporadic +`MtuExceeded` spikes during topology changes) requires inspecting +per-link MTU, per-session MTU, and the per-destination +`path_mtu_lookup` table. See +[../how-to/diagnose-mtu-issues.md](../how-to/diagnose-mtu-issues.md) +for the operator recipes. The relevant control-socket queries are +`fipsctl show sessions` (per-session MTU), `fipsctl show transports` +(per-link MTU), and `fipsctl show identity-cache` (with adapter MTU +context). + +## See also + +- [fips-transport-layer.md](fips-transport-layer.md) — the `mtu()` / + `link_mtu()` trait surface and per-transport defaults +- [fips-mesh-layer.md](fips-mesh-layer.md) — SessionDatagram and the + MtuExceeded error signal +- [fips-session-layer.md](fips-session-layer.md) — session-layer + PathMtuNotification echo, applied with hysteresis +- [fips-ipv6-adapter.md](fips-ipv6-adapter.md) — TUN-side ICMPv6 PTB + generation, MSS clamping integration, IPv6-specific overhead table +- [../reference/wire-formats.md](../reference/wire-formats.md) — + SessionDatagram, LookupResponse, MtuExceeded, PathMtuNotification + byte layouts diff --git a/docs/design/fips-nostr-discovery.md b/docs/design/fips-nostr-discovery.md index 4b44eaf..e4553fd 100644 --- a/docs/design/fips-nostr-discovery.md +++ b/docs/design/fips-nostr-discovery.md @@ -9,15 +9,14 @@ For peers behind UDP NAT, the same relay channel carries an encrypted offer/answer exchange, and STUN supplies the reflexive address used for a coordinated hole-punch. -The feature is compiled into FIPS by default on all supported platforms -(Linux, macOS, Windows) and ships in every stock packaging artifact +Nostr discovery is unconditionally compiled into the `fips` binary on +every supported platform and ships in every stock packaging artifact (`.deb`, AUR, systemd tarball, OpenWrt `.ipk`, macOS `.pkg`, Windows `.zip`). It is runtime-opt-in: the YAML configuration defaults to -disabled, so shipping the feature is a no-op until an operator enables -it. When disabled, nodes behave exactly as before: only the static -`peers[]` addresses are used. See -[Build configuration](#build-configuration) for details on opting out -at build time. +disabled (`node.discovery.nostr.enabled: false`), so the discovery +runtime stays dormant — and opens no relay connections — until an +operator flips the flag and supplies a relay list. When disabled, nodes +behave exactly as before: only the static `peers[]` addresses are used. ## Role @@ -59,301 +58,27 @@ relay dependencies, STUN round-trips for NAT cases, and a small ambient background of relay traffic; none of that is useful when you already know where peers are. -## Build configuration - -`nostr-discovery` is a default Cargo feature. Plain `cargo build ---release` produces a binary with the feature compiled in, and every -stock packaging artifact under `packaging/` ships with it enabled. -There is no extra `--features` flag to remember, on any platform. - -Shipping the feature is runtime-safe: Nostr discovery is **off by -default in the YAML configuration** -(`node.discovery.nostr.enabled: false` in every stock config). An -operator opts in per-node by flipping the flag and providing a relay -list; until then the feature is dormant and does not open connections -to any relay. - -To build a binary **without** the feature — for example, to reduce -the dependency footprint on a minimal build — use -`--no-default-features`: - -```bash -cargo build --release --no-default-features -``` - -The `nostr` and `nostr-sdk` crates are then omitted from the -dependency tree entirely, and `node.discovery.nostr` config blocks -fail at startup validation. - ## Scenarios and configuration -Each scenario below gives the minimal YAML fragment that enables it. -Only keys relevant to Nostr discovery are shown; surrounding node, -transport, TUN, DNS, and peer configuration follows the usual shape -described in [fips-configuration.md](fips-configuration.md). - -All scenarios assume `node.identity` is set to a persistent key — an -ephemeral identity would invalidate any advert the moment the node -restarts. - -### Scenario 1: Advertise a directly-reachable UDP node - -The node has a public IP (or a stable port-forward) and binds UDP on a -known port. It publishes `udp:host:port` to the advert relays. Any peer -that knows this node's npub and has Nostr discovery enabled can dial it -without knowing the address out-of-band. - -```yaml -node: - identity: - persistent: true - discovery: - nostr: - enabled: true - advertise: true - -transports: - udp: - bind_addr: "0.0.0.0:2121" - advertise_on_nostr: true - public: true -``` - -What this achieves: the node publishes a single `udp::2121` -endpoint to the three default advert relays -(`wss://relay.damus.io`, `wss://nos.lol`, `wss://offchain.pub`). - -What the other side needs: either a static `addresses` entry for this -peer, or a peer entry with `via_nostr: true` and an empty (or omitted) -`addresses` list — the advert-resolved endpoint will be used at dial -time. Static and Nostr-resolved addresses can also be combined: when -both are present, static addresses are tried first and Nostr-resolved -endpoints are appended as fallback. - -### Scenario 2: Advertise a Tor onion node - -The node runs a Tor onion service in directory mode (Tor-managed -`HiddenServiceDir`) and advertises the `.onion` address. Peers dial via -their local Tor SOCKS5 proxy without ever knowing the onion string -out-of-band. - -```yaml -node: - identity: - persistent: true - discovery: - nostr: - enabled: true - advertise: true - -transports: - tor: - mode: directory - socks5_addr: "127.0.0.1:9050" - directory_service: - hostname_file: "/var/lib/tor/fips/hostname" - bind_addr: "127.0.0.1:8444" - advertise_on_nostr: true -``` - -What this achieves: the node publishes a `tor:.onion:8443` -endpoint alongside any other advertised transports. The advert itself -is still published over clearnet WebSocket relays — Tor protects the -data plane, not the discovery plane. See -[Security and threat model](#security-and-threat-model) for the trade-off. - -### Scenario 3: Lookup a configured peer by npub (no advertising) - -The node does not publish any advert of its own. It only consumes -adverts for peers it has explicitly listed with `via_nostr: true`. This -is the right shape for a client that wants Nostr-mediated resolution -without becoming a rendezvous target itself. - -```yaml -node: - identity: - persistent: true - discovery: - nostr: - enabled: true - advertise: false - policy: configured_only - -transports: - udp: - bind_addr: "0.0.0.0:2121" - -peers: - - npub: "npub1peer..." - alias: "remote-node" - addresses: - - transport: udp - addr: "203.0.113.45:2121" - priority: 10 - via_nostr: true - connect_policy: auto_connect -``` - -What this achieves: on dial, the static address is tried first; if the -peer has published a newer advert (for example, its public IP has -changed), those addresses are appended as additional candidates. -`configured_only` is the default — it is shown here for clarity. - -If you have no static address for the peer at all, omit `addresses` -entirely (or leave it empty) — `via_nostr: true` is sufficient on its -own and dial endpoints are taken from the advert. - -### Scenario 4: UDP NAT hole-punch with a configured peer - -Neither side has a stable public UDP endpoint. Both sides advertise -`udp:nat`, run the STUN + offer/answer exchange, and punch through -their NATs to establish a direct UDP link. This is the full -NAT-traversal path. - -```yaml -node: - identity: - persistent: true - discovery: - nostr: - enabled: true - advertise: true - dm_relays: - - "wss://relay.damus.io" - - "wss://nos.lol" - stun_servers: - - "stun:stun.l.google.com:19302" - - "stun:stun.cloudflare.com:3478" - -transports: - udp: - bind_addr: "0.0.0.0:2121" - advertise_on_nostr: true - public: false - -peers: - - npub: "npub1peer..." - alias: "nat-peer" - addresses: - - transport: udp - addr: "nat" - priority: 1 - via_nostr: true - connect_policy: auto_connect - auto_reconnect: true -``` - -What this achieves: the node publishes a `udp:nat` endpoint plus its -signaling relays and STUN server list in the advert. The peer side runs -the same configuration. When either side initiates, an encrypted offer -is sealed to the peer's npub, a matching answer comes back, and both -sides punch at the negotiated time. On success, the punch socket is -adopted as an FMP UDP transport and Noise IK proceeds normally. - -> **Validation:** `advertise_on_nostr: true` with `public: false` on UDP -> requires both `dm_relays` and `stun_servers` to be non-empty. The -> node fails startup with a config validation error if either list is -> empty. This is enforced because a `udp:nat` advert without signaling -> relays or STUN servers is unreachable by construction. - -Works best with full-cone NAT on at least one side. Symmetric NAT on -both sides is not reliably traversable with this protocol and will time -out after `punch_duration_ms`; fall back to a Tor or TCP transport in -that case. - -### Scenario 5: Open discovery — no pre-configured peers - -Under `policy: open`, any node that publishes an advert under the same -`app` namespace becomes a candidate. Discovered peers are queued for -connection attempts subject to `open_discovery_max_pending`. - -```yaml -node: - identity: - persistent: true - discovery: - nostr: - enabled: true - advertise: true - policy: open - open_discovery_max_pending: 32 - app: "my-experiment.v1" - -transports: - udp: - bind_addr: "0.0.0.0:2121" - advertise_on_nostr: true - public: true - -peers: [] -``` - -What this achieves: peers are discovered entirely through ambient advert -traffic on the configured relays. Setting a non-default `app` value -(replacing `fips-overlay-v1`) scopes the discovery set to participants -who opt into the same experiment and avoids being joined to unrelated -overlays that happen to share the default namespace. - -> **Scope warning:** Open discovery is an admission-free mode. Any node -> that publishes on the same `app` name and passes the peer-ACL check -> becomes a connection candidate. If you rely on peer ACLs for admission -> control, verify that list is set correctly before enabling this mode. - -## Operational knobs - -All fields below live under `node.discovery.nostr.*`. Defaults are -defined in `src/config/node.rs`. - -| Field | Type | Default | Purpose | -| --- | --- | --- | --- | -| `enabled` | bool | `false` | Master switch. When false, the discovery runtime is not started. | -| `advertise` | bool | `true` | If true, publish this node's own overlay advert. | -| `advert_relays` | list | `["wss://relay.damus.io", "wss://nos.lol", "wss://offchain.pub"]` | Relays used to publish and fetch overlay adverts (kind 37195). | -| `dm_relays` | list | same as `advert_relays` | Relays used for encrypted offer/answer signaling (kind 21059). | -| `stun_servers` | list | `["stun:stun.l.google.com:19302", "stun:stun.cloudflare.com:3478", "stun:global.stun.twilio.com:3478"]` | STUN servers used to observe the local reflexive address before a punch. Peer-advertised STUN values are not used. | -| `share_local_candidates` | bool | `false` | If true, include this node's RFC 1918 / ULA interface addresses as host candidates in the traversal offer. Off by default — sharing private host candidates is only useful when peers are on the same physical LAN, and tends to cause misleading punch successes when an asymmetric L3 path (corporate VPN, Tailscale subnet route, overlapping address space) makes a peer's private IP one-way reachable. Enable per-node only when same-LAN punching is wanted. | -| `app` | string | `"fips-overlay-v1"` | Application namespace. Included in the advert identifier; only peers with the same value cross-resolve. | -| `policy` | enum | `configured_only` | Advert consumption policy: `disabled`, `configured_only`, or `open`. | -| `signal_ttl_secs` | u64 | `120` | TTL on the encrypted offer/answer events. Also caps the wait for an answer. | -| `advert_ttl_secs` | u64 | `3600` | NIP-40 expiration set on this node's published advert. | -| `advert_refresh_secs` | u64 | `1800` | Interval between re-publishes. Must be less than `advert_ttl_secs`. | -| `attempt_timeout_secs` | u64 | `10` | Overall timeout for a single punch attempt (STUN + signal + punch). | -| `punch_start_delay_ms` | u64 | `2000` | Delay between receiving the answer and sending the first punch packet. Gives the remote side time to arrive at the same point. | -| `punch_interval_ms` | u64 | `200` | Gap between successive punch probes. | -| `punch_duration_ms` | u64 | `10000` | How long to keep probing before declaring the attempt failed. | -| `replay_window_secs` | u64 | `300` | How long a session id stays in the replay-detection cache. | -| `max_concurrent_incoming_offers` | usize | `16` | Semaphore cap on inbound offers being processed simultaneously. Excess offers are dropped with a warn log. | -| `advert_cache_max_entries` | usize | `2048` | Max cached peer adverts (LRU by expiry). | -| `seen_sessions_max_entries` | usize | `2048` | Max tracked session ids for replay detection. | -| `open_discovery_max_pending` | usize | `64` | Max peers queued for connection attempts under `policy: open`. | - -The per-transport keys are: - -| Key | Type | Where | Default | Purpose | -| --- | --- | --- | --- | --- | -| `advertise_on_nostr` | bool | `transports.{udp,tcp,tor}` | `false` | Include this transport's endpoint in the overlay advert. | -| `public` | bool | `transports.udp` | `false` | When `advertise_on_nostr` is true: `true` publishes `udp:host:port`, `false` publishes `udp:nat`. | -| `via_nostr` | bool | `peers[]` | `false` | Append advert-resolved endpoints to this peer's dial list. | - -## Validation rules at startup - -The following combinations are rejected with `ConfigError::Validation`: - -- Any transport sets `advertise_on_nostr: true` while - `node.discovery.nostr.enabled` is `false` or absent. -- Any peer sets `via_nostr: true` while - `node.discovery.nostr.enabled` is `false` or absent. -- A UDP transport sets `advertise_on_nostr: true` with `public: false` - (a `udp:nat` advert) but `dm_relays` is empty. -- A UDP transport sets `advertise_on_nostr: true` with `public: false` - but `stun_servers` is empty. +For end-to-end operator recipes — each of the five activation scenarios +(advertise a directly-reachable UDP node, advertise a Tor onion node, +look up a configured peer by npub without advertising, NAT hole-punch +between two configured peers, and open discovery within an `app` +namespace) — see +[../how-to/enable-nostr-discovery.md](../how-to/enable-nostr-discovery.md). +The full configuration knob tables, per-transport keys, and startup +validation rules live in +[../reference/configuration.md](../reference/configuration.md) under +`node.discovery.nostr.*`. The Kind 37195 advert event format is in +[../reference/nostr-events.md](../reference/nostr-events.md). The rest +of this document covers the design of the discovery runtime itself. ## Under the covers The rest of this document describes how the feature works inside the -node. For the on-the-wire event format and NIP references, see the -protocol reference at -[../proposals/nostr-udp-hole-punch-protocol.md](../proposals/nostr-udp-hole-punch-protocol.md). +node. For the generic protocol shape (event tags, NIP usage, on-the- +wire offer/answer schema, failure-suppression machinery), see +[port-advertisement-and-nat-traversal.md](port-advertisement-and-nat-traversal.md). ### Overview @@ -403,32 +128,18 @@ configuration and by connection attempts made by the rest of the node. Adverts are published as Nostr kind `37195` parameterized replaceable events (FIPS-specific, in the application-defined replaceable range `30000–39999`; the digits visually spell `FIPS` — 7=F, 1=I, 9=P, 5=S). -The `d` tag is set to the `app` value (default `fips-overlay-v1`), so -each node has a single, in-place-updatable advert under its identity. -The event is signed with the node's FIPS identity key; there is no -separate Nostr key. A NIP-40 `expiration` tag is set to now + -`advert_ttl_secs`. - -The advert content is a JSON document shaped as `OverlayAdvert`: - -```json -{ - "identifier": "fips-overlay-v1", - "version": 1, - "endpoints": [ - {"transport": "udp", "addr": "203.0.113.45:2121"}, - {"transport": "tor", "addr": "xxxxx.onion:8443"}, - {"transport": "udp", "addr": "nat"} - ], - "signalRelays": ["wss://relay.damus.io", "wss://nos.lol"], - "stunServers": ["stun:stun.l.google.com:19302"] -} -``` - -`signalRelays` and `stunServers` are only present when at least one -endpoint is `udp:nat`; for advert shapes that cannot involve punching -they are omitted to reduce advert size and keep the relay and STUN -lists private to the nodes that need them. +The `d` tag is hardcoded to the wire-format identifier +`fips-overlay-v1` (or `fips-overlay-v1-next` on the `next` branch), +so each node has a single, in-place-updatable advert under its +identity. The configurable `app` value populates a separate +`protocol` tag, which scopes adverts within a relay set without +splitting them across multiple `d`-tag streams. The event is signed +with the node's FIPS identity key; there is no separate Nostr key. A +NIP-40 `expiration` tag is set to now + `advert_ttl_secs`, and a +`version` tag carries the protocol version. The advert content is a +JSON document shaped as `OverlayAdvert` (see +[../reference/nostr-events.md](../reference/nostr-events.md) for the +schema). Publication happens on startup, again whenever the set of advertised endpoints changes (for example, when a Tor onion hostname first @@ -438,13 +149,23 @@ deleted using a NIP-9 kind 5 delete event. Advert publication is fan-out: the same event is sent to every relay in `advert_relays` with no explicit failover — relay redundancy is implicit. +For a UDP or TCP transport with `public: true`, the address advertised +follows a fixed precedence: an operator-supplied `external_addr` wins; +otherwise a non-wildcard bound `local_addr` is used directly; +otherwise — only for UDP — the runtime asks `stun_servers` for the +reflexive address of the bound socket and advertises that. TCP has no +STUN equivalent, so wildcard-bound TCP without `external_addr` +produces a loud WARN and the endpoint is omitted from the advert. + ### Phase 2 — Lookup When the node decides to dial a peer that is eligible for Nostr resolution (a `via_nostr` peer, or any peer under `policy: open`), it issues a Nostr REQ filtered by `author = peer_pubkey`, `kind = 37195`, -`#d = `. The fetch is time-bounded (~2 s) and runs against all -configured `advert_relays` in parallel. The first valid advert wins. +`#d = fips-overlay-v1`. The fetch is time-bounded (~2 s) and runs +against all configured `advert_relays` in parallel. The first valid +advert wins; adverts whose `protocol` tag does not match the local +`app` value are rejected at validation. Results are kept in an in-memory cache keyed by author npub. Cache entries carry the advert's expiration time; a periodic prune drops @@ -477,7 +198,7 @@ The initiator performs STUN first (see Phase 4), then builds a The offer is sealed to the recipient's npub and published to the peer's preferred signaling relays — the node first tries to resolve the peer's -NIP-65 inbox relay list (kind 10002), and falls back to `dm_relays` if +NIP-17 DM relay list (kind 10050), and falls back to `dm_relays` if the inbox-relays fetch fails. Each side also publishes its own inbox relay list on startup so dialers can discover it. @@ -580,11 +301,22 @@ machinery: | Signal TTL (`signal_ttl_secs`) | 120 s | Indefinite in-flight offers on relays. | Expired offers rejected at validation. | | Open discovery queue (`open_discovery_max_pending`) | 64 | Unbounded retry queue under ambient advert load. | New candidates skipped until the queue drains. | | Punch window (`punch_duration_ms`) | 10 s | Endless probe traffic after one side has given up. | Attempt declared failed; sockets discarded. | +| Failure-streak threshold (`failure_streak_threshold`) | 5 | Repeated traversal attempts against a peer that keeps failing. | Peer enters extended cooldown. | +| Extended cooldown (`extended_cooldown_secs`) | 1800 s | Tight retry loops after a failure streak. | Per-peer suppression for the cooldown window. | +| WARN log throttle (`warn_log_interval_secs`) | 300 s | Log floods from a peer that fails on every attempt. | One WARN per peer per interval; the rest demote to debug. | +| Failure-state cap (`failure_state_max_entries`) | 4096 | Memory growth from per-peer failure tracking. | LRU eviction. | -Only one of these (`max_concurrent_incoming_offers`) is a load-shedding -mechanism — the rest are capacity bounds. The load-shedding threshold -is deliberately conservative so that a misbehaving relay cannot flood -the node with offers fast enough to starve legitimate traffic. +The load-shedding mechanisms (`max_concurrent_incoming_offers` and the +failure-streak / extended-cooldown pair) are deliberately conservative +so that a misbehaving relay cannot flood the node with offers and a +chronically unreachable peer cannot keep the traversal pipeline +saturated. The remaining rows are capacity bounds. + +Adverts also undergo a stale-advert sweep: cached entries whose +`expiresAt` has passed are evicted on the periodic prune tick. Inbound +signaling tolerates ±60 s of clock skew between sender and receiver, +and the runtime maintains an NTP-style skew estimate per remote so +that consistently-skewed relays don't trip the freshness check. ### Relay model @@ -595,12 +327,12 @@ relay selection. Redundancy is implicit — a downed relay simply means its copy of the advert or signal is unavailable, while other relays still serve the same data. -For signaling specifically, the node prefers the recipient's NIP-65 -inbox relays when available (the recipient publishes its inbox list as -a kind 10002 event to its own DM relays on startup) and falls back to +For signaling specifically, the node prefers the recipient's NIP-17 +DM relays when available (the recipient publishes its DM relay list as +a kind 10050 event to its own DM relays on startup) and falls back to the local `dm_relays` list otherwise. This keeps the common case off the sender's DM relays when those are different from the -recipient's, at the cost of one extra NIP-65 fetch per offer. +recipient's, at the cost of one extra NIP-17 fetch per offer. There is no per-relay rate limiting or health check. The relay model assumes that an operator chooses relays they trust to be best-effort @@ -646,14 +378,29 @@ semaphore and replay-cache layers downstream. ## See also -- [fips-configuration.md](fips-configuration.md) — full configuration - reference, including all surrounding keys elided from the scenarios - above. +- [../how-to/enable-nostr-discovery.md](../how-to/enable-nostr-discovery.md) + — operator activation recipes grouped under three capabilities + (resolve, advertise, open) across five scenarios. +- [../tutorials/resolve-peers-via-nostr.md](../tutorials/resolve-peers-via-nostr.md), + [../tutorials/advertise-your-node.md](../tutorials/advertise-your-node.md), + and [../tutorials/open-discovery.md](../tutorials/open-discovery.md) + — hand-held walkthroughs of the three capabilities, in + pedagogical order. +- [../reference/configuration.md](../reference/configuration.md) — full + configuration reference, including all surrounding keys elided from + the scenarios above. +- [../reference/nostr-events.md](../reference/nostr-events.md) — Kind + 37195 (overlay advert), Kind 21059 (gift-wrapped traversal + signaling), Kind 10050 (NIP-17 inbox relay list). +- [../reference/security.md](../reference/security.md) — consolidated + security reference, including how the FIPS identity key signs both + adverts and Noise handshakes. - [fips-transport-layer.md](fips-transport-layer.md) — UDP, TCP, and Tor transport mechanics; the punch socket is adopted as a normal UDP transport after handoff. - [fips-mesh-layer.md](fips-mesh-layer.md) — FMP Noise IK handshake that runs on the adopted socket. -- [../proposals/nostr-udp-hole-punch-protocol.md](../proposals/nostr-udp-hole-punch-protocol.md) - — protocol-level reference for event tags, NIP usage, and the - on-the-wire offer/answer schema. +- [port-advertisement-and-nat-traversal.md](port-advertisement-and-nat-traversal.md) + — generic protocol reference (event tags, NIP usage, on-the-wire + offer/answer schema, failure-suppression machinery), with the + FIPS-specific values called out as worked examples. diff --git a/docs/design/fips-prior-work.md b/docs/design/fips-prior-work.md new file mode 100644 index 0000000..be9195b --- /dev/null +++ b/docs/design/fips-prior-work.md @@ -0,0 +1,341 @@ +# FIPS Prior Work and References + +FIPS builds on proven designs rather than inventing new cryptography or +routing algorithms. Nearly every major design decision has deployed +precedent. This document collects the relevant prior art, organized by +the FIPS subsystem that draws on it, and gathers the academic and +standards references cited from the per-subsystem design docs. + +## Spanning Tree Self-Organization + +The idea that distributed nodes can build a spanning tree through +purely local decisions — each node selecting a parent based on +announcements from its neighbors — dates to the +[IEEE 802.1D Spanning Tree Protocol](https://en.wikipedia.org/wiki/Spanning_Tree_Protocol) +(STP, 1985). STP demonstrated that a network-wide tree emerges from a +simple deterministic rule (lowest bridge ID wins root election) +applied independently at each node. FIPS uses the same principle — +lowest node address determines the root — adapted from an Ethernet +bridging context to a general-purpose overlay mesh. + +## Tree Coordinate Routing + +The spanning tree coordinates, bloom filter candidate selection, and +greedy routing algorithms are adapted from +[Yggdrasil v0.5](https://yggdrasil-network.github.io/2023/10/22/upcoming-v05-release.html) +and its [Ironwood](https://github.com/Arceliar/ironwood) routing +library. Yggdrasil's key insight was using the tree path from root to +node as a routable coordinate, enabling greedy forwarding without +global routing tables. FIPS adapts these algorithms for +multi-transport operation, Nostr identity integration, and constrained +MTU environments. + +The theoretical foundation for greedy routing on tree embeddings draws +on [Kleinberg's work](https://www.cs.cornell.edu/home/kleinber/swn.pdf) +on navigable small-world networks, which showed that greedy forwarding +succeeds in O(log² n) steps when the network has hierarchical +structure. Thorup-Zwick compact routing schemes separately demonstrated +that sublinear routing state is achievable with bounded stretch, +motivating the use of tree coordinates rather than full routing tables. + +## Split-Horizon Bloom Filter Propagation + +FIPS distributes reachability information using bloom filters computed +with a split-horizon rule: when advertising to a peer, exclude that +peer's own contributions. This technique is borrowed from +distance-vector routing protocols — +[RIP](https://en.wikipedia.org/wiki/Routing_Information_Protocol) +(1988) and [Babel](https://www.irif.fr/~jch/software/babel/) use +split-horizon to prevent routing loops by not advertising a route back +to the neighbor it was learned from. FIPS applies the same principle +to probabilistic set advertisements rather than distance-vector tables. + +## Cryptographic Identity as Network Address + +FIPS nodes are identified by their Nostr public keys (secp256k1). The +network address *is* the cryptographic identity — there is no separate +address assignment or registration step. +[CJDNS](https://github.com/cjdelisle/cjdns) pioneered this approach in +overlay meshes, deriving IPv6 addresses from the double-SHA-512 of +each node's public key. Tor [.onion +addresses](https://spec.torproject.org/rend-spec-v3) and the IETF +[Host Identity Protocol](https://en.wikipedia.org/wiki/Host_Identity_Protocol) +(HIP) follow the same principle. FIPS uses Nostr's existing key +infrastructure rather than introducing a new identity scheme. + +## Dual-Layer Encryption + +FIPS encrypts traffic twice: FMP provides hop-by-hop link encryption +(protecting against transport-layer observers), while FSP provides +independent end-to-end session encryption (protecting against +intermediate FIPS nodes). This layered approach mirrors +[Tor](https://www.torproject.org/), where each relay peels one layer +of encryption (hop-by-hop) while the innermost layer protects +end-to-end payload. [I2P](https://geti2p.net/) uses a similar garlic +routing scheme with tunnel-layer and end-to-end encryption. Unlike Tor +and I2P, FIPS does not provide anonymity — its dual encryption +protects confidentiality and integrity rather than hiding traffic +patterns. + +## Noise Protocol Framework + +FIPS uses the [Noise Protocol Framework](https://noiseprotocol.org/) +at both protocol layers, with different handshake patterns chosen for +each layer's threat model. FMP link encryption uses **Noise IK**, +providing mutual authentication with a single round trip where the +initiator knows the responder's static key in advance. +[WireGuard](https://www.wireguard.com/) uses the same IK base pattern +(extended with a pre-shared key as IKpsk2) for VPN tunnels. FSP +session encryption uses **Noise XK**, the same pattern used by the +[Lightning Network](https://github.com/lightning/bolts/blob/master/08-transport.md), +where the initiator's static key is transmitted in a third message +rather than the first. XK provides stronger initiator identity hiding +at the cost of an additional round trip — a worthwhile tradeoff for +session-layer traffic that traverses untrusted intermediate nodes. At +the link layer, where both peers are configured and directly +connected, IK's single round trip is preferred. + +Specific Noise references and adapted constructions: + +- Perrin, T. ["The Noise Protocol Framework"](https://noiseprotocol.org/noise.html). + Revision 34, 2018. *Framework for building crypto protocols using + Diffie-Hellman key agreement and AEAD ciphers. FSP uses the XK + handshake pattern.* + +- Donenfeld, J.A. ["WireGuard: Next Generation Kernel Network Tunnel"](https://www.wireguard.com/papers/wireguard.pdf). + NDSS 2017. *Transport-independent cryptographic sessions bound to + identity keys rather than network addresses; AEAD-only authentication + model.* + +## Index-Based Session Dispatch + +FIPS uses locally-assigned 32-bit session indices to demultiplex +incoming packets to the correct cryptographic session in O(1) time, +without parsing source addresses or performing expensive lookups. +This directly follows +[WireGuard's](https://www.wireguard.com/papers/wireguard.pdf) receiver +index approach, where each peer assigns a random index during +handshake and the remote side includes it in every packet header. + +## Replay Protection Over Unreliable Transports + +FSP and FMP both use explicit per-packet counters with a sliding +bitmap window for replay protection — the standard DTLS approach, +chosen because implicit nonce counters desynchronize permanently under +UDP packet loss or reordering. + +- Rescorla, E., Modadugu, N. [RFC 6347](https://datatracker.ietf.org/doc/html/rfc6347): + "Datagram Transport Layer Security Version 1.2". 2012. *Explicit + sequence numbers with sliding bitmap window for replay protection + over unreliable transports.* + +## Transport-Agnostic Overlay Mesh + +FIPS is designed to operate over any datagram-capable transport — UDP, +raw Ethernet, Bluetooth, radio, serial — through a uniform transport +abstraction. Several mesh overlays have demonstrated transport-agnostic +design: [CJDNS](https://github.com/cjdelisle/cjdns) runs over UDP and +Ethernet, [Yggdrasil](https://yggdrasil-network.github.io/) supports +TCP and TLS transports, and [Tor](https://www.torproject.org/) can use +pluggable transports to tunnel through various media. FIPS extends +this pattern to shared-medium transports (radio, BLE) with +per-transport MTU and discovery capabilities. + +## Metrics Measurement Protocol + +MMP's design assembles well-established measurement techniques into a +unified per-link protocol. The SenderReport/ReceiverReport exchange +structure follows [RTCP](https://www.rfc-editor.org/rfc/rfc3550) +(RFC 3550), which uses the same report pairing for media stream +quality monitoring in RTP sessions. MMP's jitter computation uses the +RTCP interarrival jitter algorithm directly. + +The smoothed RTT estimator uses the Jacobson/Karels algorithm +([RFC 6298](https://www.rfc-editor.org/rfc/rfc6298)), the same SRTT +computation used in TCP for retransmission timeout calculation since +1988. MMP derives RTT from timestamp-echo in ReceiverReports with +dwell-time compensation, rather than from packet round-trips. + +The spin bit in the FMP frame header follows the +[QUIC](https://www.rfc-editor.org/rfc/rfc9000) spin bit +([RFC 9312](https://www.rfc-editor.org/rfc/rfc9312)) — a single bit +that alternates each round trip, enabling passive latency measurement. +FIPS implements the spin bit state machine but relies on +timestamp-echo for SRTT, as irregular mesh traffic makes spin bit RTT +unreliable. + +The Expected Transmission Count (ETX) metric, computed from +bidirectional delivery ratios, was introduced by +[De Couto et al. (2003)](https://pdos.csail.mit.edu/papers/grid:mobicom03/paper.pdf) +for wireless mesh routing and is used in protocols including +[OLSR](https://en.wikipedia.org/wiki/Optimized_Link_State_Routing_Protocol) +and [Babel](https://www.irif.fr/~jch/software/babel/). FIPS computes +ETX per-link from MMP loss measurements for future use in candidate +ranking. + +The CE (Congestion Experienced) echo flag provides hop-by-hop +[ECN](https://en.wikipedia.org/wiki/Explicit_Congestion_Notification) +signaling, following the TCP/IP ECN echo pattern (RFC 3168). Transit +nodes detect congestion via MMP loss/ETX metrics or kernel buffer +drops and set the CE flag on forwarded frames; destination nodes mark +ECN-capable IPv6 packets accordingly. + +## Path MTU Discovery + +FSP adapts RFC 1191 Path MTU Discovery for overlay networks. The +classic ICMP Packet Too Big mechanism is replaced by a transit-node +`min()` propagation in SessionDatagram and LookupResponse plus an +end-to-end PathMtuNotification echo back to the source. + +- Mogul, J., Deering, S. [RFC 1191](https://datatracker.ietf.org/doc/html/rfc1191): + "Path MTU Discovery". 1990. *End-to-end path MTU discovery; FSP + adapts this for overlay networks using transit-node min() + propagation.* + +## Session Restart and Simultaneous Initiation + +FSP's epoch-based peer restart detection mirrors IKEv2's +INITIAL_CONTACT notification, and its lowest-address-wins +simultaneous-initiation tie-breaker mirrors IKEv2's resolution rule. + +- Kaufman, C., Hoffman, P., Nir, Y., Eronen, P., Kivinen, T. + [RFC 7296](https://datatracker.ietf.org/doc/html/rfc7296): + "Internet Key Exchange Protocol Version 2 (IKEv2)". 2014. + *Simultaneous initiation resolution (§2.8) and INITIAL_CONTACT peer + restart detection (§2.4).* + +## Hybrid Coordinate Warmup + +FSP's hybrid coordinate warmup (CP flag piggybacking + standalone +CoordsWarmup) draws on Yggdrasil's approach of embedding coordinates +in session traffic to keep transit caches populated. + +- [Yggdrasil Network](https://yggdrasil-network.github.io/). + *Coordinate-based overlay routing with session traffic used to warm + transit node coordinate caches.* + +## Cryptographic Primitives + +FIPS reuses [Nostr's](https://github.com/nostr-protocol/nips) +cryptographic stack — secp256k1 for identity keys, Schnorr signatures +for authentication, SHA-256 for hashing, and ChaCha20-Poly1305 for +authenticated encryption. This is the same primitive set used across +Bitcoin, Nostr, and a growing ecosystem of self-sovereign identity +systems. No novel cryptography is introduced. + +## Spanning-Tree Dynamics: Foundations + +The CRDT framing, gossip dissemination, failure detection, link +metrics, and route stability mechanisms in +[spanning-tree-dynamics.md](spanning-tree-dynamics.md) draw on a body +of academic and standards work, summarized below. + +### Virtual Coordinate Routing + +- Rao, A., Ratnasamy, S., Papadimitriou, C., Shenker, S., Stoica, I. + ["Geographic Routing without Location Information"](https://people.eecs.berkeley.edu/~sylvia/papers/p327-rao.pdf). + MobiCom 2003. *Established virtual coordinate routing using network + topology.* + +### Greedy Embedding Theory + +- Kleinberg, R. + ["Geographic Routing Using Hyperbolic Space"](https://www.semanticscholar.org/paper/Geographic-Routing-Using-Hyperbolic-Space-Kleinberg/f506b2ddb142d2ec539400297ba53383d958abef). + IEEE INFOCOM 2007. *Proved every connected graph has a greedy + embedding in hyperbolic space; showed spanning trees enable + coordinate assignment.* + +- Cvetkovski, A., Crovella, M. + ["Hyperbolic Embedding and Routing for Dynamic Graphs"](https://www.cs.bu.edu/faculty/crovella/paper-archive/infocom09-hyperbolic.pdf). + IEEE INFOCOM 2009. *Dynamic embedding for nodes joining/leaving; + introduced Gravity-Pressure routing for failure recovery.* + +- Crovella, M. et al. + ["On the Choice of a Spanning Tree for Greedy Embedding"](https://www.cs.bu.edu/faculty/crovella/paper-archive/networking-science13.pdf). + Networking Science 2013. *Analysis of how tree structure affects + routing stretch.* + +- Bläsius, T. et al. + ["Hyperbolic Embeddings for Near-Optimal Greedy Routing"](https://dl.acm.org/doi/10.1145/3381751). + ACM Journal of Experimental Algorithmics 2020. *Achieved 100% + success ratio with 6% stretch on Internet graph.* + +### Link Metrics + +- De Couto, D., Aguayo, D., Bicket, J., Morris, R. + "A High-Throughput Path Metric for Multi-Hop Wireless Routing". + MobiCom 2003. *Introduced ETX (Expected Transmission Count) as a + link quality metric for wireless mesh networks.* + +### Routing Protocol Stability + +- IEEE 802.1D. "IEEE Standard for Local and Metropolitan Area + Networks: Media Access Control (MAC) Bridges". *Spanning Tree + Protocol (STP) — root election via bridge ID, BPDU exchange.* + +- Moy, J. [RFC 2328](https://datatracker.ietf.org/doc/html/rfc2328): + "OSPF Version 2". 1998. *Link-state routing with cumulative path + costs and SPF computation. FIPS's local-only cost approach is + contrasted with OSPF's cumulative model in + [spanning-tree-dynamics.md §8](spanning-tree-dynamics.md#8-parent-selection).* + +### Distributed Systems Primitives + +- Shapiro, M., Preguiça, N., Baquero, C., Zawirski, M. + "Conflict-free Replicated Data Types". SSS 2011. *Formal definition + of CRDTs enabling coordination-free consistency.* + +- Das, A., Gupta, I., Motivala, A. + ["SWIM: Scalable Weakly-consistent Infection-style Process Group Membership"](https://www.cs.cornell.edu/projects/Quicksilver/public_pdfs/SWIM.pdf). + IPDPS 2002. *O(1) failure detection, O(log N) dissemination via + gossip.* + +- Kermarrec, A-M. + ["Gossiping in Distributed Systems"](https://www.distributed-systems.net/my-data/papers/2007.osr.pdf). + ACM SIGOPS Operating Systems Review 2007. *Framework for + gossip-based protocols achieving O(log N) propagation.* + +## FIPS Contributions + +The protocol builds on these foundations and adds several new elements: + +- Cost-aware parent selection using local-only link metrics + (`effective_depth = depth + link_cost`), replacing Yggdrasil's + depth-only selection +- Combined ETX + SRTT link cost formula with MMP-measured components +- Flap dampening with mandatory switch bypass +- Announcement suppression for transient state changes +- Tree-only bloom filter merge with split-horizon exclusion +- Hybrid coordinate warmup (CP flag piggybacking plus standalone + CoordsWarmup) layered on top of SessionSetup self-bootstrapping +- Bloom-guided tree routing for discovery (vs. flooding) +- Reverse-path routing for LookupResponse via `recent_requests` + +## External Reference Index + +| Reference | Used by | +| --------- | ------- | +| [IEEE 802.1D STP](https://en.wikipedia.org/wiki/Spanning_Tree_Protocol) | spanning tree, root election | +| [Yggdrasil v0.5](https://yggdrasil-network.github.io/2023/10/22/upcoming-v05-release.html) | tree coordinates, greedy routing | +| [Ironwood](https://github.com/Arceliar/ironwood) | tree coordinates, candidate ranking | +| [Kleinberg, Small-world](https://www.cs.cornell.edu/home/kleinber/swn.pdf) | greedy routing on tree embeddings | +| [CJDNS](https://github.com/cjdelisle/cjdns) | cryptographic-identity-as-address | +| [Tor](https://www.torproject.org/) | onion address scheme, dual-layer encryption | +| [I2P](https://geti2p.net/) | dual-layer encryption (garlic routing) | +| [HIP](https://en.wikipedia.org/wiki/Host_Identity_Protocol) | identity-as-address | +| [Babel](https://www.irif.fr/~jch/software/babel/) | split-horizon, ETX | +| [RIP](https://en.wikipedia.org/wiki/Routing_Information_Protocol) | split-horizon | +| [Noise Framework](https://noiseprotocol.org/) | FMP IK, FSP XK | +| [WireGuard](https://www.wireguard.com/) | IK pattern, receiver-index dispatch, identity-bound sessions | +| [Lightning BOLT #8](https://github.com/lightning/bolts/blob/master/08-transport.md) | XK pattern | +| [QUIC (RFC 9000)](https://www.rfc-editor.org/rfc/rfc9000) | spin bit, transport design | +| [QUIC Spin Bit (RFC 9312)](https://www.rfc-editor.org/rfc/rfc9312) | passive RTT measurement | +| [RTCP (RFC 3550)](https://www.rfc-editor.org/rfc/rfc3550) | sender/receiver report structure, jitter algorithm | +| [TCP SRTT/RTO (RFC 6298)](https://www.rfc-editor.org/rfc/rfc6298) | Jacobson/Karels SRTT | +| [ECN (RFC 3168)](https://www.rfc-editor.org/rfc/rfc3168) | CE echo | +| [DTLS 1.2 (RFC 6347)](https://datatracker.ietf.org/doc/html/rfc6347) | replay window | +| [IKEv2 (RFC 7296)](https://datatracker.ietf.org/doc/html/rfc7296) | INITIAL_CONTACT, simultaneous-initiation tie-breaker | +| [PMTUD (RFC 1191)](https://datatracker.ietf.org/doc/html/rfc1191) | adapted PMTUD | +| [ETX paper, De Couto et al.](https://pdos.csail.mit.edu/papers/grid:mobicom03/paper.pdf) | ETX metric | +| [OLSR](https://en.wikipedia.org/wiki/Optimized_Link_State_Routing_Protocol) | ETX in mesh routing | +| [Nostr](https://github.com/nostr-protocol/nips) | identity stack | diff --git a/docs/fips-security.md b/docs/design/fips-security.md similarity index 54% rename from docs/fips-security.md rename to docs/design/fips-security.md index 23f9a89..b8de87d 100644 --- a/docs/fips-security.md +++ b/docs/design/fips-security.md @@ -1,39 +1,43 @@ # FIPS Mesh-Interface Security -This document describes the operator-facing security posture of the -`fips0` mesh interface on Linux: the threat model, the default-deny -nftables baseline shipped as `/etc/fips/fips.nft`, how to enable it, -and how to extend it with per-host allowances. +This document describes the threat model and design rationale for the +operator-facing security posture of the `fips0` mesh interface on Linux. +The default-deny nftables baseline shipped as `/etc/fips/fips.nft` is the +artifact discussed below; for the operator activation steps and drop-in +extension recipes, see [enable-mesh-firewall.md](../how-to/enable-mesh-firewall.md). The baseline is a documented operator conffile, not an auto-loaded package side-effect. Activation is an explicit one-liner. The -rationale for that design and the operator workflow follow. +rationale for that design follows. ## Threat Model for `fips0` -The mesh is a flat layer-3 segment. Every authenticated peer on the -mesh can route packets to every other peer's `fips0` address. Identity -on the mesh is the peer's npub — the FMP link layer authenticates that -identity with Noise IK and the FSP session layer authenticates +The mesh is a flat layer-3 segment. Every mesh node that can route to +you can deliver packets to your `fips0` address — your direct peers +forward traffic from non-peer mesh nodes onto your `fips0` the same +way any router forwards transit traffic. Identity on the mesh is the +originating node's npub — the FMP link layer authenticates direct +peers with Noise IK and the FSP session layer authenticates session endpoints with Noise XK — but identity is **not** authorization. Knowing who sent a packet does not, by itself, decide whether the local host should accept it. That means: any service on a mesh host that binds to a wildcard address (`0.0.0.0`, `[::]`, or any IPv6 address that includes the -`fips0` interface in its scope) is reachable from every peer in the -mesh by default. There is no NAT, no perimeter firewall, no -"local-only" address space between you and an arbitrary peer. The -mesh is closer to a shared LAN than to the public internet. +`fips0` interface in its scope) is reachable from every mesh node +that can route to you by default, not only from your direct peers. +There is no NAT, no perimeter firewall, no "local-only" address +space between you and an arbitrary mesh node. The mesh is closer +to a shared LAN than to the public internet. Compare to the corresponding internet trust assumptions: | Surface | Public internet | FIPS mesh (no baseline) | |---|---|---| -| Reachability from arbitrary peer | Mediated by NAT, firewalls, ISPs | Direct | -| Default identity | None | Peer npub (authenticated) | +| Reachability from arbitrary mesh node | Mediated by NAT, firewalls, ISPs | Direct | +| Default identity | None | Originating node's npub (authenticated) | | Default authorization | None | None | -| Accidental exposure cost | Low (NAT hides you) | High (every peer sees you) | +| Accidental exposure cost | Low (NAT hides you) | High (every mesh node sees you) | The third row is the gap this document closes. The default-deny baseline removes "accidental exposure" from the failure modes an @@ -70,36 +74,7 @@ operator on upgrade rather than silently overwriting local changes. The canonical artifact is the file itself; read it for the inline documentation that the rest of this document references. -## Loading the Baseline - -The package ships `fips-firewall.service`, a systemd oneshot unit -that runs `nft -f /etc/fips/fips.nft` on start and removes the -`inet fips` table on stop. It is **not** enabled by default. To -activate the baseline: - -```sh -sudo systemctl enable --now fips-firewall.service -``` - -This loads the table now and arranges for it to load on every -subsequent boot. To disable and tear it down: - -```sh -sudo systemctl disable --now fips-firewall.service -``` - -To reload after editing `/etc/fips/fips.nft` or adding a drop-in -under `/etc/fips/fips.d/`: - -```sh -sudo systemctl reload-or-restart fips-firewall.service -``` - -(or equivalently `sudo nft -f /etc/fips/fips.nft`, since the file is -idempotent — it begins with `add table inet fips; flush table inet -fips;` so re-running it replaces the live ruleset atomically.) - -### Why no auto-load on package install +## Why no auto-load on package install The `postinst` script does **not** enable `fips-firewall.service`. This is deliberate. Quietly mutating host firewall state on package @@ -115,7 +90,7 @@ rationale is documented in the file's inline header and in this document. That is enough; auto-loading would trade discoverability for no real gain. -### Coexistence with other firewalls +## Coexistence with other firewalls The `inet fips` table only matches packets arriving on `fips0`. Anything else returns from the chain on the first rule. Specifically: @@ -140,128 +115,6 @@ Anything else returns from the chain on the first rule. Specifically: mesh addresses. It is a separate concern owned by the gateway binary and is unrelated to this baseline. See the section below. -If you prefer to fold the baseline into your existing -`/etc/nftables.conf` instead of using the systemd unit, you can: - -```nft -# in /etc/nftables.conf -include "/etc/fips/fips.nft" -``` - -In that case do not enable `fips-firewall.service` — let the host's -main nftables setup own the loading. The two are mutually exclusive. - -## Operator Extension via `/etc/fips/fips.d/*.nft` - -The baseline drops everything inbound on `fips0` except conntrack -replies and ICMPv6 echo. To open specific services to specific peers, -drop a file into `/etc/fips/fips.d/` ending in `.nft`. Each file is -included inline into the `inbound` chain at the marked point and may -contain any nftables rule lines valid in that context. - -Reload after editing: - -```sh -sudo systemctl reload-or-restart fips-firewall.service -# or: sudo nft -f /etc/fips/fips.nft -``` - -Common patterns follow. - -### Allow inbound SSH from a specific peer - -```nft -# /etc/fips/fips.d/ssh-from-bastion.nft -ip6 saddr fd97:1234:5678:9abc:def0:1234:5678:9abc tcp dport 22 accept -``` - -The source filter is the peer's mesh address. To find a peer's mesh -address, look in their `fips.pub` (which contains the npub) and -derive the `fd97:...` address from it, or query the running daemon: - -```sh -fipsctl show identity-cache -fipsctl show peers -``` - -### Allow inbound HTTP from one /64 of peers - -If you operate a logical group of peers under a shared address -prefix, source-filter by the prefix: - -```nft -# /etc/fips/fips.d/http-from-cluster.nft -ip6 saddr fd97:1234:5678:9abc::/64 tcp dport 80 accept -``` - -The mesh address space is `fd00::/8`, so `/64` filters carve out -manageable subgroups. Plan your prefixes before deploying many peers. - -### Allow inbound DNS broadly - -Some services need to be reachable from any mesh peer (a public DNS -resolver, a public bootstrap node): - -```nft -# /etc/fips/fips.d/dns-public.nft -udp dport 53 accept -tcp dport 53 accept -``` - -Omit the source filter only when the service is intended to be -universally reachable on the mesh. The baseline's purpose is to make -"universally reachable" an explicit decision rather than the default. - -### Multiple peers, one service - -```nft -# /etc/fips/fips.d/git-from-trusted.nft -ip6 saddr { - fd97:1111:2222:3333:4444:5555:6666:7777, - fd97:8888:9999:aaaa:bbbb:cccc:dddd:eeee -} tcp dport 9418 accept -``` - -Set syntax keeps multi-peer rules readable and is more efficient than -a chain of individual rules. - -## Drop Visibility and Debugging - -The baseline counter increments on every dropped packet. Inspect it: - -```sh -sudo nft list table inet fips -``` - -Look for the `counter packets N bytes M drop` line at the bottom of -the `inbound` chain. A non-zero counter means peers are sending -traffic that hits the default-deny — usually benign (probes, -neighbor discovery) but occasionally a misconfigured drop-in. - -To see which packets are being dropped, uncomment the `log` line -near the bottom of `/etc/fips/fips.nft`: - -```nft -log prefix "fips drop: " level info limit rate 10/minute -``` - -Reload: - -```sh -sudo nft -f /etc/fips/fips.nft -``` - -Then tail the kernel log: - -```sh -sudo journalctl -k -f -g "fips drop:" -``` - -The rate-limit prevents flooding the journal under sustained probing. -Adjust the rate, log level, or prefix as needed for the situation. -Re-comment the rule when you are done; production hosts do not need -the log line on by default. - ## Coexistence with `inet fips_gateway` When `fips-gateway` is running, it manages a separate nftables @@ -279,8 +132,8 @@ The two tables do not interfere: They operate on different interfaces and at different hook points (`input` filter vs. `prerouting`/`postrouting` NAT). Both can be loaded simultaneously on a gateway host, and that is the intended -deployment shape. See `docs/design/fips-gateway.md` for the gateway -table's structure. +deployment shape. See [fips-gateway.md](fips-gateway.md) for the +gateway table's structure. ## What the Baseline Does Not Cover @@ -292,20 +145,28 @@ explicitly not: can send to the mesh, add rules to a separate chain hooked at `output` — out of scope for the baseline. - **Application-layer authorization.** The baseline decides whether - a packet reaches a service. It does not decide whether the peer - npub on the other end is allowed to use that service. That is the - application's responsibility (e.g., an `authorized_keys` file for - SSH, an ACL in the application's configuration). -- **ACL on the mesh handshake.** The FMP Noise IK handshake currently - authenticates the peer's npub but does not consult an allowlist - before establishing a link. A peer with a known npub can connect - and become a routing peer regardless of operator intent. Mesh- - level ACLs are tracked under IDEA-0047 / PR #50 and are a separate - concern from the inbound packet filter described here. + a packet reaches a service. It does not decide whether the + originating mesh node's npub is allowed to use that service. That + is the application's responsibility (e.g., an `authorized_keys` + file for SSH, an ACL in the application's configuration). +- **ACL on the mesh handshake.** The FMP Noise IK handshake + authenticates the peer's npub and, on both inbound and outbound + paths, consults the peer ACL (`peers.allow` / `peers.deny`) before + promoting the connection. The ACL evaluates in TCP-Wrappers order: + an `allow` match permits, otherwise a `deny` match rejects, + otherwise the connection is permitted. A strict allowlist posture + therefore requires an explicit `ALL` entry in `peers.deny`; a + populated `peers.allow` alone does not turn the ACL into a strict + allowlist. Mesh-level ACLs are a separate concern from the inbound + packet filter described here; see the peer ACL section in + [../reference/security.md](../reference/security.md). - **Compromised peers.** A peer whose key has been stolen or whose host has been taken over is, by mesh-level identity, still that - peer. Source-address filtering in drop-ins limits damage, but the - baseline cannot revoke trust on its own. + peer. Source-address filtering in drop-ins operates on the source + mesh address of inbound traffic regardless of whether that source + is a direct peer or a multi-hop mesh node, and so can limit damage + from a known-compromised mesh address; but the baseline cannot + revoke trust on its own. Treat the baseline as removing the "wide-open by default" failure mode. Higher-layer authorization decisions are the operator's and @@ -336,9 +197,19 @@ The current baseline is Linux-only. Parallel work for other targets: netlink API directly. macOS gateway support requires a PF-backed equivalent behind a shared backend trait. This is a larger lift than the static baseline and is tracked separately under the same - IDEA thread. + cross-OS thread. When those land, this document will grow per-OS sections describing each baseline's load mechanism and extension points. The threat model and the operator-extension principle are the same on every OS; only the filter syntax and the activation gesture differ. + +## See also + +- [enable-mesh-firewall.md](../how-to/enable-mesh-firewall.md) — operator + activation steps, drop-in recipes, drop visibility and debugging +- [../reference/security.md](../reference/security.md) — consolidated + security reference (cryptographic primitives, peer ACL format, + filesystem permissions, default network exposures) +- [fips-gateway.md](fips-gateway.md) — `fips-gateway` service and the + separate `inet fips_gateway` table diff --git a/docs/design/fips-session-layer.md b/docs/design/fips-session-layer.md index 7892032..faa37ee 100644 --- a/docs/design/fips-session-layer.md +++ b/docs/design/fips-session-layer.md @@ -1,10 +1,10 @@ # FIPS Session Protocol (FSP) -The FIPS Session Protocol is the top protocol layer in the FIPS stack. It sits -above the FIPS Mesh Protocol (FMP) and below applications (native FIPS API or -IPv6 adapter). FSP provides end-to-end authenticated, encrypted datagram -delivery between any two FIPS nodes, regardless of how many intermediate hops -separate them. +The FIPS Session Protocol is the topmost layer of the FIPS protocol stack. +It sits above the FIPS Mesh Protocol (FMP) and below applications (native +FIPS API or IPv6 adapter). FSP provides end-to-end authenticated, encrypted +datagram delivery between any two FIPS nodes, regardless of how many +intermediate hops separate them. ## Role @@ -99,9 +99,9 @@ FMP signals routing failures asynchronously: a standalone CoordsWarmup (rate-limited), re-discovering the destination's current coordinates, and resetting the warmup counter. - **MtuExceeded**: A transit node cannot forward a SessionDatagram because - the packet exceeds the next-hop link MTU. FSP uses the reported bottleneck - MTU to adjust its session-layer path MTU estimate. MtuExceeded is the - reactive complement to the proactive `path_mtu` field in SessionDatagram. + the packet exceeds the next-hop link MTU; FSP adjusts its + session-layer path MTU estimate from the reported bottleneck. See + [fips-mtu.md](fips-mtu.md) for the full forward/reverse MTU model. All three signals are generated by transit nodes (not the destination) and travel back to the source inside a new SessionDatagram. They are plaintext @@ -123,9 +123,10 @@ a destination with no existing session. FSP uses Noise XK for session key agreement (Noise Protocol Framework; Perrin 2018). The initiator knows the destination's npub (required for XK's pre-message `s` token); the responder learns the initiator's -identity from msg3 (not msg1, unlike IK at the link layer). This provides stronger initiator identity hiding -— the initiator's static key is encrypted under the established shared -secret rather than under only the responder's static key. +identity from msg3 (not msg1, unlike IK at the link layer). This +provides stronger initiator identity hiding — the initiator's static +key is encrypted under the established shared secret rather than under +only the responder's static key. The handshake is a three-message flow carried in SessionSetup, SessionAck, and SessionMsg3: @@ -244,17 +245,9 @@ operations) rather than under only the responder's static key. ### Cryptographic Primitives -| Component | Choice | Notes | -| --------- | ------ | ----- | -| Curve | secp256k1 | Nostr-native | -| DH | ECDH on secp256k1 | Standard EC Diffie-Hellman | -| Cipher | ChaCha20-Poly1305 | AEAD, same as NIP-44 | -| Hash | SHA-256 | Nostr-native | -| Key derivation | HKDF-SHA256 | Standard Noise KDF | - -These choices prioritize compatibility with the Nostr cryptographic stack — -secp256k1 + ChaCha20-Poly1305 + SHA-256 aligns with the NIP-44 encrypted -messaging standard. +FSP uses ChaCha20-Poly1305 with secp256k1 ECDH; see +[../reference/security.md](../reference/security.md) for the full +primitive table shared with the link layer. ### secp256k1 Parity Normalization @@ -391,22 +384,10 @@ signal generation (100ms per destination). ## Identity Cache -The identity cache maps FIPS address prefix (15 bytes, the `fd00::/8` IPv6 -address minus the `fd` prefix) to `(NodeAddr, PublicKey)`. This cache is -needed only when using the IPv6 adapter — the native FIPS API provides the -public key directly. - -The mapping is deterministic (derived from the public key via SHA-256) and -never becomes stale. The cache uses LRU-only eviction bounded by a -configurable size (default 10K entries). There is no TTL — entries are evicted -only when the cache is full and space is needed for a new entry. - -Cache population mechanisms: - -- **DNS lookup**: The primary path. Resolving `npub1xxx...xxx.fips` derives - the IPv6 address and populates the identity cache. -- **Inbound traffic**: Authenticated sessions from other nodes populate the - cache with their identity information. +The IPv6 adapter requires an identity cache to map `fd00::/8` addresses +back to `(NodeAddr, PublicKey)` for routing; see +[fips-ipv6-adapter.md](fips-ipv6-adapter.md#identity-cache) for the +cache rationale, eviction policy, and population mechanics. ## Coordinate Cache @@ -451,84 +432,30 @@ node caches (still within their 300s TTL) are re-warmed. ## Session-Layer MMP -Each established session runs its own Metrics Measurement Protocol instance, -providing end-to-end quality metrics independent of the number of hops. +FSP runs an MMP instance per established session for end-to-end metrics +independent of hop count. Reports are encrypted and forwarded through +every transit link, so bandwidth cost is proportional to path length; +the session-layer report intervals are correspondingly higher than the +link-layer intervals (clamped to `[500ms, 10s]` vs. `[1s, 5s]`). -### Relationship to Link-Layer MMP +The session-layer instance shares its algorithms (SRTT, jitter, loss, +ETX) and report wire format with link-layer MMP. The differences — +configuration namespace (`node.session_mmp.*`), routing scope, +send-failure backoff, idle-timeout interaction, and the +PathMtuNotification mechanism — are documented in the unified MMP +treatment at [fips-mmp.md](fips-mmp.md). For the end-to-end path-MTU +echo specifically, see [fips-mtu.md](fips-mtu.md). Reports and +PathMtuNotification do **not** reset the session idle timer, so a +session carrying only MMP traffic still tears down at the configured +idle threshold. -Session-layer MMP uses the same report wire format (SenderReport 0x11, -ReceiverReport 0x12) and identical algorithms as link-layer MMP, but with -two key differences: +### MtuExceeded Handling -1. **End-to-end routing**: Session reports are encrypted and forwarded through - every transit link. A 3-hop session generates report traffic on all 3 links, - making bandwidth cost proportional to path length. -2. **Independent configuration**: The `node.session_mmp.*` parameters are - separate from `node.mmp.*`, allowing operators to run a lighter mode for - sessions (e.g., Lightweight) while keeping Full mode on links. - -### Metrics - -The same metrics as link-layer MMP: SRTT, loss rate, jitter, goodput, OWD -trend, ETX, and dual EWMA trends. Session MMP additionally tracks observed -path MTU. - -### Report Intervals - -Session-layer report intervals are higher than link-layer to account for -bandwidth cost: clamped to [500ms, 10s] with a cold-start interval of 1s -(vs. link-layer [100ms, 2s] with 500ms cold-start). - -### Path MTU Tracking - -PathMtuNotification (message type 0x13) provides end-to-end path MTU -feedback, adapting RFC 1191 Path MTU Discovery for overlay networks — the -transit-node `min()` propagation replaces ICMP Packet Too Big: - -1. The source sets `path_mtu` in each SessionDatagram envelope to its - outbound link MTU. -2. Each transit node applies `min(current, transport.link_mtu(addr))` before - forwarding. -3. The destination receives the forward-path minimum and sends a - PathMtuNotification (2-byte body: u16 LE path_mtu) back to the source. -4. The source applies the notification with hysteresis: - - **Decrease**: immediate (take lower value). - - **Increase**: requires 3 consecutive higher-value notifications spanning - at least 2 × notification interval. -5. Notifications are sent on first measurement, on any decrease, and - periodically at `max(10s, 5 × SRTT)`. - -### Send Failure Backoff - -When a session MMP report cannot be delivered (destination unreachable, no -route), the sender applies exponential backoff to the probe interval (a -standard distributed systems pattern for transient failure handling): - -- Each consecutive failure doubles the interval: 2x, 4x, 8x, 16x, 32x -- Backoff caps at 32x the base interval (5 consecutive failures) -- A successful send resets to the normal SRTT-based interval -- Debug logging is suppressed after 3 consecutive failures; a summary is - logged when the destination becomes reachable again - -This prevents wasted CPU and log noise when a session's remote endpoint has -departed the network but the local session has not yet timed out. - -### Idle Timeout Interaction - -MMP reports (SenderReport, ReceiverReport) and PathMtuNotification do **not** -reset the session idle timer. Only application data (DataPacket, type 0x10) -resets `last_activity`. This ensures sessions with no application traffic -tear down after `node.session.idle_timeout_secs` (default 90s), while MMP -continues providing measurement data up to the teardown moment. - -### Operator Logging - -Session metrics are logged at info level (configurable via -`node.session_mmp.log_interval_secs`, default 30s): - -```text -MMP session metrics session=npub1tdwa...84le rtt=4.3ms loss=0.6% jitter=0.2ms goodput=71.3MB/s mtu=1472 tx_pkts=1234 rx_pkts=5678 -``` +When FMP signals MtuExceeded (a transit node could not forward a +SessionDatagram because it exceeded the next-hop link MTU), FSP uses +the reported bottleneck MTU to adjust its session-layer path MTU +estimate immediately. See [fips-mtu.md](fips-mtu.md) for the full +reactive PMTUD mechanism. ## Implementation Status @@ -559,36 +486,27 @@ MMP session metrics session=npub1tdwa...84le rtt=4.3ms loss=0.6% jitter=0.2ms go ### FIPS Internal Documentation -- [fips-intro.md](fips-intro.md) — Protocol overview and architecture +- [fips-concepts.md](fips-concepts.md) — Protocol overview +- [fips-architecture.md](fips-architecture.md) — Layer architecture and + identity model - [fips-mesh-layer.md](fips-mesh-layer.md) — FMP specification (below FSP) -- [fips-ipv6-adapter.md](fips-ipv6-adapter.md) — IPv6 adaptation layer (above FSP) -- [fips-mesh-operation.md](fips-mesh-operation.md) — Routing, discovery, and - error recovery -- [fips-wire-formats.md](fips-wire-formats.md) — Wire format reference for all - session message types +- [fips-ipv6-adapter.md](fips-ipv6-adapter.md) — IPv6 adaptation layer + (above FSP) +- [fips-mesh-operation.md](fips-mesh-operation.md) — Routing, discovery, + and error recovery +- [fips-mmp.md](fips-mmp.md) — Metrics Measurement Protocol (link + session) +- [fips-mtu.md](fips-mtu.md) — Path MTU model (PathMtuNotification, + MtuExceeded, hysteresis) +- [fips-prior-work.md](fips-prior-work.md) — Noise XK, WireGuard, + DTLS replay window, IKEv2 simultaneous initiation, hybrid coordinate + warmup citations +- [../reference/wire-formats.md](../reference/wire-formats.md) — Wire + format reference for all session message types +- [../reference/security.md](../reference/security.md) — Cryptographic + primitives and rekey defaults ### External References -- Perrin, T. ["The Noise Protocol Framework"](https://noiseprotocol.org/noise.html). - Revision 34, 2018. *Framework for building crypto protocols using Diffie-Hellman - key agreement and AEAD ciphers. FSP uses the XK handshake pattern.* - -- Donenfeld, J.A. ["WireGuard: Next Generation Kernel Network Tunnel"](https://www.wireguard.com/papers/wireguard.pdf). - NDSS 2017. *Transport-independent cryptographic sessions bound to identity keys - rather than network addresses; AEAD-only authentication model.* - -- Rescorla, E., Modadugu, N. [RFC 6347](https://datatracker.ietf.org/doc/html/rfc6347): - "Datagram Transport Layer Security Version 1.2". 2012. *Explicit sequence numbers - with sliding bitmap window for replay protection over unreliable transports.* - -- Kaufman, C., Hoffman, P., Nir, Y., Eronen, P., Kivinen, T. - [RFC 7296](https://datatracker.ietf.org/doc/html/rfc7296): - "Internet Key Exchange Protocol Version 2 (IKEv2)". 2014. *Simultaneous - initiation resolution (§2.8) and INITIAL_CONTACT peer restart detection (§2.4).* - - Mogul, J., Deering, S. [RFC 1191](https://datatracker.ietf.org/doc/html/rfc1191): "Path MTU Discovery". 1990. *End-to-end path MTU discovery; FSP adapts this for overlay networks using transit-node min() propagation.* - -- [Yggdrasil Network](https://yggdrasil-network.github.io/). *Coordinate-based - overlay routing with session traffic used to warm transit node coordinate caches.* diff --git a/docs/design/fips-spanning-tree.md b/docs/design/fips-spanning-tree.md index 72d68f9..c7b5d11 100644 --- a/docs/design/fips-spanning-tree.md +++ b/docs/design/fips-spanning-tree.md @@ -71,8 +71,11 @@ quality. 2. Compute **effective depth** for each candidate peer: `effective_depth = peer.depth + link_cost`, where `link_cost = etx * (1.0 + srtt_ms / 100.0)` using locally measured MMP - metrics. When MMP metrics have not yet converged, `link_cost` defaults to - 1.0, preserving pure depth-based behavior as a graceful fallback. + metrics. During cold start (no peer has MMP data yet), candidates without + measurements default to `link_cost = 1.0`, preserving pure depth-based + behavior. Once any peer has MMP data, unmeasured candidates are excluded + so that a freshly connected peer cannot win parent selection on the + default cost alone. 3. Apply **hysteresis**: switch parents only when the best candidate's effective depth is significantly better than the current parent's: `best_eff_depth < current_eff_depth * (1.0 - parent_hysteresis)` @@ -109,6 +112,9 @@ immediate parent reselection: (ETX and SRTT). No cumulative path costs are propagated, avoiding the trust problems inherent in self-reported cost metrics in a permissionless network. +- **Loop rejection**: Candidates whose advertised ancestry already contains + the local node are skipped, preventing two nodes from selecting each + other as parent and entering an alternating coordinate loop. ### After Parent Change @@ -180,9 +186,14 @@ A node re-announces (propagates) only when its own state changes: - **Root changed**: Always propagate — this is a significant topology event - **Depth changed**: Always propagate — affects routing distance calculations +- **Mid-chain ancestor swap**: A reroute that replaces an interior ancestor + without changing the root or the path length still alters the node's + coordinate path, so it propagates. Without this, downstream peers would + route into a phantom intermediate that no longer appears on the parent's + tree. - **Sequence-only refresh**: Does NOT propagate beyond depth 1 — peers that - receive a sequence-only update do not re-announce, because their own root - and depth have not changed + receive a sequence-only update do not re-announce, because their own root, + depth, and address path have not changed This means TreeAnnounce cascades through the tree proportional to depth, not network size. A change at depth D affects at most D nodes along the @@ -303,12 +314,15 @@ Example: In a 1000-node network with depth 10 and 5 peers, a node stores | Rate limiting (500ms per peer) | **Implemented** | | Coord cache flush on parent change | **Implemented** | | Flap dampening (extended hold-down on rapid switches) | **Implemented** | +| Loop rejection (ancestry self-check in `evaluate_parent`) | **Implemented** | +| Mid-chain ancestor swap propagation | **Implemented** | | Per-ancestry-entry signatures | Future direction | ## References - [fips-mesh-operation.md](fips-mesh-operation.md) — How the spanning tree fits into mesh routing -- [fips-wire-formats.md](fips-wire-formats.md) — TreeAnnounce wire format +- [../reference/wire-formats.md](../reference/wire-formats.md) — + TreeAnnounce wire format - [spanning-tree-dynamics.md](spanning-tree-dynamics.md) — Convergence scenario walkthroughs diff --git a/docs/design/fips-transport-layer.md b/docs/design/fips-transport-layer.md index 4a3ff28..4e6ace9 100644 --- a/docs/design/fips-transport-layer.md +++ b/docs/design/fips-transport-layer.md @@ -1,5 +1,7 @@ # FIPS Transport Layer + + The transport layer is the bottom of the FIPS protocol stack. It delivers datagrams between transport-specific endpoints over arbitrary physical or logical media. Everything above — peer authentication, routing, encryption, @@ -50,7 +52,7 @@ determine how much payload can fit in a single packet after link-layer encryption overhead. MTU is fundamentally a per-link property. A transport with a fixed MTU -(Ethernet: 1500, UDP configured at 1472) returns the same value for every +(Ethernet effective 1499, UDP default 1280) returns the same value for every link — this is the degenerate case. Transports that negotiate MTU per-connection (e.g., BLE ATT_MTU) report the negotiated value for each link individually. @@ -68,7 +70,7 @@ forwarding and LookupResponse transit annotation. ### Connection Lifecycle For connection-oriented transports, manage the underlying connection: TCP -handshake, Tor circuit establishment, Bluetooth pairing. FMP cannot begin +handshake, Tor circuit establishment, BLE pairing. FMP cannot begin the Noise IK link handshake until the transport-layer connection is established. @@ -117,7 +119,6 @@ for internet connectivity: | --------- | ---------- | --- | ----------- | ----- | | UDP/IP | host:port | 1280–1472 | Unreliable | Primary internet transport | | TCP/IP | host:port | Stream | Reliable | Requires length-prefix framing | -| WebSocket | URL | Stream | Reliable | Browser-compatible | | Tor | .onion | Stream | Reliable | High latency, strong anonymity | **Shared medium transports** operate over broadcast- or multicast-capable @@ -127,7 +128,6 @@ media: | --------- | ---------- | --- | ----------- | ----- | | Ethernet | MAC | 1500 | Unreliable | Raw AF_PACKET frames | | WiFi | MAC | 1500 | Unreliable | Infrastructure mode = Ethernet | -| Bluetooth | BD_ADDR | 672–64K | Reliable | L2CAP | | BLE | BD_ADDR | 23–517 | Reliable | Negotiated ATT_MTU | | Radio | Device addr | 51–222 | Unreliable | Low bandwidth, long range | @@ -157,9 +157,9 @@ require connection setup before FMP can begin the Noise IK link handshake, adding startup latency. **Stream vs. datagram**: Datagram transports have natural packet boundaries. -Stream transports (TCP, WebSocket, Tor) require framing to delineate FIPS -packets within the byte stream. The FMP common prefix includes a payload -length field that provides this framing directly, replacing the need for a +Stream transports (TCP, Tor) require framing to delineate FIPS packets +within the byte stream. The FMP common prefix includes a payload length +field that provides this framing directly, replacing the need for a separate length-prefix layer. **Addressing opacity**: Transport addresses are opaque byte vectors. FMP @@ -189,9 +189,7 @@ proceed. | Transport | Connection Setup | | --------- | ---------------- | | TCP/IP | TCP three-way handshake | -| WebSocket | HTTP upgrade + TCP | -| Tor | Circuit establishment (500ms–5s) | -| Bluetooth | L2CAP connection | +| Tor | Circuit establishment (typically 10–60s, default timeout 120s) | | BLE | L2CAP CoC or GATT connection | | Serial | Physical connection (static) | @@ -203,9 +201,9 @@ Connected → Disconnected. Failure can occur during connection setup, adding error handling paths that connectionless transports don't have. **Startup latency**: Connection-oriented transports add delay before a peer -becomes usable. This ranges from milliseconds (TCP) to seconds (Tor -circuit). Peer timeout configuration must account for transport-specific -setup times. +becomes usable. This ranges from milliseconds (TCP) to tens of seconds +(Tor circuit). Peer timeout configuration must account for +transport-specific setup times. **Framing**: Stream transports must delimit FIPS packets within the byte stream. The FMP common prefix includes a payload length field that provides @@ -229,45 +227,29 @@ NAT devices and firewalls, limiting deployment to networks without NAT. ### Socket Buffer Sizing -The default Linux UDP receive buffer (`net.core.rmem_default`, typically -212 KB) is insufficient for high-throughput forwarding. At ~85 MB/s, a 212 KB -buffer fills in ~2.5 ms; any stall in the async receive loop (decryption, -routing, forwarding overhead) causes the kernel to silently drop incoming -datagrams. +The default Linux UDP receive buffer (`net.core.rmem_default`, +typically 212 KB) is insufficient for high-throughput forwarding. At +~85 MB/s, a 212 KB buffer fills in ~2.5 ms; any stall in the async +receive loop (decryption, routing, forwarding overhead) causes the +kernel to silently drop incoming datagrams. -FIPS uses `socket2::Socket` wrapped in `tokio::io::unix::AsyncFd` for the -UDP receive path. This replaces `tokio::UdpSocket` and enables direct -`libc::recvmsg()` calls with ancillary data parsing — specifically the -`SO_RXQ_OVFL` socket option, which delivers a cumulative kernel receive -buffer drop counter on every received packet. The drop counter feeds into -the ECN congestion detection system (see -[fips-mesh-layer.md](fips-mesh-layer.md#ecn-congestion-signaling)). +FIPS uses `socket2::Socket` wrapped in `tokio::io::unix::AsyncFd` for +the UDP receive path. This replaces `tokio::UdpSocket` and enables +direct `libc::recvmsg()` calls with ancillary data parsing — +specifically the `SO_RXQ_OVFL` socket option, which delivers a +cumulative kernel receive buffer drop counter on every received +packet. The drop counter feeds into the ECN congestion detection +system (see [fips-mmp.md](fips-mmp.md#ecn-congestion-signaling)). -Socket buffers are configured at bind time via `socket2`: - -| Parameter | Default | Description | -| ---------------- | ------- | ------------------------------------ | -| `recv_buf_size` | 2 MB | `SO_RCVBUF` — kernel receive buffer | -| `send_buf_size` | 2 MB | `SO_SNDBUF` — kernel send buffer | - -Linux internally doubles the requested value (to account for kernel -bookkeeping overhead), so requesting 2 MB yields 4 MB actual buffer space. -The kernel silently clamps to `net.core.rmem_max` if the request exceeds it. - -**Host requirement**: `net.core.rmem_max` and `net.core.wmem_max` must be -set to at least the requested buffer size on the host. For Docker containers, -this must be configured on the Docker host (containers share the host kernel). -Verify with: - -```text -sysctl net.core.rmem_max net.core.wmem_max -``` - -Actual buffer sizes are logged at startup: - -```text -UDP transport started local_addr=0.0.0.0:2121 recv_buf=4194304 send_buf=4194304 -``` +Socket buffers (`recv_buf_size`, `send_buf_size`) are configured at +bind time via `socket2`. Linux internally doubles the requested value +(to account for kernel bookkeeping overhead) and silently clamps to +`net.core.rmem_max` / `net.core.wmem_max` if the request exceeds the +host kernel limits. The full UDP transport configuration is in +[../reference/configuration.md](../reference/configuration.md). The +host-side sysctl requirements and how to set them persistently live +in +[../how-to/tune-udp-buffers.md](../how-to/tune-udp-buffers.md). ## Ethernet: The Local Network Transport @@ -317,18 +299,16 @@ x-only public key. Receiving nodes extract the MAC source address from the frame and the public key from the payload, then report the discovered peer to FMP. -Four configuration flags control discovery behavior: +Four configuration flags control discovery behavior — `discovery` +(listen for beacons), `announce` (broadcast beacons), `auto_connect` +(initiate handshakes to discovered peers), and `accept_connections` +(accept inbound handshakes). The flag table and per-flag defaults +live in [../reference/configuration.md](../reference/configuration.md) +under `transports.ethernet.*`. -| Flag | Default | Description | -| ---- | ------- | ----------- | -| `discovery` | true | Listen for beacons from other nodes | -| `announce` | false | Broadcast beacons periodically | -| `auto_connect` | false | Initiate handshakes to discovered peers | -| `accept_connections` | false | Accept inbound handshake attempts | - -A typical discoverable node sets `announce: true`, `auto_connect: true`, and -`accept_connections: true`. A passive listener uses just `discovery: true` to -observe the network without announcing itself. +A typical discoverable node sets `announce`, `auto_connect`, and +`accept_connections` all true. A passive listener uses just +`discovery: true` to observe the network without announcing itself. ### WiFi Compatibility @@ -343,10 +323,12 @@ Startup logging: Ethernet transport started name=eth0 interface=eth0 mac=aa:bb:cc:dd:ee:ff mtu=1499 if_mtu=1500 ``` -## TCP/IP: Firewall Traversal Transport +## TCP/IP: Transport for UDP-Filtered Networks -For networks where UDP is blocked but TCP port 443 is open, the TCP -transport provides an alternative path. +For peers whose networks filter outbound UDP, the TCP transport +provides an alternative datagram path between public endpoints. TCP +is not a NAT-traversal mechanism — there is no `tcp:nat` analogue to +the UDP hole-punch flow. FIPS protocols (FMP, FSP, MMP) are all unreliable datagrams. Running them over TCP introduces head-of-line blocking, which adds latency jitter. MMP @@ -425,21 +407,12 @@ removes it from the pool and aborts its receive task. ### Configuration -```yaml -transports: - tcp: - bind_addr: "0.0.0.0:8443" # Listen address (omit for outbound-only) - mtu: 1400 # Default MTU - connect_timeout_ms: 5000 # Outbound connect timeout - nodelay: true # TCP_NODELAY (disable Nagle) - keepalive_secs: 30 # TCP keepalive interval (0 = disabled) - recv_buf_size: 2097152 # SO_RCVBUF (2 MB) - send_buf_size: 2097152 # SO_SNDBUF (2 MB) - max_inbound_connections: 256 # Resource protection limit -``` - -If `bind_addr` is configured, the transport accepts inbound connections. -Without it, the transport operates in outbound-only mode (no listener +The TCP transport configuration block (`transports.tcp.*` — bind +address, MTU, connect timeout, TCP_NODELAY, keepalive, socket buffer +sizes, max inbound connections) is documented in +[../reference/configuration.md](../reference/configuration.md). If +`bind_addr` is configured, the transport accepts inbound connections; +without it, the transport operates in outbound-only mode (no listener socket is created). ## Tor: The Anonymity Transport @@ -531,22 +504,13 @@ connections arrive from `127.0.0.1` (Tor daemon's local forwarding); peer identity is resolved during the Noise IK handshake, not from the transport address. -Configuration requires coordinating `torrc` and `fips.yaml`: - -```text -# torrc -HiddenServiceDir /var/lib/tor/fips -HiddenServicePort 8443 127.0.0.1:8444 - -# fips.yaml tor section -mode: "directory" -directory_service: - hostname_file: "/var/lib/tor/fips/hostname" - bind_addr: "127.0.0.1:8444" -``` - -The `HiddenServicePort` external port (8443) is what peers connect to. -The bind_addr must match the `HiddenServicePort` target address. +Configuration requires coordinating `torrc` and `fips.yaml`. The +operator setup — torrc directives, `fips.yaml` `tor` section, +HiddenServiceDir permissions, and `Sandbox 1` notes — is in +[../how-to/deploy-tor-onion.md](../how-to/deploy-tor-onion.md). In +brief: the `HiddenServicePort` external port is what peers connect +to, and `tor.directory_service.bind_addr` must match the +`HiddenServicePort` target address. ### Session Independence @@ -569,10 +533,9 @@ anonymous node's IP. ### Latency Characteristics -Tor adds 200ms–2s RTT per circuit. First-packet latency after connection -is higher (~2.8s) due to circuit warm-up. MMP measures this elevated -latency, and cost-based parent selection penalizes Tor links (high SRTT -→ high link cost). ETX is 1.0 since TCP handles retransmission. +Tor adds 200ms–2s RTT per circuit. MMP measures this elevated latency, +and cost-based parent selection penalizes Tor links (high SRTT → high +link cost). ETX is 1.0 since TCP handles retransmission. Tor throughput is typically 1–5 Mbps — adequate for control plane and moderate data transfer, not for bulk transfer. @@ -600,37 +563,22 @@ connections (`/run/tor/control`) are preferred over TCP for security. ### Configuration -```yaml -transports: - tor: - mode: "socks5" # "socks5", "control_port", or "directory" - socks5_addr: "127.0.0.1:9050" # SOCKS5 proxy address - connect_timeout_ms: 120000 # Connect timeout (120s for Tor circuits) - mtu: 1400 # Default MTU - # control_port mode: monitoring via Tor control port (no inbound) - # control_addr: "/run/tor/control" # Unix socket (preferred) or host:port - # control_auth: "cookie" # "cookie" or "password:" - # cookie_path: "/var/run/tor/control.authcookie" - # directory mode: inbound via Tor-managed HiddenServiceDir - # directory_service: - # hostname_file: "/var/lib/tor/fips/hostname" - # bind_addr: "127.0.0.1:8444" - # max_inbound_connections: 64 -``` - -Three modes are available: +The Tor transport block (`transports.tor.*`) is documented in +[../reference/configuration.md](../reference/configuration.md). Three +modes are available: - **`socks5`** (default): Outbound-only through a SOCKS5 proxy. No control port, no inbound connections. - **`control_port`**: Outbound via SOCKS5 plus control port connection for Tor daemon monitoring. No inbound connections. - **`directory`** (recommended for inbound): Outbound via SOCKS5 plus - inbound via Tor-managed `HiddenServiceDir` onion service. Optionally - connects to the control port for monitoring when `control_addr` is set. - Enables Tor's `Sandbox 1` for maximum security. + inbound via Tor-managed `HiddenServiceDir` onion service. + Optionally connects to the control port for monitoring when + `control_addr` is set. Enables Tor's `Sandbox 1` for maximum + security. -The Tor transport requires an external Tor daemon. Named instances are -supported for multiple proxy endpoints. +The Tor transport requires an external Tor daemon. Named instances +are supported for multiple proxy endpoints. ### Implementation Roadmap @@ -645,21 +593,11 @@ supported for multiple proxy endpoints. ### Statistics -The transport tracks per-instance statistics: - -| Counter | Description | -| ------- | ----------- | -| `packets_sent` / `bytes_sent` | Successful sends | -| `packets_recv` / `bytes_recv` | Successful receives | -| `send_errors` / `recv_errors` | Send/receive failures | -| `connections_established` | Successful SOCKS5 connections | -| `connect_timeouts` | Connection timeout count | -| `connect_refused` | Connection refused count | -| `socks5_errors` | SOCKS5 protocol errors | -| `mtu_exceeded` | Packets rejected for MTU violation | -| `connections_accepted` | Accepted inbound connections via onion service | -| `connections_rejected` | Rejected inbound connections (limit exceeded) | -| `control_errors` | Tor control port errors | +The Tor transport exposes per-instance counters covering successful +send/receive, send/receive errors, connection establishment, +SOCKS5-level errors, MTU rejections, accepted/rejected inbound +connections, and Tor control-port errors. The full counter table +lives in [../reference/transports.md](../reference/transports.md). ## Discovery @@ -731,8 +669,8 @@ Key properties: > control whether discovered peers are connected automatically or require > explicit configuration. TCP and Tor have no built-in discovery mechanism. > Nostr relay discovery and STUN-assisted UDP hole punching are -> implemented behind the `nostr-discovery` cargo feature; see -> [fips-configuration.md](fips-configuration.md) for the +> implemented and toggled via configuration; see +> [../reference/configuration.md](../reference/configuration.md) for the > `node.discovery.nostr.*` configuration tree. ## Transport Interface @@ -817,8 +755,9 @@ on all forwarded datagrams. ### Transport Addresses Transport addresses (`TransportAddr`) are opaque byte vectors. The transport -layer interprets them (e.g., UDP/TCP resolve "host:port" strings (IP fast path, DNS fallback with 60s cache for UDP)); all layers above -treat them as opaque handles passed back to the transport for sending. +layer interprets them — e.g. UDP and TCP resolve `host:port` strings (IP +fast path, DNS fallback with a 60s cache on UDP). All layers above treat +them as opaque handles passed back to the transport for sending. ### Transport State Machine @@ -839,9 +778,9 @@ transitions through `Starting` to `Up` (operational). `stop()` moves to | UDP/IP | **Implemented** | Primary transport, AsyncFd/recvmsg, SO_RXQ_OVFL kernel drop detection | | TCP/IP | **Implemented** | FMP header-based framing, non-blocking connect, per-connection MSS MTU | | Ethernet | **Implemented** | AF_PACKET SOCK_DGRAM, EtherType 0x2121, beacon discovery, Linux only | -| WiFi | Future direction | Infrastructure mode = Ethernet driver | +| WiFi | **Implemented** (via Ethernet transport, infrastructure mode) | mac80211 translates 802.11↔802.3; broadcast beacons unreliable through APs | | Tor | **Implemented** | Outbound SOCKS5, inbound via onion service, .onion and clearnet addressing | -| BLE | Future direction | ATT_MTU negotiation, per-link MTU | +| BLE | **Implemented** (Linux/glibc only; experimental) | L2CAP CoC, ATT_MTU negotiation, per-link MTU; musl/macOS/Windows skip | | Radio | Future direction | Constrained MTU (51–222 bytes) | | Serial | Future direction | SLIP/COBS framing, point-to-point | @@ -849,7 +788,7 @@ transitions through `Starting` to `Up` (operational). `stop()` moves to ### TCP-over-TCP Avoidance -Running TCP application traffic over a reliable transport (TCP, WebSocket) +Running TCP application traffic over a reliable transport (TCP, Tor) creates a layering violation where retransmission and congestion control operate at both levels. When the inner TCP detects loss (which may just be transport-layer retransmission delay), it retransmits, creating more traffic @@ -884,6 +823,15 @@ quality difference is significant. Link cost is not yet used in ## References -- [fips-intro.md](fips-intro.md) — Protocol overview and layer architecture -- [fips-mesh-layer.md](fips-mesh-layer.md) — FMP specification (the layer above) -- [fips-wire-formats.md](fips-wire-formats.md) — Transport framing details +- [fips-concepts.md](fips-concepts.md) — Protocol overview +- [fips-architecture.md](fips-architecture.md) — Layer architecture +- [fips-mesh-layer.md](fips-mesh-layer.md) — FMP specification (the + layer above) +- [fips-mtu.md](fips-mtu.md) — How transport-reported `link_mtu` + feeds the unified path-MTU model +- [../reference/wire-formats.md](../reference/wire-formats.md) — + Transport framing details +- [../reference/configuration.md](../reference/configuration.md) — + Per-transport configuration blocks +- [../reference/transports.md](../reference/transports.md) — + Per-transport statistics counter inventory diff --git a/docs/design/port-advertisement-and-nat-traversal.md b/docs/design/port-advertisement-and-nat-traversal.md new file mode 100644 index 0000000..24020eb --- /dev/null +++ b/docs/design/port-advertisement-and-nat-traversal.md @@ -0,0 +1,594 @@ +# Port Advertisement and NAT Traversal via Nostr + +## Abstract + +This document describes two related-but-independent mechanisms that an +application protocol can build on top of Nostr relays: + +1. **Port advertisement.** A node publishes a parameterized replaceable + event describing the application protocol it speaks, the version, and + the endpoint(s) at which it can be reached. Other nodes discover the + advert by querying relays. +2. **NAT traversal.** When the advertised endpoint indicates that the + responder is behind NAT, the two peers exchange ephemeral + gift-wrapped offer/answer events through Nostr relays, run STUN + against a public server to learn their reflexive addresses, and + coordinate UDP hole punching so they can exchange application traffic + over a direct UDP path. + +The two mechanisms compose naturally — an advert that includes a +`:nat` endpoint signals "reach me by running the traversal +protocol" — but they are independently useful. An advert with only +public-IP endpoints needs no traversal. A pair of peers that already +know each other's pubkeys but want to coordinate a traversal can do so +without ever publishing a public advert. + +The protocol described here is generic. Any application protocol can +adopt it by picking its own kind number, `d`-tag scope, and endpoint +schema. [FIPS](https://github.com/jmcorgan/fips) (the Free +Internetworking Peering System) is used throughout the document as an +example implementation; FIPS-specific values appear in clearly marked +example blocks and do not affect the generic protocol shape. + +No WebRTC, DTLS, or ICE stack is required. The protocol operates at +the raw UDP level, using Nostr solely for ephemeral signaling and STUN +solely for reflexive address discovery. + +--- + +## Terminology + +- **Application protocol.** The protocol that runs on top of the + punched UDP channel after this document's procedures complete. +- **Initiator.** The peer that discovers the responder's advert and + begins the traversal exchange. +- **Responder.** The peer that publishes a service advertisement and + is willing to be dialled. +- **Reflexive address.** The public `IP:port` tuple that a STUN server + observes for a UDP socket — i.e., the NAT's external mapping for + that socket. +- **Punch socket.** The single UDP socket a peer uses for STUN, for + the offer/answer exchange's address fields, for the punch packets + themselves, and for the application traffic that follows. The same + socket must be used across all phases of one traversal attempt. + +### Socket lifecycle + +The protocol assumes **per-peer, per-attempt punch sockets**: + +- Each outbound traversal attempt allocates a fresh UDP socket bound + to `0.0.0.0:0` (OS-assigned port). +- That socket is owned by exactly one remote peer and exactly one + traversal session. +- STUN, the offer/answer reflexive-address fields, the punch packets, + and the eventual adopted application transport all share that + socket for the lifetime of the attempt. +- If the attempt fails, the socket is discarded. A retry allocates a + new socket and obtains a fresh reflexive address. +- A long-lived application listener (for example, a fixed UDP port + shared across peers) must **not** be reused as the punch socket — + doing so couples NAT mappings and retry state across peers. + +This rule is not optional: closing or rebinding the socket between +phases invalidates the NAT mapping that the rest of the protocol +depends on. + +--- + +## Part 1: Service Advertisement + +### Event shape + +The advert is a NIP-01 parameterized replaceable event whose kind +falls in the application-defined replaceable range +`30000–39999`. The event carries: + +- A `d` tag scoping the advert (so the same pubkey can publish + multiple distinct adverts under different scopes). +- A `protocol` tag carrying the application protocol's name, used as + a discovery filter for peers that don't already know the + responder's pubkey. +- A `version` tag carrying the application protocol version. +- An optional `expiration` tag (NIP-40) so a relay garbage-collects + the advert when the responder goes offline without explicitly + deleting it. +- An optional `relays` tag listing relays where the responder + subscribes for incoming signaling messages (used by Part 2). +- An optional `stun` tag listing STUN servers the responder + recommends. +- A `content` field carrying the application-specific payload — + typically the endpoint set, capability flags, and any encryption + keys the application layer needs. The content may be plaintext or + NIP-44-encrypted; encryption requires the consumer to already know + the responder's pubkey. + +The replaceable semantics let the responder update the advert in +place under the same `d` tag. A NIP-09 deletion event removes the +advert when the responder permanently retires. + +```json +{ + "kind": , + "pubkey": "", + "created_at": , + "tags": [ + ["d", ""], + ["protocol", ""], + ["version", ""], + ["relays", "wss://relay1.example.com", "wss://relay2.example.com"], + ["stun", "stun.l.google.com:19302"], + ["expiration", ""] + ], + "content": "", + "sig": "" +} +``` + +### Endpoint schema + +The `content` field is application-defined. Its structure typically +includes a list of endpoints describing how the responder can be +reached. Endpoint entries should distinguish: + +- **Direct public endpoints** (transport + address + port) where any + initiator can connect without traversal. +- **NAT-mapped endpoints** that signal "I can be reached by running + the traversal protocol against this transport on my pubkey." +- **Anonymity-network endpoints** (e.g. Tor onion services) where + the addressing scheme implies its own connection semantics. + +#### FIPS example: kind 37195 advertisement + +FIPS uses **kind `37195`** (the digits visually spell `FIPS` — +7=F, 1=I, 9=P, 5=S). The `d` tag is hardcoded to +`fips-overlay-v1`; the configurable `app` value populates the +separate `protocol` tag, scoping adverts within a relay set +without splitting them across multiple `d`-tag streams. + +The advert content is a JSON document carrying a list of endpoint +entries, each shaped as `{transport, addr}`. The transport string +takes one of: + +- `udp:host:port` — direct public UDP endpoint. +- `udp:nat` — NAT-mapped UDP endpoint; reach via Part 2 traversal. +- `tcp:host:port` — direct public TCP endpoint, for peers whose + networks filter outbound UDP. Public-only; there is no + `tcp:nat` analogue. +- `tor::` — Tor onion-service endpoint. + +FIPS publishes the advert with `expiration` set to `now + +advert_ttl_secs` (default 1 hour) and refreshes it every +`advert_refresh_secs` (default 30 minutes). + +### Public-IP discovery on advertisement + +A responder behind a NAT or wildcard-bound to a non-routable address +needs to determine what external address to put in its advert. The +responder uses a fixed precedence: + +1. An operator-supplied external address override (FIPS: + `transports.{udp,tcp}.external_addr`) wins. +2. A non-wildcard `local_addr` is used directly. +3. For a wildcard-bound UDP listener with an explicit "publish this" + flag (FIPS: `public: true`), the runtime queries STUN against + the configured servers and publishes the reflexive address. +4. For a wildcard-bound TCP listener, no STUN equivalent exists. + Implementations should refuse to silently advertise an unreachable + endpoint; FIPS emits a loud WARN and omits the endpoint. + +This precedence keeps adverts honest: an endpoint that appears in +the published content is one the responder believes is reachable. + +### Discovery (consumer side) + +A consumer queries one or more relays for an advert it can act on. +Two filter shapes are typical: + +By author, when the responder's pubkey is already known: + +```json +["REQ", "", { + "kinds": [], + "authors": [""], + "#d": [""] +}] +``` + +By application protocol, for "open discovery" of any peer running +the same application: + +```json +["REQ", "", { + "kinds": [], + "#protocol": [""] +}] +``` + +Adverts whose `protocol` tag does not match the consumer's expected +value, or whose `expiration` tag has elapsed, are rejected at +validation. Consumers cache adverts in memory keyed by author npub +and respect the embedded expiration. + +#### FIPS example: discovery filters + +The FIPS daemon issues both filter shapes: by-author for peers it +intends to dial directly, and by-`#protocol` when an operator has +opted into open discovery against the same application namespace. +Cached adverts persist until their `expiration` lapses; a periodic +prune drops expired entries. + +--- + +## Part 2: NAT Traversal + +The traversal protocol coordinates UDP hole punching between two +peers via gift-wrapped Nostr signaling. It is invoked when the +initiator decides to dial a NAT-mapped endpoint advertised by the +responder. + +### Signaling event shape + +Signaling messages are ephemeral kinds in the range `20000–29999`, +NIP-44-encrypted to the recipient, and NIP-59 gift-wrapped so the +outer event is signed by an ephemeral keypair rather than the +sender's long-term identity. The wrap carries a `p` tag pointing at +the recipient's pubkey and an NIP-40 `expiration` tag bounding how +long the relay should retain it. + +#### FIPS example: signaling kind 21059 + +FIPS signaling uses **kind `21059`**. Wraps are addressed by `p` +tag and published to the responder's NIP-17 inbox relay list (kind +`10050`) when one is available, falling back to the local +`dm_relays` configuration otherwise. Each side publishes its own +inbox relay list on startup so dialers can discover it. + +### Phase 1: Initiator STUN binding + +Before constructing any signaling message, the initiator: + +1. Allocates a fresh UDP punch socket bound to `0.0.0.0:0`. +2. Sends a STUN Binding Request (RFC 8489) to one of its locally + configured STUN servers. +3. Parses the Binding Response, extracts the + `XOR-MAPPED-ADDRESS` attribute, and records that as its + reflexive address. Other STUN attributes are ignored. +4. Records local-candidate addresses for the same socket port: + active private non-loopback interface addresses (RFC1918 IPv4, + IPv6 ULA) and probed local egress addresses. + +The punch socket must remain open across all subsequent phases. +Closing or rebinding it discards the NAT mapping. + +### Phase 2: Initiator sends offer + +The initiator constructs an offer payload containing its reflexive +address, its local-candidate addresses, an opaque session +identifier, freshness timestamps, and any application-specific +parameters. The payload is NIP-44-encrypted to the responder's +pubkey, wrapped with NIP-59, and published to the responder's +signaling relays. The initiator also subscribes by `p` tag on +those relays to receive the answer. + +```json +{ + "type": "offer", + "sessionId": "", + "issuedAt": , + "expiresAt": , + "nonce": "", + "senderNpub": "", + "recipientNpub": "", + "reflexiveAddress": {"protocol":"udp","ip":"","port":}, + "localAddresses": [{"protocol":"udp","ip":"","port":}], + "stunServer": ":", + "app_params": { ... } +} +``` + +- `sessionId` is a random identifier correlating offer and answer. +- `reflexiveAddress` is the address STUN observed in Phase 1. +- `localAddresses` enables a same-LAN fast path when both peers + happen to share a private subnet. +- `stunServer` is informational, recording which server the + initiator used. +- `issuedAt` / `expiresAt` bound the freshness window — the + responder rejects stale offers, since a NAT mapping that has not + been refreshed in tens of seconds may already be gone. + +### Phase 3: Responder validates and answers + +The responder maintains a standing `p`-tagged subscription on its +advertised signaling relays. On receiving an offer: + +1. Decrypts the wrap and recovers the offer payload. +2. Validates freshness (rejects if outside the configured window; + see *Skew tolerance* below). +3. Rejects replays — if the `sessionId` is in a recently-seen + cache, drop the offer. +4. Allocates its own punch socket (`0.0.0.0:0`) and runs its own + STUN query. +5. Constructs an answer payload that echoes `sessionId`, carries + the responder's reflexive and local addresses, includes a + `PunchHint { startAtMs, intervalMs, durationMs }` telling both + sides when to begin probing and how aggressively, and is + wrapped, encrypted, and published the same way as the offer. + +```json +{ + "type": "answer", + "sessionId": "", + "issuedAt": , + "expiresAt": , + "nonce": "", + "senderNpub": "", + "recipientNpub": "", + "inReplyTo": "", + "accepted": true, + "reflexiveAddress": {"protocol":"udp","ip":"","port":}, + "localAddresses": [{"protocol":"udp","ip":"","port":}], + "stunServer": ":", + "punch": {"startAtMs": , "intervalMs": , "durationMs": }, + "offerReceivedAt": , + "app_params": { ... } +} +``` + +If the responder has no usable addresses, it returns +`accepted: false` with an explanatory `reason` and no `punch`. + +The optional `offerReceivedAt` field carries the responder's +wall-clock at the moment the offer arrived. The initiator can +combine its own `T1` (offer-publish time), `T2 = offerReceivedAt`, +`T3` (answer's `issuedAt`), and `T4` (answer-receive time) into the +NTP-style estimate `((T2 − T1) + (T3 − T4)) / 2`, giving a per-peer +clock-skew measurement that's useful for tuning freshness windows +and for telemetry. + +**Immediately after publishing the answer**, the responder begins +Phase 4 punching without waiting for any acknowledgement that the +initiator received the answer. NAT mappings are decaying and time +is the binding constraint. + +The responder must bind the inner JSON `senderNpub` / +`recipientNpub` fields to the actual Nostr pubkeys that delivered +the gift wrap, rather than treating those JSON fields as +independently trustworthy. The wrap pubkey is the authentication +ground-truth. + +### Phase 4: Hole punching + +Both peers now know each other's reflexive and local addresses. +Both begin sending UDP packets from their respective punch sockets: + +1. Send punch packets every **`intervalMs`** (typically 200 ms) + across each planned target path: + - reflexive-to-reflexive + - private-subnet local-address paths (when subnet-compatible) + - mixed local/reflexive fallbacks +2. Each punch packet carries a fixed magic header so transit and + peer code can distinguish it from stray UDP traffic: + + ```text + Bytes 0–3: (application-defined u32) + Bytes 4–7: sequence number (u32, big-endian, starting at 0) + Bytes 8–23: first 16 bytes of SHA-256(sessionId) + ``` + +3. On receiving a valid punch packet (magic matches, session-id + hash matches), the peer records the source address as the + confirmed peer address and replies with an acknowledgement + packet under a different magic value: + + ```text + Bytes 0–3: (application-defined u32) + Bytes 4–7: echoed sequence number + Bytes 8–23: first 16 bytes of SHA-256(sessionId) + ``` + +4. On receiving an acknowledgement, the peer considers the path + punched and transitions to Phase 5. + +If both peers advertised compatible local-subnet candidates, the +local-address path will typically punch through faster than the +reflexive path. The first path to acknowledge wins. + +### Phase 5: Application protocol takeover + +Once the path has acknowledged in both directions: + +- The application protocol takes over the punch socket. +- The signaling subscription can be closed. +- The application is responsible for sending keepalive traffic at + least every 15 seconds to refresh the NAT mapping. A flow that + goes idle longer risks losing its mapping and having to retraverse. + +### Phase 6: Cleanup + +After the attempt completes (success or failure): + +1. Close the relay subscription used for signaling. +2. Optionally publish a NIP-09 deletion event referencing any + signaling events the peer published. Because the wraps were + ephemeral kinds with NIP-40 expiration tags, well-behaved relays + will discard them automatically without explicit deletion. +3. Discard the per-attempt punch socket if the attempt failed; a + retry must allocate a new socket and a fresh reflexive address. + +If the responder is going offline permanently it should also +delete its kind-37195 (or equivalent) advert. + +### Timeouts and retries + +- If the initiator publishes an offer and receives no answer + within a configured window (e.g. 10 s from offer publish), the + attempt has failed. Causes: responder offline, advert stale, + responder relay unreachable. +- If the answer arrives but no valid punch acknowledgement is + observed within `durationMs` (typically 10 s), the attempt has + failed. Causes: symmetric NAT on either side, firewall + interference, stale reflexive addresses. + +The initiator may retry with a fresh STUN query, a fresh punch +socket, and a new offer. Repeated failures against the same +responder should be suppressed by the application layer; see +*Application-specific failure handling* below. + +--- + +## Security + +### Authentication + +Offer and answer payloads are NIP-44-encrypted to the recipient and +NIP-59 gift-wrapped, so only the intended recipient can decrypt. +Authentication of the sender comes from the inner-wrap signature +(the rumour signed by the sender's long-term identity inside the +NIP-59 seal), **not** from the outer wrap signature (which is the +ephemeral pubkey). + +The inner JSON `senderNpub` / `recipientNpub` fields must be bound +to the actual signing pubkey of the inner rumour. Treating those +JSON fields as independently trustworthy is a vulnerability — +implementations must compare them against the unwrapped signature. + +Once the UDP path is punched, the raw UDP channel has **no inherent +authentication or encryption**. The application layer is responsible +for establishing its own security on the punched channel — for +example, a Noise Protocol handshake keyed from the Nostr identity, +or an application-specific authenticated-encryption layer. FIPS +runs its FMP Noise IK handshake immediately after adoption; the +identity proven by the Noise handshake is the same Nostr pubkey +that signed the inner offer/answer rumour, so a man-in-the-middle on +the relay cannot impersonate the responder. + +### Replay protection + +The `sessionId` and `issuedAt` / `expiresAt` fields together +defeat replays at the signaling layer. The responder must keep a +bounded cache of recently-seen `sessionId` values and reject +duplicates within the freshness window. + +### Skew tolerance + +Strict freshness checks fail under modest clock skew between +peers. Implementations should accept offers and answers whose +timestamps are off by a small absolute amount (FIPS uses ±60 s), +and feed observed skew into a per-peer estimate for telemetry and +tuning. Outright rejection should be reserved for grossly stale or +future-dated messages. + +### Metadata exposure + +Even though signaling content is encrypted, the gift-wrap metadata +reveals that the initiator's ephemeral pubkey contacted the +responder's pubkey at a particular time, through a particular +relay. The advert itself is public and reveals the responder's +pubkey and the application protocol it speaks. + +If metadata privacy is required, the advert content can be +encrypted (consumers must already know the responder's pubkey), +both peers can use ephemeral Nostr identities rather than their +long-term keys, and the operator can run a private relay. + +### NAT mapping integrity + +If too much wall-clock time elapses between STUN discovery and the +hole-punch attempt, the reflexive address goes stale. Both peers +should complete the entire signaling exchange within tens of +seconds of their respective STUN queries. Relay latency is the +primary risk factor. Implementations targeting flaky relays should +prefer relays known to deliver ephemeral events sub-second. + +--- + +## Relay requirements + +The protocol works best with relays that: + +- Support ephemeral event kinds (`20000–29999`) and do not persist + them. +- Honor NIP-40 `expiration` tags and garbage-collect expired + events. +- Deliver events with low latency (sub-second WebSocket push). +- Support NIP-09 deletion requests. + +Relays that do not support ephemeral kinds will store the +signaling events as regular events. The encrypted content remains +opaque, but persisted wraps are wasteful and expose metadata +unnecessarily. Operators deploying this protocol at scale should +prefer relays that handle ephemeral kinds correctly, or run their +own. + +--- + +## Failure modes + +| Failure | Symptom | Mitigation | +| --- | --- | --- | +| Symmetric NAT (one side) | Punch timeout | Retry with port-prediction heuristics; otherwise fall back to a relay or different transport | +| Symmetric NAT (both sides) | Punch timeout | Application-level relay required | +| Relay latency > 60 s | Stale reflexive address | Use low-latency relays; consider self-hosted relay | +| Relay does not support ephemeral kinds | Signaling events persist | Use NIP-40 expiration + NIP-09 deletion as fallback | +| Responder offline | No answer received | Initiator times out after configurable period | +| Stale advert (responder no longer up) | Offer reaches no listener | Application-level failure suppression (see below) | +| STUN server unreachable | No reflexive address | Fall back to alternate STUN server; fail if none reachable | +| Firewall blocks outbound UDP | STUN fails entirely | NAT-traversal does not apply; reachable peers are limited to those that publish a non-UDP transport (e.g. TCP) and accept inbound | + +### Application-specific failure handling + +Repeated traversal failures against the same responder are common +in practice — the responder may be offline, the advert may be +stale, or the responder may be on a network that doesn't admit +incoming UDP. A naive implementation that retries on every dial +attempt floods the relay layer and the operator's logs. + +Implementations should layer per-peer suppression on top of the +basic retry. The shape of that suppression is application-specific. + +#### FIPS example: failure suppression + +FIPS layers the following suppression machinery on the basic retry +loop: + +- **Per-npub WARN log rate-limit** (`warn_log_interval_secs`, + default 5 minutes). Subsequent failures inside the window log + at debug level instead. +- **Per-npub consecutive-failure counter and extended cooldown.** + After `failure_streak_threshold` (default 5) consecutive + failures, the per-peer retry deadline is pushed past + `extended_cooldown_secs` (default 30 minutes). Open-discovery + sweeps consult the cooldown so they don't immediately re-enqueue + the same peer. +- **Stale-advert eviction on streak transition.** When a peer + hits the failure-streak threshold, the daemon actively + re-fetches its advert from the configured advert relays. If the + advert has been removed or replaced, the cache entry is evicted + and the streak resets; if the advert is unchanged, the cooldown + applies. +- **Per-peer skew estimate.** The NTP-style skew computed from + `offerReceivedAt` is recorded so consistently-skewed peers don't + trip the freshness check on every attempt. +- **Bounded failure-state cache** (`failure_state_max_entries`, + default 4096) with LRU eviction so the suppression machinery + itself does not grow unbounded. + +These knobs are documented in +[FIPS configuration reference](https://github.com/jmcorgan/fips/blob/master/docs/reference/configuration.md) +under `node.discovery.nostr`. + +--- + +## References + +- **RFC 8489** — Session Traversal Utilities for NAT (STUN) +- **RFC 8445** — Interactive Connectivity Establishment (ICE) +- **RFC 4787** — NAT Behavioral Requirements for Unicast UDP +- **NIP-01** — Basic Nostr protocol flow +- **NIP-09** — Event deletion request +- **NIP-17** — Inbox relay list (kind `10050`) for direct-message + routing +- **NIP-40** — Expiration timestamp +- **NIP-44** — Versioned encryption +- **NIP-59** — Gift wrap +- **NIP-78** — Application-specific data diff --git a/docs/design/spanning-tree-dynamics.md b/docs/design/spanning-tree-dynamics.md index 1cb4933..0372222 100644 --- a/docs/design/spanning-tree-dynamics.md +++ b/docs/design/spanning-tree-dynamics.md @@ -1,14 +1,20 @@ # FIPS Spanning Tree Protocol Dynamics -A detailed study of the gossip-based spanning tree protocol, focusing on -operational behavior under various mesh conditions. This document complements -[fips-intro.md](fips-intro.md) with step-by-step walkthroughs of protocol -dynamics rather than message formats and data structures. +A detailed study of the gossip-based spanning tree protocol, focusing +on operational behavior under various mesh conditions. This document +complements [fips-concepts.md](fips-concepts.md) and +[fips-architecture.md](fips-architecture.md) with step-by-step +walkthroughs of protocol dynamics rather than message formats and +data structures. -For wire formats, see [fips-wire-formats.md](fips-wire-formats.md) (TreeAnnounce section). -For spanning tree algorithms and data structures, see -[fips-spanning-tree.md](fips-spanning-tree.md). For how the spanning tree fits -into mesh routing, see [fips-mesh-operation.md](fips-mesh-operation.md). +For wire formats, see +[../reference/wire-formats.md](../reference/wire-formats.md) +(TreeAnnounce section). For spanning tree algorithms and data +structures, see [fips-spanning-tree.md](fips-spanning-tree.md). For +how the spanning tree fits into mesh routing, see +[fips-mesh-operation.md](fips-mesh-operation.md). For the academic +foundations and references that underpin this document, see +[fips-prior-work.md](fips-prior-work.md). ## Contents @@ -93,7 +99,7 @@ When a node starts with no peers, it bootstraps as a single-node network. **T0: Node A starts.** - Generates or loads keypair `(npub_A, nsec_A)` -- Computes `node_addr_A = SHA-256(npub_A)` +- Computes `node_addr_A = SHA-256(pubkey_A)[..16]` (128 bits) - Initializes empty TreeState - Sets `parent = self` (A is its own root), `sequence = 1` - Records current timestamp @@ -518,6 +524,11 @@ converge to the same "link failed" state, though B detects it up to ## 8. Parent Selection Parent selection determines tree structure and routing efficiency. +The algorithm itself (effective-depth ranking, hold-down, hysteresis, +mandatory-switch bypass) is canonically documented in +[fips-spanning-tree.md](fips-spanning-tree.md); this section walks +through what re-selection looks like under specific dynamic +conditions and the rationale for the local-only cost metric. ### Cost-Based Selection with Effective Depth @@ -537,10 +548,15 @@ purely by tree depth without link quality consideration. 2. **Compute effective depth for each candidate.** For every peer whose announced root matches the smallest root, the algorithm calculates `effective_depth = peer.depth + link_cost`, where `link_cost` comes from - `peer_costs` (MMP-derived) or defaults to 1.0 when metrics have not yet - converged. The best candidate is the peer with the lowest effective depth, - with ties broken by numerically smallest `NodeAddr`. If the best candidate - is already the current parent, no switch is needed. + `peer_costs` (MMP-derived). During cold start, when no peer has MMP data + yet (`peer_costs` is empty), unmeasured candidates default to 1.0; once + any peer has MMP data, unmeasured candidates are skipped so a freshly + connected peer cannot win on its default cost. Candidates whose ancestry + already contains the local node are also rejected, preventing an + alternating two-node loop. The best candidate is the peer with the + lowest effective depth, with ties broken by numerically smallest + `NodeAddr`. If the best candidate is already the current parent, no + switch is needed. 3. **Check for mandatory switches.** Two conditions bypass all stability mechanisms and trigger an immediate parent change: the current parent is no @@ -571,10 +587,10 @@ re-evaluation independent of TreeAnnounce traffic). Where ETX (Expected Transmission Count, from De Couto et al., "A High-Throughput Path Metric for Multi-Hop Wireless Routing", 2003) comes from -bidirectional MMP delivery ratios and SRTT (Smoothed Round-Trip Time) from MMP -timestamp-echo. When MMP -metrics have not yet converged, `link_cost` defaults to 1.0, preserving -depth-only behavior as a graceful fallback. +bidirectional MMP delivery ratios and SRTT (Smoothed Round-Trip Time) from +MMP timestamp-echo. During cold start, before any peer has MMP data, the +default cost of 1.0 is used and the algorithm reduces to depth-only +selection. **What this means for tree structure**: The algorithm can prefer a deeper parent with a better link over a shallower parent with a poor link, when the effective @@ -942,28 +958,16 @@ costs to form efficient tree structures. ### Prior Art and FIPS Contributions -The protocol builds on established foundations and adds several new elements: - -**Derived from prior work**: - -- Spanning tree coordinate routing (Yggdrasil/Ironwood, building on Kleinberg - 2007 and Cvetkovski/Crovella 2009) -- Deterministic root discovery via smallest identifier (Yggdrasil; echoes - IEEE 802.1D STP bridge ID selection) -- CRDT-based distributed state (Shapiro et al. 2011) -- Gossip dissemination (epidemic model; Kermarrec 2007) -- Heartbeat-based failure detection (SWIM; Das et al. 2002) -- ETX link metric (De Couto et al. 2003) -- Hysteresis and hold-down for route stability (OSPF, BGP, IS-IS) - -**FIPS additions**: - -- Cost-aware parent selection using local-only link metrics (effective depth = - tree depth + link cost), replacing Yggdrasil's depth-only selection -- Combined ETX + SRTT link cost formula with MMP-measured components -- Flap dampening with mandatory switch bypass -- Announcement suppression for transient state changes -- Tree-only bloom filter merge with split-horizon exclusion +The protocol builds on established foundations (Yggdrasil/Ironwood +tree-coordinate routing, IEEE 802.1D STP root election, CRDT-based +distributed state, SWIM-style failure detection, ETX, OSPF-style +hysteresis and hold-down) and adds several new elements (cost-aware +parent selection on local-only metrics, the combined ETX + SRTT cost +formula, flap dampening with mandatory-switch bypass, announcement +suppression, and tree-only bloom filter merge with split-horizon). +Both the prior-art map and the FIPS contributions list are +consolidated in +[fips-prior-work.md](fips-prior-work.md#fips-contributions). --- @@ -971,75 +975,17 @@ The protocol builds on established foundations and adds several new elements: ### FIPS Internal Documentation -- [fips-spanning-tree.md](fips-spanning-tree.md) — Spanning tree algorithms and data structures -- [fips-mesh-operation.md](fips-mesh-operation.md) — How the spanning tree fits into mesh routing -- [fips-wire-formats.md](fips-wire-formats.md) — TreeAnnounce wire format +- [fips-spanning-tree.md](fips-spanning-tree.md) — Spanning tree + algorithms and data structures +- [fips-mesh-operation.md](fips-mesh-operation.md) — How the spanning + tree fits into mesh routing +- [../reference/wire-formats.md](../reference/wire-formats.md) — + TreeAnnounce wire format -### Yggdrasil Documentation +### Prior Art and Academic Foundations -- [Yggdrasil v0.5 Release Notes](https://yggdrasil-network.github.io/2023/10/22/upcoming-v05-release.html) -- [Ironwood Routing Library](https://github.com/Arceliar/ironwood) -- [The World Tree (Yggdrasil Blog)](https://yggdrasil-network.github.io/2018/07/17/world-tree.html) -- [Yggdrasil Implementation Overview](https://yggdrasil-network.github.io/implementation.html) - -### Academic Foundations - -#### Virtual Coordinate Routing - -- Rao, A., Ratnasamy, S., Papadimitriou, C., Shenker, S., Stoica, I. - ["Geographic Routing without Location Information"](https://people.eecs.berkeley.edu/~sylvia/papers/p327-rao.pdf). - MobiCom 2003. *Established virtual coordinate routing using network topology.* - -#### Greedy Embedding Theory - -- Kleinberg, R. - ["Geographic Routing Using Hyperbolic Space"](https://www.semanticscholar.org/paper/Geographic-Routing-Using-Hyperbolic-Space-Kleinberg/f506b2ddb142d2ec539400297ba53383d958abef). - IEEE INFOCOM 2007. *Proved every connected graph has a greedy embedding in - hyperbolic space; showed spanning trees enable coordinate assignment.* - -- Cvetkovski, A., Crovella, M. - ["Hyperbolic Embedding and Routing for Dynamic Graphs"](https://www.cs.bu.edu/faculty/crovella/paper-archive/infocom09-hyperbolic.pdf). - IEEE INFOCOM 2009. *Dynamic embedding for nodes joining/leaving; introduced - Gravity-Pressure routing for failure recovery.* - -- Crovella, M. et al. - ["On the Choice of a Spanning Tree for Greedy Embedding"](https://www.cs.bu.edu/faculty/crovella/paper-archive/networking-science13.pdf). - Networking Science 2013. *Analysis of how tree structure affects routing stretch.* - -- Bläsius, T. et al. - ["Hyperbolic Embeddings for Near-Optimal Greedy Routing"](https://dl.acm.org/doi/10.1145/3381751). - ACM Journal of Experimental Algorithmics 2020. *Achieved 100% success ratio - with 6% stretch on Internet graph.* - -#### Link Metrics - -- De Couto, D., Aguayo, D., Bicket, J., Morris, R. - "A High-Throughput Path Metric for Multi-Hop Wireless Routing". - MobiCom 2003. *Introduced ETX (Expected Transmission Count) as a link - quality metric for wireless mesh networks.* - -#### Routing Protocol Stability - -- IEEE 802.1D. "IEEE Standard for Local and Metropolitan Area - Networks: Media Access Control (MAC) Bridges". *Spanning Tree - Protocol (STP) — root election via bridge ID, BPDU exchange.* - -- Moy, J. [RFC 2328](https://datatracker.ietf.org/doc/html/rfc2328): - "OSPF Version 2". 1998. *Link-state routing with cumulative path - costs and SPF computation. FIPS's local-only cost approach is - contrasted with OSPF's cumulative model in §8.* - -#### Distributed Systems Primitives - -- Shapiro, M., Preguiça, N., Baquero, C., Zawirski, M. - "Conflict-free Replicated Data Types". SSS 2011. - *Formal definition of CRDTs enabling coordination-free consistency.* - -- Das, A., Gupta, I., Motivala, A. - ["SWIM: Scalable Weakly-consistent Infection-style Process Group Membership"](https://www.cs.cornell.edu/projects/Quicksilver/public_pdfs/SWIM.pdf). - IPDPS 2002. *O(1) failure detection, O(log N) dissemination via gossip.* - -- Kermarrec, A-M. - ["Gossiping in Distributed Systems"](https://www.distributed-systems.net/my-data/papers/2007.osr.pdf). - ACM SIGOPS Operating Systems Review 2007. *Framework for gossip-based - protocols achieving O(log N) propagation.* +The Yggdrasil documentation and the academic-foundations bibliography +(virtual coordinate routing, greedy embedding theory, link metrics, +routing-protocol stability, and distributed systems primitives) are +collected in +[fips-prior-work.md](fips-prior-work.md#spanning-tree-dynamics-foundations). diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..3ac2e50 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,241 @@ +# Getting Started with FIPS + +FIPS (Free Internetworking Peering System) is a self-organizing +encrypted mesh network built on Nostr identities. Your machine +becomes a node in the mesh with a self-generated cryptographic +identity, and existing networking software — SSH, web servers, +file transfer, anything IPv6-native — runs over the mesh +unchanged. + +There are two common ways to deploy FIPS, and the rest of this +guide and the linked docs branch accordingly: + +- **As an overlay** on top of existing IP networks (Ethernet, + WiFi, the public internet, Tor), FIPS lets your node reach + any other peer regardless of NAT, ISP, or physical location. +- **From the ground up** over non-IP transports — raw Ethernet, + WiFi, Bluetooth — FIPS provides a complete permissionless + network without any pre-existing IP infrastructure, ISP, or + DNS. + +The two paths share a lot of common ground — install, identity, +configuration. They diverge mainly in transport setup and the +deployment topology you choose. + +There is no central server. Any node can run; any pair of +running nodes can mesh. + +## What you'll need + +- A Linux, macOS, or Windows host. Linux is the most exercised + platform; macOS and Windows installers are available. +- The pre-built installer for your platform (see the + [Releases page](../README.md#installation)), **or** a source + checkout if you want to build the installer yourself. +- For the source-build path only: a working Rust toolchain (the + version pinned in `rust-toolchain.toml` is auto-installed by + rustup), and the platform-specific build dependencies listed in + [packaging/README.md](../packaging/README.md). + +## Install + +FIPS is installed by running a binary installer for your +platform. The installer drops the daemon and CLI tools into +system locations, installs systemd / launchd / Windows-service +unit files, places a default `fips.yaml`, and creates the `fips` +system group. There is no `cargo install` path: the daemon needs +more than just binaries copied into place. + +You can either build the installer yourself from source, or +download a pre-built one from the release distribution. Both +paths produce the same installer artifacts and the same +post-install state. + +### From the release distribution + +The most direct path. The release distribution carries a +per-platform installer: + +- Debian/Ubuntu — `.deb` package +- Arch Linux — `fips` AUR package +- OpenWrt — `.ipk` package +- macOS — `.pkg` installer +- Windows — `.zip` with service-install scripts +- Generic systemd Linux — `.tar.gz` with an `install.sh` script + +See the [project README's Installation section](../README.md#installation) +for download links and per-platform invocations. + +### From source + +For development, custom builds, or unsupported architectures. +The `packaging/` tree builds the same installer formats locally; +you then apply the resulting installer the same way you would a +downloaded one. + +```sh +git clone https://github.com/jmcorgan/fips.git +cd fips/packaging +make deb # or: tarball, ipk, aur, pkg, zip, all +``` + +The resulting installer lands in `deploy/` at the project root. +Apply it the same way you would a downloaded one (for example +`sudo dpkg -i deploy/fips_*.deb` on Debian/Ubuntu). + +See [packaging/README.md](../packaging/README.md) for per-format +build details, cross-target options, and the full `make` target +list. + +## What's installed and running + +Here's what the installer leaves on your machine, what's +running, and what you'll need to set up yourself. + +**Binaries installed system-wide:** + +- `fips` (daemon) +- `fipsctl` (control-socket client) +- `fipstop` (live-status TUI) +- `fips-gateway` + +**Files placed on disk:** + +- `/etc/fips/fips.yaml` — default daemon config (preserved on + upgrade). +- `/etc/fips/fips.nft` — mesh-interface nftables baseline (used + only when the firewall service is enabled). +- `/etc/fips/fips.d/` — empty drop-in directory for operator + nftables additions. +- Systemd, launchd, or Windows-service unit files for the four + fips services. + +**System changes:** + +- A `fips` system group is created. Add your user to it + (`sudo usermod -aG fips $USER`, then re-login) to run + `fipsctl` and `fipstop` without `sudo`. +- The runtime directory `/run/fips/` exists with mode + `0750 root:fips`. + +**Services enabled and started on boot:** + +- `fips.service` — the daemon. Brings up the `fips0` TUN + adapter, listens on the configured transports, and exposes + the control socket at `/run/fips/control.sock`. +- `fips-dns.service` — wires `.fips` hostname resolution into + the host resolver (a `/etc/systemd/resolved.conf.d/` drop-in + pointing at `[::1]:5354` on systemd hosts). + +**Services installed but not enabled** (operator opt-in): + +- `fips-firewall.service` — applies `/etc/fips/fips.nft` to + the mesh interface. See + [how-to/enable-mesh-firewall.md](how-to/enable-mesh-firewall.md). + +**What's working out of the box:** + +- The daemon is running with a fresh **ephemeral** identity — + a new Nostr keypair is generated on every start. +- The `fips0` TUN adapter exists with the daemon's mesh address. +- The daemon's transport listeners are up: UDP `0.0.0.0:2121` + and TCP `0.0.0.0:8443`. They are inert at this point because + no other node knows your daemon's npub yet — see "What's not + yet configured" below. +- `.fips` hostname resolution is plumbed into the host + resolver. + +**What's not yet configured** — these are what guide your next +steps: + +- **No peers.** The daemon has nobody to talk to until you add + a static peer entry, enable Nostr-mediated discovery, or + bring up a transport (Ethernet, Bluetooth) where peers find + each other automatically on the same physical link. +- **Ephemeral identity.** Your node's npub changes every + restart. The + [persistent-identity tutorial](tutorials/persistent-identity.md) + walks through pinning the daemon to a stable Nostr keypair + for any node others will reference by name. +- **Mesh firewall not active.** Inbound exposure on `fips0` + follows the host's existing firewall rules until you enable + the baseline service. + +## Reaching mesh nodes by name + +A FIPS node is identified by its Nostr public key (`npub1...`). +For ordinary IP software running over the mesh — SSH, web +browsers, `ping`, file transfer — use the form `.fips` +as the destination; the local `.fips` resolver translates that +to the corresponding mesh IPv6 address so the FIPS node can be +found. The resolver runs entirely on your machine and does not +generate any external DNS traffic. + +For shorter forms, the resolver also consults two host maps +before falling back to direct npub lookup: `/etc/fips/hosts` +(shipped pre-populated with the public test mesh roster, and +freely editable for your own entries) and the `alias:` field +on configured peers in `fips.yaml`. So `test-us01.fips`, +`my-laptop.fips`, or any other shortname you map resolves the +same way `.fips` does. See +[how-to/host-aliases.md](how-to/host-aliases.md) for the full +mechanics. + +## Join the test mesh + +The fastest way to see FIPS in action is to connect your daemon +to the public FIPS test mesh. The +[Join the Test Mesh](tutorials/join-the-test-mesh.md) tutorial +walks through adding a single static peer entry, watching the +link come up, and reaching both that peer and a second mesh node +forwarded through it — a ten-minute exercise that demonstrates +the central FIPS guarantee that one good peer connects you to +the rest of the mesh. + +## Where to go next + +Documentation is organised into four sections, each with a different +job. Pick the one that matches what you want to do. + +### [Tutorials](tutorials/) + +Step-by-step lessons that take you from zero to a working setup. +Read these end-to-end. Start with +[Join the Test Mesh](tutorials/join-the-test-mesh.md) and follow +with +[ipv6-adapter-walkthrough](tutorials/ipv6-adapter-walkthrough.md) +to understand what each piece does, then move on to +[persistent-identity](tutorials/persistent-identity.md) and +the three Nostr-discovery tutorials — +[resolve-peers-via-nostr](tutorials/resolve-peers-via-nostr.md), +[advertise-your-node](tutorials/advertise-your-node.md), and +[open-discovery](tutorials/open-discovery.md) — to give your +node a stable npub, look up peer endpoints, publish your +own, and join the ambient discovery namespace. Then [host-a-service](tutorials/host-a-service.md) for hosting +a service on your node, and [ground-up-mesh](tutorials/ground-up-mesh.md) +for the second deployment mode where two devices peer over +Ethernet, WiFi, or Bluetooth with no IP between them. + +### [How-To Guides](how-to/) + +Task-oriented recipes for operators with a specific goal: enable a +firewall, deploy the LAN gateway, set up Bluetooth peering, +diagnose an MTU problem, configure persistent identity. Each guide +takes the shortest correct path from "I want to do X" to "X is done". + +### [Reference](reference/) + +Lookup material consulted on demand: wire formats, configuration +keys, command-line flags, control-socket commands. Austere by +design; no guidance on when to use a feature. + +### [Design](design/) + +Architectural and protocol-level explanations: the mesh layer, the +session layer, the spanning tree, Bloom-filter discovery, the +unified MTU model, the IPv6 adapter. Read these to understand *why* +FIPS makes the choices it does. + +The design section's +[fips-concepts.md](design/fips-concepts.md) is a good entry point if +you want the mental model before touching any commands. diff --git a/docs/how-to/README.md b/docs/how-to/README.md new file mode 100644 index 0000000..cbfa300 --- /dev/null +++ b/docs/how-to/README.md @@ -0,0 +1,27 @@ +# How-To Guides + +Task-oriented, step-by-step recipes for operators with a specific +goal in mind. Each guide assumes the reader already knows what FIPS +is and wants to get a particular thing done — enable a feature, +deploy a component, troubleshoot a class of problem. + +How-to guides do not teach concepts (that is the role of design/) +and do not enumerate options (that is the role of reference/). They +take the reader along the shortest correct path from "I want to do +X" to "X is done". + +## Available Guides + +| Guide | Goal | +| ----- | ---- | +| [enable-mesh-firewall.md](enable-mesh-firewall.md) | Activate the default-deny nftables baseline on `fips0` | +| [enable-nostr-discovery.md](enable-nostr-discovery.md) | Turn on Nostr-mediated discovery (3 capabilities — resolve, advertise, open — across 5 scenarios) | +| [deploy-tor-onion.md](deploy-tor-onion.md) | Run a Tor onion service for inbound FIPS connections | +| [tune-udp-buffers.md](tune-udp-buffers.md) | Set host sysctls so FIPS UDP sockets don't get clamped | +| [run-as-unprivileged-user.md](run-as-unprivileged-user.md) | Run the daemon under a dedicated unprivileged service account (drops the default-root posture) | +| [deploy-gateway.md](deploy-gateway.md) | Manually deploy `fips-gateway` on a non-OpenWrt Linux host (LAN-to-mesh outbound + mesh-to-LAN inbound port-forwards). For the OpenWrt path, see the gateway tutorial. | +| [troubleshoot-gateway.md](troubleshoot-gateway.md) | Diagnostic recipes for the gateway, organised by half (outbound, inbound, common) | +| [persistent-identity.md](persistent-identity.md) | Provision a stable Nostr keypair so the node keeps the same npub across restarts | +| [host-aliases.md](host-aliases.md) | Use shortnames (`test-us01.fips`, `my-laptop.fips`) instead of full npubs by editing `/etc/fips/hosts` or setting peer aliases | +| [set-up-bluetooth-peer.md](set-up-bluetooth-peer.md) | Configure a Bluetooth Low Energy peer link | +| [diagnose-mtu-issues.md](diagnose-mtu-issues.md) | Triage MTU-shaped failures and rule out their imposters (bufferbloat, transport saturation) | diff --git a/docs/how-to/deploy-gateway.md b/docs/how-to/deploy-gateway.md new file mode 100644 index 0000000..75f266f --- /dev/null +++ b/docs/how-to/deploy-gateway.md @@ -0,0 +1,450 @@ +# Deploy `fips-gateway` (Manual Linux-Host Setup) + +`fips-gateway` is a separate service that runs alongside the FIPS +daemon and bridges a non-FIPS LAN to the FIPS mesh in two +independent directions: **outbound** (LAN clients reach mesh +services through DNS proxy + virtual-IP NAT) and **inbound** (mesh +peers reach LAN services through 1:1 port forwards on `fips0`). +This guide covers the **manual Linux-host** deployment path — +wiring DNS forwarding, route distribution, and firewall integration +on a server or non-OpenWrt router by hand. + +> **Running OpenWrt?** Use the +> [tutorial](../tutorials/deploy-fips-gateway.md) instead. The OpenWrt +> ipk ships with the `gateway:` block pre-populated and the init +> script automates dnsmasq forwarding, RA route distribution, and the +> global IPv6 prefix on `br-lan`. The OpenWrt path is the canonical +> deployment of this feature; this how-to is the secondary path for +> operators with a different LAN-edge box (a Linux server already +> serving DHCP/DNS, a custom router distribution, etc.). + +For the gateway design (NAT pipeline, virtual IP pool lifecycle, DNS +resolution flow), see [../design/fips-gateway.md](../design/fips-gateway.md). +For the full `gateway.*` configuration block, see the +[Gateway section](../reference/configuration.md#gateway-gateway) of +the configuration reference. For the `fips-gateway` binary's CLI +flags, see [../reference/cli-fips-gateway.md](../reference/cli-fips-gateway.md). + +## The two halves + +The gateway exposes two independent features that share a common +control plane (the same binary, the same nftables table `inet +fips_gateway`, the same control socket `/run/fips/gateway.sock`, the +same `gateway.*` config block). You can configure either half on its +own or both together. + +- **Outbound gateway** (LAN → mesh). Non-FIPS LAN workstations resolve + `.fips` names against the gateway's DNS listener and receive + AAAA answers from the gateway's virtual-IP pool. Outbound traffic + to those addresses is DNAT'd to the real mesh address and SNAT'd + (masqueraded) onto `fips0` under the gateway's mesh identity. The + audience is unmodified LAN clients. + +- **Inbound gateway** (mesh → LAN). A static `(listen_port, proto) + → [target_addr]:target_port` table — configured in + `gateway.port_forwards[]` — exposes selected LAN services to the + mesh as `.fips:`. Mesh peers connect to + the gateway's mesh address; the gateway DNATs to the LAN target + and masquerades on the LAN side so return traffic flows through + conntrack. The audience is mesh peers reaching a service that + happens to live on this LAN. + +The two halves are independent. Configure the outbound half if you +want LAN clients to *reach* the mesh; configure the inbound half if +you want mesh peers to *reach into* the LAN; configure both if you +want both. + +## Common gateway-host setup + +Both halves require the same host preparation. Work through this +section first, then jump to whichever half (or both) you need. + +### FIPS daemon prerequisites + +The gateway runs alongside a `fips` daemon on the same host: + +- The daemon must be running with the TUN adapter enabled (the + `fips0` interface must exist). +- The daemon's DNS resolver must be enabled (`dns.enabled: true`, + default) and reachable from `fips-gateway`. By default that means + `[::1]:5354` (IPv6 loopback). The gateway's default + `dns.upstream` matches this; a v4 upstream like `127.0.0.1:5354` + cannot reach a daemon bound on `[::1]:5354` because Linux IPv6 + sockets bound to explicit `::1` do not accept v4-mapped traffic. + +If the daemon is not yet running with these features, set up the +daemon first — see [persistent-identity.md](persistent-identity.md) +and [../reference/configuration.md](../reference/configuration.md). + +### Kernel sysctls + +```sh +sudo sysctl -w net.ipv6.conf.all.forwarding=1 +sudo sysctl -w net.ipv6.conf.all.proxy_ndp=1 +``` + +`forwarding` lets the host route IPv6 packets between the LAN +interface and `fips0`. `proxy_ndp` lets the gateway answer Neighbor +Solicitation requests for virtual-pool addresses so LAN clients can +resolve their link-layer addresses (only relevant for the outbound +half, but harmless if you only run the inbound half). + +Persist via a drop-in: + +```sh +sudo tee /etc/sysctl.d/60-fips-gateway.conf <<'EOF' +net.ipv6.conf.all.forwarding = 1 +net.ipv6.conf.all.proxy_ndp = 1 +EOF +sudo sysctl --system +``` + +### Capability + +`fips-gateway` requires `CAP_NET_ADMIN` to manage its nftables table +(`inet fips_gateway`) and proxy-NDP entries. The packaged systemd +unit (`fips-gateway.service`) runs as root, which satisfies this. For +non-package installs, set the file capability: + +```sh +sudo setcap cap_net_admin+ep /usr/bin/fips-gateway +``` + +### Pool route + +At startup `fips-gateway` adds `local dev lo` to the +local routing table. This tells the kernel to accept packets +destined for pool addresses as locally-owned, enabling the NAT +processing path. The route is cleaned up on shutdown. You do not +need to install it manually; if you see "destination unreachable" +errors for pool addresses on the gateway host, verify the route is +present: + +```sh +ip -6 route show table local | grep +``` + +### Minimum configuration + +In `/etc/fips/fips.yaml`, populate the `gateway` block with at minimum +`enabled: true`, `pool`, and `lan_interface`: + +```yaml +gateway: + enabled: true + pool: "fd01::/112" + lan_interface: "enp3s0" +``` + +Pick a pool CIDR that does **not** overlap with any address space in +use on the LAN or in the mesh (the FIPS mesh occupies `fd00::/8`; +pick a different `fdXX::/N`). The `/112` size yields 65 536 virtual +IPs, which is the gateway's hard cap regardless of CIDR width. + +This minimum config is enough to start the gateway. The `dns.*` block +is optional and defaults to `listen: "[::]:53"` and +`upstream: "[::1]:5354"`. The full block — including `dns.*`, +`pool_grace_period`, `conntrack.*`, and `port_forwards[]` — is +documented in +[../reference/configuration.md#gateway-gateway](../reference/configuration.md#gateway-gateway). + +### Start the service + +```sh +sudo systemctl enable --now fips-gateway +``` + +Verify the unit came up: + +```sh +sudo systemctl status fips-gateway +sudo journalctl -u fips-gateway -e +``` + +The startup log will report `Gateway config loaded`, +`DNS upstream is reachable`, `Created nftables table 'fips_gateway'`, +and finally `fips-gateway running`. The unit's `ExecStartPre` waits up +to 30 s for `fips0` to appear, which covers the cold-boot race where +the daemon is still bringing up its TUN. + +## Configure the outbound half + +The outbound half lets LAN clients resolve `.fips` names and reach +mesh destinations. Three operator decisions are involved: pool CIDR, +DNS listen address, and how LAN clients learn the route to the pool +and the resolver address. + +### Choose the pool CIDR + +```yaml +gateway: + pool: "fd01::/112" +``` + +Constraints: + +- Must not overlap with `fd00::/8` (the FIPS mesh address space). +- Must not overlap with any LAN-side IPv6 prefix already in use. +- `/112` is the practical width — wider just wastes address space + because the pool is hard-capped at 65 536 entries. Narrower is + fine if you want a smaller pool, but you'll reject DNS lookups + faster under churn. + +### Choose the DNS listen address + +```yaml +gateway: + dns: + listen: "[::]:53" + upstream: "[::1]:5354" + ttl: 60 +``` + +Common cases: + +- **No other resolver on the host:** `listen: "[::]:53"` is the + default and works. +- **systemd-resolved is on port 53:** either disable its stub + listener (`DNSStubListener=no` in + `/etc/systemd/resolved.conf`) or move the gateway to a different + port (e.g., `[::]:5353`) and put a forwarder on 53 that delegates + `.fips` to the gateway. See + [troubleshoot-gateway.md](troubleshoot-gateway.md#port-conflict-on-the-dns-listen-port). +- **Bind on the LAN address only:** `listen: "192.168.1.1:53"` + exposes the resolver only to LAN clients, not loopback. + +The gateway returns `REFUSED` for any non-`.fips` query — clients +that point at it directly need a fallback resolver, or you should +front it with a stub forwarder. + +### Distribute the route to LAN clients + +Each LAN client must route the gateway's pool CIDR to the gateway's +LAN-side IPv6 address. Three options, in order of preference for +production: + +- **RA Route Information Option** (RFC 4191). If the LAN's RA daemon + (`radvd`, `dnsmasq --enable-ra`, OpenWrt's `odhcpd`) supports + publishing route options, configure it to advertise the pool CIDR + with the gateway as next-hop. Clients pick this up automatically. + +- **Static route on the LAN router**. If clients route through a + central LAN router, add a static route entry there — the router + then handles forwarding to the gateway. The exact syntax depends + on the router OS. + +- **Per-host static route** (testing or single-client deployments): + + ```sh + sudo ip -6 route add fd01::/112 via fe80::% + # or, if the gateway has a stable global LAN address: + sudo ip -6 route add fd01::/112 via + ``` + +### Distribute the resolver to LAN clients + +LAN clients also need to send `.fips` queries to the gateway. Two +patterns: + +- **Forward `.fips` from the LAN's main resolver.** If the LAN runs + Pi-hole, Unbound, dnsmasq, or systemd-resolved as the central + resolver, configure a conditional forward for `fips.`. Unbound + example: + + ```text + forward-zone: + name: "fips." + forward-addr: @53 + ``` + + dnsmasq example: + + ```text + server=/fips/ + ``` + + Clients keep their existing DNS settings; only `.fips` queries are + diverted. + +- **Point clients directly at the gateway.** Simpler for testing, + but the gateway returns `REFUSED` for non-`.fips` queries, so each + client must also have a fallback resolver configured. + +### Verify the outbound path + +From a LAN client: + +```sh +dig @ hostname.fips AAAA +# Expect an AAAA from the pool CIDR + +ping6 hostname.fips +# Should succeed via the gateway +``` + +If either step fails, see +[troubleshoot-gateway.md](troubleshoot-gateway.md#outbound-half-diagnostics). + +## Configure the inbound half + +The inbound half exposes a LAN-side service to mesh peers. Configured +under `gateway.port_forwards[]`: + +```yaml +gateway: + port_forwards: + - listen_port: 8080 + proto: tcp + target: "[fd12:3456::10]:80" + - listen_port: 2222 + proto: tcp + target: "[fd12:3456::20]:22" + - listen_port: 5353 + proto: udp + target: "[fd12:3456::10]:53" +``` + +Field reference: + +- `listen_port` — port on the gateway's `fips0` mesh-side address + that mesh peers connect to. Must be non-zero. Each + `(listen_port, proto)` pair must be unique across the list (the + same port on TCP and UDP is allowed; the same port twice on the + same proto is rejected at config-load time). +- `proto` — `tcp` or `udp`. +- `target` — IPv6 LAN destination as `[addr]:port`. IPv4 targets are + rejected at parse time by the YAML deserializer (the field is + typed `SocketAddrV6`). If the LAN host is reachable only by IPv4, + put a small IPv6-aware reverse proxy in front of it on the gateway + itself. + +### Worked example: HTTP and DNS + +Suppose the gateway runs on a LAN with an HTTP server at +`[fd12:3456::10]:80` and a recursive resolver at +`[fd12:3456::10]:53`, and you want mesh peers to reach them as +`.fips:8080` (HTTP) and `.fips:5353` +(DNS). Add to the gateway's `fips.yaml`: + +```yaml +gateway: + port_forwards: + - listen_port: 8080 + proto: tcp + target: "[fd12:3456::10]:80" + - listen_port: 5353 + proto: udp + target: "[fd12:3456::10]:53" +``` + +Reload: + +```sh +sudo systemctl restart fips-gateway +``` + +From any mesh peer (the host name `gateway` is whatever the gateway's +npub maps to in the local `hosts` file or via Nostr advert): + +```sh +curl http://gateway.fips:8080/ +dig @gateway.fips -p 5353 example.com A +``` + +Each mesh-side request enters `fips0` on the listen port, gets DNAT'd +to the LAN target, and the LAN-side masquerade rule rewrites the +source to the gateway's LAN address so return traffic flows back +through conntrack. + +### Compose with the mesh firewall + +`gateway.port_forwards[]` opens *mesh-side* listeners on `fips0`. If +the host's mesh firewall is enabled (see +[enable-mesh-firewall.md](enable-mesh-firewall.md)), inbound TCP/UDP +on `fips0` for these ports must be permitted in the baseline or via +a drop-in. The default baseline allows established/related and +ICMPv6 only, so without an explicit allow rule, mesh peers will see +TCP RSTs or silent drops on the listen port. + +A typical drop-in for the worked example: + +```nft +# /etc/fips/fips.d/gateway-inbound.nft +tcp dport 8080 accept +udp dport 5353 accept +``` + +Reload the firewall: + +```sh +sudo systemctl reload-or-restart fips-firewall.service +``` + +If the inbound half doesn't need access control beyond the listen +port itself, no source filter is needed. To restrict to specific +mesh peers, follow the `ip6 saddr tcp dport accept` +pattern from the firewall guide. + +### Verify the inbound path + +From a mesh peer (any FIPS node): + +```sh +curl -v http://.fips:8080/ +``` + +A successful response confirms the full path: mesh ingress on +`fips0`, DNAT to the LAN target, LAN-side masquerade, and conntrack- +tracked return. If it fails, see +[troubleshoot-gateway.md](troubleshoot-gateway.md#inbound-half-diagnostics). + +## Operate and verify + +`fips-gateway` exposes its own control socket at +`/run/fips/gateway.sock`, separate from the daemon's +`/run/fips/control.sock`. There is no `fipsctl gateway` subcommand — +talk to it directly: + +```sh +echo '{"command":"show_gateway"}' | sudo nc -U /run/fips/gateway.sock +echo '{"command":"show_mappings"}' | sudo nc -U /run/fips/gateway.sock +``` + +`show_gateway` returns pool counters (`pool_total`, `pool_allocated`, +`pool_active`, `pool_draining`, `pool_free`), `nat_mappings`, +`dns_listen`, `uptime_secs`, and the active config snapshot. +`show_mappings` returns the per-allocation list with virtual IP, mesh +address, npub-derived `node_addr`, dns name, state (`Allocated`, +`Active`, `Draining`), session count, and ages. For the full schema +see [../reference/control-socket.md#gateway-command-catalog](../reference/control-socket.md#gateway-command-catalog). + +The journal is the other primary signal: + +```sh +sudo systemctl status fips-gateway +sudo journalctl -u fips-gateway -e +``` + +Expect `MappingCreated`/`MappingRemoved` debug lines as DNS-driven +allocations come and go (run with `--log-level debug` to see them), +and `Final pool status` on shutdown. Errors in adding NAT rules or +proxy-NDP entries surface here. + +## See also + +- [../tutorials/deploy-fips-gateway.md](../tutorials/deploy-fips-gateway.md) — + the canonical, package-driven OpenWrt deployment path. +- [../design/fips-gateway.md](../design/fips-gateway.md) — gateway + design, NAT pipeline, virtual IP pool lifecycle, security + considerations. +- [Gateway section](../reference/configuration.md#gateway-gateway) of + the configuration reference — full `gateway.*` block. +- [../reference/cli-fips-gateway.md](../reference/cli-fips-gateway.md) — + `fips-gateway` binary CLI flags. +- [Gateway command catalog](../reference/control-socket.md#gateway-command-catalog) + in the control-socket reference — JSON schema for `show_gateway` + and `show_mappings`. +- [troubleshoot-gateway.md](troubleshoot-gateway.md) — diagnostic + recipes grouped by half. +- [enable-mesh-firewall.md](enable-mesh-firewall.md) — mesh-firewall + baseline and drop-ins (needed when exposing inbound ports). diff --git a/docs/how-to/deploy-tor-onion.md b/docs/how-to/deploy-tor-onion.md new file mode 100644 index 0000000..b32965c --- /dev/null +++ b/docs/how-to/deploy-tor-onion.md @@ -0,0 +1,217 @@ +# Deploy a Tor Onion Service for FIPS + +This guide covers running a Tor onion service that accepts inbound +FIPS peer connections. + +For the Tor transport's design and the bridge-node pattern (running +Tor and UDP simultaneously), see +[../design/fips-transport-layer.md](../design/fips-transport-layer.md). +For the full `transports.tor.*` config knob inventory, see +[../reference/configuration.md](../reference/configuration.md). + +## Inbound modes + +FIPS supports two inbound Tor modes. (A third mode, `socks5`, is +outbound-only and not covered here.) + +- **`directory` mode** *(recommended)*. Tor manages the onion + service via `HiddenServiceDir` and `HiddenServicePort` directives + in `torrc`. FIPS reads the resulting `.onion` hostname from a + file and binds a local TCP listener for Tor to forward inbound + connections to. No control-port interaction is required, which + makes this mode compatible with Tor's `Sandbox 1` seccomp-bpf + hardening. + - **`torrc` requires:** `HiddenServiceDir` + `HiddenServicePort`. +- **`control_port` mode**. FIPS speaks to Tor's control port to + create an ephemeral onion service at startup (`ADD_ONION`). The + onion key lives only for the lifetime of the FIPS daemon's + control-port session. This mode is **incompatible** with + `Sandbox 1` — the sandbox forbids control-port-driven onion + service management. + - **`torrc` requires:** `ControlPort` (typically the Unix socket + `/run/tor/control`) and a usable auth method + (`CookieAuthentication 1` is the common choice). + +Pick `directory` unless you have a specific reason to prefer +`control_port`. The rest of this guide covers `directory` mode +end-to-end. + +## Prerequisites + +- Tor daemon installed and running (Debian/Ubuntu: `apt install tor`) +- FIPS daemon configured and able to start +- Operator access to `/etc/tor/torrc` (or a drop-in under + `/etc/tor/torrc.d/`) + +## Step 1: Configure Tor's HiddenServiceDir + +Add the following to `/etc/tor/torrc`: + +```text +HiddenServiceDir /var/lib/tor/fips +HiddenServicePort 8443 127.0.0.1:8444 +``` + +`HiddenServiceDir` tells Tor where to store the onion service's +private key and `hostname` file. `HiddenServicePort` declares that +inbound TCP traffic to port 8443 of the onion address should be +forwarded to `127.0.0.1:8444` on the local host — that is where FIPS +will bind its listener. + +The external port (`8443` here) is what peers will connect to over +Tor; the internal target (`127.0.0.1:8444`) is purely local and is +not directly reachable from the network. + +## Step 2: Reload Tor and read the onion hostname + +```sh +sudo systemctl reload tor@default # or `tor` on systems without instance support +``` + +After Tor processes the new config, the hostname file appears: + +```sh +sudo cat /var/lib/tor/fips/hostname +# xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.onion +``` + +Tor regenerates the onion key only on first run (or if you remove +`HiddenServiceDir`). The `hostname` value is stable across daemon +restarts as long as `HiddenServiceDir` is preserved. + +## Step 3: Verify HiddenServiceDir permissions + +The directory must be readable only by the Tor user (Tor refuses to +start otherwise): + +```sh +ls -la /var/lib/tor/fips +# drwx------ debian-tor debian-tor ... +``` + +With the shipped Debian systemd unit, FIPS runs as root and reads +the `hostname` file directly — no permission adjustment is needed. + +### Non-default deployments + +If you run FIPS as an unprivileged user (custom packaging, +hardened deployment, etc.), the FIPS daemon user needs read access +to `hostname`. Options: + +- Add the FIPS user to the `debian-tor` group and loosen group + read on `HiddenServiceDir` (Tor still requires the directory + itself to be `0700`, so this typically means making `hostname` + itself group-readable rather than the directory). +- Read `hostname` once at startup as root, then drop privileges. +- Copy the hostname into a path the FIPS user can read, refreshed + whenever the onion key changes. + +## Step 4: Configure the FIPS Tor transport + +In `/etc/fips/fips.yaml`, configure `transports.tor` with `mode: +directory`: + +```yaml +transports: + tor: + mode: directory + socks5_addr: "127.0.0.1:9050" + connect_timeout_ms: 120000 + mtu: 1400 + advertised_port: 8443 + directory_service: + hostname_file: "/var/lib/tor/fips/hostname" + bind_addr: "127.0.0.1:8444" +``` + +The `bind_addr` must match the *target* of the `HiddenServicePort` +directive in `torrc`. The `hostname_file` path must match +`HiddenServiceDir` plus `/hostname`. + +`advertised_port` is the *virtual* onion port peers dial — i.e. the +first number on the `HiddenServicePort` line, **not** the local +target. The default is `443`; this guide uses `8443` on both sides +to match the `HiddenServicePort 8443 127.0.0.1:8444` example +above. Setting this explicitly is important if you ever flip +`advertise_on_nostr: true`: the published advert otherwise +defaults to `tor:.onion:443`, which won't match the actual +onion port. + +The `socks5_addr` is the Tor SOCKS5 proxy used for *outbound* +connections to other onion services or clearnet endpoints (separate +from inbound onion service handling). + +Optional monitoring knobs: `control_addr` and `control_auth` (e.g. +`/run/tor/control` and `cookie`) let the daemon read Tor's status +through the control port even in `directory` mode. They are +non-fatal on failure — the onion service still works without them. +See [../reference/configuration.md](../reference/configuration.md) +for the full key list and examples. + +## Step 5: Reload the FIPS daemon + +```sh +sudo systemctl reload-or-restart fips +``` + +At startup the daemon reads the `.onion` hostname from +`hostname_file`, binds `127.0.0.1:8444`, and announces the onion +endpoint internally. From this point inbound connections to +`.onion:8443` arrive at FIPS over Tor. + +## Step 6: Verify + +Check that the FIPS daemon log shows the onion endpoint at startup: + +```sh +sudo journalctl -u fips -e | grep -i 'onion\|directory' +``` + +You should see a line indicating the onion address FIPS will accept +inbound connections on, and that the local bind on `127.0.0.1:8444` +succeeded. + +From another node configured with the Tor transport in `socks5` or +`directory` mode, attempt to dial: + +```sh +fipsctl connect .onion:8443 tor +``` + +A successful `fipsctl show peers` afterwards on the inbound side +shows the new peer with `transport=tor`. + +## Optional: advertise the onion endpoint via Nostr discovery + +If `node.discovery.nostr.enabled: true`, set +`transports.tor.advertise_on_nostr: true` so the onion endpoint +appears in this node's published advert. See +[enable-nostr-discovery.md](enable-nostr-discovery.md) Scenario 2. + +## Troubleshooting + +- **Tor refuses to start with `Sandbox 1` and onion-service errors.** + `Sandbox 1` requires `directory` mode and forbids creating onion + services through the control port. Verify your `torrc` uses + `HiddenServiceDir` (this guide), not `ADD_ONION` via control port. +- **FIPS daemon fails to bind `127.0.0.1:8444`.** Another process is + already bound to that port. Either stop the conflicting process or + pick a different port and update both `torrc`'s + `HiddenServicePort` target and `fips.yaml`'s `bind_addr` to match. +- **Onion hostname is empty or missing.** Check `journalctl -u tor` + for permission errors on `HiddenServiceDir`. The directory must be + owned by the Tor user with mode `0700`. +- **FIPS daemon cannot read `hostname_file`.** File is owned by the + Tor user and not readable by the FIPS daemon user. Adjust + permissions, or copy the hostname into a path the FIPS user can + read. + +## See also + +- [../design/fips-transport-layer.md](../design/fips-transport-layer.md) + — Tor transport design, three modes (`socks5`, `control_port`, + `directory`), bridge-node pattern +- [../reference/configuration.md](../reference/configuration.md) — + full `transports.tor.*` configuration knob table +- [enable-nostr-discovery.md](enable-nostr-discovery.md) — Scenario 2 + for advertising the onion endpoint to peers via Nostr diff --git a/docs/how-to/diagnose-mtu-issues.md b/docs/how-to/diagnose-mtu-issues.md new file mode 100644 index 0000000..c81f727 --- /dev/null +++ b/docs/how-to/diagnose-mtu-issues.md @@ -0,0 +1,211 @@ +# Diagnose MTU Issues + +MTU symptoms in FIPS look like ordinary network failures: handshakes +succeed but bulk transfers hang, ssh connects but stalls after the +banner, an HTTP request times out on the first response. This guide +walks through the diagnostic surfaces that FIPS exposes so you can +distinguish a real MTU problem from its frequent imposters +(bufferbloat, transport saturation, transient packet loss). + +For the underlying model — encapsulation overhead, proactive vs +reactive PMTUD, the per-destination MTU storage layout — read +[../design/fips-mtu.md](../design/fips-mtu.md) first. + +## Symptom map + +| Application symptom | Likely cause | +| ------------------- | ------------ | +| `iperf3 -c ` control socket closes immediately after `Connecting to host`. | Forward-path MTU smaller than the negotiated MSS on the control connection. | +| `ssh user@` shows the SSH banner then hangs forever. | First post-banner exchange exceeds the path MTU; SYN MSS clamp did not engage in time, or the path narrowed mid-session. | +| `curl http:///` connects, then times out before the first response byte. | Same shape as the SSH-banner case, applied to the first server-to-client large packet. | +| Throughput bursts then drops to zero, recovers, drops again, in seconds-long cycles. | Bufferbloat masquerading as MTU failure — usually the upload of the underlay link is saturated. See [Distinguishing bufferbloat](#distinguishing-bufferbloat-from-mtu-drops). | +| `MtuExceeded` counters tick up under topology change but settle in seconds. | Normal: the reactive MTU mechanism doing its job. No action needed. | +| `MtuExceeded` counters tick continuously under steady state. | Forward-path MTU smaller than what the source learned via `path_mtu` echo. After `mmp.path_mtu` has settled, this is a bug — see [File a bug](#file-a-bug). | + +The first three are MTU candidates; the fourth is usually not. The +fifth is benign. The sixth is the bug shape worth filing. + +## Diagnostic toolkit + +### `fipsctl show sessions` + +The authoritative end-to-end MTU for an established session: + +```sh +fipsctl show sessions | jq '.sessions[] | {display_name, state, mmp: .mmp.path_mtu}' +``` + +`mmp.path_mtu` is the value the session-layer MMP currently believes +is in force end-to-end. It updates on each PathMtuNotification echo +from the destination — immediately on decrease, with hysteresis on +increase. A field that starts at `1280` (the IPv6 floor) and then +climbs to a higher value as echoes arrive is healthy; one that +oscillates between two values may indicate a flapping path. + +### `fipsctl show transports` + +Per-transport MTU. The `mtu` field reports the transport-wide +default; for BLE, individual links may have a smaller negotiated +ATT_MTU. + +```sh +fipsctl show transports | jq '.transports[] | {type, mtu}' +``` + +### `fipsctl show cache` + +The coordinate cache carries reverse-path-annotated MTU per +destination — the freshest "what fit on the way back from the +discovery target" estimate, consulted before the session has any +PathMtuNotification feedback. + +```sh +fipsctl show cache | jq '.entries[] | {display_name, depth, path_mtu}' +``` + +Entries without a `path_mtu` field are pre-discovery or were +populated through a path that did not annotate the MTU. + +### `fipsctl show peers` + +Per-peer link state, including the link-layer MMP metrics. Useful +mostly for ruling out underlying loss (loss rate near zero, SRTT +sane) before chasing an MTU explanation. + +```sh +fipsctl show peers | jq '.peers[] | {display_name, mmp: .mmp}' +``` + +### Trace logging + +Module-scoped trace logging on the TUN reader and the MMP handler +shows the per-packet decisions. The `tracing` macros default the +target to the emitting module path, so the filter targets are the +fully-qualified module paths under the `fips` crate. + +```sh +sudo systemctl edit fips +# Add: +# [Service] +# Environment=RUST_LOG=info,fips::upper::tun=trace,fips::node::handlers::mmp=debug +sudo systemctl restart fips +sudo journalctl -u fips -f +``` + +### tcpdump on `fips0` + +Capturing on the TUN reveals the IPv6 packets the daemon hands the +kernel and vice-versa. Two important caveats live in the design doc +and are worth restating here: + +- TX direction (outbound from a local app): tcpdump sees the packet + **before** the daemon's TCP MSS clamp at the TUN boundary. The + packet may be larger than the daemon will let leave the node. +- RX direction (inbound to a local app): tcpdump sees the packet + **after** the daemon's MSS clamp on inbound SYN-ACKs. The clamp + fires only when `max_mss < kernel-natural-MSS`; otherwise it is a + silent no-op. + +```sh +sudo tcpdump -ni fips0 -w /tmp/fips0.pcap port 22 or port 80 +# in another terminal, reproduce the symptom, then Ctrl-C +``` + +Open the pcap in Wireshark and check segment sizes against what the +session's `path_mtu` reports. + +## Distinguishing bufferbloat from MTU drops + +WAN bufferbloat (sustained upload saturation on a cable or DSL link) +produces a retransmit signature that looks remarkably like +oversized-packet drops. Both manifest as long stalls in TCP flows, +both clear when you stop pushing data, both can ramp the loss-rate +counter without obvious cause. + +Two ways to disambiguate: + +1. **Saturate the underlay first.** Run a reference upload outside + FIPS (`iperf3 -c `) until it stabilises, then + measure latency to the underlay's first hop with a separate `ping`. + If RTT shoots up by hundreds of ms during the upload, the + underlay buffer is the culprit, not FIPS MTU. Apply CAKE / fq_codel + on the underlay router before continuing. + +2. **Watch the FIPS counters during the symptom.** A real MTU + problem ticks `MtuExceeded` (visible in `fipsctl show routing`'s + `error_signals` block) and shifts the session's `mmp.path_mtu` + downward. Bufferbloat ticks loss rate and RTT but leaves + `path_mtu` and `MtuExceeded` alone. + +If both signatures fire together, you have both problems. + +## Cold-flow first-SYN + +The MMP echo populates path-MTU state only after the first +end-to-end exchange, but the TUN reader has to size the very first +SYN before any echo has arrived. The cold-flow ceiling is the +1143-byte conservative fallback derived from the 1280-byte IPv6 +floor. The first SYN may therefore be smaller than what the path +ultimately supports; once MMP echoes arrive, subsequent flows use +the larger learned value. + +If the first SYN of a flow is still oversized relative to the path, +the receiving transit node generates an `MtuExceeded`, the source +shrinks immediately, and the next packet of the flow fits. This is +expected for one round trip; it becomes a problem only if it +persists. + +## Fixes + +The operator's choices, in rough order of preference: + +### Pin a per-transport MTU floor in config + +If a known link in the path has a small MTU that discovery does not +pick up promptly (e.g., a Tor hop with an unusually tight cap), set +a transport-level MTU floor on the relevant `transports.*` block. +See [../reference/configuration.md](../reference/configuration.md) +for the per-transport MTU keys. + +### Tune host UDP buffers + +For UDP transports specifically, undersized kernel buffers can drop +oversized datagrams in a way that looks identical to MTU failure. +See [tune-udp-buffers.md](tune-udp-buffers.md). + +### Accept the floor on intrinsically small links + +Tor and BLE link MTUs are properties of the medium, not tunables. +For sessions that cross those links, the path MTU will be small; the +fix is to design applications around it (smaller TCP windows, fewer +large RTTs) rather than fight the transport. + +### File a bug + +The bug shape worth filing is session `mmp.path_mtu` itself +oscillating, or `MtuExceeded` ticking *within* an established +session after `mmp.path_mtu` has settled. The TCP-clamp mirror +(`path_mtu_lookup`) is now updated on every successful proactive +`PathMtuNotification` apply (tighter-only) as well as by the +reactive `MtuExceeded` handler, so a steady-state divergence +between the per-session `mmp.path_mtu` and the mirror used for +new TCP flows is itself a defect, not an expected behavior. + +Capture `fipsctl show sessions`, `fipsctl show cache`, `fipsctl +show routing` (for the `error_signals` block), and a tcpdump from +`fips0` covering the symptom window. See +[../design/fips-mtu.md](../design/fips-mtu.md#per-destination-mtu-storage) +for the per-destination MTU storage layout. + +## See also + +- [../design/fips-mtu.md](../design/fips-mtu.md) — encapsulation + overhead, the proactive `path_mtu` field, the reactive + `MtuExceeded` mechanism, MSS clamping, the no-fragmentation + policy. +- [../design/fips-mmp.md](../design/fips-mmp.md) — what the MMP + metrics mean and how they are computed. +- [../design/fips-ipv6-adapter.md](../design/fips-ipv6-adapter.md) — + TUN-side ICMPv6 PTB generation and the MSS clamp. +- [tune-udp-buffers.md](tune-udp-buffers.md) — host sysctl recipes + that rule out kernel-buffer drops as a confounder. diff --git a/docs/how-to/enable-mesh-firewall.md b/docs/how-to/enable-mesh-firewall.md new file mode 100644 index 0000000..93cf665 --- /dev/null +++ b/docs/how-to/enable-mesh-firewall.md @@ -0,0 +1,158 @@ +# Enable the Mesh-Interface Firewall + +FIPS ships a default-deny nftables baseline at `/etc/fips/fips.nft` that +restricts inbound traffic on the `fips0` mesh interface to conntrack +replies and ICMPv6 echo. The baseline is **not** enabled by default — see +[../design/fips-security.md](../design/fips-security.md) for the threat +model and the rationale behind keeping activation explicit. This guide +covers the operator steps to load the baseline, extend it with per-host +allowances, and inspect drops. + +## Activate the baseline + +The package ships `fips-firewall.service`, a systemd oneshot that runs +`nft -f /etc/fips/fips.nft` on start and removes the `inet fips` table +on stop. To activate: + +```sh +sudo systemctl enable --now fips-firewall.service +``` + +This loads the table now and arranges for it to load on every subsequent +boot. To disable and tear it down: + +```sh +sudo systemctl disable --now fips-firewall.service +``` + +To reload after editing `/etc/fips/fips.nft` or adding a drop-in under +`/etc/fips/fips.d/`: + +```sh +sudo systemctl reload-or-restart fips-firewall.service +``` + +The file is idempotent — it begins with `add table inet fips; flush +table inet fips;` so re-running it replaces the live ruleset atomically. +Equivalently: + +```sh +sudo nft -f /etc/fips/fips.nft +``` + +## Folding the baseline into the host's main nftables + +If you prefer to load the baseline from your existing +`/etc/nftables.conf` rather than via the systemd unit, include it +directly: + +```nft +# in /etc/nftables.conf +include "/etc/fips/fips.nft" +``` + +In that case do **not** enable `fips-firewall.service` — the host's main +nftables setup owns the loading. The two paths are mutually exclusive. + +## Extend with per-host allowances via drop-ins + +The baseline drops everything inbound on `fips0` except conntrack +replies and ICMPv6 echo. To open specific services to specific mesh +nodes, drop a file into `/etc/fips/fips.d/` ending in `.nft`. Each +file is included inline into the `inbound` chain at the marked point +and may contain any nftables rule lines valid in that context. + +Reload after editing: + +```sh +sudo systemctl reload-or-restart fips-firewall.service +# or: sudo nft -f /etc/fips/fips.nft +``` + +### Allow inbound SSH from a specific mesh node + +```nft +# /etc/fips/fips.d/ssh-from-bastion.nft +ip6 saddr fd97:1234:5678:9abc:def0:1234:5678:9abc tcp dport 22 accept +``` + +The source filter is the node's mesh address. To find a node's mesh +address, look in their `fips.pub` (which contains the npub) and derive +the `fd97:...` address from it, or query the running daemon: + +```sh +fipsctl show identity-cache +fipsctl show peers +``` + +### Allow inbound DNS broadly + +Some services need to be reachable from any mesh node (a public DNS +resolver, a public bootstrap node): + +```nft +# /etc/fips/fips.d/dns-public.nft +udp dport 53 accept +tcp dport 53 accept +``` + +Omit the source filter only when the service is intended to be +universally reachable on the mesh. The baseline's purpose is to make +"universally reachable" an explicit decision rather than the default. + +### Multiple nodes, one service + +```nft +# /etc/fips/fips.d/git-from-trusted.nft +ip6 saddr { + fd97:1111:2222:3333:4444:5555:6666:7777, + fd97:8888:9999:aaaa:bbbb:cccc:dddd:eeee +} tcp dport 9418 accept +``` + +Set syntax keeps multi-node rules readable and is more efficient than a +chain of individual rules. + +## Inspect drops + +The baseline counter increments on every dropped packet. Inspect it: + +```sh +sudo nft list table inet fips +``` + +Look for the `counter packets N bytes M drop` line at the bottom of the +`inbound` chain. A non-zero counter means mesh nodes are sending +traffic that hits the default-deny — usually benign (probes, neighbor +discovery) but occasionally a misconfigured drop-in. + +To see which packets are being dropped, uncomment the `log` line near +the bottom of `/etc/fips/fips.nft`: + +```nft +log prefix "fips drop: " level info limit rate 10/minute +``` + +Reload: + +```sh +sudo nft -f /etc/fips/fips.nft +``` + +Then tail the kernel log: + +```sh +sudo journalctl -k -f -g "fips drop:" +``` + +The rate-limit prevents flooding the journal under sustained probing. +Adjust the rate, log level, or prefix as needed for the situation. +Re-comment the rule when you are done; production hosts do not need +the log line on by default. + +## See also + +- [../design/fips-security.md](../design/fips-security.md) — threat + model, baseline design, and coexistence with other firewalls +- [../reference/security.md](../reference/security.md) — consolidated + security reference diff --git a/docs/how-to/enable-nostr-discovery.md b/docs/how-to/enable-nostr-discovery.md new file mode 100644 index 0000000..1ad5deb --- /dev/null +++ b/docs/how-to/enable-nostr-discovery.md @@ -0,0 +1,347 @@ +# Enable Nostr-Mediated Discovery and NAT Traversal + +Nostr-mediated discovery lets FIPS nodes find each other (and punch +through UDP NAT) using public Nostr relays as the signaling channel. +The feature ships in every stock packaging artifact but is **off by +default** — it activates when an operator sets +`node.discovery.nostr.enabled: true` and provides a relay list. See +[../design/fips-nostr-discovery.md](../design/fips-nostr-discovery.md) +for the design and rationale; see +[../reference/configuration.md](../reference/configuration.md) for the +full knob inventory. + +Nostr discovery provides three independent capabilities. They can be +enabled separately; most deployments end up using two or three of +them together. + +1. **Resolve a known peer's address by npub.** Your daemon consumes + adverts from the relays to look up the current network endpoint + for a peer you have configured by npub. You don't have to know + their IP / port / transport in advance. +2. **Publish your own endpoint so others can resolve you.** Your + daemon publishes a signed advert listing the transports it will + accept connections on. Has two sub-shapes depending on your + network topology: UDP (using NAT traversal if needed) or TCP. + Running a Tor onion service is a separate deployment mode, + covered in its own section below. +3. **Discover peers without prior configuration.** Your daemon + subscribes to all adverts on a chosen application namespace and + treats any publisher as a connection candidate. The most + permissive posture; useful for ambient mesh participation. + +Each capability is covered below as one or more scenarios with the +minimal YAML fragment that enables it. Only keys relevant to Nostr +discovery are shown; surrounding node, transport, TUN, DNS, and peer +configuration follows the usual shape. + +All scenarios assume `node.identity` is set to a persistent key — an +ephemeral identity would invalidate any advert the moment the node +restarts. See [persistent-identity.md](persistent-identity.md) for +the persistent-key setup. + +For hand-held walkthroughs of each capability, see the +[resolve-peers-via-nostr](../tutorials/resolve-peers-via-nostr.md), +[advertise-your-node](../tutorials/advertise-your-node.md), +and [open-discovery](../tutorials/open-discovery.md) +tutorials. + +## Capability 1: Resolve a known peer's address by npub + +The node does not publish any advert of its own. It only consumes +adverts for peers it has explicitly listed with `via_nostr: true`. +This is the right shape for a client that wants Nostr-mediated +resolution without becoming a rendezvous target itself. + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: false + policy: configured_only + +transports: + udp: + bind_addr: "0.0.0.0:2121" + +peers: + - npub: "npub1peer..." + alias: "remote-node" + via_nostr: true + connect_policy: auto_connect +``` + +What this achieves: dial endpoints for this peer are taken from +the peer's published Nostr advert. `configured_only` is the +default — it is shown here for clarity. + +> **Note:** You can also supply a static address alongside +> `via_nostr: true` (for example, while testing, or as a +> known-good fallback if the advert is stale). Add an `addresses` +> block to the peer entry; static addresses are tried first on +> dial and Nostr-resolved endpoints are appended as additional +> candidates. + +## Capability 2: Publish your own endpoint so others can resolve you + +This capability has three sub-scenarios depending on the network +shape your node sits behind. + +### Sub-scenario 2a: UDP (using NAT traversal if needed) + +The node has a public IP (or a stable port-forward) and binds UDP on +a known port. It publishes `udp:host:port` to the advert relays. Any +peer that knows this node's npub and has Nostr discovery enabled can +dial it without knowing the address out-of-band. + +When UDP is wildcard-bound (`0.0.0.0:2121`, the default), the daemon +needs help knowing what IP to put in the advert. There are two ways: +STUN auto-discovery (`public: true`) or an explicit override +(`external_addr`). Both are first-class options; pick the one that +fits the deployment. + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: true + +transports: + udp: + bind_addr: "0.0.0.0:2121" + advertise_on_nostr: true + public: true # ← STUN auto-discovery +``` + +Or, when the public IP is known up front (static residential IP, +cloud Elastic IP behind 1:1 NAT, etc.): + +```yaml +transports: + udp: + bind_addr: "0.0.0.0:2121" + advertise_on_nostr: true + external_addr: "203.0.113.45:2121" # ← explicit +``` + +`external_addr` accepts a bare IP (combined with the bind port) or a +full `host:port`. Setting both `public: true` and `external_addr` +together is allowed — the explicit override wins, with STUN as a +logging cross-check. If UDP is bound directly to a public IP rather +than to a wildcard, neither flag is needed; the daemon advertises +the bound endpoint. + +What this achieves: the node publishes a single +`udp::2121` endpoint to the three default advert relays +(`wss://relay.damus.io`, `wss://nos.lol`, `wss://offchain.pub`). + +What the other side needs: either a static `addresses` entry for this +peer, or a peer entry with `via_nostr: true` and an empty (or +omitted) `addresses` list — the advert-resolved endpoint will be used +at dial time. Static and Nostr-resolved addresses can also be +combined: when both are present, static addresses are tried first and +Nostr-resolved endpoints are appended as fallback. + +#### When the node is behind NAT + +If this node doesn't have a stable public UDP endpoint, advertise +`udp:nat`. The daemon runs the STUN + offer/answer exchange with +the peer and punches through the NAT to establish a direct UDP +link. The peer can either have a public endpoint of its own or +also be behind NAT — both shapes work, as long as at least one +side has a NAT type compatible with hole-punching. + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: true + dm_relays: # overrides the default three-relay + - "wss://relay.damus.io" # set with two for demonstration; + - "wss://nos.lol" # omit this block to keep the defaults + stun_servers: + - "stun:stun.l.google.com:19302" + - "stun:stun.cloudflare.com:3478" + +transports: + udp: + bind_addr: "0.0.0.0:2121" + advertise_on_nostr: true + public: false + +peers: + - npub: "npub1peer..." + alias: "nat-peer" + addresses: + - transport: udp + addr: "nat" + via_nostr: true + connect_policy: auto_connect +``` + +What this achieves: the node publishes a `udp:nat` endpoint plus its +signaling relays in the advert. When either side initiates, an +encrypted offer is sealed to the peer's npub, a matching answer +comes back, and both sides punch at the negotiated time. On success, +the punch socket is adopted as an FMP UDP transport and Noise IK +proceeds normally. + +> **Validation:** `advertise_on_nostr: true` with `public: false` on +> UDP requires `dm_relays` and `stun_servers` to be non-empty. Both +> ship with non-empty defaults (three relays and three STUN servers +> respectively), so the default config passes. The node fails +> startup only if the operator has explicitly emptied either list — +> a `udp:nat` advert without signaling relays or STUN servers is +> unreachable by construction. + +Works best with full-cone NAT on at least one side. Symmetric NAT on +both sides is not reliably traversable with this protocol and will +time out after `punch_duration_ms`. + +### Sub-scenario 2b: TCP + +The node has a public IP (or a stable port-forward) and accepts +inbound TCP. It publishes `tcp:host:port` to the advert relays. + +TCP endpoints exist to serve peers whose networks filter outbound +UDP (corporate LANs, restrictive guest WiFi). NAT traversal does +not apply: the publishing node is publicly reachable on TCP, and +the dialing peer's network only needs to permit outbound TCP to +the advertised port. + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: true + +transports: + tcp: + bind_addr: "0.0.0.0:8443" + advertise_on_nostr: true + external_addr: "203.0.113.45:8443" +``` + +`external_addr` is typically required on cloud setups (AWS Elastic +IP, etc.) where binding directly to the public IP returns +`EADDRNOTAVAIL`. When TCP is bound directly to a public IP, the +override is unnecessary. + +What this achieves: the node publishes a `tcp::8443` +endpoint to the advert relays. Peers with Nostr discovery enabled +dial by npub without out-of-band address exchange. + +### Tor onion node + +A separate deployment mode for nodes that want anonymity and +censorship-resistance properties on the data plane. Functionally +this still uses Capability 2 (publishing an endpoint to advert +relays) — the difference is that the published endpoint is a Tor +hidden service rather than a public IP. + +The node runs a Tor onion service in directory mode (Tor-managed +`HiddenServiceDir`) and advertises the `.onion` address. Peers dial +via their local Tor SOCKS5 proxy without ever knowing the onion +string out-of-band. For the Tor daemon side of this setup, including the inbound-mode +trade-offs and the `torrc` directives each requires, see +[deploy-tor-onion.md](deploy-tor-onion.md). + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: true + +transports: + tor: + mode: directory + socks5_addr: "127.0.0.1:9050" + directory_service: + hostname_file: "/var/lib/tor/fips/hostname" + bind_addr: "127.0.0.1:8444" + advertise_on_nostr: true +``` + +What this achieves: the node publishes a `tor:.onion:8443` +endpoint alongside any other advertised transports. The advert itself +is still published over clearnet WebSocket relays — Tor protects the +data plane, not the discovery plane. See the security and threat +model section in +[../design/fips-nostr-discovery.md](../design/fips-nostr-discovery.md#security-and-threat-model) +for the trade-off and how to route relay traffic through Tor as well. + +## Capability 3: Discover peers without prior configuration + +Under `policy: open`, any node that publishes an advert under the +same `app` namespace becomes a candidate. Discovered peers are queued +for connection attempts subject to `open_discovery_max_pending`. + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: true + policy: open + open_discovery_max_pending: 64 + app: "my-experiment.v1" + +transports: + udp: + bind_addr: "0.0.0.0:2121" + advertise_on_nostr: true + public: true + +peers: [] +``` + +What this achieves: peers are discovered entirely through ambient +advert traffic on the configured relays. Setting a non-default `app` +value (replacing `fips-overlay-v1`) scopes the discovery set to +participants who opt into the same experiment and avoids being joined +to unrelated overlays that happen to share the default namespace. + +> **Scope warning:** Open discovery is an admission-free mode. Any +> node that publishes on the same `app` name and passes the peer-ACL +> check becomes a connection candidate. If you rely on peer ACLs for +> admission control, verify that list is set correctly before +> enabling this mode. See +> [../reference/security.md](../reference/security.md) for the peer +> ACL format. + +## See also + +- [../tutorials/resolve-peers-via-nostr.md](../tutorials/resolve-peers-via-nostr.md) + — hand-held walkthrough of capability 1 +- [../tutorials/advertise-your-node.md](../tutorials/advertise-your-node.md) + — hand-held walkthrough of capability 2 (publish, plus a + short section on `udp:nat` NAT traversal) +- [../tutorials/open-discovery.md](../tutorials/open-discovery.md) + — hand-held walkthrough of capability 3 (open ambient + discovery, the additive policy: open mode) +- [../design/fips-nostr-discovery.md](../design/fips-nostr-discovery.md) + — discovery runtime design, security model +- [../reference/configuration.md](../reference/configuration.md) — + full `node.discovery.nostr.*` and per-transport + `advertise_on_nostr`/`public` table +- [../reference/nostr-events.md](../reference/nostr-events.md) — Kind + 37195 advert format, Kind 21059 traversal signaling, Kind 10050 + inbox relay list +- [deploy-tor-onion.md](deploy-tor-onion.md) — Tor daemon-side setup + for advertising onion endpoints diff --git a/docs/how-to/host-aliases.md b/docs/how-to/host-aliases.md new file mode 100644 index 0000000..7445224 --- /dev/null +++ b/docs/how-to/host-aliases.md @@ -0,0 +1,173 @@ +# Use Shortnames Instead of Long Npubs + +A FIPS node's canonical address is `.fips`. The npub is +63 characters of bech32 — fine for the daemon, awkward to type +or fit in a docs example. The local DNS resolver consults a +host map before falling back to direct-npub resolution, so +short names like `test-us01.fips` work as substitutes wherever +`.fips` would. + +This guide covers the two ways to populate that map and when +to use which. + +## When to use which + +Two independent mechanisms feed the same DNS responder: + +| Mechanism | Source | Scope | Reload | +|-----------|--------|-------|--------| +| Hosts file | `/etc/fips/hosts` | Node-local, intended for shared rosters | Auto on mtime change | +| Peer alias | `alias:` field on a `peers:` entry | Node-local, scoped to configured peers | Daemon restart | + +Pick the hosts file when: + +- The shortname refers to a peer your operator-team agrees to + call by that name across machines (the public test mesh + ships this way). +- You want the destination's name to resolve in DNS or appear + in `fipsctl show peers` display even though it isn't in your + `peers:` block — e.g., a mesh node you reach transitively + through your direct peers. The hosts-file entry is for name + resolution and display only; it does not stand in for the + npub a peer-config entry requires. + +Pick the peer alias when: + +- The shortname is just a label *you* use locally for a peer + that's already in your `peers:` block. +- You want the alias to live with the rest of the peer config + (one place to look) rather than in a separate file. + +The two coexist. If both reference the same shortname, +`/etc/fips/hosts` wins — the file is treated as the +authoritative shared roster. + +## What ships in the default `/etc/fips/hosts` + +The installer drops `/etc/fips/hosts` populated with the +public test mesh roster: + +```text +test-us01 npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98 +test-us02 npub10yffd020a4ag8zcy75f9pruq3rnghvvhd5hphl9s62zgp35s560qrksp9u +test-us03 npub136yqae6na688fs75g95ppps3lxe07fvxefj77938zf47uhm6074sxw8ctm +test-us03-next npub15m6c4ghuegx4pcde6tra8f7smn8vfv2wundyxwhkjynuerkrzmgsy09sh3 +test-us04 npub1gd7ye2qp2lphhzx75fynnjzaxx4dqanddecet0wtt5ss5ek8h9ps62wdkf +test-de01 npub1260n42s06vzc7796w0fh3ny7zcpw6tlk4gq3940gmfrzl5c9pv2s3657q8 +test-es01 npub17lpmzulpc98d8ff727k6e98atxn3phzupzsqqwe54ytduym747ws4tw5zm +test-uk01 npub1u0z26dc4qeneu5rvwvmpfhtwh3522ed6rlgxr9jarrfnjrc6ew4qxjysrs +``` + +These resolve out of the box — `ping6 test-us01.fips` works +even before you've added any peer to your config, as long as +the destination is reachable through your mesh links. + +If you don't intend to interact with the public test mesh, +the entries are safe to comment out or delete. They are +plain hosts-file lines, not protocol participants — removing +them only changes name resolution on your machine. + +## Add an entry to `/etc/fips/hosts` + +Append a line to `/etc/fips/hosts`: + +```text +my-laptop npub1abc...xyz +``` + +Format rules: + +- One hostname and one npub per line, separated by + whitespace. +- Hostnames are lowercase letters, digits, and hyphens; max + 63 characters. +- Comments start with `#` and continue to end of line; blank + lines are ignored. +- On duplicate hostnames, the last entry wins. + +The daemon picks up the change on the next DNS query — no +restart required (the file's mtime is checked on each query). +Verify: + +```sh +dig my-laptop.fips AAAA +short +``` + +Expect one `fd97:...` AAAA record. + +`/etc/fips/hosts` is shipped as a dpkg conffile (and the AUR +equivalent), so package upgrades preserve your edits. The +file is `0644 root:root` — readable by anyone, writable by +root. + +## Add a peer alias + +In `/etc/fips/fips.yaml`, set the `alias:` field on the peer +entry: + +```yaml +peers: + - npub: "npub1abc...xyz" + alias: "my-laptop" + addresses: + - transport: udp + addr: "192.0.2.10:2121" + connect_policy: auto_connect +``` + +Restart the daemon for the alias to take effect: + +```sh +sudo systemctl restart fips +dig my-laptop.fips AAAA +short +``` + +The alias also shows up in `fipsctl show peers` `display_name` +column, so log entries and CLI output reference the peer by +shortname instead of truncated npub. + +## Resolution order + +When the DNS responder receives a query for `.fips`: + +1. **Hosts file lookup.** If `` matches an entry in + `/etc/fips/hosts`, the daemon returns the AAAA record + derived from that entry's npub. +2. **Peer alias lookup.** If `` matches the `alias` + field on a configured peer, return that peer's AAAA. +3. **Direct npub resolution.** If `` is itself a valid + bech32 npub (the canonical 63-char `npub1...` form), the + daemon returns the AAAA derived from that npub directly. +4. **NXDOMAIN.** If none of the above match, the query + returns no answer. + +The order means the hosts file overrides peer aliases on +conflict. That's deliberate: the file represents +operator-shared naming, the peer alias is a node-local label. + +## Cross-references and ACLs + +Aliases interact with the peer ACL — if you maintain +`peers.allow` or `peers.deny` lists keyed on hostnames rather +than npubs, those names go through the same hosts-file +resolution. See +[../reference/security.md](../reference/security.md) for the +ACL format and the alias-resolution semantics. + +`fipsctl connect` and `fipsctl disconnect` accept a shortname +where they expect an npub. Resolution for these commands goes +through `/etc/fips/hosts` only — peer-config `alias:` entries +are not loaded by `fipsctl`, so a shortname that exists only as +a peer alias must still be referenced by full npub on the CLI. +See [../reference/cli-fipsctl.md](../reference/cli-fipsctl.md). + +## See also + +- [../reference/configuration.md § Host Mapping](../reference/configuration.md#host-mapping) + — minimal reference entry for the host-map mechanism. +- [../reference/cli-fipsctl.md](../reference/cli-fipsctl.md) + — `fipsctl` arguments that accept shortnames. +- [../reference/security.md](../reference/security.md) + — peer ACL semantics with aliased entries. +- [../design/fips-ipv6-adapter.md](../design/fips-ipv6-adapter.md) + — the DNS resolver design and the npub-to-IPv6 derivation. diff --git a/docs/how-to/persistent-identity.md b/docs/how-to/persistent-identity.md new file mode 100644 index 0000000..81934f7 --- /dev/null +++ b/docs/how-to/persistent-identity.md @@ -0,0 +1,232 @@ +# Provision a Persistent Identity + +A FIPS node's identity is a Nostr keypair. Its public key (npub) +determines the node's `fd00::/8` mesh address; peers and configs +reference the node by that npub. Out of the box the daemon generates +a fresh identity on every start (`node.identity.persistent: false`), +which is fine for one-off testing but useless when other nodes need +to refer to this one across restarts. + +This guide covers the three ways to give a node a stable identity. +For the configuration keys involved, see +[../reference/configuration.md](../reference/configuration.md). + +> **First time?** If you have just installed FIPS and want a +> hand-held walkthrough of the package-default path (set +> `persistent: true`, restart, observe the keys land), the +> [persistent-identity tutorial](../tutorials/persistent-identity.md) +> is the gentler entry point. This guide assumes an operator +> picking among Options A/B/C for a deployment. + +## When to use + +Use a persistent identity for any node that: + +- Other operators reference by npub (in their `peers` lists, `hosts` + files, or ACL allow-lists). +- Acts as a discoverable bootstrap or rendezvous (Nostr advert, + static peer entry, gateway). +- Is expected to keep its `fd00::/8` mesh address across restarts. + +Stay with the ephemeral default for throw-away clients, sandbox +nodes, and tests where you actively want a fresh identity per run. + +## Option A: Let the package do it + +The Debian/Ubuntu `.deb` and the Arch `fips` AUR package both ship a +default `/etc/fips/fips.yaml` with `node.identity.persistent` left as +the upstream default (false), so the daemon writes a fresh keypair to +`/etc/fips/fips.{key,pub}` on every start until you set +`persistent: true`. To pin the current keypair: + +1. Install the package and start the daemon once so it generates + `fips.key` / `fips.pub`: + + ```sh + sudo systemctl start fips + sudo systemctl status fips # confirm it came up + ``` + +2. Edit `/etc/fips/fips.yaml` and set: + + ```yaml + node: + identity: + persistent: true + ``` + +3. Restart the daemon and verify the identity is reused: + + ```sh + sudo systemctl restart fips + fipsctl show status | grep -E '"npub"|"node_addr"' + cat /etc/fips/fips.pub + ``` + + The npub printed by `fipsctl show status` should match + `/etc/fips/fips.pub` and remain stable across subsequent restarts. + +The package's `postinst` script does **not** generate the keypair — +the daemon does, on first start. This means the keypair is only +present after the first successful daemon start. If the daemon never +came up cleanly (config error, permission problem), the key files +will be missing. + +### File layout and permissions + +| Path | Mode | Owner | Contents | +| ---- | ---- | ----- | -------- | +| `/etc/fips/fips.key` | `0600` | `root:root` | Bech32 `nsec` (one line). | +| `/etc/fips/fips.pub` | `0644` | `root:root` | Bech32 `npub` (one line). | + +Both files live next to the highest-priority `fips.yaml` the daemon +loaded. For non-systemd installs that use a different config path, +the key files are placed in that config's directory. + +## Option B: Generate manually + +For from-source installs, custom config paths, or any deployment +where you want to mint the keypair before the daemon ever runs. + +### With `fipsctl keygen` + +```sh +sudo fipsctl keygen --dir /etc/fips +``` + +This writes `/etc/fips/fips.key` (mode `0600`) and +`/etc/fips/fips.pub` (mode `0644`), prints the new npub on stderr, +and reminds you to set `persistent: true`. Add `--force` to overwrite +an existing `fips.key`. Add `--stdout` to print `nsec` then `npub` +to stdout instead of writing files. + +To put the keypair in a non-default directory (e.g., a per-deployment +config tree), pass `--dir` and point your `fips.yaml` search at the +matching directory. + +### Without the daemon installed + +If you cannot run `fipsctl` (e.g., scripting on a build host), any +nostr-tools-equivalent that emits a bech32 `nsec` works. Write the +nsec to `fips.key` (mode `0600`) and the corresponding `npub` to +`fips.pub` (mode `0644`). + +### Hooking the keypair into the config + +```yaml +node: + identity: + persistent: true +``` + +`persistent: true` plus a `fips.key` next to the loaded config is the +intended steady-state setup. + +## Option C: Provision from an existing nsec + +To migrate an existing Nostr identity into a FIPS node — for example, +re-using a personal npub for a node you operate. + +1. Obtain the bech32 `nsec` for the identity. +2. Write it to the config-adjacent key file: + + ```sh + sudo install -m 0600 -o root -g root /dev/null /etc/fips/fips.key + sudo bash -c 'printf "%s\n" nsec1... > /etc/fips/fips.key' + ``` + +3. Derive the matching `npub` and write `fips.pub`: + + ```sh + # compute the npub with any nostr tool, then: + sudo bash -c 'printf "%s\n" npub1... > /etc/fips/fips.pub' + sudo chmod 0644 /etc/fips/fips.pub + ``` + +4. Set `persistent: true` and restart: + + ```yaml + node: + identity: + persistent: true + ``` + + ```sh + sudo systemctl restart fips + fipsctl show status | grep '"npub"' + ``` + +The reported npub should match the one you wrote to `fips.pub`. + +## Verifying + +The daemon prints the resolved identity at startup; the same value is +queryable via the control socket: + +```sh +fipsctl show status | jq '{npub, node_addr, ipv6_addr}' +cat /etc/fips/fips.pub +``` + +The `npub` field of `show status` and the contents of `fips.pub` +should match. The `node_addr` is the SHA-256 prefix used internally +by FMP/FSP; the `ipv6_addr` is the routable `fd00::/8` mesh address +derived from the node addr. Together they are stable for the lifetime +of the keypair. + +The journal also records the source on every start: + +```text +INFO Loaded persistent identity from key file path=/etc/fips/fips.key +``` + +(`Generated persistent identity, saved to key file` on the first +start; `Using ephemeral identity (new keypair each start)` when +persistence is off.) + +## Rotating + +Key rotation is a destructive operation: every cached +`(node_addr → npub)` mapping on every other node points at the old +key, every Nostr advert and every static peer entry references the +old npub, and every existing FSP session was authenticated under the +old keypair. There is no in-protocol "key change" message. + +To rotate: + +1. Stop the daemon. + + ```sh + sudo systemctl stop fips + ``` + +2. Remove the existing key files. + + ```sh + sudo rm /etc/fips/fips.key /etc/fips/fips.pub + ``` + +3. Start the daemon. With `persistent: true`, the daemon generates a + new keypair and writes new `fips.key` / `fips.pub`. + + ```sh + sudo systemctl start fips + cat /etc/fips/fips.pub # the new npub + ``` + +4. Update every downstream reference: peer configs that name this + node by npub, `hosts` files, ACL allow-lists, Nostr adverts + pinned by other operators. + +There is no recovery from a lost `fips.key` — the npub is gone with +the secret. Treat key rotation as a coordinated event; do not rotate +production identities ad hoc. + +## See also + +- [../reference/configuration.md](../reference/configuration.md) — + `node.identity.*` keys. +- [../reference/cli-fipsctl.md](../reference/cli-fipsctl.md) — + `fipsctl keygen`. +- [../design/fips-architecture.md](../design/fips-architecture.md) — + identity model, npub-to-NodeAddr derivation. diff --git a/docs/how-to/run-as-unprivileged-user.md b/docs/how-to/run-as-unprivileged-user.md new file mode 100644 index 0000000..f6f04a3 --- /dev/null +++ b/docs/how-to/run-as-unprivileged-user.md @@ -0,0 +1,195 @@ +# Run the FIPS Daemon as an Unprivileged User + +By default, the FIPS daemon runs as `root` — the shipped Debian +systemd unit configures this, and no further setup is required. +The trade-off is that the daemon has full root authority, +including outside its actual network needs. Acceptable for +single-purpose hosts; less desirable for shared hosts. + +This guide covers the alternative: drop privileges and run the +daemon under a dedicated unprivileged user account. The TUN +device that the FIPS IPv6 adapter creates requires +`CAP_NET_ADMIN` on Linux; the recipe below grants that privilege +via a file capability on the binary, plus everything else the +daemon needs to keep working without root: a service user +account, file permissions on the config directory, and a systemd +unit override to drop privileges. + +For the design context (why the adapter needs a TUN, how the +adapter integrates with the kernel routing table), see +[../design/fips-ipv6-adapter.md](../design/fips-ipv6-adapter.md). + +## Prerequisites + +- FIPS package installed (the postinst already creates the `fips` + system group used for control-socket access). +- `setcap` available (`apt install libcap2-bin` on Debian/Ubuntu; + it is a standard utility on most distributions). +- Operator access to systemd unit overrides (`systemctl edit`). + +## Step 1: Create a `fips` system user + +The package creates a `fips` system *group* but no matching user. +Add a system user that belongs to the `fips` group: + +```sh +sudo useradd --system --gid fips --no-create-home --shell /usr/sbin/nologin fips +``` + +The user has no home directory and no login shell — this account +exists only to run the daemon. + +## Step 2: Grant `CAP_NET_ADMIN` to the binary + +Apply the file capability so the daemon can create the TUN device +without root authority: + +```sh +sudo setcap cap_net_admin+ep /usr/bin/fips +``` + +Verify: + +```sh +getcap /usr/bin/fips +# /usr/bin/fips cap_net_admin=ep +``` + +The binary can now create TUN devices when run by any user. + +**File-capability caveats:** + +- The capability is attached to the binary file. **Re-applying + the capability after every package upgrade is required**, + because package upgrades replace the binary file and lose the + cap. The systemd override in Step 4 includes an `ExecStartPre` + line that automates this. +- File capabilities are stripped when the binary is copied across + most filesystems and when it is downloaded via web tooling. If + you build from source and install manually, remember to + re-`setcap` after each rebuild. +- `LD_LIBRARY_PATH` and similar environment-driven loader + controls are stripped at exec time when file capabilities are + present; this is normally what you want, but development + workflows that rely on custom library paths may be surprised. + +## Step 3: Adjust config-file permissions + +The shipped `/etc/fips/fips.yaml` is mode `0600` and owned by +`root:root`. The daemon needs to read it and, if persistent +identity is enabled, write `/etc/fips/fips.key` into the same +directory. + +```sh +sudo chown -R fips:fips /etc/fips +sudo chmod 0640 /etc/fips/fips.yaml +``` + +If `node.identity.persistent: true` is set and `fips.key` does +not exist yet, leave `/etc/fips` itself writable by the `fips` +user so the daemon can create it on first start. After the key +file exists, you can tighten further: + +```sh +sudo chmod 0600 /etc/fips/fips.key +``` + +## Step 4: Drop privileges in the systemd unit + +Create an override: + +```sh +sudo systemctl edit fips.service +``` + +Add: + +```ini +[Service] +User=fips +Group=fips +AmbientCapabilities=CAP_NET_ADMIN +NoNewPrivileges=no +ExecStartPre=/sbin/setcap cap_net_admin+ep /usr/bin/fips +``` + +`User=` / `Group=` set the service identity. +`AmbientCapabilities=` ensures the file capability granted in +Step 2 actually carries into the daemon's process tree. +`NoNewPrivileges=no` is required for file-capability execution +to work — systemd defaults this to `yes` for hardened units, +which would block the `setcap` from taking effect. +`ExecStartPre=` re-applies the capability before each start, +which makes the package-upgrade path self-heal. + +The unit's `RuntimeDirectory=fips` directive already arranges +for `/run/fips/` to be created with the right ownership at +service start, now as `fips:fips 0750` instead of +`root:fips 0750`. + +Reload and restart: + +```sh +sudo systemctl daemon-reload +sudo systemctl restart fips +``` + +## Step 5: Verify + +Confirm the daemon is running as `fips`: + +```sh +ps -eo user,cmd | grep '[/]usr/bin/fips' +# fips /usr/bin/fips --config /etc/fips/fips.yaml +``` + +Confirm the TUN device came up (the `setcap` worked): + +```sh +ip link show fips0 +# fips0: mtu 1280 ... +``` + +Confirm the control socket is bound and accessible to the `fips` +group: + +```sh +ls -la /run/fips/control.sock +# srwxrwx--- 1 fips fips ... /run/fips/control.sock +``` + +Add yourself to the `fips` group so you can use `fipsctl` / +`fipstop` without `sudo`: + +```sh +sudo usermod -aG fips $USER +# log out and back in for the group change to take effect +``` + +Then: + +```sh +fipsctl show node +``` + +## Caveats + +- **`fips-firewall.service` still runs as root.** Loading nftables + rules into the kernel requires root regardless. The firewall + unit is intentionally separate from the daemon unit. +- **Bluetooth peers (`transports.ble.*`)** require additional + privileges the `CAP_NET_ADMIN` setcap doesn't cover. If you use + the BLE transport, you'll likely need to keep running as root + or layer additional capability/D-Bus configuration; that path + is not covered here. + +## See also + +- [persistent-identity.md](persistent-identity.md) — how the + daemon manages `/etc/fips/fips.key` +- [../design/fips-ipv6-adapter.md](../design/fips-ipv6-adapter.md) + — IPv6 adapter design, TUN interface architecture +- [../reference/security.md](../reference/security.md) — + consolidated security surface +- [../reference/configuration.md](../reference/configuration.md) + — `tun.*` configuration block diff --git a/docs/how-to/set-up-bluetooth-peer.md b/docs/how-to/set-up-bluetooth-peer.md new file mode 100644 index 0000000..f60198b --- /dev/null +++ b/docs/how-to/set-up-bluetooth-peer.md @@ -0,0 +1,299 @@ +# Set Up a Bluetooth (BLE) Peer Link + +FIPS supports Bluetooth Low Energy as a transport for short-range +mesh extension — same room, same building, no IP infrastructure +between the two endpoints. The BLE transport runs as L2CAP +Connection-Oriented Channels on a configurable PSM and reports +per-link MTU back to the mesh layer for path-MTU computation. + +For the design rationale and per-link MTU model, see +[../design/fips-transport-layer.md](../design/fips-transport-layer.md). +For all `transports.ble.*` configuration keys, see +[../reference/configuration.md](../reference/configuration.md). + +> **Experimental.** The BLE transport works but is still maturing. +> Expect rougher edges than UDP or TCP — particularly around link +> stability under interference and MTU negotiation on older +> controllers. Treat it as you would any experimental transport in a +> production deployment. + +## When to use + +BLE is the right transport when: + +- Two nodes are within roughly 10 metres line-of-sight (more with + external antennas, less through walls). +- You want a self-contained mesh segment with no shared WiFi or + Ethernet between the participants. +- You can work within practical L2CAP CoC throughput (1-2 Mbps in + good conditions, often substantially less under interference or + at range) and the higher latency variance compared to WiFi. + +It is **not** the right transport for backbone links between rooms +where WiFi or Ethernet exists, for high-throughput data, or for any +deployment where range matters more than infrastructure-freedom. + +## Platform support + +The BLE transport is **Linux-only** in the current implementation. +The runtime depends on BlueZ via the `bluer` crate, which in turn +needs `glibc` (musl builds skip BLE; the build script gates the +crate accordingly). + +| Platform | BLE transport | +| -------- | -------------- | +| Linux (glibc) | Supported. | +| Linux (musl, OpenWrt) | Disabled at build time. | +| macOS | Not supported. | +| Windows | Not supported. | + +The Debian package `Recommends: bluez`; install it explicitly if you +opted out: + +```sh +sudo apt install bluez +``` + +## Prerequisites + +Both endpoints need: + +1. A BLE-capable HCI adapter visible to BlueZ. Confirm with: + + ```sh + sudo bluetoothctl show + ``` + + Note the controller name (typically `hci0`). + +2. The `bluetoothd` service running and the adapter powered on: + + ```sh + sudo systemctl enable --now bluetooth + sudo bluetoothctl power on + ``` + +3. Sufficient privileges for the FIPS daemon. There are two + independent privilege concerns; the BLE-only deployment case + (mesh router with `tun.enabled: false`) needs only the second. + + - **TUN adapter (always required when `tun.enabled: true`).** + The daemon needs `CAP_NET_ADMIN` to create and configure the + TUN device. The shipped systemd unit handles this by running + as root; if you prefer to drop privileges, see + [run-as-unprivileged-user.md](run-as-unprivileged-user.md). + + - **BLE access (required for this how-to).** BlueZ exposes + L2CAP and D-Bus paths under either group membership or + `CAP_NET_RAW`. Pick one: + + - Run the daemon as root. The shipped systemd unit takes + this route. + - Run as an unprivileged user that is a member of the + `bluetooth` group. No additional capability is needed for + the BLE side. + - Run as an unprivileged user with no group membership, and + grant the binary `CAP_NET_RAW`: + + ```sh + sudo setcap cap_net_raw+ep $(which fips) + ``` + + This bypasses BlueZ's polkit/group check by holding + `CAP_NET_RAW` directly. If you also need `CAP_NET_ADMIN` + for TUN, combine them: + + ```sh + sudo setcap cap_net_admin,cap_net_raw+ep $(which fips) + ``` + +4. The same L2CAP PSM on both endpoints. The default is `0x0085` + (133); override only if you need to coexist with another L2CAP + service on that PSM. + +## Configuration + +Add a `ble` block under `transports` in `fips.yaml`. A minimum BLE- +active node looks like this: + +```yaml +transports: + ble: + adapter: "hci0" + advertise: true + scan: true + auto_connect: true + accept_connections: true +``` + +Note: `auto_connect: true` is intentionally non-default (the default +is `false`). For a symmetric ground-up discovery flow where either +side may dial, both ends must opt in explicitly. + +| Key | Purpose | +| --- | ------- | +| `adapter` | HCI controller name. Default: `hci0`. | +| `psm` | L2CAP PSM. Default: `0x0085` (must match on both ends). | +| `mtu` | Default L2CAP CoC MTU. Default: `2048`. The kernel may negotiate lower per link. | +| `max_connections` | Concurrent BLE connections. Default: `7` (Bluetooth controllers typically support up to ~7 simultaneous L2CAP CoCs). | +| `advertise` | Broadcast our BLE adverts so other FIPS nodes discover us. Default: `true`. | +| `scan` | Listen for other FIPS nodes' BLE adverts. Default: `true`. | +| `auto_connect` | Initiate a BLE connection to discovered FIPS adverts. Default: `false`. | +| `accept_connections` | Accept inbound L2CAP connections. Default: `true`. | +| `connect_timeout_ms` | Outbound L2CAP connect timeout. Default: `10000`. | +| `probe_cooldown_secs` | After probing a BD_ADDR (success or failure), wait this long before probing it again. Default: `30`. | + +Two pairing patterns are common: + +**Symmetric auto-discovery.** Both nodes advertise, scan, and +auto-connect. Whichever side completes the L2CAP connection first +wins; the other side aborts its in-flight attempt. This is the +"toss two devices in the same room" setup. + +```yaml +# Both nodes +transports: + ble: + adapter: "hci0" + advertise: true + scan: true + auto_connect: true + accept_connections: true +``` + +**Asymmetric peripheral / central.** One node only listens +(peripheral), the other actively dials (central). Useful when one +endpoint is a dedicated bootstrap and the other is mobile. + +```yaml +# Listener +transports: + ble: + adapter: "hci0" + advertise: true + scan: false + auto_connect: false + accept_connections: true +``` + +```yaml +# Dialer +transports: + ble: + adapter: "hci0" + advertise: false + scan: true + auto_connect: true + accept_connections: false +``` + +After editing, restart the daemon on each side: + +```sh +sudo systemctl restart fips +``` + +## Verify + +On each endpoint, confirm the transport came up: + +```sh +fipsctl show transports +``` + +Look for an entry of type `ble` in the `state: Running` (or +equivalent) state. The `mtu` field reports the configured default; +per-link MTU is reported separately. + +Confirm the link is established: + +```sh +fipsctl show peers +``` + +The peer entry for the BLE-attached neighbour should report +`transport_type: "ble"` and a non-zero `last_seen_ms`. + +BLE peering is auto-discovery only: there is no `fipsctl connect` +path for BLE (the command accepts `udp`, `tcp`, `tor`, and +`ethernet` only). Links come up via advert/scan; if you don't see +the peer here, the configuration above is the only knob. + +To watch the link in real time, use `fipstop`'s **Peers** and +**Transports** tabs: + +```sh +fipstop +``` + +The Performance tab reports the per-link MMP metrics — SRTT, loss +rate, ETX — which on BLE typically run an order of magnitude worse +than over UDP, with much higher jitter. + +## Troubleshooting + +### Transport never comes up + +Check the BlueZ side first: + +```sh +systemctl status bluetooth +sudo bluetoothctl show +``` + +If `bluetoothctl show` reports `Powered: no`, fix that before +debugging FIPS. The FIPS daemon will log a warning if it cannot +acquire the adapter. + +If the FIPS log contains `bluer` D-Bus errors, the daemon usually +lacks permission. Run as root or grant `CAP_NET_ADMIN` and add the +fips user to the `bluetooth` group. + +### Peers see each other but never connect + +Verify `accept_connections` is true on at least one side and +`auto_connect` is true on at least one side. Two listen-only nodes +will discover each other but never establish an L2CAP connection. + +Check `psm` matches on both ends. A mismatch presents as adverts +visible (in `fipstop` discovery counters) but every connect attempt +fails. + +### Link comes up but throughput is poor + +Practical L2CAP CoC throughput in good conditions reaches +1-2 Mbps, but interference, range, and controller capability all +push it lower. If throughput is well below that range, check the +negotiated ATT_MTU — a small ATT_MTU (default 23 bytes when +extended ATT MTU is not negotiated) caps per-PDU payload +regardless of radio conditions. The per-link MTU reported in +`fipsctl show transports` reveals what was negotiated. + +If MTU is unexpectedly low, both endpoints must support and have +negotiated the BlueZ L2CAP `cocmode=2` extension. Older Bluetooth +controllers cap MTU regardless. + +### Unstable links / repeated reconnects + +Bluetooth in busy 2.4 GHz environments suffers from WiFi +interference. Switch the adapter to a less crowded channel (kernel +side, not configurable from FIPS) or add an external antenna. The +`probe_cooldown_secs` tunable backs off retry attempts; raise it if +the daemon log shows many short-lived probes. + +### Permission errors on socket open + +Most modern systemd installs do not allow non-root processes to +open raw L2CAP sockets without an explicit policy. Run the daemon +as root (the shipped systemd unit does this) or add a `polkit` +rule for the `bluetooth` group. + +## See also + +- [../design/fips-transport-layer.md](../design/fips-transport-layer.md) + — per-transport MTU reporting and the BLE row of the supported- + transports table. +- [../reference/configuration.md](../reference/configuration.md) — + full `transports.ble.*` reference. +- [run-as-unprivileged-user.md](run-as-unprivileged-user.md) — + adjacent privilege handling for the daemon process. diff --git a/docs/how-to/troubleshoot-gateway.md b/docs/how-to/troubleshoot-gateway.md new file mode 100644 index 0000000..324735f --- /dev/null +++ b/docs/how-to/troubleshoot-gateway.md @@ -0,0 +1,458 @@ +# Troubleshoot `fips-gateway` + +Diagnostic recipes for `fips-gateway`, grouped by which half of the +gateway is failing. For gateway design and deployment, see +[../design/fips-gateway.md](../design/fips-gateway.md) and +[deploy-gateway.md](deploy-gateway.md). For OpenWrt-specific +deployment problems, see the +[OpenWrt deployment tutorial](../tutorials/deploy-fips-gateway.md); +most of the recipes below apply on OpenWrt as well, but paths and +service names differ. + +## Inspect gateway state via the control socket + +Before digging into nftables or conntrack, ask the gateway directly +whether it has the mapping or session you expect. `fips-gateway` +exposes a separate control socket (`/run/fips/gateway.sock`) with its +own command set; there is no `fipsctl gateway` subcommand — talk to +the socket directly with `nc -U`. Each request is a single line of +JSON terminated with a newline; the connection is closed after one +response. + +Pool summary, listen address, NAT counters, uptime, and the loaded +config snapshot: + +```sh +echo '{"command":"show_gateway"}' | sudo nc -U /run/fips/gateway.sock +``` + +Per-mapping virtual-IP state (allocated, active, draining): + +```sh +echo '{"command":"show_mappings"}' | sudo nc -U /run/fips/gateway.sock +``` + +If either command returns `gateway not yet initialized`, the gateway +is still in early startup; wait a moment and retry. If a mapping you +expect is not in the list, the DNS path didn't allocate it — fall +through to the outbound DNS recipes below. If the mapping exists in +`state: Active` but mesh traffic still fails, the problem is +downstream of the allocation (firewall, route, masquerade); see the +recipes that follow. + +For the full command catalog and JSON shapes, see +[../reference/control-socket.md#gateway-command-catalog](../reference/control-socket.md#gateway-command-catalog). + +## Common (either-half) issues + +These break both halves at once because they affect the gateway +process itself or the shared NAT machinery. + +### "No gateway section in configuration" + +`fips-gateway` is normally launched by the systemd unit shipped with +the package: + +```sh +sudo systemctl restart fips-gateway +sudo journalctl -u fips-gateway -e +``` + +The unit reads the standard FIPS config search paths (typically +`/etc/fips/fips.yaml`). If the unit logs "no gateway section in +configuration" or "Gateway section exists but is not enabled", confirm +the section is present and `enabled: true`: + +```sh +grep -A1 '^gateway:' /etc/fips/fips.yaml +``` + +For one-off debugging outside systemd, run the binary directly and +point it at a specific config file: + +```sh +sudo fips-gateway --config /etc/fips/fips.yaml --log-level debug +``` + +This is useful to capture stderr in a terminal, but the systemd unit +is the supported entry point in production. See +[../reference/cli-fips-gateway.md](../reference/cli-fips-gateway.md) +for the full flag list. + +### Port conflict on the DNS listen port + +Symptom: gateway fails to start with "address already in use" on +port 53 (or whatever `gateway.dns.listen` is set to). + +Another DNS server (systemd-resolved, dnsmasq, BIND) is bound to +the port. Identify it: + +```sh +sudo ss -tulnp | grep ':53' +``` + +Two options: + +- **Use an alternate listen address.** Pick a non-conflicting port + and update the gateway config: + + ```yaml + gateway: + dns: + listen: "192.168.1.1:5353" + ``` + + Then either point LAN clients at the alternate port directly, or + run a forwarding stub on port 53 that delegates `.fips` queries to + the gateway. + +- **Relocate the conflicting resolver.** Move it to a different port + (or disable it if not needed) and let the gateway bind 53. + Practical for systemd-resolved (set `DNSStubListener=no` in + `/etc/systemd/resolved.conf`); rarely worth it for production + resolvers. + +### IPv6 forwarding disabled + +Symptom: gateway exits at startup with +"IPv6 forwarding is disabled. Enable with: sysctl -w +net.ipv6.conf.all.forwarding=1". + +The gateway is completely non-functional without forwarding — packets +cannot traverse the NAT pipeline. Enable it: + +```sh +sudo sysctl -w net.ipv6.conf.all.forwarding=1 +``` + +Persist via the drop-in shown in +[deploy-gateway.md](deploy-gateway.md#kernel-sysctls). The same +section lists `proxy_ndp`, which is also required for the outbound +half. + +### nftables table missing or not loaded + +Symptom: `show_gateway` reports an active gateway but +`nft list table inet fips_gateway` errors with "No such file or +directory". + +The table is created by the gateway at startup and rebuilt atomically +on every mapping change and on every `set_port_forwards` call. If the +table is missing while the gateway claims to be running, something +else (a host firewall script, a `nft flush ruleset` from another +service) deleted it after creation. Restart the gateway to recreate +it: + +```sh +sudo systemctl restart fips-gateway +``` + +If a peer service is repeatedly clobbering the table, switch that +service to use `add table` / `flush table ` for its own table +rather than `flush ruleset`, which destroys every table on the host. + +### Control socket permission errors + +Symptom: `nc -U /run/fips/gateway.sock` fails with "Permission +denied" or "No such file or directory". + +The socket is owned by root with mode `0660` (group `fips`). Either +run `nc` as root (`sudo nc -U ...`) or add your user to the `fips` +group and re-login. If the file does not exist at all, the gateway +either failed to start (check `journalctl -u fips-gateway`) or +failed to bind the socket and continued without it (the warning +`Failed to bind gateway control socket — continuing without it` is +in the journal in that case). + +## Outbound-half diagnostics + +Symptoms in this section all involve a LAN client trying to reach a +mesh destination through the gateway and failing. + +### DNS queries fail + +Symptom: LAN clients get `SERVFAIL` or no response when querying +`.fips` names; or the gateway log shows DNS upstream timeouts. + +**Step 1.** Verify the daemon resolver is running and reachable from +the gateway host: + +```sh +dig @::1 -p 5354 hostname.fips AAAA +``` + +If this returns no answer or fails, the FIPS daemon's DNS resolver is +not running or not enabled. Check that the daemon config has +`dns.enabled: true` (the default) and the daemon is healthy: +`fipsctl show status`. + +**Step 2.** Verify the gateway is listening on its DNS port: + +```sh +sudo ss -tulnp | grep -E ':53\b' +``` + +If nothing is listening on the configured `dns.listen` address, the +gateway either failed to start or is bound to a different address. +Check the gateway log: `sudo journalctl -u fips-gateway -e`. + +**Step 3.** Verify the LAN client can reach the gateway's DNS port: + +```sh +# from the LAN client +dig @ hostname.fips AAAA +``` + +If this hangs, the LAN-side firewall is blocking DNS, or the LAN +route to the gateway is missing. + +### Ping works but TCP does not + +Symptom: `ping6 ` succeeds from a LAN client, but TCP +connections (SSH, HTTP) hang or time out. + +This usually means the `fips0`-side masquerade rule is missing or +misconfigured. Inspect the gateway's nftables table: + +```sh +sudo nft list table inet fips_gateway +``` + +In the `postrouting` chain, look for a rule matching +`oifname "fips0"` with a `masquerade` verdict. Without masquerade, +the destination mesh node sees a source address (from the virtual +pool) it cannot route replies to, and return packets are +black-holed. + +If the rule is missing, restart the gateway — the table is rebuilt +atomically on every mapping change and on startup. + +### Connection timeout to a virtual IP + +Symptom: any traffic to a virtual pool address times out, including +ping. + +**Step 1.** Verify IPv6 forwarding is still enabled: + +```sh +sysctl net.ipv6.conf.all.forwarding +# Expect: net.ipv6.conf.all.forwarding = 1 +``` + +**Step 2.** Verify the pool route exists: + +```sh +ip -6 route show table local | grep +``` + +If the route is missing, the kernel does not recognize pool +addresses as locally-owned and drops the packets before NAT can +process them. The gateway adds this route at startup; if it's +missing, check the gateway log for startup errors. + +**Step 3.** Verify the destination mesh address actually exists in +the FIPS daemon's identity cache: + +```sh +fipsctl show identity-cache | grep +``` + +If the entry is missing, the DNS-side mapping never primed the +identity cache, which means the daemon resolver did not actually +resolve the `.fips` name. Re-test the DNS path: + +```sh +dig @::1 -p 5354 hostname.fips AAAA +``` + +### Virtual IP unreachable from a LAN client + +Symptom: client cannot reach the virtual IP at all (no ping, no +ARP/ND response). + +**Step 1.** Verify the client has a route to the pool via the +gateway: + +```sh +# from the LAN client +ip -6 route get +``` + +The output should show the gateway as the next-hop. If it shows +something else (or "unreachable"), fix the LAN-side route — see +[deploy-gateway.md](deploy-gateway.md#distribute-the-route-to-lan-clients). + +**Step 2.** On the gateway, verify proxy NDP entries exist for +allocated virtual IPs: + +```sh +ip -6 neigh show proxy +``` + +If proxy NDP entries are missing, the gateway cannot answer Neighbor +Solicitation requests for virtual IPs on the LAN, so clients cannot +resolve the link-layer address and packets never leave the client's +NIC. + +The gateway adds these entries when a mapping is created (i.e., when +a `.fips` DNS query allocates a virtual IP). If they're absent, +trigger a DNS query first: + +```sh +dig @ hostname.fips AAAA +``` + +Then re-check `ip -6 neigh show proxy`. + +**Step 3.** Verify `proxy_ndp` is enabled in the kernel: + +```sh +sysctl net.ipv6.conf.all.proxy_ndp +# Expect: net.ipv6.conf.all.proxy_ndp = 1 +``` + +If 0, enable it (see +[deploy-gateway.md](deploy-gateway.md#kernel-sysctls)). + +## Inbound-half diagnostics + +Symptoms in this section all involve a mesh peer trying to reach a +LAN-side service through the gateway and failing. + +### Mesh peer can't reach `.fips:` + +Walk the path from the mesh-side ingress to the LAN target: + +**Step 1.** Verify the port-forward rule is loaded. On the gateway: + +```sh +sudo nft list table inet fips_gateway +``` + +Look in the `prerouting` chain for a rule of the form + +```text +iif "fips0" meta nfproto ipv6 meta l4proto \ + dport dnat ip6 to []: +``` + +and, in the `postrouting` chain, a rule of the form + +```text +iif "fips0" oif "" meta nfproto ipv6 masquerade +``` + +The port-forward DNAT and the LAN-side masquerade come from +`gateway.port_forwards[]` and the active `lan_interface` setting. +The masquerade is emitted only when at least one port-forward exists. +If either rule is missing, restart the gateway — the table is rebuilt +atomically on config load. + +**Step 2.** Verify the mesh firewall is not blocking the listen +port. If `fips-firewall.service` is enabled, the default baseline +drops everything inbound on `fips0` except established/related and +ICMPv6. Add an explicit allow rule under `/etc/fips/fips.d/`: + +```nft +# /etc/fips/fips.d/gateway-inbound.nft +tcp dport accept +``` + +(See [enable-mesh-firewall.md](enable-mesh-firewall.md) for the +full drop-in pattern, including source-address restrictions.) +Without an allow rule, mesh peers see TCP RSTs (the firewall drops +on the way in) or silent UDP loss. + +**Step 3.** Verify the LAN target is reachable from the gateway +itself: + +```sh +ping6 +curl -v http://[]:/ # for TCP HTTP +``` + +If the target is unreachable from the gateway, the DNAT rule will +fire but the inner connection attempt will fail. Fix LAN-side +routing or the target service before going further. + +**Step 4.** Verify conntrack is tracking the inbound flow. Try the +connection from a mesh peer once: + +```sh +curl -v http://.fips:/ +``` + +Then on the gateway: + +```sh +sudo conntrack -L | grep -E '|' +``` + +You should see a flow tuple in both directions (orig and reply) with +the mesh peer's source on `fips0` and the gateway's LAN address as +the masqueraded source on the LAN side. No conntrack entry suggests +the prerouting DNAT didn't match — recheck step 1. + +**Step 5.** Check the gateway log for nftables or rule install +errors: + +```sh +sudo journalctl -u fips-gateway -e | grep -E 'port_forward|nftables' +``` + +A "Failed to install port-forward rules" log line at startup means +the rule batch was rejected by netlink — usually a transient +condition during a config edit, but persistent failures warrant +inspecting the rule with `nft -d`. + +### Config rejected: IPv4 target + +Symptom: `fips-gateway` exits at startup with a deserialization +error referencing `port_forwards[N].target` and an invalid IPv6 +literal. + +The `target` field is typed as `SocketAddrV6` and rejects IPv4 +literals at parse time: + +```yaml +# fails at config load +- listen_port: 8080 + proto: tcp + target: "192.168.1.10:80" +``` + +Either re-address the LAN service to be reachable on IPv6, or front +it with a small IPv6-aware reverse proxy on the gateway and point +the `target` at that proxy. + +### Config rejected: zero or duplicate listen_port + +Symptom: `fips-gateway` exits at startup with +"Invalid gateway.port_forwards: …". + +`validate_port_forwards()` enforces: + +- `listen_port` must be non-zero. +- The pair `(listen_port, proto)` must be unique across the list + (the same port on TCP and UDP simultaneously is allowed; the same + port twice on the same proto is not). + +Fix the offending entry and reload. + +## See also + +- [../tutorials/deploy-fips-gateway.md](../tutorials/deploy-fips-gateway.md) — + canonical OpenWrt deployment. +- [../design/fips-gateway.md](../design/fips-gateway.md) — gateway + design, NAT pipeline, virtual IP pool lifecycle, security + considerations. +- [deploy-gateway.md](deploy-gateway.md) — manual Linux-host setup. +- [Gateway section](../reference/configuration.md#gateway-gateway) of + the configuration reference — full `gateway.*` block. +- [../reference/cli-fips-gateway.md](../reference/cli-fips-gateway.md) — + `fips-gateway` binary CLI flags. +- [Gateway command catalog](../reference/control-socket.md#gateway-command-catalog) + in the control-socket reference — JSON schema for `show_gateway` + and `show_mappings`. +- [enable-mesh-firewall.md](enable-mesh-firewall.md) — mesh-firewall + baseline and drop-ins. diff --git a/docs/how-to/tune-udp-buffers.md b/docs/how-to/tune-udp-buffers.md new file mode 100644 index 0000000..f043625 --- /dev/null +++ b/docs/how-to/tune-udp-buffers.md @@ -0,0 +1,125 @@ +# Tune Host UDP Socket Buffers for FIPS + +The FIPS UDP transport requests larger send and receive socket +buffers (default 2 MB each, doubled by the kernel to 4 MB actual) +than the Linux defaults provide. The kernel silently clamps the +request to `net.core.rmem_max` and `net.core.wmem_max` if those +sysctls are smaller than the requested size — which causes silent +packet drops under high throughput. For the design context (why FIPS +requests larger buffers and how `SO_RXQ_OVFL` feeds ECN congestion +detection), see +[../design/fips-transport-layer.md](../design/fips-transport-layer.md#socket-buffer-sizing). + +This guide covers the host-side sysctl setup needed before deploying +a high-throughput FIPS node. + +## Why this matters + +The default Linux UDP receive buffer (`net.core.rmem_default`, +typically 212 KB) fills in roughly 2.5 ms at ~85 MB/s. Any stall in +the FIPS receive loop (decryption, routing, forwarding) causes the +kernel to drop incoming datagrams without notification — they don't +appear in `recv` errors, they don't trigger any application-visible +event. The drops show up only in `SO_RXQ_OVFL` on subsequent +packets, where FIPS surfaces them as congestion-detection events. + +Setting `rmem_max` and `wmem_max` to at least the requested buffer +size prevents the kernel clamp and the silent drop loss it causes. + +## Step 1: Check current limits + +```sh +sysctl net.core.rmem_max net.core.wmem_max +``` + +Typical defaults on stock Linux distributions are 212992 bytes +(212 KB). FIPS requests 2 MB by default, which the kernel doubles +internally to 4 MB; for the request to succeed without clamping, both +sysctls must be at least 4194304 (4 MB). + +## Step 2: Set the limits temporarily + +```sh +sudo sysctl -w net.core.rmem_max=4194304 +sudo sysctl -w net.core.wmem_max=4194304 +``` + +Verify: + +```sh +sysctl net.core.rmem_max net.core.wmem_max +``` + +These changes take effect immediately for new socket binds but do +not survive a reboot. + +## Step 3: Make the limits persistent + +Drop a file under `/etc/sysctl.d/`: + +```sh +sudo tee /etc/sysctl.d/60-fips.conf <<'EOF' +# FIPS UDP transport requests 2 MB socket buffers, kernel doubles to 4 MB. +# Avoid silent receive-buffer drops under load. +net.core.rmem_max = 4194304 +net.core.wmem_max = 4194304 +EOF +``` + +Apply: + +```sh +sudo sysctl --system +``` + +The drop-in is loaded automatically on every boot. + +## Step 4: Restart FIPS and verify the actual buffer size + +After raising the host limits, restart the FIPS daemon so the next +socket bind picks up the new ceiling: + +```sh +sudo systemctl restart fips +``` + +The daemon logs the actual buffer sizes at startup: + +```text +UDP transport started local_addr=0.0.0.0:2121 recv_buf=4194304 send_buf=4194304 +``` + +If `recv_buf` or `send_buf` shows a smaller number than expected, the +host sysctl is still clamping. Recheck `sysctl net.core.rmem_max +net.core.wmem_max` and confirm the drop-in file is being loaded +(`sudo sysctl --system` prints the loaded files). + +## Docker and other container hosts + +Containers share the host kernel, so sysctls apply to the host, not +the container. If you run FIPS inside Docker, set +`net.core.rmem_max` / `net.core.wmem_max` on the **Docker host**, not +inside the container. Container privileges (cap_sys_admin) and +`--sysctl` flags do not let you raise these particular limits from +inside a container — they are global to the host network namespace. + +For Kubernetes deployments, the host-level sysctl tuning is the same; +node-level configuration (DaemonSet with `privileged: true`, or a +node-init script) is the typical mechanism. + +## Tuning higher + +The 4 MB ceiling is a conservative starting point. For very high +throughput (multi-gigabit per second), raise both sysctls and the +corresponding `transports.udp.recv_buf_size` / +`transports.udp.send_buf_size` config values together. Setting a config value +larger than the host ceiling silently clamps to the ceiling, so both +must move in lockstep. + +## See also + +- [../design/fips-transport-layer.md](../design/fips-transport-layer.md) + — UDP transport design, why FIPS requests larger buffers, + `SO_RXQ_OVFL` and ECN integration +- [../reference/configuration.md](../reference/configuration.md) — + `transports.udp.recv_buf_size` and `send_buf_size` defaults diff --git a/docs/proposals/README.md b/docs/proposals/README.md deleted file mode 100644 index 93a887f..0000000 --- a/docs/proposals/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Proposals - -Protocol proposals and specifications for features under consideration or -recently implemented. Documents marked Proposed describe mechanisms that -have not yet been implemented and may evolve significantly during review. - -Implemented proposals are retained as protocol references until they are -superseded by dedicated design documents. - -| Document | Status | Description | -|----------------------------------------------------------------------|-------------|----------------------------------------------------------------------------------------| -| [nostr-udp-hole-punch-protocol.md](nostr-udp-hole-punch-protocol.md) | Implemented | Nostr-signaled UDP hole punching protocol (behind the `nostr-discovery` cargo feature) | diff --git a/docs/proposals/nostr-udp-hole-punch-protocol.md b/docs/proposals/nostr-udp-hole-punch-protocol.md deleted file mode 100644 index 5641209..0000000 --- a/docs/proposals/nostr-udp-hole-punch-protocol.md +++ /dev/null @@ -1,388 +0,0 @@ -# Nostr-Signaled UDP Hole Punching Protocol - -> **Status**: Implemented behind the `nostr-discovery` cargo feature. -> This proposal is retained as a protocol reference and will be -> superseded by a dedicated design document. - -## Abstract - -This document describes a protocol for establishing direct UDP connectivity between two peers behind NAT, using Nostr relays as the signaling channel and public STUN servers for reflexive address discovery. The protocol assumes a **responder** that advertises a UDP service via a Nostr replaceable event, and an **initiator** that discovers the responder and negotiates a direct UDP connection. - -No WebRTC, DTLS, or ICE stack is required. The protocol operates at the raw UDP level, using Nostr solely for ephemeral signaling and STUN solely for reflexive address discovery. - ---- - -## Terminology - -- **Initiator**: The peer that discovers the responder's service advertisement and begins the connection process. -- **Responder**: The peer running a UDP service, which has published a replaceable event advertising its availability. -- **Reflexive address**: The public `IP:port` tuple as observed by a STUN server (i.e., the NAT's external mapping). -- **Punch socket**: The single UDP socket a peer uses for both STUN queries and subsequent hole-punching traffic within one connection attempt. This socket **must not** change between phases of that attempt. - -### Socket lifecycle - -This protocol assumes **per-peer, per-attempt punch sockets**: - -- Each outbound traversal attempt creates a fresh UDP socket bound to `0.0.0.0:0`. -- That socket is owned by exactly one remote peer and exactly one traversal session. -- STUN, offer/answer metadata, punch packets, and the eventual adopted UDP transport all use that same socket for the lifetime of the attempt. -- If the attempt fails, the socket is discarded. A retry allocates a new socket and obtains a fresh reflexive address. -- The long-lived application UDP listener (for example, a fixed port such as `2121`) is **not** reused as the punch socket. - -This choice avoids cross-peer state coupling and keeps NAT mappings, retry state, and adopted traversal transports isolated per peer. - ---- - -## Nostr Event Kinds - -| Purpose | Kind | Persistence | -|---|---|---| -| Service advertisement | `37195` (parameterized replaceable) | Persistent, updated by responder | -| Signaling messages | `21059` (ephemeral gift-wrap) | Ephemeral, not stored by relays | - -The service advertisement uses kind `37195`, an application-specific parameterized replaceable event in the NIP-01 replaceable range `30000–39999`. The digits visually spell `FIPS` (7=F, 1=I, 9=P, 5=S). The replaceable semantics let the responder update the advert in place under the same `d` tag. Signaling messages use ephemeral gift-wrapped events (kind `21059`, combining NIP-59 gift wrap with the ephemeral kind range `20000–29999`) to avoid relay storage. - ---- - -## Phase 0: Service Advertisement (Responder) - -The responder publishes a parameterized replaceable event advertising its UDP service. This event is long-lived and updated as the responder's parameters change. - -```json -{ - "kind": 37195, - "pubkey": "", - "created_at": , - "tags": [ - ["d", "fips-overlay-v1"], - ["protocol", ""], - ["version", ""], - ["relays", "wss://relay1.example.com", "wss://relay2.example.com"], - ["stun", "stun.l.google.com:19302", "stun1.l.google.com:19302"], - ["expiration", ""] - ], - "content": "", - "sig": "" -} -``` - -### Tag semantics - -- **`d`**: Namespaced identifier. `fips-overlay-v1` scopes this to FIPS overlay endpoint adverts. -- **`protocol`**: Application protocol name for filtering (e.g., `myapp-file-sync`). -- **`version`**: Protocol version string for compatibility checking. -- **`relays`**: One or more relay URLs where the responder subscribes for incoming signaling messages. The initiator **must** send signaling events to at least one of these relays. -- **`stun`**: STUN server(s) both peers should use. Using the same STUN server improves the chance that NAT mappings are allocated from similar port ranges, though this is not strictly required. -- **`expiration`** (NIP-40): Optional. Allows the advertisement to expire if the responder goes offline without explicitly deleting it. -- **`content`**: Optional application-specific parameters (e.g., supported features, capabilities, public encryption keys for the application layer). Encrypted with NIP-44 if privacy is required, or plaintext if the parameters are non-sensitive. - -The responder should update this event (same `d` tag, new `created_at`) whenever its parameters change, and publish a kind `5` deletion event (NIP-09) when it goes offline permanently. - -Current in-tree defaults: - -- `node.discovery.nostr.advert_ttl_secs = 3600` (1 hour) -- `node.discovery.nostr.advert_refresh_secs = 1800` (30 minutes) - ---- - -## Phase 1: Discovery (Initiator) - -The initiator queries one or more relays for the responder's service advertisement: - -```json -["REQ", "", { - "kinds": [37195], - "authors": [""], - "#d": ["fips-overlay-v1"] -}] -``` - -If the responder's pubkey is not known in advance, the initiator can discover peers by filtering on the `protocol` tag: - -```json -["REQ", "", { - "kinds": [37195], - "#protocol": [""] -}] -``` - -Upon receiving the advertisement, the initiator extracts the relay list and any -advertised STUN metadata. In the current in-tree implementation, advertised -STUN entries are informational only; outbound STUN is driven by the initiator's -own configured allowlist. -The current in-tree STUN parser handles IPv4 and IPv6 mapped-address -responses. Offer/answer local-address candidates include active private -non-loopback interface addresses (RFC1918 IPv4 and IPv6 ULA) plus probed local -egress addresses. - ---- - -## Phase 2: STUN Binding (Initiator) - -Before sending any signaling message, the initiator binds a UDP socket and performs a STUN Binding Request: - -1. Bind a fresh UDP socket to `0.0.0.0:0` (OS-assigned port). This becomes the **punch socket** for this peer and this traversal attempt. -2. Send a STUN Binding Request (RFC 8489) to one of the initiator's locally configured STUN servers. -3. Parse the Binding Response and extract the `XOR-MAPPED-ADDRESS` attribute — this is the initiator's reflexive address. -4. Record the reflexive address and local interface candidates for the same - punch-socket port. - -**Critical**: The punch socket must remain open and must be reused for all subsequent protocol phases of the same attempt. Closing or rebinding it invalidates the NAT mapping. - ---- - -## Phase 3: Signaling — Offer (Initiator → Responder) - -The initiator constructs a signaling message containing its reflexive address and sends it to the responder as an ephemeral NIP-44 encrypted, NIP-59 gift-wrapped event. - -### Signaling payload (JSON, before encryption) - -```json -{ - "app": "", - "eventKind": 21059, - "type": "offer", - "sessionId": "", - "issuedAt": , - "expiresAt": , - "nonce": "", - "senderNpub": "", - "recipientNpub": "", - "reflexiveAddress": {"protocol":"udp","ip":"","port":}, - "localAddresses": [{"protocol":"udp","ip":"","port":}], - "stunServer": ":", - "app_params": { ... } -} -``` - -- **`sessionId`**: A random 32-character hex string identifying this connection attempt. Both peers use this to correlate signaling messages belonging to the same session. -- **`reflexiveAddress`**: The initiator's reflexive address as reported by STUN. -- **`localAddresses`**: Candidate local interface addresses for the same socket - port. Useful if both peers happen to share a private subnet (the responder - can attempt direct local paths in parallel). -- **`stunServer`**: Which STUN server the initiator used, so the responder can use the same one if desired. -- **`issuedAt`/`expiresAt`**: Freshness window. The responder should reject stale offers since the NAT mapping may have expired. -- **`app_params`**: Optional application-specific handshake parameters. - -### Wrapping and delivery - -1. Encrypt the JSON payload using NIP-44 to the responder's pubkey. -2. Wrap in a NIP-59 gift-wrap event using an ephemeral kind (`21059`). -3. Add an `expiration` tag (NIP-40) set to `now + 120s`. -4. Publish to the relay(s) listed in the responder's service advertisement. - -```json -{ - "kind": 21059, - "pubkey": "", - "created_at": , - "tags": [ - ["p", ""], - ["expiration", ""] - ], - "content": "", - "sig": "" -} -``` - -The initiator also opens a subscription on the same relay(s) to listen for the responder's answer: - -```json -["REQ", "", { - "kinds": [21059], - "#p": [""], - "since": -}] -``` - ---- - -## Phase 4: Signaling — Answer (Responder → Initiator) - -The responder maintains a standing subscription on its advertised relay(s): - -```json -["REQ", "", { - "kinds": [21059], - "#p": [""], - "since": -}] -``` - -Upon receiving and decrypting an offer, the responder: - -1. Validates the timestamp (rejects if older than 60 seconds). -2. Validates the `session_id` is not a replay of a previously seen session. -3. Binds its own fresh **punch socket** to `0.0.0.0:0`. -4. Performs a STUN Binding Request using one of the responder's locally configured STUN servers. -5. Extracts its own reflexive address. -6. Constructs and sends an answer: - -### Signaling payload (JSON, before encryption) - -```json -{ - "app": "", - "eventKind": 21059, - "type": "answer", - "sessionId": "", - "issuedAt": , - "expiresAt": , - "nonce": "", - "senderNpub": "", - "recipientNpub": "", - "inReplyTo": "", - "accepted": true, - "reflexiveAddress": {"protocol":"udp","ip":"","port":}, - "localAddresses": [{"protocol":"udp","ip":"","port":}], - "stunServer": ":", - "app_params": { ... } -} -``` - -This is encrypted and gift-wrapped identically to the offer, but addressed to the initiator's pubkey and published to the same relay(s). - -Implementations should also bind the JSON sender/recipient identity fields to -the actual Nostr pubkeys that delivered the gift-wrapped events, rather than -treating those JSON fields as independently trustworthy. - -**Immediately after publishing the answer**, the responder begins Phase 5 (punching) without waiting for confirmation that the initiator received it. Time is critical — the NAT mappings are decaying. - ---- - -## Phase 5: Hole Punching - -Both peers now know each other's reflexive address and local-address -candidates. Both begin sending UDP packets from their punch socket. - -### Procedure - -1. Both peers send punch packets every **200ms** across planned target paths: - - reflexive-to-reflexive - - private-subnet local-address paths (when subnet-compatible) - - mixed local/reflexive fallbacks -2. Each punch packet contains a fixed magic header to distinguish it from stray traffic: - -``` -Bytes 0–3: 0x4E505443 ("NPTC" — Nostr P2P Tunnel Connect) -Bytes 4–7: sequence number (u32, big-endian, starting at 0) -Bytes 8–23: first 16 bytes of SHA-256(session_id) -``` - -3. Upon receiving a valid punch packet (magic header matches, session hash matches), the peer records the source address as the confirmed peer address and sends an **acknowledgment punch**: - -``` -Bytes 0–3: 0x4E505441 ("NPTA" — Nostr P2P Tunnel Ack) -Bytes 4–7: echoed sequence number from the received punch -Bytes 8–23: first 16 bytes of SHA-256(session_id) -``` - -4. Upon receiving an acknowledgment, the peer considers the hole punched and transitions to the application protocol. - -### Timeout - -If no valid punch packet is received within **10 seconds**, the attempt has failed. Possible causes include symmetric NAT on one or both sides, firewall interference, or stale reflexive addresses. The initiator may retry with a fresh STUN query, a fresh punch socket, and a new offer, or fall back to an application-level relay. - -### LAN optimization - -If both peers advertise compatible private-subnet candidates (e.g., -`192.168.1.x`), they should simultaneously attempt punching via both reflexive -and local-address paths. The first path to complete wins. - ---- - -## Phase 6: Application Protocol Handoff - -Once both peers have exchanged acknowledgments, the connection is established. The punch socket for that attempt is now a live UDP channel between the two peers. From this point: - -- The application protocol takes over the socket. -- The Nostr signaling channel is no longer needed for this session. -- Both peers should send application-level keepalive packets at least every **15 seconds** to prevent the NAT mapping from expiring. - ---- - -## Phase 7: Cleanup - -After the hole punch succeeds (or fails), both peers perform cleanup: - -1. **Close the relay subscription** used for signaling. -2. **Publish a kind `5` deletion event** (NIP-09) referencing any signaling events they published, requesting relays delete them: - -```json -{ - "kind": 5, - "tags": [ - ["e", ""] - ], - "content": "session concluded" -} -``` - -3. Because the signaling events used an ephemeral kind (`21059`) with an `expiration` tag, well-behaved relays will discard them automatically even without explicit deletion. - -If the responder is going offline permanently, it should also delete its kind `37195` service advertisement. - ---- - -## Security Considerations - -### Authentication - -The offer and answer are NIP-44 encrypted and NIP-59 gift-wrapped, ensuring that only the intended recipient can decrypt the signaling payload. The Nostr signatures authenticate both peers by their pubkeys. - -However, once the UDP hole is punched, the raw UDP channel has **no inherent authentication or encryption**. The application layer is responsible for establishing its own security (e.g., Noise Protocol handshake, DTLS, or application-level encryption keyed from the Nostr identity). - -### Replay protection - -The `session_id` and `timestamp` fields protect against replay attacks on the signaling layer. The responder must track recently seen `session_id` values and reject duplicates within a window. - -### Metadata exposure - -Even though signaling content is encrypted, the gift-wrap metadata reveals that the initiator's ephemeral pubkey contacted the responder's pubkey at a particular time, through a particular relay. The service advertisement (kind `37195`) is public and reveals the responder's pubkey and that it is running a particular service. - -If metadata privacy is required, the `content` of the service advertisement can be encrypted (requiring the initiator to already know the responder's pubkey), and both peers can use ephemeral Nostr identities rather than their long-term keys. - -### NAT mapping integrity - -If the time between STUN discovery and hole-punch initiation exceeds the NAT's mapping timeout, the reflexive address becomes stale. Both peers should complete the entire signaling exchange within **60 seconds** of their respective STUN queries. Relay latency is the primary risk factor here. - ---- - -## Relay Requirements - -This protocol works best with relays that: - -- Support ephemeral event kinds (`20000–29999`) and do not persist them. -- Honor NIP-40 expiration tags and garbage-collect expired events. -- Deliver events with low latency (sub-second WebSocket push). -- Support NIP-09 deletion requests. - -Relays that do not support ephemeral kinds will store the signaling events as regular events. While the encrypted content remains opaque, this is wasteful and exposes metadata unnecessarily. Operators deploying this protocol at scale should run or select relays known to handle ephemeral events correctly. - ---- - -## Failure Modes - -| Failure | Symptom | Mitigation | -|---|---|---| -| Symmetric NAT (one side) | Punch timeout | Retry with port prediction heuristics, or fall back to relay/TURN | -| Symmetric NAT (both sides) | Punch timeout | Application-level relay required | -| Relay latency > 60s | Stale reflexive address | Use low-latency relays; consider self-hosted relay | -| Relay does not support ephemeral kinds | Signaling events persist | Use NIP-40 expiration + NIP-09 deletion as fallback | -| Responder offline | No answer received | Initiator times out after configurable period (e.g., 30s) | -| STUN server unreachable | No reflexive address | Fall back to alternate STUN server; fail if none reachable | -| Firewall blocks outbound UDP | STUN fails entirely | Protocol cannot proceed; application must use TCP/WebSocket fallback | - ---- - -## References - -- **RFC 8489** — Session Traversal Utilities for NAT (STUN) -- **RFC 8445** — Interactive Connectivity Establishment (ICE) -- **RFC 4787** — NAT Behavioral Requirements for Unicast UDP -- **NIP-01** — Basic Nostr protocol flow -- **NIP-09** — Event deletion request -- **NIP-40** — Expiration timestamp -- **NIP-44** — Versioned encryption -- **NIP-59** — Gift wrap -- **NIP-78** — Application-specific data diff --git a/docs/reference/README.md b/docs/reference/README.md new file mode 100644 index 0000000..1127216 --- /dev/null +++ b/docs/reference/README.md @@ -0,0 +1,25 @@ +# Reference + +Information-oriented technical descriptions for lookup on demand. +Reference content describes *what is*: wire formats, configuration +keys, command-line flags, control-socket commands, default values, +file paths, exit codes. It is consulted, not read end-to-end. + +Reference is austere by design: minimal narrative, no opinions, no +guidance on when to use a feature. The "why" lives in design/; the +"how do I accomplish X" lives in how-to/. + +## Available Reference + +| Document | Scope | +| -------- | ----- | +| [wire-formats.md](wire-formats.md) | All FMP and FSP message byte layouts, encapsulation walkthrough | +| [configuration.md](configuration.md) | Full YAML configuration reference for the daemon and gateway | +| [security.md](security.md) | nftables baseline, peer ACL, cryptographic primitives, rekey defaults, threat-resistance matrix | +| [nostr-events.md](nostr-events.md) | Kind 37195 advert, Kind 21059 traversal signaling, Kind 10050 inbox relays | +| [transports.md](transports.md) | Per-transport statistics counter inventory | +| [control-socket.md](control-socket.md) | Line-delimited JSON control protocol for the daemon and gateway | +| [cli-fips.md](cli-fips.md) | `fips` daemon CLI: options, exit codes, environment, files | +| [cli-fipsctl.md](cli-fipsctl.md) | `fipsctl` control-client: subcommands, options, exit codes | +| [cli-fipstop.md](cli-fipstop.md) | `fipstop` live-status TUI: tabs, keybindings | +| [cli-fips-gateway.md](cli-fips-gateway.md) | `fips-gateway` service CLI: options, exit codes, files | diff --git a/docs/reference/cli-fips-gateway.md b/docs/reference/cli-fips-gateway.md new file mode 100644 index 0000000..5a764fc --- /dev/null +++ b/docs/reference/cli-fips-gateway.md @@ -0,0 +1,125 @@ +# `fips-gateway` + +Long-running service that bridges a LAN segment into the FIPS mesh. + +## Synopsis + +```text +fips-gateway [-c FILE] [-l LEVEL] +``` + +## Description + +`fips-gateway` runs alongside `fips` on the same host, reads the same +`fips.yaml`, and exposes two complementary functions to the LAN it +fronts: + +- **Outbound (LAN -> mesh).** Allocates a virtual IPv6 from a managed + pool when a LAN client resolves `.fips`, installs nftables + DNAT/SNAT/masquerade rules so the client's traffic is rewritten and + carried into the mesh through the daemon's `fips0` adapter. +- **Inbound (mesh -> LAN).** Installs nftables DNAT and LAN-side + masquerade rules so mesh-side traffic arriving on `fips0` for the + configured listen ports is rewritten to a LAN `host:port`, per the + `gateway.port_forwards[]` block. + +The service runs alongside `fips`, not as a replacement for it: +the daemon must be running on the same host with the TUN adapter +and DNS resolver enabled. The gateway is read-only with respect to the +daemon's state, and connects to the daemon's resolver only — it is +not a peer. For the architecture, see +[../design/fips-gateway.md](../design/fips-gateway.md). + +`fips-gateway` is **Linux-only**. The binary errors out and exits with +status `1` on any other platform, since the NAT pipeline is built on +nftables and proxy NDP. See +[Configuration](#configuration) for the platform notes that follow +from this. + +## Options + +| Flag | Argument | Default | Description | +| ---- | -------- | ------- | ----------- | +| `-c`, `--config` | `FILE` | *(default search paths)* | Use `FILE` as the configuration. Skips the default search paths. | +| `-l`, `--log-level` | `LEVEL` | `info` | Tracing level: `trace`, `debug`, `info`, `warn`, `error`. Overridden by `RUST_LOG` if set (see [Environment](#environment)). | +| `-V` | — | — | Print the short version. | +| `--version` | — | — | Print the long version (short version plus build target triple). | +| `-h`, `--help` | — | — | Print usage and exit. | + +## Configuration + +`fips-gateway` reads the same `fips.yaml` as `fips`; the gateway is +configured under the top-level `gateway:` block. The block must +include at minimum `enabled: true`, `pool`, and `lan_interface`. For +each field — pool, LAN interface, DNS listener, conntrack overrides, +and inbound `port_forwards[]` — see the +[Gateway section](configuration.md#gateway-gateway) of the +configuration reference. + +The same default search paths apply as for `fips` +(see [`fips`](cli-fips.md#files)); `-c FILE` overrides the search. +The gateway must be able to read the same configuration file the +daemon is reading, or the two will disagree about pool, DNS port, +and LAN interface. + +For deployment recipes, see +[../how-to/deploy-gateway.md](../how-to/deploy-gateway.md) (manual +Linux host) and +[../tutorials/deploy-fips-gateway.md](../tutorials/deploy-fips-gateway.md) +(OpenWrt walk-through). + +## Exit Codes + +| Code | Meaning | +| ---- | ------- | +| `0` | Clean shutdown after `SIGINT` / `SIGTERM`. | +| `1` | Non-Linux platform, configuration load failure, missing or invalid `gateway:` block, NAT/network setup failure, or control-socket bind failure. The reason is printed to stderr or the log before exit. | + +## Environment + +| Variable | Description | +| -------- | ----------- | +| `RUST_LOG` | Tracing filter directive. Takes precedence over `--log-level`. Examples: `info`, `debug`, `fips=trace,fips::gateway=debug`. | + +## Files + +| Path | Purpose | +| ---- | ------- | +| `/etc/fips/fips.yaml` | Gateway configuration (top-level `gateway:` block). Same file the daemon reads. | +| `/run/fips/gateway.sock` | Gateway control socket. Hardcoded path; chowned to group `fips` (mode `0770`) at startup so members of that group can query without sudo. | +| `inet fips_gateway` (nftables) | NAT table the gateway installs and tears down. View with `nft list table inet fips_gateway`. | + +The gateway also adds and removes a `local dev lo` route +in the local routing table so the kernel accepts pool addresses as +locally-owned. + +## Control Socket + +`fips-gateway` exposes a JSON line-protocol control socket separate +from the daemon's. The command set (`show_gateway`, `show_mappings`) +and JSON shapes are documented in the +[Gateway Command Catalog](control-socket.md#gateway-command-catalog). + +There is no `fipsctl` subcommand for the gateway — query the socket +directly with `nc -U`, or watch the **Gateway** tab in +[`fipstop`](cli-fipstop.md), which polls the gateway socket +automatically. + +## See also + +- [`fips`](cli-fips.md) — the daemon. Required to be running on the + same host. +- [`fipstop`](cli-fipstop.md) — the live-status TUI; its Gateway tab + polls the gateway control socket. +- [configuration.md § Gateway](configuration.md#gateway-gateway) — + full `gateway.*` block reference. +- [control-socket.md § Gateway Command Catalog](control-socket.md#gateway-command-catalog) + — wire protocol for the gateway socket. +- [../design/fips-gateway.md](../design/fips-gateway.md) — design, + NAT pipeline, virtual IP pool lifecycle. +- [../how-to/deploy-gateway.md](../how-to/deploy-gateway.md) — manual + Linux deployment. +- [../how-to/troubleshoot-gateway.md](../how-to/troubleshoot-gateway.md) + — diagnostic recipes. +- [../tutorials/deploy-fips-gateway.md](../tutorials/deploy-fips-gateway.md) + — OpenWrt walk-through. diff --git a/docs/reference/cli-fips.md b/docs/reference/cli-fips.md new file mode 100644 index 0000000..40b5844 --- /dev/null +++ b/docs/reference/cli-fips.md @@ -0,0 +1,93 @@ +# `fips` + +The FIPS mesh network daemon. + +## Synopsis + +```text +fips [-c FILE] +``` + +On Windows the same binary additionally accepts `--install-service`, +`--uninstall-service`, and (used internally by the service control +manager) `--service`. + +## Description + +`fips` is the FIPS daemon. It loads a YAML configuration, resolves an +identity, brings up the TUN adapter, listens on configured transports, +authenticates peers, maintains the spanning tree, and forwards mesh +traffic. There is one daemon per node. + +The daemon stays in the foreground, logging to stderr, until it +receives `SIGINT` or `SIGTERM`. On Windows, the service variant is +controlled through the standard service control manager. + +## Options + +| Flag | Argument | Description | +| ---- | -------- | ----------- | +| `-c`, `--config` | `FILE` | Use `FILE` as the configuration. Skips the default search paths. | +| `-V` | — | Print the short version (e.g. `0.3.0-dev (rev abcdef1)`). | +| `--version` | — | Print the long version: short version plus build target triple. | +| `-h`, `--help` | — | Print usage and exit. | +| `--install-service` | — | (Windows only) Install `fips` as a Windows service. Requires Administrator. | +| `--uninstall-service` | — | (Windows only) Uninstall the Windows service. Requires Administrator. | +| `--service` | — | (Windows only, internal) Run as a Windows service. Invoked by the service control manager — not for direct use. | + +There are no other CLI flags; all daemon behaviour is governed by the +YAML configuration. See [configuration.md](configuration.md). + +## Exit Codes + +| Code | Meaning | +| ---- | ------- | +| `0` | Clean shutdown after `SIGINT` / `SIGTERM`. | +| `1` | Failed to load configuration, resolve identity, construct the node, or start the node. The reason is printed to stderr before exit. | + +## Environment + +| Variable | Description | +| -------- | ----------- | +| `RUST_LOG` | Tracing filter directive. Overrides `node.log_level` from the config. Examples: `info`, `debug`, `fips=trace,fips::node::handlers::mmp=debug`. | +| `XDG_RUNTIME_DIR` | Used to derive the default control-socket path when `/run/fips` does not exist. See [control-socket.md](control-socket.md). | +| `FIPS_CONFIG` | (Windows service mode only) Path to the configuration file when the daemon runs under the service control manager. | + +The daemon also clamps the `nostr_relay_pool`, `nostr_sdk`, and `nostr` +log targets to `info` whenever the effective log level is below +`trace`, so that `RUST_LOG=debug` does not flood the journal with raw +relay frames. To see those frames, set the level to `trace`. + +## Files + +`fips` looks for `fips.yaml` in the following locations, lowest to +highest priority. All present files are merged in priority order; the +highest-priority value wins. + +| Priority | Path | Purpose | +| -------- | ---- | ------- | +| 1 | `/etc/fips/fips.yaml` | System-wide defaults | +| 2 | `~/.config/fips/fips.yaml` | User preferences | +| 3 | `~/.fips.yaml` | Legacy user config | +| 4 | `./fips.yaml` | Deployment-specific overrides | + +Adjacent to the highest-priority config file the daemon reads (or +writes, on first start) the identity files: + +| File | Mode | Purpose | +| ---- | ---- | ------- | +| `fips.key` | `0600` | Bech32 nsec for the persistent identity (Unix only; Windows inherits parent ACLs). | +| `fips.pub` | `0644` | Bech32 npub corresponding to `fips.key`. | + +When `node.identity.persistent` is `false` (the default), a fresh +keypair is written to these files on every start. + +The control socket path is derived per +[control-socket.md](control-socket.md). + +## See also + +- [`fipsctl`](cli-fipsctl.md) — control-socket client. +- [`fipstop`](cli-fipstop.md) — live-status TUI. +- [configuration.md](configuration.md) — YAML reference. +- [control-socket.md](control-socket.md) — control-socket protocol. diff --git a/docs/reference/cli-fipsctl.md b/docs/reference/cli-fipsctl.md new file mode 100644 index 0000000..c80867e --- /dev/null +++ b/docs/reference/cli-fipsctl.md @@ -0,0 +1,148 @@ +# `fipsctl` + +Command-line client for the FIPS daemon's control socket. + +## Synopsis + +```text +fipsctl [-s SOCKET] [args...] +``` + +## Description + +`fipsctl` connects to a running daemon over its control socket +(Unix domain socket on Linux/macOS, TCP loopback on Windows), sends +one JSON request, and pretty-prints the response. Exits with a +non-zero status if the socket cannot be reached, the daemon returns an +error, or the request times out. + +`fipsctl keygen` is a special case: it does not contact the daemon and +operates purely on local files. + +For the line-delimited JSON wire protocol, see +[control-socket.md](control-socket.md). For the YAML configuration +that defines the socket location, see +[configuration.md](configuration.md). + +## Global Options + +| Flag | Argument | Description | +| ---- | -------- | ----------- | +| `-s`, `--socket` | `PATH` | Override the control-socket path (Linux/macOS) or TCP port (Windows). | +| `-V`, `--version` | — | Print the short version. | +| `--version` | — | Print the long version. | +| `-h`, `--help` | — | Print usage and exit. Per-subcommand help via `fipsctl --help`. | + +## Subcommands + +### `show ` + +Read-only queries against the daemon. Each subcommand maps 1:1 to a +control-socket query (see [control-socket.md](control-socket.md)) and +prints the response's `data` object as pretty JSON. + +| Subcommand | Control-socket command | Returns | +| ---------- | ---------------------- | ------- | +| `show status` | `show_status` | Node-level status: identity, version, peer/link/session counts, TUN state, recent sparklines. | +| `show peers` | `show_peers` | Authenticated peer list with link IDs, transport addresses, MMP metrics, Noise/rekey state. | +| `show links` | `show_links` | Active links (one per FMP-authenticated peer): direction, state, byte counters. | +| `show tree` | `show_tree` | Spanning-tree state: root, my coordinates, parent, peer declarations. | +| `show sessions` | `show_sessions` | End-to-end FSP sessions: state, traffic counters, session-MMP metrics, path MTU. | +| `show bloom` | `show_bloom` | Bloom-filter state: own filter sequence, leaf dependents, per-peer filter summaries. | +| `show mmp` | `show_mmp` | MMP metrics summary: per-peer link-layer metrics and per-session session-layer metrics. | +| `show cache` | `show_cache` | Coordinate cache: TTL, fill ratio, per-destination coords and path MTU. | +| `show connections` | `show_connections` | Pending handshake connections: state, idle time, resend count. | +| `show transports` | `show_transports` | Transport instances: type, state, MTU, local address, per-transport stats. | +| `show routing` | `show_routing` | Routing summary: pending lookups, retry state, forwarding/discovery/error/congestion counters. | +| `show identity-cache` | `show_identity_cache` | Cached `(node_addr → npub)` entries with last-seen timestamps. | + +### `acl ` + +| Subcommand | Control-socket command | Returns | +| ---------- | ---------------------- | ------- | +| `acl show` | `show_acl` | Loaded peer-ACL state: allow/deny files, effective mode, default decision, entry counts. | + +### `stats ` + +Time-series metrics from the in-process history rings. + +| Subcommand | Control-socket command | Description | +| ---------- | ---------------------- | ----------- | +| `stats list` | `show_stats_list` | Enumerate available metrics, their units, and the per-ring retention windows. | +| `stats peers` | `show_stats_peers` | List peers tracked in stats history (active or recently active). | +| `stats history [options]` | `show_stats_history` | Fetch a time-series window for one metric. | + +`stats history` options: + +| Flag | Argument | Default | Description | +| ---- | -------- | ------- | ----------- | +| `--peer` | `npub` or hostname | *(none)* | Required for per-peer metrics; resolves through `/etc/fips/hosts` if not an npub. | +| `--window` | `s` / `m` / `h` | `10m` | Window duration. | +| `--granularity` | `1s` or `1m` | `1s` | Ring resolution. `1s` uses the fast ring; `1m` uses the slow ring. | +| `--plot` | — | off | Render a Unicode-block sparkline to stdout instead of JSON. | + +### `keygen [options]` + +Generate a new FIPS identity keypair locally. Does not contact the +daemon. + +| Flag | Argument | Default | Description | +| ---- | -------- | ------- | ----------- | +| `-d`, `--dir` | `DIR` | `/etc/fips` (Unix), `%APPDATA%\fips` (Windows) | Output directory for `fips.key` and `fips.pub`. | +| `-f`, `--force` | — | off | Overwrite an existing `fips.key`. | +| `-s`, `--stdout` | — | off | Print `nsec` then `npub` to stdout instead of writing files. | + +`fips.key` is written with mode `0600` and `fips.pub` with mode `0644` +on Unix. After running `keygen`, set `node.identity.persistent: true` +in `fips.yaml` or the daemon will overwrite the keys on next start. + +### `connect
` + +Tell the daemon to dial a peer over a specific transport. + +| Argument | Description | +| -------- | ----------- | +| `peer` | npub (bech32) or hostname from `/etc/fips/hosts`. | +| `address` | Transport endpoint, e.g. `192.168.1.10:2121`, `[2001:db8::1]:2121`, or a Tor onion. FIPS-mesh ULAs (`fd00::/8`) are rejected for the IP-based transports (udp, tcp, ethernet). | +| `transport` | One of `udp`, `tcp`, `tor`, `ethernet`. | + +### `disconnect ` + +Tell the daemon to drop a peer link. + +| Argument | Description | +| -------- | ----------- | +| `peer` | npub (bech32) or hostname from `/etc/fips/hosts`. | + +## Exit Codes + +| Code | Meaning | +| ---- | ------- | +| `0` | Daemon returned `{"status":"ok",...}`. | +| `1` | Argument parse failure, control-socket connection failure, daemon returned `{"status":"error",...}`, or local I/O failure (keygen). The error message is printed to stderr. | + +## Environment + +| Variable | Description | +| -------- | ----------- | +| `XDG_RUNTIME_DIR` | Used to derive the default control-socket path when `/run/fips` is absent. | + +`fipsctl` does not consume `RUST_LOG`; logging is for the daemon. + +## Files + +| Path | Purpose | +| ---- | ------- | +| `/etc/fips/hosts` | Maps hostnames to npubs for the `connect`, `disconnect`, and `--peer` arguments. See [configuration.md](configuration.md). | +| Control socket (default) | Same resolution as the daemon: `/run/fips/control.sock` if present, else `$XDG_RUNTIME_DIR/fips/control.sock`, else `/tmp/fips-control.sock` (Unix); TCP `localhost:21210` (Windows). | + +If you get `Permission denied` connecting to the socket on Linux, +add your user to the `fips` group (`sudo usermod -aG fips $USER`) +and log out and back in. + +## See also + +- [`fips`](cli-fips.md) — the daemon. +- [`fipstop`](cli-fipstop.md) — live-status TUI. +- [control-socket.md](control-socket.md) — wire protocol. +- [configuration.md](configuration.md) — YAML reference. diff --git a/docs/reference/cli-fipstop.md b/docs/reference/cli-fipstop.md new file mode 100644 index 0000000..7ea8a99 --- /dev/null +++ b/docs/reference/cli-fipstop.md @@ -0,0 +1,123 @@ +# `fipstop` + +Live-status terminal UI for a running FIPS daemon. + +## Synopsis + +```text +fipstop [-s SOCKET] [--gateway-socket PATH] [-r SECONDS] +``` + +## Description + +`fipstop` is a `ratatui`-based dashboard. It opens the daemon control +socket, polls a small set of `show_*` queries on a timer, and renders +the state in a tabbed full-screen UI. A separate poll runs against the +gateway control socket when the Gateway tab is active. + +`fipstop` is read-only — it cannot mutate daemon state. Use +[`fipsctl`](cli-fipsctl.md) for `connect` / `disconnect` and friends. + +## Options + +| Flag | Argument | Default | Description | +| ---- | -------- | ------- | ----------- | +| `-s`, `--socket` | `PATH` | (auto) | Daemon control-socket path / port. Same default as `fipsctl`. | +| `--gateway-socket` | `PATH` | (auto) | `fips-gateway` control-socket path / port. Default: `/run/fips/gateway.sock` (Unix), TCP port `21211` (Windows). | +| `-r`, `--refresh` | `SECONDS` | `2` | Poll interval. | +| `-V`, `--version` | — | — | Print short version. | +| `--version` | — | — | Print long version. | +| `-h`, `--help` | — | — | Print usage and exit. | + +## Tabs + +Tabs cycle in this order. Each tab issues the listed control-socket +query on its first activation and on every refresh tick while active. + +| Tab | Query | Shows | +| --- | ----- | ----- | +| **Node** | `show_status` | Identity, version, uptime, peer/link/session counts, sparklines for mesh size, tree depth, peer count, bytes, loss. | +| **Peers** | `show_peers` (+ `show_links`, `show_transports` cross-refs) | Authenticated peers in a table. Selecting a row and pressing Enter opens a detail view. | +| **Transports** | `show_transports` (+ `show_links`, `show_peers` cross-refs) | Tree of transport instances with per-link children when expanded. | +| **Sessions** | `show_sessions` | End-to-end FSP sessions. | +| **Tree** | `show_tree` | Spanning-tree state and per-peer coordinates. | +| **Filters** | `show_bloom` | Per-peer Bloom-filter state. | +| **Performance** | `show_mmp` | Link-layer and session-layer MMP metrics. | +| **Routing** | `show_routing` (+ `show_cache` cross-ref) | Forwarding/discovery counters, pending lookups, retry state. | +| **Graphs** | `show_stats_history` family + `show_stats_peers` | Stacked time-series plots. Three modes: node-level metrics, one metric across peers, all metrics for one peer. | +| **Gateway** | `show_gateway` and `show_mappings` against the gateway socket | Pool utilisation and per-mapping state when `fips-gateway` is running. Empty when the gateway socket is unreachable. | + +The cycle order in the UI is: Node → Peers → Transports → Sessions → +Tree → Filters → Performance → Routing → Graphs → Gateway. The Links +and Cache tabs are not in the cycle but are fetched as cross-references +to populate Peers, Transports, and Routing detail views. + +## Keybindings + +### Global + +| Key | Action | +| --- | ------ | +| `q`, `Ctrl-C` | Quit. | +| `Tab` | Next tab. | +| `Shift-Tab` | Previous tab. | +| `g` | Jump to the Graphs tab. | +| `Esc` | Close detail view (if open). | + +### Table tabs (Peers, Sessions, Transports, Gateway) + +| Key | Action | +| --- | ------ | +| `Up`, `Down` | Move row selection. | +| `Enter` | Open detail view for the selected row. | + +### Transports tab (extra) + +| Key | Action | +| --- | ------ | +| `Right`, `Space` | Expand the selected transport row to show its links. | +| `Left` | Collapse the selected transport row. | +| `e` | Expand all transports. | +| `c` | Collapse all transports. | + +### Graphs tab (extra) + +| Key | Action | +| --- | ------ | +| `Up`, `Down` | Scroll within the stacked plots. | +| `Right`, `Space` | Next time window. Cycles `1m / 1s` → `10m / 1s` → `1h / 1s` → `24h / 1m`. | +| `Left` | Previous time window. | +| `m` | Cycle view mode: `Node` (stacked node metrics) → `MetricByPeer` (one per-peer metric across all peers) → `PeerByMetric` (all per-peer metrics for one peer). | +| `n` | Next selector (next per-peer metric in MetricByPeer; next peer in PeerByMetric). | +| `Shift-N` | Previous selector. | + +## Exit Codes + +| Code | Meaning | +| ---- | ------- | +| `0` | Normal quit. | +| `1` | Failed to initialise the terminal. The reason is printed to stderr. | + +A failure to reach the daemon socket is **not** fatal: the dashboard +displays "Disconnected" in the status bar and retries on every refresh +tick. + +## Environment + +| Variable | Description | +| -------- | ----------- | +| `XDG_RUNTIME_DIR` | Used to derive the default control-socket and gateway-socket paths when `/run/fips` is absent. | + +## Files + +Same control-socket resolution rules as +[`fipsctl`](cli-fipsctl.md#files). The gateway socket follows the same +pattern with `gateway.sock` in place of `control.sock`, falling back +to `/tmp/fips-gateway.sock` if neither system path nor +`XDG_RUNTIME_DIR` is available. + +## See also + +- [`fipsctl`](cli-fipsctl.md) — issue mutating commands. +- [`fips`](cli-fips.md) — the daemon. +- [control-socket.md](control-socket.md) — wire protocol fipstop polls. diff --git a/docs/design/fips-configuration.md b/docs/reference/configuration.md similarity index 76% rename from docs/design/fips-configuration.md rename to docs/reference/configuration.md index 686dc82..1692ca3 100644 --- a/docs/design/fips-configuration.md +++ b/docs/reference/configuration.md @@ -54,7 +54,7 @@ peers: # Static peer list | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `node.control.enabled` | bool | `true` | Enable the control socket | -| `node.control.socket_path` | string | *(auto)* | **Linux:** Socket file path. Default: `$XDG_RUNTIME_DIR/fips/control.sock`, then `/run/fips/control.sock` (if root), then `/tmp/fips-control.sock`. **Windows:** TCP port number (default: `21210`); the control socket listens on `localhost` at this port. | +| `node.control.socket_path` | string | *(auto)* | **Linux:** Socket file path. Resolved at daemon startup: `$XDG_RUNTIME_DIR/fips/control.sock` if `XDG_RUNTIME_DIR` is set, else `/run/fips/control.sock` if `/run/fips` can be created (typical when running under the shipped systemd unit), else `/tmp/fips-control.sock`. (Note: the `fipsctl` / `fipstop` clients use a different fallback order — `/run/fips` first if it already exists, then `XDG_RUNTIME_DIR`, then `/tmp` — so when both schemes apply, set this field explicitly to avoid mismatch.) **Windows:** TCP port number (default: `21210`); the control socket listens on `127.0.0.1` at this port. | The control socket provides access to node state and runtime management via the `fipsctl` command-line tool. In addition to read-only status @@ -109,6 +109,7 @@ to the highest-priority config file for operator visibility, even in ephemeral m | `node.base_rtt_ms` | u64 | `100` | Initial RTT estimate for new links before measurements converge | | `node.heartbeat_interval_secs` | u64 | `10` | Heartbeat send interval per peer for liveness detection | | `node.link_dead_timeout_secs` | u64 | `30` | No-traffic timeout before a peer is declared dead and removed | +| `node.log_level` | string | `"info"` | Tracing filter default. Case-insensitive; one of `trace`, `debug`, `info`, `warn`, `error`. Overridden by the `RUST_LOG` environment variable when set | ### Resource Limits (`node.limits.*`) @@ -181,8 +182,9 @@ offer/answer + punch-through, after which the established UDP socket is handed into the normal FIPS transport/session stack. Inbox-relay discovery falls back to the local DM relay list if remote relay metadata cannot be fetched. -This support is compiled behind the crate feature `nostr-discovery`; builds -without that feature ignore `udp:nat` bootstrap configuration. +The Nostr discovery runtime is compiled into every build of the crate; it +is enabled at runtime via `node.discovery.nostr.enabled: true` and stays +inert otherwise. | Parameter | Type | Default | Description | |-----------|------|---------|-------------| @@ -196,7 +198,8 @@ without that feature ignore `udp:nat` bootstrap configuration. | `node.discovery.nostr.advert_relays` | list[string] | `["wss://relay.damus.io", "wss://nos.lol", "wss://offchain.pub"]` | Relays used for service adverts | | `node.discovery.nostr.dm_relays` | list[string] | `["wss://relay.damus.io", "wss://nos.lol", "wss://offchain.pub"]` | Relays used for encrypted signaling events | | `node.discovery.nostr.stun_servers` | list[string] | `["stun:stun.l.google.com:19302", "stun:stun.cloudflare.com:3478", "stun:global.stun.twilio.com:3478"]` | STUN servers used for local reflexive address discovery | -| `node.discovery.nostr.app` | string | `"fips-overlay-v1"` | Traversal application namespace and advert identifier suffix | +| `node.discovery.nostr.share_local_candidates` | bool | `false` | Whether to advertise local (RFC 1918 / ULA) interface addresses as host candidates in the traversal offer. Off by default: in most deployments peers aren't on the same broadcast domain, and sharing private host candidates causes misleading punch successes when an asymmetric L3 path (VPN, Tailscale subnet route, overlapping address space) makes a peer's private IP one-way reachable. Enable only when peers are on the same physical LAN | +| `node.discovery.nostr.app` | string | `"fips-overlay-v1"` | Traversal application namespace, published in the advert's `protocol` tag (the `d` tag itself is hardcoded to `fips-overlay-v1`) | | `node.discovery.nostr.signal_ttl_secs` | u64 | `120` | Signaling TTL in seconds | | `node.discovery.nostr.attempt_timeout_secs` | u64 | `10` | Overall traversal attempt timeout in seconds | | `node.discovery.nostr.replay_window_secs` | u64 | `300` | Replay tracking retention window in seconds | @@ -207,6 +210,11 @@ without that feature ignore `udp:nat` bootstrap configuration. | `node.discovery.nostr.advert_refresh_secs` | u64 | `1800` | How often adverts are refreshed in seconds | | `node.discovery.nostr.startup_sweep_delay_secs` | u64 | `5` | Settle delay after Nostr discovery starts before the one-shot startup advert sweep runs (only used under `policy: open`). Allows the relay subscription backlog to populate the in-memory advert cache before the sweep fires | | `node.discovery.nostr.startup_sweep_max_age_secs` | u64 | `3600` | Maximum advert age (`now - created_at`) considered by the one-shot startup sweep (only used under `policy: open`). Adverts older than this are skipped on startup; the per-tick sweep still considers them up to `valid_until_ms` | +| `node.discovery.nostr.failure_streak_threshold` | u32 | `5` | Consecutive NAT-traversal failures against a peer before an extended cooldown is applied. At this threshold the daemon also actively re-fetches the peer's advert from `advert_relays` to evict cache entries for peers that have gone away | +| `node.discovery.nostr.extended_cooldown_secs` | u64 | `1800` | Cooldown applied to a peer once `failure_streak_threshold` is hit. Suppresses both open-discovery sweep enqueues and per-attempt retry firings until elapsed (30 minutes default) | +| `node.discovery.nostr.warn_log_interval_secs` | u64 | `300` | Minimum interval between `NAT traversal failed` WARN log lines for the same peer. Subsequent failures inside the window log at DEBUG to reduce log spam on public-test nodes with many cache-learned peers | +| `node.discovery.nostr.failure_state_max_entries` | usize | `4096` | Maximum entries retained in the per-npub failure-state map. Bounds memory under high cache turnover; oldest entries (by last failure time) are evicted when the cap is exceeded | +| `node.discovery.nostr.protocol_mismatch_cooldown_secs` | u64 | `86400` | Cooldown applied after observing a fatal protocol mismatch on a Nostr-adopted bootstrap transport (e.g. `Unknown FMP version` from a peer running a different FMP-protocol version). Independent of `extended_cooldown_secs` and much longer (24 hours default) because the mismatch is structural — re-traversing is wasted effort until one side upgrades | If `stun_servers` is omitted, the built-in default list above is used. If it is specified in YAML, the configured list fully overrides the defaults. @@ -246,6 +254,7 @@ Controls tree construction and parent selection. | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `node.bloom.update_debounce_ms` | u64 | `500` | Debounce interval for filter update propagation | +| `node.bloom.max_inbound_fpr` | f64 | `0.05` | Antipoison cap: reject inbound `FilterAnnounce` frames whose advertised false-positive rate exceeds this value. Valid range `(0.0, 1.0)`. The default `0.05` corresponds to fill 0.549 at k=5 (≈3,200 entries on the 1 KB filter) | Bloom filter size (1 KB), hash count (5), and size classes are protocol constants and not configurable. @@ -302,7 +311,7 @@ configurable. ### Link-Layer MMP (`node.mmp.*`) Metrics Measurement Protocol for per-peer link measurement. See -[fips-mesh-layer.md](fips-mesh-layer.md) for behavioral details. +[../design/fips-mesh-layer.md](../design/fips-mesh-layer.md) for behavioral details. | Parameter | Type | Default | Description | |-----------|------|---------|-------------| @@ -350,7 +359,7 @@ with the node for routing. | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `dns.enabled` | bool | `true` | Enable DNS responder | -| `dns.bind_addr` | string | `"127.0.0.1"` | Bind address | +| `dns.bind_addr` | string | `"::1"` | Bind address. Default is IPv6 loopback only; the shipped `fips-dns-setup` configures systemd-resolved to forward `.fips` queries to `[::1]:5354`. To expose the responder to mesh peers (or to the gateway over IPv4), override (e.g., `"::"` for all interfaces). | | `dns.port` | u16 | `5354` | Listen port | | `dns.ttl` | u32 | `300` | AAAA record TTL in seconds | @@ -367,21 +376,30 @@ The host map is populated from two sources: 2. **Hosts file** — `/etc/fips/hosts`, one `hostname npub1...` per line. Blank lines and `#` comments are allowed. -The hosts file is auto-reloaded on modification (mtime change) without +On conflict, hosts-file entries take precedence over peer aliases. The +hosts file is auto-reloaded on modification (mtime change) without restarting the daemon. Hostnames are case-insensitive. +The installer ships `/etc/fips/hosts` pre-populated with the public test +mesh roster (`test-us01` … `test-uk01`). Operator-style guide for +adding entries and the precedence rules: +[../how-to/host-aliases.md](../how-to/host-aliases.md). + ## Transports (`transports.*`) ### UDP (`transports.udp.*`) | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `transports.udp.bind_addr` | string | `"0.0.0.0:2121"` | UDP bind address and port | +| `transports.udp.bind_addr` | string | `"0.0.0.0:2121"` | UDP bind address and port. Ignored when `outbound_only: true` (kernel-assigned ephemeral port is used regardless). | | `transports.udp.mtu` | u16 | `1280` | Transport MTU | | `transports.udp.recv_buf_size` | usize | `2097152` | UDP socket receive buffer size in bytes (2 MB). Linux kernel doubles the requested value internally. Host `net.core.rmem_max` must be >= this value. | | `transports.udp.send_buf_size` | usize | `2097152` | UDP socket send buffer size in bytes (2 MB). Host `net.core.wmem_max` must be >= this value. | -| `transports.udp.advertise_on_nostr` | bool | `false` | Include this UDP transport in Nostr endpoint adverts | +| `transports.udp.advertise_on_nostr` | bool | `false` | Include this UDP transport in Nostr endpoint adverts. Implicitly forced false when `outbound_only: true`. | | `transports.udp.public` | bool | `false` | If advertised: `true` publishes direct `host:port`; `false` publishes `udp:nat` rendezvous | +| `transports.udp.external_addr` | string | *(none)* | Explicit advertise-as override. Bare IP (`"203.0.113.45"` — bind port is appended) or full `host:port`. Takes precedence over the bound address and STUN autodiscovery. Useful when the public IP isn't on a local interface (cloud 1:1 NAT, EIP) or to skip STUN for a deterministic value. | +| `transports.udp.outbound_only` | bool | `false` | Pure-client posture. When `true`, the transport binds to `0.0.0.0:0` (kernel-assigned ephemeral port) regardless of `bind_addr`, refuses inbound handshake msg1, and is never advertised on Nostr regardless of `advertise_on_nostr`. | +| `transports.udp.accept_connections` | bool | `true` | Accept inbound handshake msg1 from new peers. Combine with `outbound_only: false` and `accept_connections: false` (plus `auto_connect` on peer entries) for a node that initiates outbound links but rejects fresh inbound handshakes. The handshake handler carves out msg1 from peers already established on this transport so rekey continues to work. | ### Ethernet (`transports.ethernet.*`) @@ -436,6 +454,8 @@ overhead. | `transports.tcp.recv_buf_size` | usize | `2097152` | Socket receive buffer size in bytes (2 MB) | | `transports.tcp.send_buf_size` | usize | `2097152` | Socket send buffer size in bytes (2 MB) | | `transports.tcp.max_inbound_connections` | usize | `256` | Maximum simultaneous inbound connections | +| `transports.tcp.advertise_on_nostr` | bool | `false` | Include this TCP transport in Nostr endpoint adverts | +| `transports.tcp.external_addr` | string | *(none)* | Explicit advertise-as override. Bare IP or full `host:port`. **Required** when `bind_addr` is wildcard (e.g. `"0.0.0.0:443"`) and `advertise_on_nostr: true`, since TCP has no STUN equivalent for autodiscovery. Common on cloud 1:1 NAT / EIP setups where the public IP isn't bindable on the host. | **Named instances.** Like other transports, multiple TCP instances can be configured with named sub-keys: @@ -559,9 +579,13 @@ HiddenServicePort 8443 127.0.0.1:8444 ### BLE (`transports.ble.*`) Bluetooth Low Energy transport using L2CAP Connection-Oriented Channels. -Requires BlueZ and the `ble` Cargo feature flag (default-on). Linux only; -guarded by `#[cfg(target_os = "linux")]`. Communicates with BlueZ via D-Bus -using the `bluer` crate. +Linux + glibc only — at build time, `build.rs` probes for the BlueZ / +`bluer` crate dependencies and sets the `bluer_available` `cfg`; the BLE +runtime is gated behind `#[cfg(bluer_available)]`. There is no Cargo +feature flag to toggle. On non-glibc Linux (musl) or non-Linux platforms, +BLE config still parses but the transport runtime is absent and config +entries become no-ops. Communicates with BlueZ via D-Bus through the +`bluer` crate. | Parameter | Type | Default | Description | |-----------|------|---------|-------------| @@ -636,13 +660,108 @@ Static peer list. Each entry defines a peer to connect to. |-----------|------|---------|-------------| | `peers[].npub` | string | *(required)* | Peer's Nostr public key (npub-encoded) | | `peers[].alias` | string | *(none)* | Human-readable name for logging | +| `peers[].addresses` | list | `[]` | Transport addresses for the peer. May be left empty (or omitted) when `via_nostr: true`, in which case the daemon resolves endpoints from the peer's Nostr advert at dial time. | | `peers[].addresses[].transport` | string | *(required)* | Transport type: `udp`, `tcp`, `ethernet`, `tor`, or `ble` | | `peers[].addresses[].addr` | string | *(required)* | Transport address. UDP/TCP: `"host:port"` (IP or DNS hostname). Ethernet: `"interface/mac"` (e.g., `"eth0/aa:bb:cc:dd:ee:ff"`). BLE: `"adapter/device_address"` (e.g., `"hci0/AA:BB:CC:DD:EE:FF"`). Tor: `".onion:port"` or `"host:port"` | | `peers[].addresses[].priority` | u8 | `100` | Address priority (lower = preferred) | -| `peers[].connect_policy` | string | `"auto_connect"` | Connection policy: `auto_connect`, `on_demand`, or `manual` | +| `peers[].connect_policy` | string | `"auto_connect"` | Connection policy: `auto_connect`, `on_demand`, or `manual`. Note: `on_demand` and `manual` are reserved for future use; the only policy currently honored at runtime is `auto_connect`. | | `peers[].auto_reconnect` | bool | `true` | Automatically reconnect after MMP link-dead removal (exponential backoff, unlimited retries) | | `peers[].via_nostr` | bool | `false` | Append Nostr advert-derived endpoints after static addresses for this peer | +## Gateway (`gateway.*`) + +The `gateway.*` block configures the optional `fips-gateway` +service, which lets unmodified LAN hosts reach mesh destinations +through DNS proxy + virtual-IP NAT (and, optionally, exposes +LAN-side services back into the mesh through inbound port forwards). +The gateway is a separate service from the FIPS daemon but reads the +same `fips.yaml` file. The block is read only when `fips-gateway` is +running; the `fips` daemon ignores it. Linux only — the field is +gated behind `#[cfg(target_os = "linux")]`. For setup, see +[../how-to/deploy-gateway.md](../how-to/deploy-gateway.md); for the +end-to-end design, see +[../design/fips-gateway.md](../design/fips-gateway.md). + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `gateway.enabled` | bool | `false` | Enable the gateway. Must be `true` for `fips-gateway` to start. | +| `gateway.pool` | string | *(required)* | Virtual IPv6 pool CIDR (e.g., `"fd01::/112"`). Must not overlap with the FIPS mesh address space (`fd00::/8`) or any address space already in use on the LAN. The `/112` size yields 65 536 virtual IPs, which is the gateway's hard cap regardless of CIDR width. | +| `gateway.lan_interface` | string | *(required)* | LAN-facing network interface name (e.g., `"enp3s0"`). Used for proxy-NDP entry installation so LAN clients can resolve the link-layer address of allocated virtual IPs. | +| `gateway.pool_grace_period` | u64 | `60` | Seconds a virtual-IP allocation is retained after its last referencing session ends, before the address is returned to the free pool. Larger values reduce churn for short-lived flows; smaller values reclaim addresses faster. | + +### Gateway DNS (`gateway.dns.*`) + +Settings for the gateway's DNS listener and its upstream link to the +FIPS daemon's `.fips` resolver. The gateway proxies `.fips` queries to +the daemon's resolver, which returns mesh addresses; the gateway then +allocates a virtual IP from the pool and rewrites the response. +Non-`.fips` queries are answered with `REFUSED`. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `gateway.dns.listen` | string | `"[::]:53"` | LAN-facing DNS listen address. Bind on the LAN-side IP (e.g., `"192.168.1.1:53"`) or on all interfaces (`"[::]:53"`) for LAN clients to query. Bind to a non-53 port if another resolver already owns 53 on the host (see [../how-to/troubleshoot-gateway.md](../how-to/troubleshoot-gateway.md)). | +| `gateway.dns.upstream` | string | `"[::1]:5354"` | Upstream FIPS daemon resolver. **Must match the daemon's `dns.bind_addr` and `dns.port`.** Defaults match the daemon defaults (`::1:5354`). A v4 upstream (`"127.0.0.1:5354"`) cannot reach a daemon bound on `[::1]:5354` — Linux IPv6 sockets bound to explicit `::1` do not accept v4-mapped traffic. If you change the daemon's `dns.bind_addr`, update this field accordingly. | +| `gateway.dns.ttl` | u32 | `60` | TTL in seconds on AAAA responses returned to LAN clients. Smaller values let the gateway recycle pool addresses faster; larger values reduce LAN-side query traffic. | + +### Conntrack (`gateway.conntrack.*`) + +Linux conntrack timeout overrides for the gateway's NAT table. These +adjust the kernel-default timeouts for NAT sessions installed by the +gateway. All values are in seconds; omit any field to inherit the +gateway's built-in default (which itself usually matches the kernel +default for that protocol). + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `gateway.conntrack.tcp_established` | u64 | `432000` | TCP established-state timeout (5 days). Long-lived TCP flows (SSH, persistent HTTP) keep their NAT mapping alive for at least this long without traffic. | +| `gateway.conntrack.udp_timeout` | u64 | `30` | UDP unreplied timeout. Applied until reply traffic is observed in the reverse direction. | +| `gateway.conntrack.udp_assured` | u64 | `180` | UDP assured (bidirectional) timeout. Applied once reply traffic has been observed. | +| `gateway.conntrack.icmp_timeout` | u64 | `30` | ICMP echo / error timeout. | + +### Inbound Port Forwards (`gateway.port_forwards[]`) + +Optional list of inbound port-forward rules. Each rule maps a TCP or +UDP port on the gateway's `fips0` mesh-side address to a `host:port` +on the LAN. Mesh peers connect to the gateway's mesh address on the +listen port; the gateway terminates the connection and forwards the +payload to the LAN target. This is the inverse of the outbound mode: +the LAN service is exposed to the mesh, not the other way around. See +[../how-to/deploy-gateway.md](../how-to/deploy-gateway.md) for the +operator recipe. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `gateway.port_forwards[].listen_port` | u16 | *(required)* | Port on `fips0` that mesh peers connect to. Must be non-zero. The `(listen_port, proto)` pair must be unique across the list. | +| `gateway.port_forwards[].proto` | string | *(required)* | Transport protocol: `tcp` or `udp`. | +| `gateway.port_forwards[].target` | string | *(required)* | LAN destination as IPv6 `[addr]:port` (e.g., `"[fd12:3456::10]:80"`). IPv4 targets are rejected at config-load time. | + +### Gateway Example + +A typical gateway with both outbound (LAN-to-mesh) and inbound +(mesh-to-LAN) modes enabled: + +```yaml +gateway: + enabled: true + pool: "fd01::/112" + lan_interface: "enp3s0" + dns: + listen: "[::]:53" + upstream: "[::1]:5354" + ttl: 60 + pool_grace_period: 60 + conntrack: + tcp_established: 432000 + udp_assured: 180 + port_forwards: + - listen_port: 8080 + proto: tcp + target: "[fd12:3456::10]:80" + - listen_port: 5353 + proto: udp + target: "[fd12:3456::10]:53" +``` + ## Minimal Example A typical node configuration enabling TUN, DNS, and a single peer: @@ -780,6 +899,7 @@ node: flap_dampening_secs: 120 # extended hold-down on flap bloom: update_debounce_ms: 500 + max_inbound_fpr: 0.05 # antipoison cap on inbound FilterAnnounce FPR session: default_ttl: 64 pending_packets_per_dest: 16 @@ -818,7 +938,7 @@ tun: dns: enabled: true - bind_addr: "127.0.0.1" + bind_addr: "::1" port: 5354 ttl: 300 @@ -859,9 +979,10 @@ transports: # # cookie_path: "/var/run/tor/control.authcookie" # # directory mode (inbound via Tor-managed onion service): # # directory_service: - # # hostname_file: "/var/lib/tor/fips/hostname" - # # bind_addr: "127.0.0.1:8444" + # # hostname_file: "/var/lib/tor/fips_onion_service/hostname" + # # bind_addr: "127.0.0.1:8443" # # max_inbound_connections: 64 + # # advertised_port: 443 # public-facing onion port for Nostr adverts # ble: # uncomment to enable BLE transport (Linux only, requires BlueZ) # adapter: "hci0" # HCI adapter name # psm: 0x0085 # L2CAP PSM (133) diff --git a/docs/reference/control-socket.md b/docs/reference/control-socket.md new file mode 100644 index 0000000..daa2df2 --- /dev/null +++ b/docs/reference/control-socket.md @@ -0,0 +1,166 @@ +# Control Socket Protocol + +The FIPS daemon and `fips-gateway` each expose a local control socket +that accepts line-delimited JSON requests and returns line-delimited +JSON responses. `fipsctl` and `fipstop` are clients of this protocol; +operators can also drive it directly with any tool that can speak +length-bounded JSON over a stream socket. + +## Connection + +### Linux / macOS + +A Unix domain socket. The default path is resolved in this order: + +1. `/run/fips/control.sock` (or `/run/fips/gateway.sock` for the + gateway), if `/run/fips` exists. This is what the `fips.service` + systemd unit creates. +2. `$XDG_RUNTIME_DIR/fips/control.sock` otherwise. +3. `/tmp/fips-control.sock` if neither of the above is available. + +The daemon `chown`s the socket file and its parent directory to the +`fips` group at bind time and sets mode `0770`. Members of the `fips` +group can therefore connect without root. Add a user with +`sudo usermod -aG fips $USER` (re-login required). + +The path can be overridden at the daemon side via +`node.control.socket_path` in the YAML config, and at the client side +via `fipsctl -s PATH` or `fipstop -s PATH`. + +### Windows + +A TCP listener bound to `127.0.0.1`. The daemon's port is `21210` by +default; the gateway's is `21211`. Only loopback connections are +accepted. Override via `node.control.socket_path` (which takes a port +number string on Windows). + +Windows TCP does not provide filesystem-level ACLs — any local user +can connect. See the security note in +[configuration.md](configuration.md#control-socket-nodecontrol). + +## Request Format + +One JSON object per line, terminated by `\n`. Maximum request size is +4096 bytes; longer requests are dropped with `request too large`. + +```json +{"command": "", "params": {}} +``` + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `command` | string | yes | Command name. See [Daemon command catalog](#daemon-command-catalog) and [Gateway command catalog](#gateway-command-catalog). | +| `params` | object | only for commands that take parameters | Parameter object. Unknown fields are ignored; missing required fields produce an error response. | + +Unknown top-level fields in the request are silently ignored. + +## Response Format + +One JSON object per line. + +```json +{"status": "ok", "data": {}} +{"status": "error", "message": ""} +``` + +| Field | Type | When present | +| ----- | ---- | ------------ | +| `status` | string | always; one of `"ok"` or `"error"`. | +| `data` | object | on `ok` responses. | +| `message` | string | on `error` responses. | + +### I/O timeouts + +The daemon enforces a 5-second timeout for both the request read and +the response write. If the connection idles longer than that, the +daemon closes it with no response. + +### Common error messages + +| Message | Cause | +| ------- | ----- | +| `empty request` | Connection closed before a newline was received. | +| `invalid request: ` | Malformed JSON or missing `command`. | +| `request too large` | Request exceeded 4096 bytes. | +| `read timeout` / `read error: ...` | Slow client or transport failure. | +| `unknown command: ` | Command not registered with this daemon. | +| `missing params for ` | Command requires `params` but none were provided. | +| `missing '' parameter` | Required parameter missing. | +| `query timeout` | Internal handler did not respond within 5 seconds. | +| `node shutting down` | Daemon is exiting. | +| `gateway not yet initialized` | (Gateway socket only) snapshot has not been published yet. | + +## Daemon Command Catalog + +Read-only queries are dispatched in `src/control/queries.rs`; +mutating commands are dispatched in `src/control/commands.rs`. The +table below lists every command currently registered. + +### Read-only queries + +| Command | Params | `data` shape (top-level keys) | +| ------- | ------ | ----------------------------- | +| `show_status` | — | `version`, `npub`, `node_addr`, `ipv6_addr`, `state`, `is_leaf_only`, `peer_count`, `session_count`, `link_count`, `transport_count`, `connection_count`, `tun_state`, `tun_name`, `effective_ipv6_mtu`, `control_socket`, `pid`, `exe_path`, `uptime_secs`, `estimated_mesh_size`, `forwarding`, `sparklines`. | +| `show_acl` | — | `allow_file`, `deny_file`, `enforcement_active`, `effective_mode`, `default_decision`, `allow_all`, `deny_all`, `allow_file_entries`, `deny_file_entries`, `allow_entries`, `deny_entries`. | +| `show_peers` | — | `peers[]` — per-peer object: `node_addr`, `npub`, `display_name`, `ipv6_addr`, `connectivity`, `link_id`, `direction`, `transport_addr`, `transport_type`, `is_parent`, `is_child`, `tree_depth`, `stats`, `noise`, `current_k_bit`, `mmp`, plus optional `nostr_traversal`, `rekey_in_progress`, `rekey_draining`. | +| `show_links` | — | `links[]` — `link_id`, `transport_id`, `remote_addr`, `direction`, `state`, `created_at_ms`, `stats`. | +| `show_tree` | — | `my_node_addr`, `root`, `is_root`, `depth`, `my_coords[]`, `parent`, `parent_display_name`, `declaration_sequence`, `declaration_signed`, `peer_tree_count`, `peers[]`, `stats`. | +| `show_sessions` | — | `sessions[]` — `remote_addr`, `npub`, `display_name`, `state` (`established`, `initiating`, `awaiting_msg3`, `unknown`), `is_initiator`, `last_activity_ms`, `stats`, optional `mmp`, `current_k_bit`, `is_draining`. | +| `show_bloom` | — | `own_node_addr`, `is_leaf_only`, `sequence`, `leaf_dependent_count`, `leaf_dependents[]`, `peer_filters[]`, `stats`. | +| `show_mmp` | — | `peers[]` (link-layer per peer), `sessions[]` (session-layer per session). Each entry includes loss/RTT/ETX/goodput, smoothed values, trends. | +| `show_cache` | — | `count`, `max_entries`, `fill_ratio`, `default_ttl_ms`, `expired`, `avg_age_ms`, `entries[]` — per-destination coords, depth, age, last-used, optional `path_mtu`. | +| `show_connections` | — | `connections[]` — pending handshakes: `link_id`, `direction`, `handshake_state`, `started_at_ms`, `idle_ms`, `resend_count`, optional `expected_peer`. | +| `show_transports` | — | `transports[]` — `transport_id`, `type`, `state`, `mtu`, `name`, `local_addr`, optional `tor_mode`, `onion_address`, `tor_monitoring`, `stats`. | +| `show_routing` | — | `coord_cache_entries`, `identity_cache_entries`, `pending_lookups[]`, `pending_tun_destinations`, `pending_tun_packets`, `recent_requests`, `retries[]`, `forwarding`, `discovery`, `error_signals`, `congestion`. | +| `show_identity_cache` | — | `entries[]`, `count`, `max_entries`. Each entry: `node_addr`, `npub`, `display_name`, `ipv6_addr`, `last_seen_ms`, `age_ms`. | +| `show_stats_list` | — | `metrics[]` (each with `name`, `unit`, `scope`), `fast_ring_seconds`, `slow_ring_minutes`, `peer_retention_seconds`. | +| `show_stats_history` | `metric` (req), `peer` (req for per-peer metrics), `window` (`s` / `m` / `h`, default `10m`), `granularity` (`1s` / `1m`, default `1s`) | A single `Series`: `metric`, `unit`, `granularity_seconds`, `values[]`. | +| `show_stats_all_history` | `peer` (optional npub), `window`, `granularity` | `granularity_seconds`, `window_seconds`, `peer`, `series[]` (one per metric). | +| `show_stats_peers` | — | `peers[]`, `count`. Each entry: `npub`, `node_addr`, `display_name`, `is_active`, `first_seen_secs_ago`, `last_contact_secs_ago`. | +| `show_stats_history_all_peers` | `metric` (req per-peer name), `window`, `granularity` | `metric`, `unit`, `granularity_seconds`, `window_seconds`, `peers[]` (each with `node_addr`, `display_name`, `is_active`, `values[]`). | + +The schema of each query response is pinned by snapshot tests in +`src/control/snapshots/`; intentional schema changes regenerate those +fixtures. + +### Mutating commands + +| Command | Required params | Behaviour | +| ------- | --------------- | --------- | +| `connect` | `npub` (bech32), `address` (transport endpoint), `transport` (`udp`, `tcp`, `tor`, `ethernet`) | Asks the node to dial the peer over the named transport. Returns the API result on success or an error string on failure. | +| `disconnect` | `npub` (bech32) | Asks the node to drop the link to the named peer. | + +Both commands run on the daemon's main task and may block briefly +while the node mutates its state. + +## Gateway Command Catalog + +`fips-gateway` exposes a separate control socket with its own command +set. Dispatch lives in `src/gateway/control.rs`. + +| Command | Params | `data` shape | +| ------- | ------ | ------------ | +| `show_gateway` | — | `pool_total`, `pool_allocated`, `pool_active`, `pool_draining`, `pool_free`, `nat_mappings`, `dns_listen`, `uptime_secs`, `pool_cidr`, `lan_interface`, `dns_upstream`, `dns_ttl`, `pool_grace_period`. | +| `show_mappings` | — | `mappings[]` — `virtual_ip`, `mesh_addr`, `node_addr`, `dns_name`, `state` (`Allocated`, `Active`, `Draining`), `sessions`, `age_secs`, `last_ref_secs`. | + +Until the first snapshot has been published (very early in startup), +both commands return `gateway not yet initialized`. + +## Driving the Socket Directly + +```sh +# Linux / macOS +echo '{"command":"show_status"}' | sudo nc -U /run/fips/control.sock + +# Windows (PowerShell with a TCP-capable tool of your choice) +``` + +The newline at the end of the request is required: the daemon reads +one line per connection. The connection is closed after the single +response is written. + +## See also + +- [`fipsctl`](cli-fipsctl.md) — full-featured client. +- [`fipstop`](cli-fipstop.md) — read-only TUI. +- [configuration.md](configuration.md) — `node.control.*` keys. diff --git a/docs/design/diagrams/ancestry-entry.svg b/docs/reference/diagrams/ancestry-entry.svg similarity index 100% rename from docs/design/diagrams/ancestry-entry.svg rename to docs/reference/diagrams/ancestry-entry.svg diff --git a/docs/design/diagrams/common-prefix.svg b/docs/reference/diagrams/common-prefix.svg similarity index 100% rename from docs/design/diagrams/common-prefix.svg rename to docs/reference/diagrams/common-prefix.svg diff --git a/docs/design/diagrams/coords-required.svg b/docs/reference/diagrams/coords-required.svg similarity index 100% rename from docs/design/diagrams/coords-required.svg rename to docs/reference/diagrams/coords-required.svg diff --git a/docs/design/diagrams/disconnect.svg b/docs/reference/diagrams/disconnect.svg similarity index 100% rename from docs/design/diagrams/disconnect.svg rename to docs/reference/diagrams/disconnect.svg diff --git a/docs/design/diagrams/established-complete-frame.svg b/docs/reference/diagrams/established-complete-frame.svg similarity index 100% rename from docs/design/diagrams/established-complete-frame.svg rename to docs/reference/diagrams/established-complete-frame.svg diff --git a/docs/design/diagrams/established-inner-header.svg b/docs/reference/diagrams/established-inner-header.svg similarity index 100% rename from docs/design/diagrams/established-inner-header.svg rename to docs/reference/diagrams/established-inner-header.svg diff --git a/docs/design/diagrams/established-outer-header.svg b/docs/reference/diagrams/established-outer-header.svg similarity index 100% rename from docs/design/diagrams/established-outer-header.svg rename to docs/reference/diagrams/established-outer-header.svg diff --git a/docs/design/diagrams/filter-announce.svg b/docs/reference/diagrams/filter-announce.svg similarity index 100% rename from docs/design/diagrams/filter-announce.svg rename to docs/reference/diagrams/filter-announce.svg diff --git a/docs/design/diagrams/fsp-complete-message.svg b/docs/reference/diagrams/fsp-complete-message.svg similarity index 100% rename from docs/design/diagrams/fsp-complete-message.svg rename to docs/reference/diagrams/fsp-complete-message.svg diff --git a/docs/design/diagrams/handshake-flow.svg b/docs/reference/diagrams/handshake-flow.svg similarity index 100% rename from docs/design/diagrams/handshake-flow.svg rename to docs/reference/diagrams/handshake-flow.svg diff --git a/docs/design/diagrams/lookup-request.svg b/docs/reference/diagrams/lookup-request.svg similarity index 100% rename from docs/design/diagrams/lookup-request.svg rename to docs/reference/diagrams/lookup-request.svg diff --git a/docs/design/diagrams/lookup-response.svg b/docs/reference/diagrams/lookup-response.svg similarity index 100% rename from docs/design/diagrams/lookup-response.svg rename to docs/reference/diagrams/lookup-response.svg diff --git a/docs/design/diagrams/mtu-exceeded.svg b/docs/reference/diagrams/mtu-exceeded.svg similarity index 100% rename from docs/design/diagrams/mtu-exceeded.svg rename to docs/reference/diagrams/mtu-exceeded.svg diff --git a/docs/design/diagrams/noise-ik-msg1.svg b/docs/reference/diagrams/noise-ik-msg1.svg similarity index 100% rename from docs/design/diagrams/noise-ik-msg1.svg rename to docs/reference/diagrams/noise-ik-msg1.svg diff --git a/docs/design/diagrams/noise-ik-msg2.svg b/docs/reference/diagrams/noise-ik-msg2.svg similarity index 100% rename from docs/design/diagrams/noise-ik-msg2.svg rename to docs/reference/diagrams/noise-ik-msg2.svg diff --git a/docs/design/diagrams/path-broken.svg b/docs/reference/diagrams/path-broken.svg similarity index 100% rename from docs/design/diagrams/path-broken.svg rename to docs/reference/diagrams/path-broken.svg diff --git a/docs/design/diagrams/path-mtu-notification.svg b/docs/reference/diagrams/path-mtu-notification.svg similarity index 100% rename from docs/design/diagrams/path-mtu-notification.svg rename to docs/reference/diagrams/path-mtu-notification.svg diff --git a/docs/design/diagrams/receiver-report.svg b/docs/reference/diagrams/receiver-report.svg similarity index 100% rename from docs/design/diagrams/receiver-report.svg rename to docs/reference/diagrams/receiver-report.svg diff --git a/docs/design/diagrams/sender-report.svg b/docs/reference/diagrams/sender-report.svg similarity index 100% rename from docs/design/diagrams/sender-report.svg rename to docs/reference/diagrams/sender-report.svg diff --git a/docs/design/diagrams/session-ack.svg b/docs/reference/diagrams/session-ack.svg similarity index 100% rename from docs/design/diagrams/session-ack.svg rename to docs/reference/diagrams/session-ack.svg diff --git a/docs/design/diagrams/session-datagram.svg b/docs/reference/diagrams/session-datagram.svg similarity index 100% rename from docs/design/diagrams/session-datagram.svg rename to docs/reference/diagrams/session-datagram.svg diff --git a/docs/design/diagrams/session-msg3.svg b/docs/reference/diagrams/session-msg3.svg similarity index 100% rename from docs/design/diagrams/session-msg3.svg rename to docs/reference/diagrams/session-msg3.svg diff --git a/docs/design/diagrams/session-setup.svg b/docs/reference/diagrams/session-setup.svg similarity index 100% rename from docs/design/diagrams/session-setup.svg rename to docs/reference/diagrams/session-setup.svg diff --git a/docs/design/diagrams/tree-announce.svg b/docs/reference/diagrams/tree-announce.svg similarity index 100% rename from docs/design/diagrams/tree-announce.svg rename to docs/reference/diagrams/tree-announce.svg diff --git a/docs/reference/nostr-events.md b/docs/reference/nostr-events.md new file mode 100644 index 0000000..3a05c4d --- /dev/null +++ b/docs/reference/nostr-events.md @@ -0,0 +1,197 @@ +# Nostr Event Reference + +The Nostr-protocol surface FIPS uses for discovery and signaling. For +the design of the discovery runtime and the rationale behind these +event shapes, see +[../design/fips-nostr-discovery.md](../design/fips-nostr-discovery.md). +For operator activation recipes, see +[../how-to/enable-nostr-discovery.md](../how-to/enable-nostr-discovery.md). + +FIPS uses three Nostr event kinds: + +| Kind | Name | Encryption | Storage | Purpose | +| ---- | ---- | ---------- | ------- | ------- | +| 37195 | Overlay advert | None (signed only) | Replaceable | Publish reachable transport endpoints | +| 21059 | Traversal signaling | NIP-44 inside NIP-59 gift wrap | Ephemeral | Carry `TraversalOffer`/`TraversalAnswer` payloads | +| 10050 | NIP-17 inbox relay list | None (signed only) | Replaceable | Tell dialers where to publish offers | + +All three are signed with the node's FIPS identity key (the same +secp256k1 keypair Nostr uses); there is no separate Nostr key. + +## Kind 37195 — Overlay Advert + +A parameterized replaceable event in the application-defined +replaceable range `30000–39999` (the digits visually spell `FIPS`: +7=F, 1=I, 9=P, 5=S). Each node has a single in-place-updatable advert +under its identity. + +### Tags + +- `d` — fixed to the literal `fips-overlay-v1` (the application + identifier baked into the binary). Together with `pubkey`, this + identifies the unique replaceable event slot. +- `protocol` — the configured `node.discovery.nostr.app` value + (default `fips-overlay-v1`). Distinct from the `d` tag so the + application string can evolve without breaking the replaceable + event slot. +- `version` — protocol version string (currently `"1"`). +- `expiration` — NIP-40 expiration timestamp set to now + + `node.discovery.nostr.advert_ttl_secs` (default 3600 seconds). + Conforming relays stop serving the event after this time. + +### Content + +The event content is a JSON document shaped as `OverlayAdvert`: + +```json +{ + "identifier": "fips-overlay-v1", + "version": 1, + "endpoints": [ + {"transport": "udp", "addr": "203.0.113.45:2121"}, + {"transport": "tor", "addr": "xxxxx.onion:8443"}, + {"transport": "udp", "addr": "nat"} + ], + "signalRelays": ["wss://relay.damus.io", "wss://nos.lol"], + "stunServers": ["stun:stun.l.google.com:19302"] +} +``` + +Field semantics: + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `identifier` | string | Application namespace; must match the `d` tag. | +| `version` | integer | Advert schema version (currently 1). | +| `endpoints` | array | List of transport endpoints. Each is `{transport, addr}` where `transport` is `"udp"`, `"tcp"`, or `"tor"`, and `addr` is `"host:port"`, `".onion:port"`, or the literal `"nat"` (for UDP NAT-punch). | +| `signalRelays` | array? | Optional. Relays the publisher prefers for offer/answer signaling. Present only when at least one endpoint is `udp:nat`. | +| `stunServers` | array? | Optional. STUN servers the publisher uses for reflexive discovery. Present only when at least one endpoint is `udp:nat`. Informational — peers do not use these to choose their own STUN targets. | + +### Signature scope + +The Nostr event signature covers the standard Nostr event ID +(serialized `[0, pubkey, created_at, kind, tags, content]`), so the +content JSON, tags, kind, and timestamp are all bound to the signing +identity. + +### Replacement and deletion + +Because kind 37195 is replaceable, publishing a new advert replaces +the prior one in the same `(pubkey, d-tag)` slot. To withdraw an +advert without publishing a successor, the node publishes a NIP-9 +kind 5 delete event referencing the prior advert. + +## Kind 21059 — Traversal Signaling + +An ephemeral event (kinds in the 20000–29999 range are not stored by +conforming relays). Used to deliver gift-wrapped, NIP-44-encrypted +`TraversalOffer` and `TraversalAnswer` payloads between dialer and +responder during a UDP NAT hole-punch. + +### Encryption envelope + +The wire shape is the standard NIP-59 gift wrap: + +1. **Rumor** — the unsigned `TraversalOffer`/`TraversalAnswer` + payload (JSON), authored by the actual sender's identity. +2. **Seal** — a kind 13 event whose content is the rumor + NIP-44-encrypted to the recipient's pubkey, signed by the sender. +3. **Gift wrap** — a kind 21059 event whose content is the seal + NIP-44-encrypted to the recipient under an ephemeral key, signed + by that ephemeral key. The outer `pubkey` of the kind 21059 event + is the ephemeral identity, not the sender's real identity. + +Only the intended recipient can decrypt the wrap to recover the seal, +and only the recipient can decrypt the seal to recover the rumor. + +### Wrapped payloads + +The `TraversalOffer` carries: + +- `type` — message-type tag. +- `sessionId` — unique identifier correlating offer and answer. +- `senderNpub` / `recipientNpub` — bech32-encoded pubkeys, repeated + inside the encrypted payload (the outer wrap pubkey is ephemeral). +- `issuedAt` / `expiresAt` — Unix-ms timestamps; `expiresAt` is + `issuedAt + signal_ttl_secs * 1000`. +- `nonce` — random per-offer value. +- `reflexiveAddress` — `{protocol, ip, port}` observed via STUN, or + `null` if STUN failed or returned no usable address. +- `localAddresses` — array of `{protocol, ip, port}` private + candidates, populated when `share_local_candidates` is enabled. +- `stunServer` — the STUN server actually used (informational). + +The `TraversalAnswer` echoes `sessionId` and carries: + +- `type`, `senderNpub`, `recipientNpub`, `issuedAt`, `expiresAt`, + `nonce` — same shape as the offer. +- `inReplyTo` — the offer's event id. +- `accepted` — boolean; false when the responder has no usable + addresses. +- `reflexiveAddress` and `localAddresses` — the responder's + candidates, in the same shape as the offer. +- `stunServer` — informational. +- `punch` — a `PunchHint { startAtMs, intervalMs, durationMs }` + telling both sides when to begin probing and how aggressively. + Absent on rejected offers. +- `reason` — optional rejection string when `accepted` is false. +- `offerReceivedAt` — optional responder wall-clock (Unix ms) at + the moment it received the offer; the initiator uses this to + derive a clock-skew estimate. + +### Relay selection + +Dialer publishes offers to the recipient's NIP-17 inbox relays (kind +10050) when available; otherwise to the local +`node.discovery.nostr.dm_relays` list. The responder publishes the +answer back through the same relay channel. + +## Kind 10050 — NIP-17 Inbox Relay List + +A standard NIP-17 event used by FIPS to advertise which relays this +node prefers for receiving direct-message-style signaling — for FIPS, +the gift-wrapped traversal offers (kind 21059). + +This is **NIP-17** (`kind 10050`, inbox relays for DM delivery), not +NIP-65 (`kind 10002`, general read/write relay list). The two serve +different purposes: + +- Kind 10002 (NIP-65) — general read/write relays for ordinary event + publication and subscription. +- Kind 10050 (NIP-17) — relays the recipient prefers for receiving + DM-shaped (NIP-59 wrapped) events. + +FIPS publishes its own kind 10050 on startup so dialers can discover +where to send traversal offers. When dialing a peer, FIPS first +fetches the peer's kind 10050 from the peer's `advert_relays`; on +fetch failure it falls back to the local `dm_relays` list. + +### Tags + +Standard NIP-17 form: each relay is encoded as an `r` tag whose +single value is the relay URL. + +```text +["r", "wss://relay.damus.io"] +["r", "wss://nos.lol"] +``` + +### Content + +Empty per NIP-17. + +## See also + +- [../design/fips-nostr-discovery.md](../design/fips-nostr-discovery.md) + — discovery runtime design and the five activation scenarios +- [../how-to/enable-nostr-discovery.md](../how-to/enable-nostr-discovery.md) + — operator recipes +- [../tutorials/resolve-peers-via-nostr.md](../tutorials/resolve-peers-via-nostr.md), + [../tutorials/advertise-your-node.md](../tutorials/advertise-your-node.md), + [../tutorials/open-discovery.md](../tutorials/open-discovery.md) + — hand-held tutorial walkthroughs of the three capabilities +- [../design/port-advertisement-and-nat-traversal.md](../design/port-advertisement-and-nat-traversal.md) + — generic protocol reference (event tags, NIP usage, on-the-wire + offer/answer schema), with FIPS values as worked examples +- [security.md](security.md) — how the FIPS identity key signs both + adverts and Noise handshakes diff --git a/docs/reference/security.md b/docs/reference/security.md new file mode 100644 index 0000000..a439b0c --- /dev/null +++ b/docs/reference/security.md @@ -0,0 +1,235 @@ +# Security Reference + +Consolidated security reference covering the nftables baseline, peer +ACL file format, cryptographic primitives, rekey defaults, replay +window, filesystem permissions, threat-resistance matrix, and default +network exposures per transport. For the threat-model design and +rationale, see [../design/fips-security.md](../design/fips-security.md). +For the operator activation steps and drop-in recipes, see +[../how-to/enable-mesh-firewall.md](../how-to/enable-mesh-firewall.md). + +## nftables Baseline + +The shipped baseline is `/etc/fips/fips.nft`. It defines a single +nftables table `inet fips` with one chain hooked at `input`, structured +as follows: + +| Step | Rule | Effect | +| ---- | ---- | ------ | +| 1 | `iifname != "fips0" return` | Match only traffic arriving on `fips0`; everything else short-circuits. | +| 2 | `ct state established,related accept` | Allow conntrack replies and related ICMPv6 errors. | +| 3 | `icmpv6 type echo-request accept` | Allow IPv6 echo (ping6 reachability). | +| 4 | `include "/etc/fips/fips.d/*.nft"` | Splice in operator drop-ins (empty matches nothing). | +| 5 | `counter drop` | Default-deny everything else; counter increments on every drop. | + +Outbound from `fips0` is unrestricted. The baseline is a documented +dpkg conffile — operator edits to `/etc/fips/fips.nft` are preserved +across upgrades. + +The systemd unit is `fips-firewall.service` (oneshot). It is **not** +enabled by default; activation is an explicit operator gesture +documented in +[../how-to/enable-mesh-firewall.md](../how-to/enable-mesh-firewall.md). + +## Drop-In File Format + +Operator extensions live under `/etc/fips/fips.d/` with the `.nft` +suffix. Each file is included inline into the `inbound` chain at the +marked point and may contain any nftables rule lines valid in that +context. + +Naming convention: `-from-.nft` keeps drop-ins easy +to scan. Examples shipped in the design discussion: + +- `ssh-from-bastion.nft` — accept TCP/22 from a single mesh-node address +- `http-from-cluster.nft` — accept TCP/80 from a `/64` mesh-address prefix +- `dns-public.nft` — accept UDP/53 and TCP/53 from any mesh node +- `git-from-trusted.nft` — accept TCP/9418 from a set of mesh-node addresses + +After editing, reload via +`sudo systemctl reload-or-restart fips-firewall.service` (or +equivalently `sudo nft -f /etc/fips/fips.nft` since the file is +idempotent). + +## Cryptographic Primitives + +| Component | Choice | Where Used | +| --------- | ------ | ---------- | +| Curve | secp256k1 | FMP IK, FSP XK, Schnorr signatures | +| Diffie-Hellman | ECDH on secp256k1 (x-only normalized) | Noise IK, Noise XK | +| AEAD | ChaCha20-Poly1305 | FMP link encryption, FSP session encryption | +| Hash | SHA-256 | NodeAddr derivation, Noise transcript | +| Key derivation | HKDF-SHA256 | Noise key schedule | +| Signatures | secp256k1 Schnorr | TreeAnnounce, LookupResponse proof, Nostr adverts | +| Noise pattern (link) | `Noise_IK_secp256k1_ChaChaPoly_SHA256` | FMP link layer (IK with epoch payload) | +| Noise pattern (session) | `Noise_XK_secp256k1_ChaChaPoly_SHA256` | FSP session layer (XK with epoch payload) | + +These choices align with the Nostr cryptographic stack +(secp256k1 + ChaCha20-Poly1305 + SHA-256) and the NIP-44 encrypted +messaging standard. + +## Rekey Defaults + +Both link-layer and session-layer Noise sessions rekey under one of +two triggers, configurable under `node.rekey.*`: + +| Parameter | Default | Description | +| --------- | ------- | ----------- | +| `enabled` | `true` | Master switch. | +| `after_secs` | `120` | Time-based rekey threshold. | +| `after_messages` | `65536` | Message-count rekey threshold. | + +In addition to the configurable triggers, the daemon retains the old +session keys for a fixed **10-second drain window** after each +cutover (compile-time constant `DRAIN_WINDOW_SECS` in +`src/node/handlers/rekey.rs`). Rekey rotates the Noise key schedule +and the session indices; old session keys are kept in +`previous_session` for the drain window so in-flight packets +encrypted under the old keys still decrypt. + +## Replay Window + +Both layers use explicit per-packet counters with a sliding bitmap +window for replay protection. The bitmap is **2048 entries** at both +layers — large enough to accommodate UDP reordering and packet loss +without false-positive replay rejection. Counters older than the +window are rejected. The same `ReplayWindow` and +`decrypt_with_replay_check()` implementation is used at both the FMP +and FSP layers. + +## Peer ACL + +Mesh-level ACL files at `/etc/fips/peers.allow` and +`/etc/fips/peers.deny` give the operator allowlist/blocklist control +over which npubs may complete the FMP Noise IK link handshake. + +File format: + +- One entry per line. An entry is either a bech32 `npub1...`, + an alias defined in `/etc/fips/hosts`, or the literal `ALL` + wildcard (case-insensitive). +- Lines beginning with `#` are comments. +- Blank lines are ignored. + +Evaluation order (first match wins, default-allow on no match): + +1. `peers.allow` — if the peer matches an entry here (or `ALL` is + in `peers.allow`), the handshake is admitted, regardless of any + `peers.deny` entry. +2. `peers.deny` — if the peer matches an entry here (or `ALL` is + in `peers.deny`), the handshake is refused. +3. Otherwise the peer is admitted. + +`peers.allow` is **not** an exclusive gate on its own: an unlisted +peer falls through to step 3 and is admitted unless it appears in +`peers.deny`. To turn `peers.allow` into a strict allowlist, place +`ALL` in `peers.deny` so every unlisted peer is rejected at step 2. + +The `ALL` wildcard makes the operator's posture explicit: + +- `ALL` in `peers.allow` admits every peer (same effect as the + default-allow behavior, but documented in the file). +- `ALL` in `peers.deny` blocks every peer except those listed in + `peers.allow` — the "allowlist-strict" posture. + +In practice this collapses to a few common postures: + +- **Default-allow with denylist**: leave `peers.allow` empty; + populate `peers.deny`. All npubs may peer except those listed. +- **Allowlist-strict**: populate `peers.allow` and put `ALL` + in `peers.deny`. Only the listed npubs may peer; everyone else + is rejected at step 2. + +A populated `peers.allow` with an empty `peers.deny` is not a +strict allowlist — it is equivalent to default-allow plus an +explicit "always-admit" set. The strict variant requires `ALL` +in `peers.deny`. + +Aliases are resolved through `/etc/fips/hosts` at file-load +time. If `peers.allow` lists `core-vm` and `/etc/fips/hosts` +maps `core-vm` to a specific npub, that npub is admitted. If +`core-vm` is later remapped to a different npub, the ACL +re-resolves on the next mtime change. Operators should be aware +that ACL semantics follow the `hosts`-file aliasing, not just +the literal npubs visible in the file. + +Both files are reloaded automatically when their mtime changes +— no daemon restart or signal is needed. ACL evaluation runs +after msg1 decryption but before any further peer-state +mutation; rate-limited msg1s never reach the ACL. + +## Filesystem Permissions + +| Path | Owner | Mode | Purpose | +| ---- | ----- | ---- | ------- | +| `/etc/fips/fips.key` | root:root | `0600` | Persistent identity private key (sensitive). | +| `/etc/fips/fips.pub` | root:root | `0644` | Public key (npub). | +| `/etc/fips/fips.yaml` | root:root | `0644` | Daemon configuration (dpkg conffile). | +| `/etc/fips/fips.nft` | root:root | `0644` | nftables baseline (dpkg conffile). | +| `/etc/fips/fips.d/` | root:root | `0755` | Operator drop-in directory. | +| `/etc/fips/hosts` | root:root | `0644` | Optional hostname → npub map (dpkg conffile). | +| `/etc/fips/peers.allow` | root:root | `0644` | Optional peer allowlist. | +| `/etc/fips/peers.deny` | root:root | `0644` | Optional peer denylist. | +| `/run/fips/control.sock` | root:fips | `0770` | Control socket (members of `fips` group can use `fipsctl`). | +| `/run/fips/` | root:fips | `0750` | Control socket parent directory. | + +Adding a user to the `fips` group grants `fipsctl` access without +requiring root. The daemon `chown`s the control socket and its parent +directory at bind time. + +## Threat-Resistance Matrix + +The link layer's threat-resistance matrix is consolidated here from +the FMP design document: + +| Threat | Mitigation | +| ------ | ---------- | +| Connection exhaustion | Token-bucket rate limit + connection count limit | +| CPU exhaustion (msg1 flood) | Rate limit before crypto operations | +| Replay attacks | Counter-based nonces with sliding window (2048 entries) | +| State confusion | Strict handshake state machine validation | +| Spoofed encrypted packets | Index lookup + AEAD verification | +| Spoofed msg2 | Index lookup + Noise ephemeral key binding | +| Address spoofing | Cryptographic authority, not address-based | +| Session correlation | Index rotation on rekey | +| Inbound exposure on `fips0` | Default-deny nftables baseline (operator opt-in) | +| Sybil identities | Discretionary peering + handshake rate limiting + optional peer ACL | +| Eclipse attack | Diverse peering across independent operators and transports | +| Unauthorized peer admission | Optional `peers.allow` allowlist consulted before handshake | + +See [../design/fips-mesh-layer.md](../design/fips-mesh-layer.md) for +the unauthenticated-attack-surface analysis (only handshake msg1 is +reachable by unauthenticated parties), and +[../design/fips-mesh-operation.md](../design/fips-mesh-operation.md#privacy-considerations) +for the metadata-privacy model and the rejection of onion routing. + +## Default Network Exposures by Transport + +| Transport | Default Inbound | Default Bind | Opt-in | +| --------- | --------------- | ------------ | ------ | +| UDP | None until `bind_addr` set | `0.0.0.0:2121` typical | Operator sets `transports.udp.bind_addr` | +| TCP | None until `bind_addr` set | None — outbound-only without bind | Operator sets `transports.tcp.bind_addr` | +| Ethernet | Listens on configured interface (raw `AF_PACKET`) | EtherType 0x2121 on selected interface | Per-flag `discovery`, `announce`, `auto_connect`, `accept_connections` | +| Tor | None until `directory_service` configured | `127.0.0.1:8443` (loopback only) | Operator sets `transports.tor.directory_service` and configures `HiddenServiceDir` in `torrc` | +| BLE | Off by default | n/a | Operator enables `transports.ble.*` | +| Nostr discovery | Off by default | n/a (relay client, not a listener) | Operator sets `node.discovery.nostr.enabled: true` | + +The mesh-layer `fips0` interface is reachable from any mesh node that +can route to you, not only direct peers — your direct peers forward +traffic from any reachable mesh node onto your `fips0`. The +default-deny nftables baseline (operator opt-in) is the recommended +way to restrict inbound traffic on `fips0`. See +[../how-to/enable-mesh-firewall.md](../how-to/enable-mesh-firewall.md). + +## See also + +- [../design/fips-security.md](../design/fips-security.md) — threat + model and design rationale for the `fips0` baseline +- [../design/fips-mesh-layer.md](../design/fips-mesh-layer.md) — FMP + link encryption, replay protection, rate limiting +- [../design/fips-session-layer.md](../design/fips-session-layer.md) + — FSP end-to-end encryption, Noise XK, replay window +- [../how-to/enable-mesh-firewall.md](../how-to/enable-mesh-firewall.md) + — operator activation and drop-in recipes +- [configuration.md](configuration.md) — full `node.rekey.*`, + `node.rate_limit.*` parameter tables diff --git a/docs/reference/transports.md b/docs/reference/transports.md new file mode 100644 index 0000000..f241722 --- /dev/null +++ b/docs/reference/transports.md @@ -0,0 +1,92 @@ +# Transport Statistics Reference + +Per-transport statistics counter inventories. Counters are exposed +through the daemon control socket (`fipsctl show transports`) and the +`fipstop` operator UI. For the transport-layer design (services +provided to FMP, transport categories, the trait surface, connection +model), see +[../design/fips-transport-layer.md](../design/fips-transport-layer.md). + +All transports report counters via `fipsctl show transports`; the +tables below are source-extracted from each transport's `stats.rs` +module. + +## UDP + +| Counter | Description | +| ------- | ----------- | +| `packets_sent` / `bytes_sent` | Successful sends | +| `packets_recv` / `bytes_recv` | Successful receives | +| `send_errors` / `recv_errors` | Send/receive failures | +| `mtu_exceeded` | Packets rejected for MTU violation | +| `kernel_drops` | Kernel `SO_RXQ_OVFL` drop count (feeds ECN congestion detection) | + +## TCP + +| Counter | Description | +| ------- | ----------- | +| `packets_sent` / `bytes_sent` | Successful sends | +| `packets_recv` / `bytes_recv` | Successful receives | +| `send_errors` / `recv_errors` | Send/receive failures | +| `mtu_exceeded` | Packets rejected for MTU violation | +| `connections_established` | Successful outbound connections | +| `connections_accepted` | Accepted inbound connections | +| `connections_rejected` | Rejected inbound connections (limit exceeded) | +| `connect_timeouts` | Connection timeout count | +| `connect_refused` | Connection refused count | + +## Ethernet + +| Counter | Description | +| ------- | ----------- | +| `frames_sent` / `frames_recv` | Successful frame send/receive | +| `bytes_sent` / `bytes_recv` | Byte counters | +| `send_errors` / `recv_errors` | Send/receive failures | +| `beacons_sent` / `beacons_recv` | Peer-discovery beacon traffic | +| `frames_too_short` | Frames below minimum length, dropped | +| `frames_too_long` | Frames above transport MTU, dropped | + +## Tor + +| Counter | Description | +| ------- | ----------- | +| `packets_sent` / `bytes_sent` | Successful sends | +| `packets_recv` / `bytes_recv` | Successful receives | +| `send_errors` / `recv_errors` | Send/receive failures | +| `connections_established` | Successful SOCKS5 connections | +| `connect_timeouts` | Connection timeout count | +| `connect_refused` | Connection refused count | +| `socks5_errors` | SOCKS5 protocol errors | +| `mtu_exceeded` | Packets rejected for MTU violation | +| `connections_accepted` | Accepted inbound connections via onion service | +| `connections_rejected` | Rejected inbound connections (limit exceeded) | +| `control_errors` | Tor control port errors | + +## Bluetooth + +| Counter | Description | +| ------- | ----------- | +| `packets_sent` / `bytes_sent` | Successful L2CAP CoC sends | +| `packets_recv` / `bytes_recv` | Successful L2CAP CoC receives | +| `send_errors` / `recv_errors` | Send/receive failures | +| `mtu_exceeded` | Packets rejected for MTU violation | +| `connections_established` | Successful outbound L2CAP connections | +| `connections_accepted` | Accepted inbound L2CAP connections | +| `connections_rejected` | Rejected inbound (limit exceeded) | +| `connect_timeouts` | Connection timeout count | +| `pool_evictions` | Connection-pool entries evicted | +| `advertisements_sent` | BLE advertisements emitted | +| `scan_results` | BLE scan results observed | + +## See also + +- [../design/fips-transport-layer.md](../design/fips-transport-layer.md) + — transport-layer design, trait surface, per-transport sections +- [configuration.md](configuration.md) — `transports.*` configuration + blocks +- [../how-to/tune-udp-buffers.md](../how-to/tune-udp-buffers.md) — + host-side `net.core.rmem_max` / `net.core.wmem_max` setup for UDP +- [../how-to/deploy-tor-onion.md](../how-to/deploy-tor-onion.md) — + Tor `directory` mode operator setup +- [../how-to/set-up-bluetooth-peer.md](../how-to/set-up-bluetooth-peer.md) + — Linux BLE peer config diff --git a/docs/design/fips-wire-formats.md b/docs/reference/wire-formats.md similarity index 89% rename from docs/design/fips-wire-formats.md rename to docs/reference/wire-formats.md index 45bd213..818d107 100644 --- a/docs/design/fips-wire-formats.md +++ b/docs/reference/wire-formats.md @@ -5,6 +5,45 @@ protocol layers. It covers transport framing, link-layer message formats, and session-layer message formats, with an encapsulation walkthrough showing how application data is wrapped through each layer. +## FMP Message Type Catalog + +The FMP link layer defines the following message types, dispatched by the +`msg_type` byte in the encrypted inner header: + +| Type | Name | Forwarding | +| ---- | ---- | ---------- | +| 0x00 | SessionDatagram | Routed hop-by-hop toward the destination | +| 0x01 | SenderReport | Peer-to-peer (MMP, link-layer instance) | +| 0x02 | ReceiverReport | Peer-to-peer (MMP, link-layer instance) | +| 0x10 | TreeAnnounce | Peer-to-peer (spanning-tree gossip) | +| 0x20 | FilterAnnounce | Peer-to-peer (bloom-filter gossip) | +| 0x30 | LookupRequest | Forwarded — bloom-guided through tree peers | +| 0x31 | LookupResponse | Forwarded — reverse-path via `recent_requests` | +| 0x50 | Disconnect | Peer-to-peer (orderly link teardown) | +| 0x51 | Heartbeat | Peer-to-peer (link liveness) | + +Handshake messages travel before encryption is established and are identified +by the FMP common-prefix `phase` field rather than a `msg_type` byte +(phase 0x1 = Noise IK msg1, phase 0x2 = Noise IK msg2). + +## Packet Type Summary + +A higher-level summary that includes typical sizes and forwarding category: + +| Message | Typical Size | When | Forwarded? | +| ------- | ------------ | ---- | ---------- | +| TreeAnnounce | Variable (depth-dependent) | Topology changes | No (peer-to-peer) | +| FilterAnnounce | ~1 KB | Topology changes | No (peer-to-peer) | +| LookupRequest | ~300 bytes | First contact, recovery | Yes (bloom-guided tree) | +| LookupResponse | ~400 bytes | Response to discovery | Yes (reverse-path) | +| SessionDatagram + SessionSetup | ~232–402 bytes | Session establishment | Yes (routed) | +| SessionDatagram + SessionAck | ~170 bytes | Session confirmation | Yes (routed) | +| SessionDatagram + Data (minimal) | 77 bytes + IPv6 payload | Bulk IPv6 traffic (compressed) | Yes (routed) | +| SessionDatagram + Data (with CP) | 77 + coords + IPv6 payload | Warmup/recovery (compressed) | Yes (routed) | +| SessionDatagram + CoordsRequired | 70 bytes | Cache miss error | Yes (routed) | +| SessionDatagram + PathBroken | 70+ bytes | Dead-end error | Yes (routed) | +| Disconnect | 2 bytes | Link teardown | No (peer-to-peer) | + ## Encoding Rules - All multi-byte integers are **little-endian** (LE) @@ -19,10 +58,10 @@ how application data is wrapped through each layer. Datagram-oriented transports (UDP, raw Ethernet, radio) preserve natural packet boundaries and require no additional framing. Stream-oriented -transports (TCP, WebSocket, Tor) must delineate FIPS packets within the -byte stream; the common prefix `payload_len` field provides this -framing directly. TCP and Tor share a common stream reader -(`tcp/stream.rs`) that implements this framing. +transports (TCP, Tor) must delineate FIPS packets within the byte +stream; the common prefix `payload_len` field provides this framing +directly. TCP and Tor share a common stream reader (`tcp/stream.rs`) +that implements this framing. **Ethernet data frame header.** The Ethernet transport prepends a 3-byte header before the FMP payload on data frames: a 1-byte frame type @@ -505,10 +544,21 @@ Message types 0x10-0x14 are carried inside the AEAD ciphertext (dispatched by the `msg_type` field in the encrypted inner header). Types 0x20-0x22 are plaintext error signals (U flag set, no encryption). -Session-layer SenderReport (0x11) and ReceiverReport (0x12) use the same -body format as their link-layer counterparts (0x01 and 0x02). The msg_type -byte in the body matches the link-layer value; dispatch to the correct layer -happens at the session level based on the FSP message type. +Session-layer SenderReport (0x11) and ReceiverReport (0x12) carry the same +metric fields as their link-layer counterparts (0x01 and 0x02), but the +body framing differs because the FSP encrypted inner header already +carries the message-type byte. The session body therefore omits the +msg_type byte and uses 2 reserved bytes (not 3) before the fields: + +| Layer | Wire size | Header inside body | +| ----- | --------- | ------------------ | +| Link SenderReport (0x01) | 48 bytes | msg_type(1) + reserved(3) + fields(44) | +| Session SenderReport (0x11) | 46 bytes | reserved(2) + fields(44) | +| Link ReceiverReport (0x02) | 68 bytes | msg_type(1) + reserved(3) + fields(64) | +| Session ReceiverReport (0x12) | 66 bytes | reserved(2) + fields(64) | + +Dispatch happens at the session level via the `msg_type` byte in the FSP +encrypted inner header. ### SessionSetup (phase 0x1) @@ -844,7 +894,7 @@ endpoint session keys). | ------- | ---- | ----- | | TreeAnnounce | 100 + 32n bytes | n = depth + 1 | | FilterAnnounce | 1,035 bytes | v1 (1KB filter) | -| LookupRequest | 303 + 16n bytes | n = origin depth + 1 | +| LookupRequest | 46 + 16n bytes | n = origin depth + 1 | | LookupResponse | 93 + 16n bytes | n = target depth + 1 | | SessionDatagram | 36 + payload bytes | Fixed 36-byte header | | Disconnect | 2 bytes | | @@ -879,8 +929,10 @@ endpoint session keys). ## References -- [fips-mesh-layer.md](fips-mesh-layer.md) — FMP behavioral specification -- [fips-session-layer.md](fips-session-layer.md) — FSP behavioral specification -- [fips-transport-layer.md](fips-transport-layer.md) — Transport framing -- [fips-mesh-operation.md](fips-mesh-operation.md) — How messages work together -- [fips-ipv6-adapter.md](fips-ipv6-adapter.md) — MTU enforcement +- [../design/fips-mesh-layer.md](../design/fips-mesh-layer.md) — FMP behavioral specification +- [../design/fips-session-layer.md](../design/fips-session-layer.md) — FSP behavioral specification +- [../design/fips-transport-layer.md](../design/fips-transport-layer.md) — Transport framing +- [../design/fips-mesh-operation.md](../design/fips-mesh-operation.md) — How messages work together +- [../design/fips-ipv6-adapter.md](../design/fips-ipv6-adapter.md) — MTU enforcement +- [../design/fips-bloom-filters.md](../design/fips-bloom-filters.md) — FilterAnnounce parameters and FPR analysis +- [../design/fips-mtu.md](../design/fips-mtu.md) — How `path_mtu` and MtuExceeded fit together diff --git a/docs/tutorials/README.md b/docs/tutorials/README.md new file mode 100644 index 0000000..25d136f --- /dev/null +++ b/docs/tutorials/README.md @@ -0,0 +1,73 @@ +# Tutorials + +If you have just installed FIPS, this is where to start. The +tutorials below take you from a freshly-installed daemon to a +node that: + +- Has joined the public test mesh and can reach other nodes on it. +- Carries a stable identity that other operators can address. +- Discovers peers — and is discoverable — over Nostr. +- Hosts and consumes real services across the mesh. + +Each tutorial is a complete, working session at the keyboard. You +configure something, restart the daemon, watch it come up, and +verify the result. The point is to build muscle memory, not to +cover every option. + +> **Read them in order.** Each tutorial assumes the state the +> previous one left you in. If you skip ahead, the cross-references +> that lead you back may not match what you have on disk. + +## The new-user progression + +| # | Tutorial | What you'll do | +| - | -------- | -------------- | +| 1 | [join-the-test-mesh.md](join-the-test-mesh.md) | Add one public test peer to your config, watch the link come up, ping that peer and a second mesh node it routes you to. The starting point for everything else. | +| 2 | [persistent-identity.md](persistent-identity.md) | Pin your daemon to a stable Nostr keypair so your address stops changing on every restart. Other operators can now add you to their `peers:` lists; the services you host get a fixed name. | +| 3 | [resolve-peers-via-nostr.md](resolve-peers-via-nostr.md) | Stop hard-coding peer addresses. Drop the address line from your peer entry and let the daemon look up the current endpoint from public Nostr relays at dial time. | +| 4 | [advertise-your-node.md](advertise-your-node.md) | Publish your own UDP endpoint to Nostr so any operator who knows your npub can reach you, with a short final section on `udp:nat` traversal for symmetric-NAT networks. | +| 5 | [open-discovery.md](open-discovery.md) | Switch to `policy: open` and let your peer list populate itself from the ambient `fips-overlay-v1` namespace. Hands-off mesh participation. | +| 6 | [reach-mesh-services.md](reach-mesh-services.md) | Drive ordinary IPv6 tools — `ping6`, `nc`, `traceroute6`, `curl`, `ssh` — at mesh nodes by `.fips`. Get a feel for the daemon's IPv6 adapter, which makes unmodified IPv6 software work over the mesh. | +| 7 | [host-a-service.md](host-a-service.md) | Bring up an HTTP server bound to `fips0` so mesh nodes can reach it, with a deliberate exposure decision (mesh-only vs every interface), and the mesh firewall as a default-deny baseline. The peer ACL (a separate, transport-layer control over which npubs may peer with your node) is briefly mentioned alongside. | +| 8 | [ground-up-mesh.md](ground-up-mesh.md) | Bring up a second deployment mode: two devices joined by Ethernet (or WiFi, or BLE) with no IP infrastructure between them. The mesh emerges from layer 2 up. Coexists with overlay peers — the same daemon can carry both. | + +After tutorial 8 you have a fully participating mesh node that +reaches services hosted by other mesh nodes and hosts services of its own, +with identity, discovery, reachability, an explicit exposure +policy, and an understanding of both deployment modes — overlay +on top of existing IP, and ground-up where the mesh is the +network. + +There is also a side trip you can take any time after tutorial 1: + +- [ipv6-adapter-walkthrough.md](ipv6-adapter-walkthrough.md) — + trace one `ssh` from DNS query through session setup to the + far-side TUN, using `fipstop` and `fipsctl` to watch each step. + Optional, but if you like seeing how the pieces fit together, + this is the doc that shows you. + +## Advanced + +These are not part of the new-user progression. They assume you +have already worked through the tutorials above and now want to +fold FIPS into a wider network deployment. + +- [deploy-fips-gateway.md](deploy-fips-gateway.md) — Stand up a + `fips-gateway` on an OpenWrt access point so unmodified LAN + hosts can reach `.fips` destinations through a DNS- + allocated virtual IPv6 pool and kernel nftables NAT, with no + per-host FIPS install. Also walks through one inbound port + forward exposing a LAN service to mesh peers. Aimed at + operators bridging a LAN segment into the overlay from the + edge router. For a non-OpenWrt host the same deployment is + in [../how-to/deploy-gateway.md](../how-to/deploy-gateway.md). + +## When to use the how-to guides instead + +The tutorials here walk through one specific path each. The +how-to guides under [../how-to/](../how-to/) are the operator +recipes — alternative provisioning paths, less-common +configurations, troubleshooting techniques. Once you have the +shape of FIPS in your head from these tutorials, the how-tos are +where you'll go to look up "how do I do X?" without being walked +through the surrounding context. diff --git a/docs/tutorials/advertise-your-node.md b/docs/tutorials/advertise-your-node.md new file mode 100644 index 0000000..abc8188 --- /dev/null +++ b/docs/tutorials/advertise-your-node.md @@ -0,0 +1,413 @@ +# Advertise Your Node on Nostr + +After +[resolve-peers-via-nostr](resolve-peers-via-nostr.md) your +daemon can look up a peer's current endpoint by npub. This +tutorial flips it around: you publish a signed advert listing +your own endpoint(s), so any other operator who knows your +npub can dial you the same way you dialed `test-us01`. + +The whole exercise should take about ten minutes if you have +a public IP or full-cone home NAT. A short final section +covers the alternative path for symmetric-NAT networks. + +## What you'll build + +```text + ┌───────────────────────┐ + │ your fips daemon │ + │ persistent npub │ + └──────────┬────────────┘ + │ signed advert (Kind 37195) + │ { udp::2121, ... } + │ refreshes every 30 min + ▼ + ┌──────────────────────────────────────────┐ + │ Nostr relays │ + │ relay.damus.io / nos.lol / offchain.pub│ + └──────────────────────┬───────────────────┘ + │ + │ "what's 's endpoint?" + │ + ┌───────────┴───────────┐ + │ another fips daemon │ + │ knows your npub, │ + │ via_nostr: true │ + └───────────────────────┘ +``` + +You will change two things in `/etc/fips/fips.yaml`: + +- Flip `discovery.nostr.advertise` from `false` to `true`. +- Add `advertise_on_nostr: true` and `public: true` under + `transports.udp`. + +After restart, your daemon publishes a Kind 37195 event tied +to your npub, listing the UDP endpoint other peers should +dial. + +## How advertising works + +> **Adverts are signed Nostr events.** Every advert is a Kind +> 37195 event signed by your daemon's secret key. Anyone +> reading it can verify the advert really came from the npub +> claiming the endpoint. The advert is the `(npub → current +> endpoints)` mapping, signed and published. + +The advert lists transports the daemon is willing to expose, +and only those: + +> **Endpoints are opt-in per transport.** Only transports +> with `advertise_on_nostr: true` are listed in your advert. +> Transports without that flag stay private — they still +> work for peers who reach you via static config, but they +> won't appear in your published advert. + +For UDP specifically, the daemon needs to know what IP and +port to put in the advert: + +> **Determining the advertised endpoint (wildcard-bound UDP).** +> With UDP bound to a wildcard like `0.0.0.0:2121`, the daemon +> doesn't know its own public IP at startup. You have two ways +> to tell it what to put in the advert: +> +> - `public: true` — daemon does a one-shot STUN observation +> against the configured STUN servers and uses the reflexive +> IPv4 it learns. Right when your public IP is dynamic or +> you'd rather not pin it in config. Works for nodes with a +> directly-bound public IP and for nodes behind full-cone +> NAT (most home routers). +> - `external_addr: "[:]"` — explicit override. +> Right when you already know your public IP — a static +> residential IP, an Elastic IP behind 1:1 NAT, a cloud +> instance whose advertised port differs from the bind +> port — and you don't want to depend on STUN reachability. +> Required for TCP on cloud setups where binding directly +> to the public IP returns `EADDRNOTAVAIL`. +> +> If you bind UDP to a specific public IP rather than +> `0.0.0.0`, neither flag is needed — the daemon advertises +> whatever it's bound to. + +Adverts don't sit on the relays forever: + +> **TTL and refresh.** Adverts have a 1-hour expiration +> (NIP-40 `expiration` tag) and the daemon re-publishes every +> 30 minutes. If your daemon goes offline, your advert decays +> from caches in roughly an hour and consumers stop trying. + +## Step 1: Confirm your starting state + +You should be coming out of +[resolve-peers-via-nostr](resolve-peers-via-nostr.md) with: + +- A persistent npub (`fipsctl show status | grep '"npub"'`). +- Nostr discovery in consume-only mode + (`discovery.nostr.enabled: true`, + `discovery.nostr.advertise: false`). +- A peer entry for `test-us01` with `via_nostr: true` and no + static address. `fipsctl show peers` shows the link + established. + +If any of those isn't true, finish the previous tutorials +first. + +Capture your npub now — you'll need it for the verification +step: + +```sh +sudo fipsctl show status | grep '"npub"' +``` + +Copy the value. + +## Step 2: Enable advertising in the config + +Open `/etc/fips/fips.yaml` and change two things. + +**Change 1: flip `advertise` to `true`.** Find the +`discovery.nostr` block under `node:` and set: + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: true +``` + +(The previous tutorial set `advertise: false`; you're flipping +that bit now.) + +**Change 2: add the UDP advert flags.** Find the `udp:` block +under `transports:`. The wildcard-bind default +(`0.0.0.0:2121`) means the daemon needs help knowing what to +advertise — pick one of the two approaches from the callout +above. + +If you want STUN auto-discovery (works for full-cone NATs and +nodes with a directly-bound public IP): + +```yaml +transports: + udp: + bind_addr: "0.0.0.0:2121" + advertise_on_nostr: true + public: true +``` + +If you already know your public IP (e.g., a static residential +IP or a cloud Elastic IP behind 1:1 NAT) and want to skip the +STUN dependency: + +```yaml +transports: + udp: + bind_addr: "0.0.0.0:2121" + advertise_on_nostr: true + external_addr: "203.0.113.45:2121" +``` + +Replace `203.0.113.45:2121` with your actual public IP and +port. The bare-IP form `external_addr: "203.0.113.45"` is also +accepted; the daemon combines it with the bind port. You may +set both `public: true` and `external_addr` together — the +explicit override wins, with STUN as a logging cross-check. + +`advertise_on_nostr: true` is the bit that says "include this +transport in my published advert" — common to both paths. + +Save the file. + +## Step 3: Restart the daemon + +```sh +sudo systemctl restart fips +sudo systemctl status fips +``` + +Status should show `active (running)`. Within a few seconds the +daemon will: + +1. Run a one-shot STUN observation against the default STUN + servers to learn its public IP. +2. Build a Kind 37195 advert listing + `udp::2121` (and any other transports you have + `advertise_on_nostr: true` on). +3. Sign the advert with the daemon's nsec. +4. Publish it to the three default advert relays. +5. Schedule a refresh every 30 minutes. + +If STUN fails (for example, if the network blocks outbound +UDP/3478), the daemon emits a WARN line in the journal and +suppresses the UDP entry from the advert rather than publishing +a wrong address. The link to `test-us01` from the previous +tutorial keeps working regardless — only the publish side is +gated on STUN. + +Quick sanity check on the journal: + +```sh +sudo journalctl -u fips -n 200 | grep -iE 'STUN|advert|warn' | head -20 +``` + +If you see `WARN` lines mentioning STUN or wildcard-bind +fallthrough, jump to [Troubleshooting](#troubleshooting); the +rest of the tutorial assumes the publish succeeded. + +## Step 4: Verify your advert is on the network + +The advert is a public Nostr event — anyone, including you, +can fetch it. With the `nak` Nostr CLI installed, query the +relays for adverts published by your npub: + +```sh +nak req -k 37195 -d "fips-overlay-v1" \ + -a $(nak decode | jq -r .pubkey) \ + --limit 1 wss://relay.damus.io +``` + +Replace `` with the npub you copied in Step 1. The +inner `nak decode` converts your bech32 npub to the hex pubkey +the relay filter expects. + +Expect one event back. The interesting fields: + +- `pubkey` — your npub in hex form. +- `tags` — includes `["d","fips-overlay-v1"]` (the namespace), + `["protocol","fips-overlay-v1"]`, and an `["expiration", …]` + tag set ~1 hour in the future. +- `content` — JSON listing the `endpoints` array. You should + see one entry like: + + ```json + {"transport":"udp","addr":":2121"} + ``` + +That `` is what STUN learned. Confirm it +matches what you'd expect for your network — for a home node, +it should be your residential IP, not a `192.168.x.x` LAN +address. + +## Step 5: Watch for inbound connections + +Your advert is now consumable by any FIPS daemon running open +discovery on the same `fips-overlay-v1` namespace. The public +test mesh nodes do exactly this — they subscribe to all +adverts in the namespace and try to dial new publishers. + +Within a minute or two of restart, run: + +```sh +sudo fipsctl show peers +``` + +In addition to your configured `test-us01` peer, you may see +an entry for `test-us03` (the open-discovery test mesh node). +It will have `connectivity` active and its own +`transport_addr`. This peering appeared without you +configuring anything — the test-mesh open-discovery node saw +your advert, dialed the endpoint, and Noise IK established +the link. + +If no inbound peers appear, that's not necessarily a failure +of advertising — it just means no one has consumed your advert +*and* dialed back yet. The advert is on the relays regardless, +verifiable in Step 4. + +## What you've learned + +- **Adverts are publish + sign.** Every running FIPS daemon + with `advertise: true` publishes a signed advert; reading it + is one Nostr event lookup. +- **Endpoint inclusion is per-transport.** Only the transports + you set `advertise_on_nostr: true` on appear in the advert. +- **`public: true` invokes STUN.** Wildcard-bound UDP with + `public: true` runs a one-shot STUN observation to learn + its public IP. +- **Refresh is automatic.** Adverts re-publish every 30 + minutes; consumers cache them with a 1-hour staleness + bound. +- **The publish side stands alone.** Once your advert is on + the relays, peers can dial you whether you're advertising + to them specifically or not. The test mesh's open-discovery + nodes will pick you up automatically. + +## If you're behind symmetric NAT + +`public: true` + STUN works on most home and office NATs (the +full-cone variety) and on nodes with a directly-bound public +IP. It does *not* work on symmetric NAT, where the NAT mapping +is keyed on (source-port, destination-host) so the IP/port +your STUN server saw isn't the IP/port a different peer would +see. + +For symmetric-NAT networks the alternative is `udp:nat` mode, +which advertises a placeholder `udp:nat` endpoint along with +the daemon's signaling-relay and STUN-server lists, and +performs UDP hole-punching at dial time. Both sides need to be +running matching configs and at least one side needs a +non-symmetric NAT for the punch to succeed; symmetric on both +sides is not reliably traversable and will time out. + +The minimal config switch: + +```yaml +transports: + udp: + bind_addr: "0.0.0.0:2121" + advertise_on_nostr: true + public: false # ← was true; change to false +``` + +And add the signaling/STUN block under `discovery.nostr`: + +```yaml +discovery: + nostr: + enabled: true + advertise: true + dm_relays: + - "wss://relay.damus.io" + - "wss://nos.lol" + stun_servers: + - "stun:stun.l.google.com:19302" + - "stun:stun.cloudflare.com:3478" +``` + +For the full setup including peer-side config and the punch- +duration knob, see +[../how-to/enable-nostr-discovery.md § Capability 2c](../how-to/enable-nostr-discovery.md#sub-scenario-2c-udp-hole-punching-for-nodes-behind-nat). + +Separately from NAT considerations, FIPS supports running a +node behind a Tor onion service as a deployment shape in its +own right — chosen for the privacy, anonymity, and +censorship-resistance properties it brings, not as a fallback +when UDP or TCP fail. If those properties are an independent +goal for your node, see +[../how-to/enable-nostr-discovery.md § Sub-scenario 2b](../how-to/enable-nostr-discovery.md#sub-scenario-2b-tor-onion-node) +and +[../how-to/deploy-tor-onion.md](../how-to/deploy-tor-onion.md). + +## Troubleshooting + +If your advert doesn't appear on the relays: + +- **STUN failed.** Check the journal for WARN lines mentioning + STUN or wildcard-bind. The most common causes are outbound + UDP/3478 blocked or DNS for `stun.l.google.com` failing. + Try: `dig stun.l.google.com` and + `nc -uvz stun.l.google.com 19302` to verify reachability. +- **Wrong public IP advertised.** If `nak` shows your advert + with a non-public address (e.g., `10.x.x.x` or + `192.168.x.x`), STUN didn't see your real public IP — likely + you're behind a CGNAT that NATs your STUN traffic too, or a + corporate firewall that proxies it. Switch to the + `external_addr` form from Step 2 with your actual public + IP, or replace `public: true` with the bound interface IP + directly under `bind_addr`. + +- **Relay reachability.** `nak req` against a relay you can + reach but no events return — possibly the publish failed + silently because the daemon couldn't connect to that + specific relay. Try the other two: + + ```sh + nak req ... wss://nos.lol + nak req ... wss://offchain.pub + ``` + +- **`advertise_on_nostr` typo.** YAML is case-sensitive and + silently ignores unknown keys. If `nak` returns no advert at + all, double-check the spelling on the UDP block and that + `discovery.nostr.advertise: true` is also set. + +## What's next + +- **Open discovery.** + [open-discovery](open-discovery.md) flips the consume side + symmetric — switch your daemon to `policy: open` and watch + your peer list populate from the ambient + `fips-overlay-v1` namespace, the same mechanism + `test-us03` is using right now to find you. + +- **Host a service of your own.** + [host-a-service](host-a-service.md) walks through bringing up + an HTTP server addressable as `.fips`, the same + way the connecting node now reaches `test-us01`. The natural + follow-on now that other operators can dial you by npub. + +For the operator-style scenario reference covering all five +shapes of Nostr discovery side-by-side (consume-only, +publish-direct, publish-Tor, NAT traversal, open): + +- [../how-to/enable-nostr-discovery.md](../how-to/enable-nostr-discovery.md) + +For the wire-format and discovery design: + +- [../reference/nostr-events.md](../reference/nostr-events.md) + — Kind 37195 advert format, Kind 21059 traversal signaling. +- [../design/fips-nostr-discovery.md](../design/fips-nostr-discovery.md) + — discovery runtime design, security and threat model. diff --git a/docs/tutorials/deploy-fips-gateway.md b/docs/tutorials/deploy-fips-gateway.md new file mode 100644 index 0000000..11f6aeb --- /dev/null +++ b/docs/tutorials/deploy-fips-gateway.md @@ -0,0 +1,468 @@ +# Deploy a `fips-gateway` on an OpenWrt AP + +In every other tutorial in this set you put FIPS *on* the host that +needs to talk to the mesh. This one is the exception. Here you stand +up a `fips-gateway` on an OpenWrt access point so the unmodified LAN +behind it — phones, laptops, smart-home gear — can reach mesh +destinations by `.fips` without any FIPS software of their own, +and so a service running on a LAN box can be exposed to mesh peers +through a port forward. Two halves, one binary, one config. + +This is **advanced** material. It assumes you have already worked +through [join-the-test-mesh](join-the-test-mesh.md) on some other +machine, you understand what `.fips` means, and now you want to +fold an existing LAN into the mesh from the edge router rather than +installing FIPS on every device. The whole exercise should take about +forty-five minutes. + +## What you'll build + +```text + ┌──────────────────────────────────────────────────────────────┐ + │ OpenWrt access point │ + │ │ + │ br-lan ┌──────────────┐ fips0 ┌──────────────┐ │ + │ (LAN side) ◀──│ fips-gateway │──────────▶│ fips daemon │ │ + │ │ (service) │ │ │──┼─▶ mesh + │ │ │ fd97:.. │ fd97:.. │ │ + │ │ fd01::/112 │ │ │ │ + │ │ pool │ │ │ │ + │ └──────┬───────┘ └──────────────┘ │ + │ │ │ + │ ▼ │ + │ nftables NAT │ + │ (inet fips_gateway) │ + └─────────────────┬────────────────────────────────────────────┘ + │ br-lan + ▼ + ┌──────────────────────────────────────────────┐ + │ LAN clients (phones, laptops, smart-home) │ + │ │ + │ no FIPS install — just IPv6 + DNS │ + │ ▲ │ + │ │ curl http://test-us01.fips/ │ + │ └────── DNS to dnsmasq ──▶ gateway DNS │ + │ │ + └──────────────────────────────────────────────┘ +``` + +By the end you will have: + +- An OpenWrt AP whose LAN clients can fetch `http://test-us01.fips/` + with no FIPS software installed on them. +- An inbound port forward exposing one LAN service to the mesh as + `.fips:`. +- An understanding of which LAN-side glue OpenWrt automates for you + (DNS forwarding, RA route, IPv6 prefix) and which the operator owns + (port forwards, mesh firewall). + +## Why an OpenWrt AP + +The gateway has very specific dependencies on the box it runs on. It +needs to own DNS for the LAN, it needs to advertise an IPv6 route to +the LAN, it needs a stable LAN-side interface, and it needs to be the +default IPv6 router for the segment. An OpenWrt-based access point +already does all of those things — it runs `dnsmasq`, it runs +`odhcpd` for IPv6 RA, it owns `br-lan`, and clients are already using +it as their gateway. The OpenWrt ipk leans into that: the `gateway:` +block in `/etc/fips/fips.yaml` is pre-populated, and the +`/etc/init.d/fips-gateway` init script wires up the LAN-side glue +automatically when you start the service. + +On a non-OpenWrt host the same integration is manual; that path is +covered by [../how-to/deploy-gateway.md](../how-to/deploy-gateway.md). + +## Prerequisites + +- An OpenWrt 22.03+ AP serving DHCP and DNS to a wired or wireless + LAN segment. +- The FIPS ipk installed and a working FIPS daemon on the AP. If you + haven't done that yet, follow + [../../packaging/openwrt-ipk/README.md](../../packaging/openwrt-ipk/README.md) + for the install, then come back here. +- The AP joined to the mesh — at least one healthy peer link. If it + isn't, work through [join-the-test-mesh](join-the-test-mesh.md) on + the AP first. +- Root SSH to the AP. Every command in this tutorial runs on the AP. +- A LAN client (phone, laptop) to test the outbound half from. + +## Step 1: Verify the FIPS daemon is up + +Confirm the daemon is running, the TUN is up, and at least one mesh +peer is reachable: + +```sh +service fips status +ip -6 addr show fips0 +fipsctl show peers +``` + +You should see: + +- `service fips status` reports `running`. +- `fips0` exists and has one `inet6 fd97:...` address. That is the + AP's mesh-side identity. +- `fipsctl show peers` lists at least one peer with active + connectivity (not `idle` / not zero bytes). + +Confirm the AP can resolve a known mesh node by name: + +```sh +ping6 -c 2 test-us01.fips +``` + +If any of these fail, fix the daemon side first — the gateway is a +separate service that runs alongside a working daemon, not a +substitute for one. + +## Step 2: Inspect the pre-populated gateway config + +The OpenWrt ipk ships `/etc/fips/fips.yaml` with the `gateway:` block +already filled in. View the relevant section: + +```sh +sed -n '/^gateway:/,$p' /etc/fips/fips.yaml +``` + +You will see roughly: + +```yaml +gateway: + enabled: true + pool: "fd01::/112" # virtual IP range (up to 65535 addresses) + lan_interface: "br-lan" # LAN-facing interface for proxy NDP + dns: + listen: "[::1]:5353" # gateway DNS listener (dnsmasq forwards here) + upstream: "[::1]:5354" # FIPS daemon DNS resolver (matches daemon default) + ttl: 60 # DNS TTL and mapping lifetime (seconds) + pool_grace_period: 60 # seconds after last session before reclaiming +``` + +Three things to notice: + +- `pool: "fd01::/112"` — the virtual-IP CIDR the gateway hands out + to LAN clients. 65 536 addresses, the gateway's hard cap. Pick a + different `fdXX::/N` prefix if `fd01::/112` collides with anything + on your network. +- `lan_interface: "br-lan"` — the OpenWrt LAN bridge. The gateway + installs proxy-NDP entries on this interface so LAN clients can + ARP-equivalent for pool addresses. +- `dns.listen: "[::1]:5353"` — the gateway's DNS listener is bound + to IPv6 loopback only. dnsmasq, which owns LAN port 53, forwards + `.fips` queries to it. The init script wires up that forwarding; + you don't bind to a LAN address yourself. + +For the full reference, see +[../reference/configuration.md § Gateway](../reference/configuration.md#gateway-gateway). + +## Step 3: Enable and start fips-gateway + +The service is shipped disabled — enable it once and start it: + +```sh +service fips-gateway enable +service fips-gateway start +``` + +Behind that single command, the init script +(`/etc/init.d/fips-gateway`) does five things: + +1. **Loads gateway sysctls.** `net.ipv6.conf.all.proxy_ndp=1` and + `net.ipv6.conf.all.forwarding=1` from + `/etc/sysctl.d/fips-gateway.conf`. +2. **Reconfigures dnsmasq via UCI** so `.fips` queries arriving at + the LAN's port 53 are forwarded to the gateway's loopback + listener on port 5353 instead of going straight to the daemon's + resolver on port 5354. (Dnsmasq still owns 53; the gateway sits + in front of the daemon for `.fips` only.) +3. **Adds a global-scope IPv6 prefix** to `br-lan`. Without a + non-ULA address on the local interface, Android and Chrome + suppress AAAA queries entirely — they assume the LAN has no + real IPv6 and don't bother. The init script adds a small + benchmarking-range prefix to convince them otherwise. +4. **Adds an RA route for the virtual pool.** A UCI `route6` entry + under `dhcp` tells `odhcpd` to advertise the pool CIDR via Router + Advertisement (RFC 4191), so LAN clients learn how to reach pool + addresses automatically. No per-client static routes needed. +5. **Spawns `fips-gateway` under procd** with `--config + /etc/fips/fips.yaml`, with crash-respawn. + +Verify it is running: + +```sh +service fips-gateway status +logread | grep fips-gateway | tail +``` + +Expect a `running` status and a startup log line of the form +`fips-gateway 0.x.y starting`, followed by entries for DNS bind, NAT +table install, and pool initialisation. + +> **What just changed on the LAN.** The AP is now offering two +> things it wasn't offering a moment ago: AAAA records under `.fips` +> that resolve to virtual IPs in `fd01::/112`, and a route to that +> CIDR in its Router Advertisements. Existing LAN clients pick both +> up the next time they re-resolve a name and the next time `odhcpd` +> sends an RA, respectively. No reboot required on the client side. + +## Step 4: Test the outbound half from a LAN client + +From a phone or laptop on the AP's LAN — anything that does IPv6 and +DNS, with no FIPS software installed — try one of the public test +mesh nodes: + +```sh +dig test-us01.fips AAAA +ping6 -c 4 test-us01.fips +curl -6 http://test-us01.fips/ +``` + +Expectations: + +- `dig` returns an AAAA in `fd01::...`, **not** `fd97:...`. The + `fd01:` address is the gateway's virtual-IP allocation; the LAN + client never sees the raw mesh address. +- `ping6` succeeds. ICMPv6 echo travels through the NAT pipeline and + back. +- `curl` fetches the page (whatever the test mesh is currently + serving on `test-us01`). + +> **What just happened end to end.** Your client asked dnsmasq for +> `test-us01.fips`. Dnsmasq forwarded the query to the gateway's +> loopback listener on port 5353. The gateway forwarded the query on +> to the daemon's resolver on port 5354. The daemon answered with +> `test-us01`'s mesh address (`fd97:...`). The gateway allocated a +> virtual IP from `fd01::/112`, installed nftables DNAT/SNAT/ +> masquerade rules pinning that virtual IP to the mesh address, +> installed a proxy-NDP entry on `br-lan` so the client could resolve +> the virtual IP at the link layer, and returned the virtual IP in +> the AAAA reply. Your client then routed traffic to the virtual IP +> via the RA-advertised pool route, the AP's kernel rewrote the +> destination to the mesh address, and the daemon's adapter carried +> the packets across the mesh. Return traffic followed conntrack +> back. The client never knew the mesh existed. + +## Step 5 (optional): Inspect the gateway state + +The gateway exposes its own control socket separate from the daemon's. +Two useful queries: + +```sh +echo '{"command":"show_gateway"}' | nc -U /run/fips/gateway.sock +echo '{"command":"show_mappings"}' | nc -U /run/fips/gateway.sock +``` + +`show_gateway` reports pool utilisation, the DNS listen address, +uptime, and the conntrack/NAT counters. `show_mappings` lists each +allocated virtual IP, the mesh address it points at, the DNS name +that triggered the allocation, and the mapping's lifecycle state +(`Allocated`, `Active`, `Draining`). + +For the full command catalog and JSON shapes, see +[../reference/control-socket.md § Gateway Command Catalog](../reference/control-socket.md#gateway-command-catalog). +The same data is rendered visually in the **Gateway** tab of +[`fipstop`](../reference/cli-fipstop.md). + +If you want to see the kernel rules the gateway installed: + +```sh +nft list table inet fips_gateway +``` + +You will see DNAT, SNAT, and masquerade chains populated with one +rule per active mapping. + +## Step 6: Add an inbound port-forward for a LAN service + +The outbound half is the steady-state use of a gateway. The inbound +half — exposing a LAN service to mesh peers — is a separate decision, +configured per service under `gateway.port_forwards[]`. + +For the worked example, run a one-page static web server on the AP +itself, bound to its `br-lan` address, and expose it to the mesh +through a port-forward. Anything would do — the point of the exercise +is the port-forward, not the service. We use what is already on the +AP: `busybox httpd`. In a real deployment the LAN-side target would +typically be a separate host (a NAS, a home server, a dev box on the +LAN); the rule shape is identical. + +Find the AP's `br-lan` IPv6 address and save it for the rest of the +step: + +```sh +BR_LAN_ADDR=$(ip -6 addr show br-lan \ + | awk '/inet6 fd|inet6 2/ && !/scope link/ {print $2}' \ + | head -1 | cut -d/ -f1) +echo "$BR_LAN_ADDR" +``` + +Pick from the global-scope benchmarking prefix the init script added +in Step 3, or your own ULA if `br-lan` has one — anything except a +link-local `fe80::/10` address. + +Set up a one-file docroot and start a foreground `busybox httpd` +bound to that LAN address on port 8000: + +```sh +mkdir -p /tmp/mesh-demo +echo '

Hello from the mesh-gateway demo

' > /tmp/mesh-demo/index.html +busybox httpd -f -p "[${BR_LAN_ADDR}]:8000" -h /tmp/mesh-demo +``` + +Leave it running in this shell. Open a second SSH session on the AP +to add the port-forward. + +Edit `/etc/fips/fips.yaml`. Inside the existing `gateway:` block, +add a `port_forwards:` list: + +```yaml +gateway: + enabled: true + pool: "fd01::/112" + lan_interface: "br-lan" + dns: + listen: "[::1]:5353" + upstream: "[::1]:5354" + ttl: 60 + pool_grace_period: 60 + port_forwards: + - listen_port: 8080 + proto: tcp + target: "[]:8000" +``` + +Substitute the real address for ``. The IPv6 form +(`[addr]:port`) is required — IPv4 targets are rejected at config +load. + +Restart the gateway so it re-reads the config: + +```sh +service fips-gateway restart +``` + +From any *other* mesh node, fetch the demo page through the gateway +using the AP's npub: + +```sh +# on the AP, get the npub: +NPUB=$(cat /etc/fips/fips.pub) +echo "$NPUB" +``` + +Then on the remote mesh node: + +```sh +curl -6 "http://${NPUB}.fips:8080/" +``` + +Expect: + +```text +

Hello from the mesh-gateway demo

+``` + +The connection landed on the gateway's `fips0` ingress on TCP/8080, +nftables DNAT rewrote the destination to `[BR_LAN_ADDR]:8000`, +LAN-side masquerade rewrote the source so `busybox httpd` saw a +LAN-routable address, and the response retraced via conntrack. + +> **What you exposed.** With the port-forward active, every mesh +> peer that can route to your AP can hit `${NPUB}.fips:8080/` and +> reach this service. That is exactly what the inbound half is +> for — but if you want to scope visibility to a specific subset +> of peers, the FIPS mesh firewall is the layer that does it; see +> [../how-to/enable-mesh-firewall.md](../how-to/enable-mesh-firewall.md). +> The port-forward rule and the firewall rule are independent: the +> port-forward installs the rewrite; the firewall decides who is +> allowed to reach the listen port. + +## Step 7: Tidy up + +In the first shell, stop `busybox httpd` with `Ctrl-C`. The demo +docroot at `/tmp/mesh-demo` can stay — it is wiped on reboot — or +remove it now (`rm -rf /tmp/mesh-demo`). + +If you want to keep the outbound half but withdraw the inbound +forward, remove the `port_forwards:` entry from `/etc/fips/fips.yaml` +and `service fips-gateway restart`. The mesh-side listener disappears +and so does the corresponding nftables rule. + +To turn the gateway off entirely: + +```sh +service fips-gateway stop +service fips-gateway disable +``` + +The init script's `stop_service` handler reverses the LAN-side +integration on the way out: dnsmasq's `.fips` forwarder is pointed +back at the daemon's port 5354, the RA route for the pool is +withdrawn from `odhcpd`, and the global-scope IPv6 prefix on +`br-lan` is removed. The LAN reverts to the state it was in before +you ran `service fips-gateway start` in Step 3. + +The daemon and the rest of `/etc/fips/` are untouched. Existing mesh +peering on the AP itself continues to work. + +## What you've learned + +- **The gateway is a niche feature for a niche box.** Most FIPS + hosts run the daemon and reach the mesh directly. The gateway + exists so an AP can fold an entire unmodified LAN behind it into + the mesh in one place. +- **Two halves of the same binary.** Outbound mode hands LAN clients + virtual IPs and NATs them onto the mesh; inbound mode listens on + `fips0` and forwards to LAN targets. They share one nftables + table, one control socket, and one config block, but each half + has its own use case. +- **OpenWrt does the LAN-side glue for you.** The init script + reconfigures dnsmasq, installs the RA route, adds the global IPv6 + prefix, and loads sysctls. On a non-OpenWrt host that integration + is manual — see [../how-to/deploy-gateway.md](../how-to/deploy-gateway.md). +- **Inbound forwards stay manual on every distro.** The + `port_forwards[]` block is uniform across hosts, and on every + distro you still own the decision of which LAN target to expose + and on which mesh-side port. +- **The mesh firewall is a separate decision.** Opening a port + forward on the gateway side does not open it on the firewall + side; if `fips-firewall.service` is enabled, you still need a + drop-in that admits the listen port. + +## Troubleshooting + +If something doesn't work as described above, the operator-recipe +guide [../how-to/troubleshoot-gateway.md](../how-to/troubleshoot-gateway.md) +groups the common failures by symptom: + +| Symptom | Where to look | +| ------- | ------------- | +| LAN client gets `fd97:...`, not `fd01:...` | DNS path: dnsmasq still pointing at port 5354. See "DNS queries fail". | +| `dig` succeeds with a pool address but `ping6` times out | Pool route or proxy NDP. See "Virtual IP unreachable from client". | +| `ping6` works but TCP times out | NAT pipeline or mesh-side firewall. See "Ping works but TCP does not". | +| Gateway service won't start | "No gateway section in configuration" recipe. | +| Inbound `curl` hits the listen port but never reaches the LAN target | Mesh-side firewall first, then the port-forward rule. | + +The first thing the troubleshoot guide does in any of these cases is +ask the gateway directly via `show_gateway` and `show_mappings`. If +the mapping you expect is not there, the failure is on the DNS path; +if it is there in `state: Active` but traffic still fails, the +failure is downstream. + +## What's next + +- [../how-to/deploy-gateway.md](../how-to/deploy-gateway.md) — + Manual deployment on a non-OpenWrt Linux host. Same gateway, + same config, but you wire up dnsmasq/Unbound/etc. yourself, + install a pool route per LAN client (or via your own RA daemon), + and manage the systemd unit instead of the procd init script. +- [../design/fips-gateway.md](../design/fips-gateway.md) — The + design doc: NAT pipeline (DNAT, SNAT, masquerade, inbound DNAT), + virtual-IP pool lifecycle (Allocated -> Active -> Draining -> + reclaimed), DNS resolution flow, conntrack integration. +- [../reference/configuration.md § Gateway](../reference/configuration.md#gateway-gateway) + — Every field of the `gateway:` block, including the conntrack + timeout overrides not used in this tutorial. +- [../reference/cli-fips-gateway.md](../reference/cli-fips-gateway.md) + — The `fips-gateway` binary's CLI options, exit codes, and + environment variables. diff --git a/docs/tutorials/ground-up-mesh.md b/docs/tutorials/ground-up-mesh.md new file mode 100644 index 0000000..2c5afd7 --- /dev/null +++ b/docs/tutorials/ground-up-mesh.md @@ -0,0 +1,518 @@ +# Build a Mesh from the Ground Up + +The earlier tutorials in this progression rode existing IP — your +daemon reached `test-us01` over the public internet through your +ISP, your ISP's upstream, and however many hops separate you from +the test node. That is the *overlay* deployment mode of FIPS: +useful, but not the new ground. + +This tutorial is about the other mode. Two devices, a wire (or a +radio link) between them, no IP between them, and FIPS daemons on +each end. The two daemons discover each other over the raw link, +peer over Noise, and bring up an end-to-end mesh with addressing, +naming, and reachability — all from layer 2 up. There is no DHCP, +no router, no upstream. The mesh is the network. + +This is the deployment mode FIPS was designed for. Overlay mode +exists because riding existing IP is a useful convenience; the +ground-up mode is what FIPS uniquely enables. + +> **The two modes are not exclusive.** A node can carry overlay +> peers and ground-up peers at the same time — different transports +> on the same daemon. If you have already worked through +> [join-the-test-mesh](join-the-test-mesh.md), the static peer to +> `test-us01` you configured there can stay in place; the Ethernet +> peer you add in this tutorial sits alongside it. Traffic flows +> through whichever path is shortest by mesh metric, and a node on +> one side can reach a node on the other through your machine +> acting as a bridge between the two. + +## What you'll build + +```text + ┌──────────────────────┐ raw Ethernet frames ┌──────────────────────┐ + │ node A │ ─────────────────────── │ node B │ + │ npub1aaa… │ EtherType 0x2121 │ npub1bbb… │ + │ fips0 fd97:..:A │ no IP between them │ fips0 fd97:..:B │ + └──────────────────────┘ └──────────────────────┘ + │ │ + │ a single Ethernet cable │ + │ (or both NICs on the same │ + │ unmanaged switch — no DHCP, │ + │ no router, no IP at all) │ + └─────────────────────────────────────────────────┘ +``` + +Two machines, each running `fips`, joined by a physical Ethernet +link. After the worked example: + +- The two daemons have discovered each other via L2 beacons on + the link, peered over Noise IK, and brought up an FMP link. +- Each `fips0` adapter has a routable mesh address; each can + ping the other by `.fips`. +- Nothing between the two machines speaks IP. The link carries + raw FIPS frames at EtherType `0x2121`. + +The whole exercise should take about twenty minutes if you have +the hardware ready. + +## Why ground-up + +Most networking tutorials assume IP is already there: an address +arrived from DHCP, a default gateway routes you onward, DNS +resolves names. FIPS does not need any of that. Two devices and +a way to deliver bytes between them at layer 2 is enough — FIPS +supplies the rest: + +- **Identity**: each daemon has an npub (the same kind you saw + in the overlay tutorials). Nothing in the ground-up case + depends on a network identity from a router; the npub is the + identity. +- **Addressing**: the `fips0` adapter takes an `fd97:...` ULA + derived from the npub. No DHCP. No SLAAC. The address is + cryptographically tied to the identity. +- **Discovery**: each daemon broadcasts a small beacon on the + link advertising its npub; the other daemon's listener picks + it up and dials in over the same link. +- **Routing**: the FIPS mesh layer builds its own spanning tree + across whatever links it has. Add a third node (peered to + either A or B) and traffic reaches it transparently. + +The point is not that ground-up replaces overlay. It's that +overlay is one of two modes the same daemon supports, and +ground-up is what unlocks the use cases overlay cannot — +ad-hoc local meshes, partitioned networks, situations where +no IP infrastructure exists or can be relied on. + +## Prerequisites + +Two devices (call them **node A** and **node B**) and a way to +join them at layer 2: + +- Ethernet (the worked example): a direct cable between two + modern NICs (auto-MDI/MDIX handles crossover for you), or + both machines on a small unmanaged switch with no DHCP + server. USB-Ethernet dongles work; a typical "USB-to-RJ45" + adapter is fine on either end. The link does **not** need + to be the machine's primary network interface — a second + NIC dedicated to the mesh is the cleanest setup. +- WiFi (a one-line variation, covered later): both machines + associated to a common AP that has client (station) + isolation **off**. +- Bluetooth LE (a separate worked example via a how-to, + covered later): two BLE-capable Linux hosts within roughly + 10 metres line of sight. + +On both nodes: + +- `fips` installed and running, per [getting-started](../getting-started.md). +- A persistent identity from + [persistent-identity](persistent-identity.md). Ephemeral + identities work, but on each restart the npub regenerates + and you'll have to re-check `fipsctl show peers` to see the + new identity. Persistent makes the lesson stick. +- The daemon running with `CAP_NET_RAW` (the shipped systemd + unit runs as root and gets this for free; running + interactively from a user account requires `setcap` — + noted at the relevant step below). + +You do **not** need: + +- An IP address on the chosen interface. The Ethernet + transport opens a raw socket directly; the kernel does not + need to assign an IP to the NIC. +- A default route. The mesh routes itself. +- DNS resolution between the machines via any external + service. The local `.fips` resolver supplies names from + the npubs the daemons exchange. + +## Step 1: Identify the link interface on each node + +On each node, list the network interfaces and pick the one that +sits on the link between the two machines. If it's a dedicated +NIC for the mesh, that NIC has no other purpose; if it's a +USB-Ethernet dongle, plug it in first so the kernel names it. + +```sh +ip link show +``` + +Pick out the interface name. Common forms: + +- `enp3s0`, `eno1` — built-in NICs under predictable naming. +- `eth0` — older or container-style naming. +- `enxAABBCCDDEEFF` — USB-Ethernet dongles often appear under + this MAC-derived form. + +Bring the interface up if it isn't: + +```sh +sudo ip link set dev up +``` + +Confirm: + +```sh +ip -br link show +``` + +You want `UP` and `LOWER_UP` in the flags. The interface does +not need an IP address — `LOWER_UP` indicates the NIC sees +carrier (cable plugged into something at the other end), and +that is all the Ethernet transport needs. + +For the rest of the tutorial we'll write the chosen interface +as ``. Substitute the actual name on each node when you +run the commands. Note that node A and node B may have +different interface names — that is normal. + +> **No IP needed.** If your chosen interface has an address +> from a previous DHCP lease, leave it alone or remove it with +> `sudo ip addr flush dev ` — the FIPS Ethernet transport +> uses raw `AF_PACKET` sockets that bypass the IP stack +> entirely. The interface needs to be `up` and `LOWER_UP`, +> nothing more. + +## Step 2: Configure the Ethernet transport on each node + +Edit `/etc/fips/fips.yaml` on **both** nodes. Under +`transports:`, add an `ethernet:` block. The key settings are +the four discovery flags — both nodes must opt in to all four, +and they default to off: + +```yaml +transports: + ethernet: + interface: "" # the name from Step 1 + announce: true # broadcast our beacon on the link + discovery: true # listen for beacons (default; shown for clarity) + auto_connect: true # dial peers we discover + accept_connections: true # accept dial-ins from peers we discover +``` + +Each flag does one thing: + +- `announce: true` — emit a small beacon every + `beacon_interval_secs` (default 30s) carrying our npub. +- `discovery: true` — listen for incoming beacons; populate a + candidate-peer list keyed by source MAC and observed npub. +- `auto_connect: true` — when we see a beacon from an npub + we have not yet peered with, initiate the outbound Noise + handshake. +- `accept_connections: true` — when a remote npub initiates + the handshake on this transport, complete it. + +If only one node sets `announce`, the other won't see it; if +only one side sets `auto_connect` or `accept_connections`, the +roles are asymmetric and the link won't establish unless both +are configured. The cleanest pattern for a ground-up tutorial +is "all four flags on both ends." + +> **Multiple Ethernet links.** If a node has more than one +> physical interface that participates in the mesh, configure +> each one as a *named instance* under `ethernet:`: +> +> ```yaml +> transports: +> ethernet: +> lan: +> interface: "eth0" +> announce: true +> discovery: true +> auto_connect: true +> accept_connections: true +> dongle: +> interface: "enx00aabbccddee" +> announce: true +> # ... +> ``` +> +> Each named instance runs its own socket and discovery state. +> A single ground-up link only needs the flat form shown +> first; named instances become useful when the same node +> bridges multiple physical segments. + +## Step 3: Grant the daemon permission to open raw sockets + +The Ethernet transport opens an `AF_PACKET` `SOCK_DGRAM` socket +bound to the chosen interface. That requires `CAP_NET_RAW`. + +If you installed FIPS via the Debian package and run via the +shipped systemd unit, the daemon runs as root and has +`CAP_NET_RAW` already — there is nothing to do here. Skip to +Step 4. + +If you are running the daemon interactively as your user (a +from-source / development setup), grant the capability once on +the binary: + +```sh +sudo setcap CAP_NET_RAW,CAP_NET_ADMIN+ep "$(which fips)" +``` + +`CAP_NET_ADMIN` is what the daemon needs for the `fips0` TUN +adapter regardless; `CAP_NET_RAW` is the ground-up addition. +The `setcap` invocation only needs to be repeated when the +binary is replaced. + +## Step 4: Restart the daemon on each node + +```sh +sudo systemctl restart fips +``` + +Or, if running interactively, restart your `fips` invocation +in whichever way you started it. + +Watch the startup logs for the Ethernet transport coming up: + +```sh +sudo journalctl -u fips -f --since="1 minute ago" +``` + +Look for landmarks like: + +- A line indicating the Ethernet transport opened the chosen + interface and started its receive loop. +- Periodic outbound beacon messages (one per + `beacon_interval_secs` window). +- After the second beacon round on the *other* node, an + inbound beacon parsed and a candidate-peer entry created. +- Once each side dials, a Noise handshake completion log + message naming the remote npub. + +Beacon interval defaults to 30s, so the first peering can take +up to a minute (one beacon window per side, plus handshake). +Lower the interval for the tutorial if you want faster +feedback: + +```yaml +transports: + ethernet: + # ... + beacon_interval_secs: 10 # minimum allowed +``` + +## Step 5: Verify the link + +On either node: + +```sh +sudo fipsctl show peers +``` + +Expect one entry whose `npub` matches the **other** node and +whose `addresses` line shows `transport: ethernet`. Your +existing overlay peers (if any from earlier tutorials) appear +alongside it. Each peer has its own row, and the link status +columns show whether the Noise session is up. + +```sh +sudo fipsctl show transports +``` + +Confirms that the Ethernet transport is running and shows the +beacon counters incrementing. Both `beacons_sent` and +`beacons_received` should be non-zero if the link is healthy. + +## Step 6: Reach the other node by name + +On node A, ping node B by `.fips` name. Get node B's npub +from its `fipsctl show status` output (it's the persistent +identity you established earlier), then: + +```sh +ping6 npub1bbb…long-string….fips +``` + +Expect ICMPv6 echo replies. The path is: + +1. The local `.fips` resolver translates the npub-form name + into an `fd97:...` mesh address (cryptographically derived + from the npub on both ends — the resolver does the + computation locally, with no network round trip). +2. The kernel routes the packet via `fips0`. +3. The FIPS daemon accepts it from the TUN, looks up the + mesh route, and hands it to the FMP link to node B. +4. The Ethernet transport on node A frames the FMP packet as + a raw EtherType `0x2121` Ethernet frame addressed to node + B's MAC, learned from B's beacons. +5. Node B's daemon receives the frame, peels off the + Ethernet/FIPS framing, and the packet emerges on node B's + `fips0`. +6. The kernel on node B sees an inbound ICMPv6 echo and + replies, and the same path runs in reverse. + +If you have a hosts file with shortnames configured (see +[host-aliases](../how-to/host-aliases.md)), substitute the +shortname for the full npub form. + +## Step 7: Try a forward composition + +If node A also has the `test-us01` overlay peer from +[join-the-test-mesh](join-the-test-mesh.md), node B can +reach `test-us01` *through* node A — even though node B has +no direct internet path of its own: + +On node B: + +```sh +ping6 npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98.fips +``` + +The packet leaves B's `fips0`, traverses the Ethernet link to +A, gets forwarded by A across the overlay UDP transport to +`test-us01`, and the reply comes back the same way. + +This is the composition the chapter intro flagged: the two +deployment modes coexist on a single daemon. Node A is +participating in the test mesh via the internet *and* in your +local Ethernet mesh. From node B's perspective, the test mesh +is reachable. From `test-us01`'s perspective, B is reachable. +The mesh handles the rest. + +## Variations + +### WiFi (AP mode), same shape as Ethernet + +Replace `` with the WiFi interface name (typically +`wlan0` or `wlp3s0`) on each node. The WiFi NIC is presented +as an Ethernet-class interface to the kernel by the +`mac80211` abstraction; the FIPS Ethernet transport opens +the same `AF_PACKET` socket on it. No FIPS-side configuration +change beyond the interface name. + +What you do need on the AP side: + +- Both nodes associated to the same SSID. +- **Client (station) isolation must be OFF** on the AP. + Most consumer routers ship with it off; many guest + networks and "secure" enterprise APs ship with it on. + When client isolation is on, the AP refuses to forward + station-to-station frames — the broadcast beacons never + arrive at the other node, and discovery fails silently. + If beacons aren't crossing, this is the first thing to + check. + +There is no FIPS-specific configuration for WiFi versus +Ethernet on the daemon side; the choice is purely the +adapter name. + +### Bluetooth LE (experimental but works) + +BLE is a separate transport (`transports.ble.*`) with its own +discovery model — L2CAP advertisements rather than raw L2 +broadcasts. The shape of the tutorial is the same (advertise + +scan + auto-connect + accept), but the prerequisites are +different: BlueZ, `bluetoothd`, an HCI adapter, and the +`bluetooth` group or capability set. + +The full operator recipe is in +[../how-to/set-up-bluetooth-peer.md](../how-to/set-up-bluetooth-peer.md). +Mark this transport as experimental: it works in most +configurations but the BLE stack has more variability than +Ethernet — adapter quirks, BlueZ version differences, and the +shorter range all matter. + +The BLE transport is **Linux-only** at present; macOS and +Windows builds skip it. + +## What you've learned + +- **Ground-up is the new ground.** FIPS does not need any IP + infrastructure between two devices to mesh them. A wire (or + a radio link), `CAP_NET_RAW`, and a few config flags on each + end are sufficient. The mesh supplies its own identity, + addressing, discovery, and routing. +- **Discovery is a four-flag opt-in.** `announce`, `discovery`, + `auto_connect`, and `accept_connections` each control one + thing; both ends must agree before a link will form. +- **The two modes coexist.** Overlay peers and ground-up peers + ride the same daemon — same FMP link layer, same FSP session + layer, same `fips0` adapter. A node can be a bridge between + the two without any extra plumbing. +- **No IP on the link.** The Ethernet transport bypasses the + kernel IP stack via `AF_PACKET`. Whether the interface has + an IP address is irrelevant; whether it has carrier is what + matters. +- **Names work the same way.** `.fips` resolves locally + via the cryptographically-derived ULA. The resolver does + not care whether the destination is reached over Ethernet, + UDP overlay, or some hop chain combining both. + +## Troubleshooting + +- **No beacons received.** On either node, `sudo fipsctl show + transports` should show `beacons_received` incrementing + every `beacon_interval_secs` once the other node is also + running. If it stays at zero: + - Confirm the chosen interface is `LOWER_UP` (carrier + present). + - Confirm the other node is announcing (its `beacons_sent` + should be non-zero). + - On WiFi: confirm AP client isolation is off. + - On a switch: confirm the switch is unmanaged or that + EtherType `0x2121` is not being filtered. Most consumer + switches forward all EtherTypes; managed switches + sometimes don't. +- **Beacons received but no peer entry.** The handshake is + failing. Tail logs (`journalctl -u fips`) for Noise + handshake errors. Common causes: peer ACL active and not + including the remote npub (out of scope for this tutorial, + but check `/etc/fips/peers.allow` if you have set one); + daemon's clock drift large enough to fail freshness + checks (rare). +- **Daemon won't start with the Ethernet transport.** Likely + a permissions error. Check `journalctl -u fips` for an + `EPERM` or "operation not permitted" message; if running + interactively, confirm the binary has `CAP_NET_RAW` + (`getcap "$(which fips)"`). +- **Beacons in both directions, peers entries on both sides, + but ping6 times out.** The handshake completed but the FSP + session is not flowing data. Check `fipsctl show peers`'s + link status columns — if the FMP link is healthy but FSP + is not, the mesh-layer side is fine and the issue is one + layer up. The + [reach-mesh-services § Troubleshooting](reach-mesh-services.md#troubleshooting) + section covers symptoms at this level. +- **`AF_PACKET` socket bind fails on a kernel-protected + interface.** Some hardened kernels (`grsec`, certain + containers, certain VMs) restrict raw-socket access even + with `CAP_NET_RAW`. The daemon log will name the failing + syscall. The fix is host-side: relax the restriction or + pick a different interface. + +## What's next + +You now have the second deployment mode of FIPS in your +hands. From here: + +- **Add a third node.** Bring up a third machine on the same + Ethernet segment, configure it identically, and watch all + three nodes form a mesh. The FIPS spanning tree picks a + root and routing converges within a few beacon intervals. +- **Mix transports.** Add an overlay peer (per + [join-the-test-mesh](join-the-test-mesh.md)) to one of + your ground-up nodes; the local mesh now reaches the test + mesh through that node, and vice versa. +- **Host services.** Anything you do on `fips0` with overlay + peers — bind an HTTP server (per + [host-a-service](host-a-service.md)), reach a service via + the daemon's IPv6 adapter (per + [reach-mesh-services](reach-mesh-services.md)) — works + identically on a ground-up mesh. The data plane is the + same. + +For more depth on the link-layer machinery: + +- [../reference/transports.md § Ethernet](../reference/transports.md) + — full Ethernet transport reference (counter inventory, + per-instance configuration, MTU model). +- [../reference/configuration.md § Ethernet](../reference/configuration.md#ethernet-transportsethernet) + — every configuration key and its default. +- [../how-to/set-up-bluetooth-peer.md](../how-to/set-up-bluetooth-peer.md) + — operator recipe for the BLE variant. +- [../design/fips-transport-layer.md](../design/fips-transport-layer.md) + — the design doc that describes the per-link MTU model and + why each transport is treated as link-layer rather than + network-layer. diff --git a/docs/tutorials/host-a-service.md b/docs/tutorials/host-a-service.md new file mode 100644 index 0000000..8bb1b10 --- /dev/null +++ b/docs/tutorials/host-a-service.md @@ -0,0 +1,477 @@ +# Host a Service of Your Own + +In [reach-mesh-services](reach-mesh-services.md) you used +ordinary IPv6 tools to reach services on other mesh nodes. This +tutorial flips the direction. You will bring up a small HTTP +server on your machine, make a deliberate choice about which +interface it binds to, and turn on the mesh firewall to keep +the exposure to what you intended. By the end you will have +hosted your first peer-reachable service and made an informed +decision about who can reach it. + +The whole exercise should take about twenty minutes. You should +have already worked through +[persistent-identity](persistent-identity.md) so that the npub +your service is reachable at does not change between restarts. + +## What you'll build + +```text + ┌──────────────────────────────────────────┐ + │ your fips node │ + │ │ + │ python3 -m http.server │ + │ --bind 8080 │ + │ │ │ + │ ▼ │ + │ fips0 fd97:….:Y ◀─── port 8080 open │ + └────────────┬─────────────────────────────┘ + │ + │ reachable as + │ http://.fips:8080/ + ▼ + any mesh peer that can route to .fips +``` + +You will have: + +- A single-page HTTP server bound to `fips0` only — not the + public internet, not your LAN, just the mesh. +- The mesh firewall baseline active, default-deny on `fips0` + inbound, with one explicit drop-in that allows TCP/8080. +- A clear understanding of which interface choice corresponds + to which audience. + +## Why bind interface matters + +The single most important decision when hosting a service is +**which interface (and therefore which audience) the service is +exposed to**. This is true for every IPv6 service, not just +FIPS — but FIPS makes it stark because your machine often has +several interfaces with very different exposure profiles. + +> **Bind interface = exposure surface.** A typical FIPS host +> has at least two distinct audiences: +> +> - `fips0` (`fd97:…`, reachable as `.fips`) — +> reachable only from FIPS peers that have a working link to +> your node. Bound by Noise authentication and (optionally) +> the peer ACL. +> - `eth0` / `wlan0` (your LAN address) — reachable from anyone +> on your local network segment, with no FIPS auth in the +> way. +> +> When you run a server, the `--bind` argument decides which of +> these audiences sees the service. Binding to a *specific* +> address opts in to one audience. Binding to `[::]` or +> `0.0.0.0` opts in to **all** of them at once — including any +> you forgot you had. +> +> **Audit what's already listening.** Bringing up `fips0` adds +> a new audience to every service on this host that was already +> bound to `0.0.0.0` or `[::]`. SSH, your web server, a database +> — if any of them was listening on all interfaces before you +> joined the mesh, they are now reachable from mesh peers too. +> A quick `ss -tulnp` will show you everything currently +> listening and on which addresses. The mesh firewall (Step 5 +> below) is one way to bring those exposures back under explicit +> control; rebinding the affected services to a specific +> non-mesh address is another. + +There is nothing FIPS-specific about this rule; it applies to +SSH, web servers, databases, anything. FIPS just gives you the +option of "mesh peers only" as a distinct audience, which most +hosts otherwise don't have. + +## Step 1: Find your node's mesh address + +You need the `fd97:...` address assigned to your `fips0` +adapter. Two equivalent ways to get it: + +```sh +ip -6 addr show fips0 +``` + +Look for the `inet6 fd97:...` line. The address up to the `/` +is what you want. + +Or via the daemon: + +```sh +sudo fipsctl show identities +``` + +The first JSON entry has `local: true` and a `ula` field — that +is your address. + +For the rest of this tutorial we will write the address as +``. Substitute the actual `fd97:...` value +when you run the commands. Save it to a shell variable for +convenience: + +```sh +FIPS0_ADDR=$(ip -6 addr show fips0 | awk '/inet6 fd97:/ {print $2}' | cut -d/ -f1) +echo "$FIPS0_ADDR" +``` + +You should also know your npub from +[persistent-identity](persistent-identity.md): + +```sh +NPUB=$(sudo cat /etc/fips/fips.pub) +echo "$NPUB" +``` + +`.fips` and `` are two names for +the same destination. + +## Step 2: Bring up an HTTP server bound to fips0 + +Make a small directory with one file in it so the server has +something to serve: + +```sh +mkdir -p /tmp/mesh-demo +echo '

Hello from the mesh

' > /tmp/mesh-demo/index.html +cd /tmp/mesh-demo +``` + +Start a Python HTTP server bound to your `fips0` address: + +```sh +python3 -m http.server --bind "$FIPS0_ADDR" 8080 +``` + +Leave the server running. The terminal will show: + +```text +Serving HTTP on fd97:... port 8080 (http://[fd97:...]:8080/) ... +``` + +Two things to notice: + +- The "Serving HTTP on …" line names your `fd97:...` address + explicitly. Python is binding only to that one address. +- The default would have been `0.0.0.0` — which is **not** + what you want here. Without `--bind`, the server would be + reachable from your LAN and any public IP this host has, + not just from the mesh. + +## Step 3: Verify the service locally + +Open a second terminal. From the same host, fetch the page: + +```sh +curl -6 "http://[$FIPS0_ADDR]:8080/" +``` + +Expect: + +```text +

Hello from the mesh

+``` + +Now fetch it by name. Both forms should work: + +```sh +curl -6 "http://${NPUB}.fips:8080/" +``` + +The daemon's local DNS responder turned the npub-form name into +the `fd97:...` address, and the request landed at your HTTP +server. + +> **What just happened.** The kernel routed your local request +> via the loopback path because the destination address is +> assigned to one of your own interfaces. You exercised the +> client side (DNS, IPv6 socket open, HTTP request) and the +> server side (HTTP listener, response). What you have not yet +> verified is reachability *from another mesh node*. That is +> the next concern. + +## Step 4: Reachability from a mesh node + +Any mesh node — a direct peer, or a node several hops away — +reaches your service the same way you reached `test-us01` in +[reach-mesh-services](reach-mesh-services.md): it looks up +`.fips`, gets back your `fd97:...` address, opens +a TCP connection to it, and the FIPS data plane carries the +packets across the mesh to you. From the remote host the curl +looks identical to yours: + +```sh +curl -6 "http://${NPUB}.fips:8080/" +``` + +If you have a second machine on the mesh — or you can ask +another operator to try it — this is the moment to confirm. +A node that can already `ping6 ${NPUB}.fips` should also be +able to fetch your page. If it can ping but the curl times +out, jump to [Troubleshooting](#troubleshooting) — but most +likely the firewall step in this tutorial has not happened +yet, so a remote attempt right now will succeed straight +through to your HTTP server. + +That is the problem. With no firewall in place, *any* mesh +node that can route to you — your direct peers, and every node +beyond them in the mesh — can reach port 8080. You have not +yet made a deliberate decision about whether you want that. +The rest of this tutorial replaces the implicit "every port +with a listener is reachable" with an explicit "only the ports +I have opened are reachable, optionally only from specific +mesh nodes." That is the firewall's job, and it is the only +mechanism in play in the rest of this tutorial. + +(There is a separate, unrelated control called the *peer ACL* +that decides which npubs may establish a peer connection with +your node at the transport layer. It is not part of the +firewall and does not affect what is described below; Step 7 +is a brief signpost to it.) + +## Step 5: Activate the mesh firewall baseline + +FIPS ships a default-deny nftables baseline at +`/etc/fips/fips.nft` that restricts inbound traffic on `fips0` +to ICMPv6 echo and conntrack replies. The baseline is **not** +enabled by default — activation is an explicit step the +operator has to take. + +Activate it: + +```sh +sudo systemctl enable --now fips-firewall.service +``` + +This loads the table immediately and arranges for it to load +on every subsequent boot. Confirm: + +```sh +sudo nft list table inet fips +``` + +You will see one chain named `inbound` hooked at `input`, +roughly: + +```text +table inet fips { + chain inbound { + type filter hook input priority filter; policy accept; + iifname != "fips0" return + ct state established,related accept + icmpv6 type echo-request accept + counter packets 0 bytes 0 drop + } +} +``` + +The chain admits ICMPv6 echo (so `ping6` from any mesh node +still works) and conntrack replies (so your *outbound* +connections still get their replies back). Everything else +inbound on `fips0` hits the final `counter ... drop`. + +> **What this changed.** Your HTTP server is still running and +> still reachable *from this same host* (same-host traffic to +> `fd97:...` goes via the loopback path, which has +> `iifname != "fips0"` and short-circuits at the first rule). +> But any mesh node trying to reach `fd97:...:8080` now has its +> TCP SYN dropped before it can reach your server. From the +> remote end the connection times out. + +So the firewall is in the right shape but in the wrong state +for our purpose: we *want* mesh nodes to reach port 8080. The +next step opens that one port. + +## Step 6: Open port 8080 via a drop-in + +Drop-ins live under `/etc/fips/fips.d/` with the `.nft` +suffix. Each file is included into the `inbound` chain at the +marked point and may contain any nftables rule lines valid in +that context. + +Create one for your HTTP service: + +```sh +sudo tee /etc/fips/fips.d/http-mesh-demo.nft >/dev/null <<'EOF' +tcp dport 8080 accept +EOF +``` + +Reload the firewall: + +```sh +sudo systemctl reload-or-restart fips-firewall.service +``` + +Confirm the rule is live: + +```sh +sudo nft list table inet fips +``` + +The `inbound` chain now contains your `tcp dport 8080 accept` +rule between the conntrack rule and the final `counter drop`. + +A curl from any mesh node will now reach the HTTP server. The +path is: remote node's mesh data plane → forwarded across the +mesh → your direct peer's link to you → `fips0` ingress → +`inbound` chain → matches `tcp dport 8080 accept` → delivered +to the HTTP server. + +If you only want to expose the service to a *specific* node +or set of nodes, source-filter the rule. The address filter +applies to the mesh-source address as it arrives on `fips0`, +which is the originating node's address — not necessarily a +direct peer. Replace the drop-in contents with something like: + +```nft +ip6 saddr fd97:1234:5678:9abc:def0:1234:5678:9abc tcp dport 8080 accept +``` + +The source address is the node's mesh address, which it +publishes in its `fips.pub` (and which you can resolve from +its npub). For multiple nodes, use a set: + +```nft +ip6 saddr { + fd97:1111:2222:3333:4444:5555:6666:7777, + fd97:8888:9999:aaaa:bbbb:cccc:dddd:eeee +} tcp dport 8080 accept +``` + +For the worked example, leave the drop-in unfiltered — any +mesh node that can route to you can fetch your page. + +## Step 7: A note on the peer ACL + +The firewall you just configured is the only control in scope +for this tutorial. There is a separate, optional control +called the *peer ACL* that you may run across in other docs; +it is unrelated to the firewall and worth a sentence here only +so you do not confuse the two. + +The peer ACL decides which npubs may establish a peer +connection with your node at the transport layer. It does not +look at ports, drop-ins, or `fips0` traffic. You do not need +it for this tutorial. + +For when you do: + +- [../how-to/enable-nostr-discovery.md](../how-to/enable-nostr-discovery.md) + — operator recipe. +- [../reference/security.md § Peer ACL](../reference/security.md#peer-acl) + — file format, evaluation order, alias handling. + +## Step 8: Stop the server and tidy up + +When you are done, stop the HTTP server in the first terminal +with `Ctrl-C`. The drop-in stays in place; remove it if you +do not want port 8080 reachable after the demo: + +```sh +sudo rm /etc/fips/fips.d/http-mesh-demo.nft +sudo systemctl reload-or-restart fips-firewall.service +``` + +The `fips-firewall.service` itself can stay enabled — +default-deny on `fips0` is a sensible posture even with no +extra services running. To turn it back off: + +```sh +sudo systemctl disable --now fips-firewall.service +``` + +## What you've learned + +- **Bind interface = audience.** Binding to a specific address + opts in to one audience; binding to wildcard + (`0.0.0.0` / `[::]`) opts in to *all* of them, including + ones you forgot you had. For mesh-only exposure, bind to + your `fd97:...` address. +- **Same-host loopback is misleading.** A local curl to your + own `fd97:...` address goes via the loopback path, not + through `fips0` ingress. To actually verify mesh-side + reachability you need a second machine, or to read what + the firewall is doing in `nft list table inet fips`. +- **The mesh firewall is opt-in.** `fips-firewall.service` is + not enabled by default. Once it is enabled, `fips0` is + default-deny except for ICMPv6 echo and conntrack replies. +- **Ports open via drop-ins.** Each file under + `/etc/fips/fips.d/*.nft` adds rules into the `inbound` + chain. Source-filter with `ip6 saddr` to scope a port to + specific mesh nodes. +- **Two independent controls at two different layers.** The + firewall is a layer-3 filter on `fips0`: it controls which + TCP/UDP ports are reachable and (optionally) which mesh + source addresses may reach them. The peer ACL is a + transport-layer admission filter on Noise handshakes: it + controls which npubs may become direct peers of your node. + They are unrelated — the ACL does not touch fips0 traffic, + and the firewall does not look at npubs. + +You now have the mental model for hosting any IPv6 service +behind a deliberate exposure policy. The mechanics generalize: +SSH on port 22, a database on port 5432, a custom protocol on +its own port — same `--bind` rule, same drop-in shape. + +## Troubleshooting + +- **A remote mesh node cannot reach the service after the + firewall reload.** Check the drop-in syntax with + `sudo nft -c -f /etc/fips/fips.nft` before reloading; a + syntax error in any drop-in causes the whole table to fail + to load and the previous rules persist. Then + `sudo nft list table inet fips` to confirm your + `tcp dport 8080 accept` rule is present in the `inbound` + chain. +- **Local curl works, remote curl times out.** The packet is + reaching `fips0` ingress and being dropped by the baseline. + Either your drop-in did not load (see above) or it has a + source filter that excludes the remote node's address. +- **Local curl fails after binding to fips0.** Double-check + that your `FIPS0_ADDR` matches the address shown in + `ip -6 addr show fips0`. The Python server message also + echoes the bound address — confirm it starts with `fd97:`, + not `127.0.0.1` or `::`. +- **`Address already in use` from Python.** Another process + holds port 8080. Pick a different port (`8081`, `9000`, …) + for both the `python3 -m http.server` invocation and the + drop-in. +- **Watch the firewall counter to confirm drops.** The + `counter ... drop` line at the bottom of the chain + increments on every dropped inbound packet. After a remote + mesh node attempts to reach a port you have not opened, + `sudo nft list table inet fips` will show the counter + packet count rising. + +## What's next + +- [ground-up-mesh.md](ground-up-mesh.md) — Bring up two devices + on a shared physical link — Ethernet, WiFi, or Bluetooth — + with no pre-existing IP infrastructure between them. The + second deployment mode of FIPS, where the mesh is the + network rather than an overlay on top of one. Coexists with + overlay peers; the same daemon carries both. + +For more depth on the firewall and ACL surface: + +- [../how-to/enable-mesh-firewall.md](../how-to/enable-mesh-firewall.md) + — operator recipes for the baseline, drop-in patterns, + and how to fold the baseline into an existing + `nftables.conf`. +- [../reference/security.md](../reference/security.md) — + consolidated security reference: nftables baseline rules, + drop-in format, peer ACL semantics, default exposures by + transport, threat-resistance matrix. +- [../design/fips-security.md](../design/fips-security.md) — + threat model, why the baseline is opt-in, the metadata- + privacy posture. + +If you want to host a service that is *not* on a FIPS node — say, +an existing HTTP server on a regular LAN box — and expose it to +mesh peers through a `fips-gateway`, that's the inbound +port-forward mode: the gateway runs a mesh-side listener on `fips0` +and forwards to a LAN target. The operator recipe is at +[../how-to/deploy-gateway.md#inbound-port-forwarding](../how-to/deploy-gateway.md#inbound-port-forwarding); +a hand-held walk-through on an OpenWrt AP is at +[deploy-fips-gateway.md](deploy-fips-gateway.md) under "Advanced" +in [README.md](README.md). diff --git a/docs/tutorials/ipv6-adapter-walkthrough.md b/docs/tutorials/ipv6-adapter-walkthrough.md new file mode 100644 index 0000000..90f88d3 --- /dev/null +++ b/docs/tutorials/ipv6-adapter-walkthrough.md @@ -0,0 +1,231 @@ +# IPv6 Adapter Walkthrough + +You have completed [join-the-test-mesh](join-the-test-mesh.md). +Your daemon is peered with `test-us01` and you can ping mesh +nodes by `.fips` name. This tutorial walks the plumbing that +makes that possible: what happens between the moment your shell +types `ssh user@.fips` and the moment a TCP SYN arrives at +sshd on the far side. Each step is something you can observe +with the running daemon from the previous tutorial. + +By the end you will be comfortable reading `fipstop` output and +you will know which design doc to consult when something looks +off. + +> **Prerequisites.** The daemon from +> [join-the-test-mesh.md](join-the-test-mesh.md) is running and +> peered with at least one test-mesh node, and your host's local +> resolver is forwarding `.fips` queries to the daemon's DNS +> responder (the system fips-dns.service drop-in does this +> automatically on systemd hosts). + +## The path we're tracing + +```text +shell ──ssh──> libc resolver ──.fips──> fips DNS ──AAAA──> fd97:...:test-us01 + │ + ▼ + kernel IPv6 stack + │ + ▼ + fips0 (TUN) + │ + ▼ + your fips daemon + (FSP session setup, + FMP forwarding) + │ + UDP / internet + ▼ + test-us01's fips daemon + │ + ▼ + fips0 (TUN) + │ + ▼ + kernel IPv6 stack + │ + ▼ + sshd +``` + +In a multi-hop mesh the middle would have additional FMP +forwarders between your daemon and the destination. For this +walkthrough you have a single direct link to `test-us01`, which +keeps the trace simple. + +## Step 1: Watch the DNS resolution + +Ask the system resolver to translate `test-us01`'s npub into its +mesh address: + +```sh +dig npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98.fips AAAA +short +``` + +You should see one AAAA record returning a `fd97:...` address. +The prefix is the FIPS ULA range (`fd00::/8`, with `fd97:...` +covering the address space derived from npubs). + +The query went through `systemd-resolved` (or your platform +equivalent), which routed `.fips` queries to the daemon's local +responder via the drop-in installed by `fips-dns.service`. To +confirm, query the daemon directly: + +```sh +dig @::1 -p 5354 npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98.fips AAAA +short +``` + +Same answer, same fast turnaround — no external DNS traffic in +either case. + +The mapping `npub → fd00::/8 address` is deterministic. The +responder hashes the public key into 16 bytes, prepends the +prefix, and returns the result. There is no shared registry; the +address space is self-allocating from the public-key namespace. + +If you ask for any non-`.fips` suffix, the responder returns +`REFUSED` — it is intentionally a stub for this single zone, not a +recursive resolver. An unknown `.fips` name returns `NXDOMAIN`. + +The full DNS integration is documented in +[../design/fips-ipv6-adapter.md](../design/fips-ipv6-adapter.md). + +## Step 2: Watch the session being created + +Open `fipstop` against your daemon's control socket: + +```sh +sudo fipstop +``` + +Press `Tab` until you reach the **Sessions** tab. Before any +TCP traffic to `test-us01`, the table is empty (or has rows from +earlier exchanges). + +In another terminal, kick off a TCP connection from your host +toward `test-us01`: + +```sh +ssh -o ConnectTimeout=5 user@test-us01.fips +``` + +(`test-us01.fips` resolves to the same address as the npub +form via the installer's `/etc/fips/hosts` entry.) + +(It is fine if the SSH attempt fails authentication or if no +sshd is exposed on the far side — what we want to observe is the +session machinery firing, not a successful login.) + +In `fipstop`'s Sessions tab you should see a new row appear with: + +- `state` cycling from `initiating` to `awaiting_msg3` to + `established` (the three FSP handshake states). +- `display_name` showing `test-us01` (the `alias` you set in your + `peers:` block in the previous tutorial). +- A non-zero `last_activity_ms`. + +Once `established`, the session row stays put until idle-timeout +expires. The traffic counters and MMP metrics tick as data flows. + +> **Watch for.** Some intermediate states may be too fast to +> see at the default `fipstop` refresh rate of 2 s. Run +> `sudo fipstop -r 1` for a faster refresh during the exercise. + +## Step 3: Watch the per-session metrics + +Switch to the **Performance** tab. Each established session has +a session-layer MMP entry showing: + +- `srtt_ms` — smoothed end-to-end round-trip time. Over a + public-internet path this typically lands in the tens of + milliseconds; for a US-coast destination from a US client you + might see 30–80 ms steady-state. +- `loss_rate` — fraction of in-flight payloads inferred lost from + counter gaps. Stays at 0 on a healthy link; small bursts during + congestion or path changes. +- `path_mtu` — the end-to-end MTU the session-layer MMP currently + believes is in force. Starts at the IPv6 floor and climbs as + PathMtuNotification echoes arrive. +- `etx` and `goodput_bps` — derived metrics, useful as + steady-state indicators. + +The same metrics are available without the TUI: + +```sh +sudo fipsctl show sessions | jq '.sessions[] | {display_name, state, mmp}' +``` + +What these numbers mean is documented in +[../design/fips-mmp.md](../design/fips-mmp.md). Briefly: SRTT is +RFC 6298-style with α = 1/8; loss is bidirectional, inferred from +counter gaps in MMP reports; path MTU is end-to-end-echoed with +hysteresis on increase. + +## Step 4: Watch the link below the session + +Switch to the **Peers** tab. Each authenticated peer has its own +link-layer MMP block, distinct from the session-layer one above. +The link-layer metrics measure a single hop (here, your daemon +↔ `test-us01` over UDP), independent of any session that +traverses it. + +Compare the link-layer SRTT for `test-us01` to the session-layer +SRTT of the session you just created. Because your reach to +`test-us01` is one direct hop, the two should be very close — +the session has no transit forwarders to add latency. + +If you reach a node that `test-us01` forwards to (try the +`test-us02` ping from the previous tutorial), the session-layer +SRTT for that destination will be measurably larger than the +link-layer SRTT to `test-us01`. The difference is the time +`test-us01` spent forwarding plus the hop from `test-us01` to +`test-us02`. + +In a deeper mesh this divergence grows: link-layer SRTT measures +the direct neighbour, session-layer SRTT measures the full +end-to-end path. + +## Step 5: Read the relevant design docs + +You have now seen the moving parts. To go from "I can read these +metrics" to "I understand why each one moves the way it does": + +- [../design/fips-ipv6-adapter.md](../design/fips-ipv6-adapter.md) + — DNS responder, identity cache, TUN reader/writer, IPv6 header + compression, MTU enforcement at the TUN boundary. +- [../design/fips-session-layer.md](../design/fips-session-layer.md) + — FSP session lifecycle: msg1 / msg2 / msg3, the rekey state + machine, the drain window for old sessions during cutover. +- [../design/fips-mmp.md](../design/fips-mmp.md) — both link-layer + and session-layer MMP: report format, SRTT estimation, + loss/jitter/ETX computation, the trend indicators. +- [../design/fips-mtu.md](../design/fips-mtu.md) — what `path_mtu` + in `show sessions` means: the proactive forward-path field, the + reactive `MtuExceeded` mechanism, the hysteresis on increase. +- [../design/fips-architecture.md](../design/fips-architecture.md) + — the two-layer encryption model: link-layer Noise IK over + each hop, end-to-end Noise XK over the session. + +## What you've learned + +- A `.fips` name resolves through a daemon-local stub responder. + The mapping from npub to `fd00::/8` address is deterministic + and needs no registry. +- The kernel IPv6 stack treats the TUN adapter as an ordinary + interface; packets to `fd00::/8` go out via that route. The + daemon reads them off the TUN, looks up an FSP session for the + destination (creating one if needed), and forwards them onward + through its peers. +- The session layer (FSP) and the link layer (FMP) each maintain + their own MMP metrics. Session-layer metrics measure the path + end-to-end; link-layer metrics measure a single hop. The two + align when the destination is your direct peer; they diverge + when traffic traverses additional hops. +- `fipstop` exposes both views in real time. `fipsctl show sessions`, + `fipsctl show peers`, and `fipsctl show transports` cover the + same ground programmatically. + +When something looks off in production, the `fipsctl show *` +queries are usually the first stop; the relevant design doc tells +you what the numbers mean and what they should do. diff --git a/docs/tutorials/join-the-test-mesh.md b/docs/tutorials/join-the-test-mesh.md new file mode 100644 index 0000000..4028a4b --- /dev/null +++ b/docs/tutorials/join-the-test-mesh.md @@ -0,0 +1,324 @@ +# Join the FIPS Test Mesh + +In this tutorial you will connect your FIPS daemon to a public +test peer over UDP, watch the link come up, and reach the peer's +mesh address from your machine. By the end you will have seen one +complete end-to-end flow — config, handshake, live link, traffic +— for a real peer somewhere out on the public internet. + +The whole exercise should take about ten minutes. If you have +already worked through [getting-started.md](../getting-started.md) +and have the `fips` daemon running on your host, you have +everything you need. + +## What you'll build + +```text + ┌────────────────────┐ UDP/IPv4 ┌──────────────────────┐ + │ your fips node │ ──────────────────────── │ test-us01 │ + │ ephemeral npub │ test-us01.fips.network │ npub1qmc3...zel98 │ + │ fips0 fd97:..:Y │ :2121 │ fips0 fd97:..:T │ + └────────────────────┘ └──────────────────────┘ +``` + +Your daemon will peer with one of the public test nodes the +project maintains. `test-us01` has a stable DNS name, listens on +UDP/2121, and is reachable from any network that permits arbitrary +outbound UDP. + +> **Peer vs. node.** In FIPS terminology, a *peer* is a node +> you have a direct link to — same Noise IK handshake, same +> transport socket. A *node* is any participant on the mesh, +> whether you peer with it directly or reach it through one or +> more hops via your peer's connections. Peering is a local +> configuration choice; reachability is mesh-wide. One good peer +> connects you to everyone the rest of the mesh connects to. + +After the link to `test-us01` establishes, your daemon's `fips0` +adapter can reach `test-us01` itself and — through `test-us01`'s +connections — any other node on the test mesh, exactly as if you +had a direct connection to each of them. + +> **About the test mesh.** The project maintains a small roster +> of public test nodes (`test-us01` through `test-uk01`) intended +> for new-user on-ramps and integration testing. They accept +> inbound peering from arbitrary npubs without prior coordination. +> A future reference doc will list the full roster; for this +> tutorial you only need `test-us01` as your peer, and `test-us02` +> later on as a second mesh destination to demonstrate +> forwarding. + +## Step 1: Confirm the daemon is running + +```sh +sudo systemctl status fips +``` + +Expect `active (running)`. If it is not running, the +[getting-started](../getting-started.md) guide covers installation +and service management. While you're checking, note your daemon's +current npub: + +```sh +sudo fipsctl show status +``` + +Look for the `npub` field. With the default ephemeral-identity +config, this regenerates on every restart — that is fine for the +tutorial. `test-us01` admits any inbound npub. + +## Step 2: Add a static peer to the daemon config + +Edit `/etc/fips/fips.yaml`. Find the line that reads `peers: []` +and replace it with: + +```yaml +peers: + - npub: "npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98" + alias: "test-us01" + addresses: + - transport: udp + addr: "test-us01.fips.network:2121" + connect_policy: auto_connect +``` + +What each field does: + +- `npub` — the canonical Nostr public key of `test-us01`. This is + who your daemon will mutually authenticate with over Noise IK. +- `alias` — a short name your daemon will use when referring to + this peer in logs and `fipsctl show peers` output. Optional. +- `addresses` — one or more transport endpoints. UDP on the + published hostname and port is the most direct path. +- `connect_policy: auto_connect` — your daemon initiates an + outbound connection rather than waiting for the peer to reach + in. + +## Step 3: Restart the daemon + +```sh +sudo systemctl restart fips +``` + +Watch the daemon's journal as it comes back up and dials the +peer: + +```sh +sudo journalctl -u fips -f +``` + +Within a few seconds you should see lines mentioning: + +- An outbound connection attempt to `test-us01` or + `test-us01.fips.network:2121` +- A handshake completion (a "Noise IK link handshake complete" + style line, or "peer authenticated" with the test-us01 npub) +- An MMP / link metrics entry naming `test-us01` + +If the handshake does not complete within roughly 30 seconds, jump +to [Troubleshooting](#troubleshooting) below. + +## Step 4: Verify the link + +```sh +sudo fipsctl show peers +``` + +Expect one entry whose `alias` is `test-us01`. Useful fields: + +- `connectivity` — should be active / authenticated. +- `transport_addr` — the resolved UDP endpoint your daemon is + using to reach `test-us01`. +- `transport_type` — `udp`. +- `mmp.srtt_ms` — appears once the first MMP report has been + exchanged. This is your round-trip time to `test-us01`. + +The transport view confirms your UDP listener and the peer +mapping: + +```sh +sudo fipsctl show transports +``` + +## Step 5: Ping your peer + +`test-us01`'s mesh address derives from its npub. Address it as +`.fips` and your daemon's local DNS responder will translate +that to its `fd97:...` mesh address. + +First see the resolved address: + +```sh +dig npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98.fips AAAA +short +``` + +You should see one `fd97:...` line. + +Now ping it: + +```sh +ping6 -c 4 npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98.fips +``` + +Expect four replies. The first packet may take noticeably longer +than subsequent ones — that round trip includes destination +discovery, FSP session establishment, and the proactive path-MTU +probe. After that, the RTT settles to a steady value reflecting +the path between your host and `test-us01`. + +This confirms the direct link works. So far, though, you have only +reached the peer you configured. The next step demonstrates the +mesh-wide reach that peering buys you. + +## Step 6: Reach a different node through the mesh + +`test-us02` is another public test node. You did **not** add it +to your `peers:` block — your daemon has no direct link to it. +But because `test-us01` participates in the same mesh and has its +own connections to other nodes, your daemon can reach `test-us02` +through `test-us01` without any additional configuration. + +```sh +ping6 -c 4 npub10yffd020a4ag8zcy75f9pruq3rnghvvhd5hphl9s62zgp35s560qrksp9u.fips +``` + +Same form, different npub. Expect replies. The packets travel +from your daemon to `test-us01` over the direct UDP link, then +onward through `test-us01` (and possibly other test-mesh nodes) +to reach `test-us02`'s `fips0` adapter. Replies retrace the path. + +This is the central FIPS guarantee: **peering is local, but +reachability is mesh-wide.** You only need one good peer to talk +to everyone else they (transitively) talk to. + +If the `test-us02` ping fails while the `test-us01` ping +succeeded, the test mesh's routing between those two nodes is +momentarily unhealthy — try again in a minute, or pick a different +test node from the roster. The link to your peer is unaffected. + +## What you've learned + +You now have a single FIPS node connected to one peer in the +public test mesh, with reach to every node that mesh routes you +to. You have seen: + +- **Identity.** Your daemon's ephemeral keypair authenticated to + `test-us01` over Noise IK without either side trusting anyone in + advance. +- **Transports.** A UDP socket on your host carries + authenticated, encrypted mesh frames to your peer. No central + server, no VPN concentrator. +- **Peering vs. reachability.** You configured one peer + (`test-us01`) and got reach to a second node (`test-us02`) for + free, through the mesh. The same shape extends to every other + node `test-us01` can reach. +- **Naming.** The local `.fips` resolver translated npub-form + hostnames into their `fd97:...` mesh addresses with no external + DNS traffic. +- **End-to-end.** ICMPv6 traffic over the FIPS data plane reached + both destinations and came back, end-to-end encrypted along + every link layer in the path. + +> **By the way: shortnames.** Those long `npub1...fips` +> destinations are the canonical addresses, but the installer +> ships an `/etc/fips/hosts` file with shortname entries for +> the public test mesh, so `test-us01.fips` and +> `test-us02.fips` resolve to the same addresses without +> typing 80 characters of bech32. You can add your own entries +> too. See +> [../how-to/host-aliases.md](../how-to/host-aliases.md). The +> rest of the tutorials use shortnames where they're available. + +## Troubleshooting + +If the handshake does not complete: + +- **Outbound UDP may be blocked.** Some networks filter + arbitrary outbound UDP or block return traffic. From a + UDP-filtered network you cannot reach peers that only + publish UDP endpoints — your reachable peers are limited + to those that accept incoming TCP (outbound TCP is + typically allowed even on networks that block UDP). The + test-mesh nodes publish a TCP endpoint on port 443 for + exactly this case; replace the `udp` entry in the peer's + `addresses:` block with the TCP equivalent: + + ```yaml + addresses: + - transport: tcp + addr: "test-us01.fips.network:443" + ``` + + Restart the daemon and re-check `fipsctl show peers`. The + link will be slower than UDP but is the supported transport + for restrictive egress environments. +- **Confirm the testnode is reachable at the IP layer.** Run + `dig +short test-us01.fips.network` to confirm DNS, then + `nc -uvz test-us01.fips.network 2121` to confirm UDP + reachability. +- **Confirm your config parsed.** `sudo journalctl -u fips -n 50` + near the daemon-start time will show config-load lines and any + parse errors. +- **Time skew.** A heavily skewed system clock can make + signature validation fail. `timedatectl status` should show + the system clock as synchronized. + +## What's next + +These are the natural follow-on tutorials in the new-user +progression. Some are still being written and will appear +alongside this one in the [tutorials/](.) directory. + +- **Make your node's identity persistent.** + [persistent-identity](persistent-identity.md) walks through + pinning the daemon to a stable Nostr keypair so your npub + does not change across restarts — the prerequisite for other + operators adding you to their `peers:` blocks. + +- **Resolve peers via Nostr.** + [resolve-peers-via-nostr](resolve-peers-via-nostr.md) is the + smallest useful step toward Nostr-mediated discovery: + configure a peer by npub alone and let the daemon look up + the current endpoint from public relays. The first of three + tutorials covering Nostr discovery; the others — + [advertise-your-node](advertise-your-node.md) and + [open-discovery](open-discovery.md) — round out the + publish and ambient-consume sides. + +- **Trace a connection end-to-end.** + [ipv6-adapter-walkthrough](ipv6-adapter-walkthrough.md) walks + the data path from a `.fips` DNS query through session setup + to the far-side TUN adapter, using `fipstop` and `fipsctl` to + observe each step. + +- **Reach services on other mesh nodes.** + [reach-mesh-services](reach-mesh-services.md) generalizes the + `ping6` you just ran to any IPv6-capable tool — `nc`, + `traceroute6`, `curl`, `ssh` — addressed by `.fips`. + The point is that the FIPS data plane is just IPv6; + applications don't need to know they're on a mesh. + +- **Host a service of your own.** + [host-a-service](host-a-service.md) walks through bringing up + a small HTTP server bound to `fips0` so mesh nodes can reach + it, with a deliberate exposure decision (mesh-only vs every + interface), the mesh firewall, and a brief signpost to the + separate, unrelated peer ACL (which controls who may peer + with your node, not what they can reach on your `fips0`). + +- [ground-up-mesh](ground-up-mesh.md) — Bring up two devices on + a shared physical link — Ethernet, WiFi, or Bluetooth — with + no pre-existing IP infrastructure. The second deployment mode + of FIPS, coexisting on the same daemon as the overlay peer to + `test-us01` you just configured. + +For "what just happened, in detail": + +- [../design/fips-architecture.md](../design/fips-architecture.md) — + the protocol stack and the two-layer encryption model. +- [../design/fips-mesh-layer.md](../design/fips-mesh-layer.md) — + Noise IK link encryption, hop-by-hop forwarding. +- [../design/fips-session-layer.md](../design/fips-session-layer.md) + — end-to-end Noise XK, session lifecycle. +- [../design/fips-ipv6-adapter.md](../design/fips-ipv6-adapter.md) — + the TUN, the local DNS responder, MTU enforcement. diff --git a/docs/tutorials/open-discovery.md b/docs/tutorials/open-discovery.md new file mode 100644 index 0000000..e17a63d --- /dev/null +++ b/docs/tutorials/open-discovery.md @@ -0,0 +1,338 @@ +# Open Discovery: Find Peers Without Configuration + +After +[advertise-your-node](advertise-your-node.md), your daemon +publishes its endpoint on Nostr and other open-discovery nodes +on the test mesh have already started dialing you. This +tutorial flips the symmetry: turn your own daemon into a +consumer of every advert in the namespace, so any operator +who's publishing becomes a candidate peer of yours. + +The whole exercise should take about ten minutes. After it, +you'll be a full participant in the ambient +`fips-overlay-v1` namespace — publishing your own advert +*and* discovering everyone else's. + +## What you'll build + +```text + ┌─────────────────────────────────────────────────────┐ + │ Nostr relays — fips-overlay-v1 namespace │ + │ adverts from: test-us01..uk01, others │ + └────────────────┬─────────────────────────▲──────────┘ + │ │ + │ subscribe to all │ your own + │ adverts in namespace │ advert + │ │ + ▼ │ + ┌────────────────────────────────────────────────────┐ + │ your fips daemon (policy: open) │ + │ │ + │ peers list grows ambient as adverts arrive: │ + │ test-us01 ← was static │ + │ test-us03 ← inbound (was already there) │ + │ test-de01, test-es01, test-uk01, test-us04... │ + │ plus any other publisher in the namespace │ + └────────────────────────────────────────────────────┘ +``` + +You will change one thing in `/etc/fips/fips.yaml`: under +`discovery.nostr`, set `policy: open` (the default is +`configured_only`). After restart, the daemon subscribes to +every Kind 37195 advert in the `fips-overlay-v1` namespace and +queues the publishers for outbound connection attempts. + +## How open discovery works + +> **Discovery policy.** `discovery.nostr.policy` decides what +> the daemon does with incoming advert data. Two values: +> +> - `configured_only` (the default): the daemon only consumes +> adverts for peers it has explicitly listed with +> `via_nostr: true`. This is what you've been running +> through the previous two Nostr tutorials. +> - `open`: the daemon subscribes to every advert in the +> configured `app` namespace. Any publisher becomes a +> candidate peer, no `peers:` list entry needed. + +Switching the policy doesn't disturb anything that was already +working: + +> **Open is additive, not exclusive.** Switching to +> `policy: open` doesn't replace your static peers — both +> mechanisms run in parallel. Configured peers stay in your +> `peers:` block and continue to be dialed via their static +> addresses or `via_nostr` lookups; open-discovered peers +> stack on top from the ambient namespace. You can run open +> discovery with a populated `peers:` list (the path this +> tutorial walks, since you're keeping `test-us01`), with +> `peers: []` for pure ambient discovery, or with a long +> `peers:` list and open layered on top to broaden reach. + +The namespace is what scopes who's visible to whom: + +> **The namespace is the scope.** `discovery.nostr.app` +> defaults to `fips-overlay-v1` — the namespace the public +> test mesh uses. Setting a different value (e.g., +> `app: "my-experiment.v1"`) carves out a private discovery +> set: only nodes that share your `app` value find each +> other. For this tutorial we stay on the default and join +> the public namespace. + +Open discovery is best-effort by design — not every +discovered peer will connect: + +> **Best-effort, by design.** Many discovered peers will fail +> to connect — they may be offline, behind incompatible NAT, +> running a different protocol version, or have peer ACLs +> that reject you. That's normal for ambient discovery; +> connection attempts are best-effort and rate-limited by +> `open_discovery_max_pending` (default 64). Your peer list +> grows over time as candidates land in the cache, not all +> at once on restart. + +The peer ACL is the admission-control surface, separate from +discovery: + +> **Open is admission-free under your peer ACL.** Open +> discovery does not bypass the peer ACL — every candidate +> still has to pass it. By default the ACL accepts everyone, +> so any publisher in the namespace becomes a connection +> candidate. If you rely on a non-default ACL for admission +> control, verify it is set the way you want *before* +> enabling `policy: open`. See +> [../reference/security.md](../reference/security.md) for +> the ACL format. + +## Step 1: Confirm your starting state + +You should be coming out of +[advertise-your-node](advertise-your-node.md) with: + +- Persistent identity, advertising enabled + (`discovery.nostr.advertise: true`), UDP advertising on + Nostr (`transports.udp.advertise_on_nostr: true`). +- A static `test-us01` peer entry that the daemon dials + outbound; possibly an inbound `test-us03` peer (the + open-discovery test mesh node that dialed in after seeing + your advert). + +Capture the current peer count for comparison: + +```sh +sudo fipsctl show peers | grep -c 'npub' +``` + +You'll likely see 1 (just `test-us01`) up to a handful, depending +on how many open-discovery test mesh nodes have already dialed +you. + +## Step 2: Switch the discovery policy to `open` + +Open `/etc/fips/fips.yaml` and find the `discovery.nostr` +block. Add (or change) the `policy` line: + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: true + policy: open +``` + +That's the only change. Notes on what you don't have to touch: + +- **You don't have to drop the `peers:` block.** Static peers + and open-discovered peers coexist; static entries get + priority for direct dialing, open-discovered ones are + layered on top. +- **You don't have to set `app`.** The default + `fips-overlay-v1` is the namespace the public test mesh + uses; staying with the default is what gets you discovered. +- **You don't have to set `open_discovery_max_pending`.** The + default of 64 is plenty for a tutorial; only tune it if you + see the daemon log + `open-discovery: max-pending reached, deferring`. + +Save the file. + +## Step 3: Restart and let discovery populate + +```sh +sudo systemctl restart fips +``` + +Give the daemon a minute or two. Open discovery doesn't fire +all at once — the daemon subscribes to the relays, accumulates +adverts as they arrive (or as relays return historical +events), and queues each publisher for a dial attempt. + +## Step 4: Inspect the discovered peer list + +```sh +sudo fipsctl show peers +``` + +You should see considerably more entries than before: + +- `test-us01` — still there, still using the configured + static dial path. +- `test-us03` — same as before (the open-discovery test + mesh node that dials you when it sees your advert). +- `test-us04`, `test-de01`, `test-es01`, `test-uk01` — the + other test mesh nodes; your daemon picked their adverts up + from the namespace and dialed them. +- Plus any other operator publishing on `fips-overlay-v1` + (community nodes, other operators' experiments). + +Each entry has its own `connectivity` state. Some will be +`active` (handshake completed). Some will show as +`connecting` and may transition to `failed` shortly after — +that's normal; the publisher might be offline, the advert +might be stale, or NAT traversal failed for that pair. + +To get a list of just the active links: + +```sh +sudo fipsctl show peers | jq '.peers[] | select(.connectivity == "active") | .npub' +``` + +The peer count will continue to drift over time as adverts +expire and new ones arrive. This is steady-state behavior, +not a transient. + +## Step 5: Confirm the mesh-wide reach + +You can now reach any of the discovered nodes the same way +you reached `test-us01` and `test-us02` in +[join-the-test-mesh](join-the-test-mesh.md). Pick one of the +new test mesh nodes from the peer list and ping it by its +shortname: + +```sh +ping6 -c 4 test-uk01.fips +``` + +(`test-uk01` is the United Kingdom test node; the installer's +`/etc/fips/hosts` entry resolves it to the corresponding npub. +Substitute any active peer's shortname or full +`.fips`.) + +Expect four replies. The packet path may go through your +direct link to that peer (if the open-discovery dial succeeded +and the link is up) or via a test-mesh forwarder (if the +direct link is down but the destination is still reachable +through the mesh). Either way, the npub-as-name addressing +works the same way. + +## What you've learned + +- **Open discovery is the consume side of full ambient + participation.** With `policy: open` plus advertising, your + daemon both publishes its advert *and* consumes everyone + else's in the namespace. +- **The namespace defines the scope.** Everyone publishing + with the same `app` value finds each other; different + namespaces are isolated discovery sets. +- **Best-effort means failure is normal.** Many discovered + peers won't actually connect; that's expected and + rate-limited by `open_discovery_max_pending`. +- **Static and open coexist.** Configured peers keep working + with their own dial paths; open-discovered peers stack on + top. +- **The peer ACL still gates everything.** Open is + admission-free relative to discovery, not relative to your + ACL — the ACL is what you'd use to restrict who can connect + if you don't want a fully open posture. + +## Custom namespaces for private experiments + +If you want to use FIPS open discovery for a private set of +nodes — colleagues, a workshop cohort, a specific deployment +— set a custom `app` value: + +```yaml +discovery: + nostr: + enabled: true + advertise: true + policy: open + app: "my-team.experiment-1" +``` + +All nodes participating in the experiment use the same +`app` string. Pick something distinctive — short identifiers +risk colliding with other operators' experiments. Once your +nodes use a custom `app`, they no longer find or are found +by the public test mesh (the public mesh uses +`fips-overlay-v1`). + +## Troubleshooting + +If your peer list doesn't grow past the inbound peers from +the previous tutorial: + +- **Wait.** Open discovery accumulates adverts; the first + pass after restart can take a couple of minutes to populate + on a new subscription. +- **Verify the namespace.** With `app:` unset, the daemon + uses `fips-overlay-v1`. If you set a custom `app:` for an + experiment, your daemon is in a different namespace than + the public test mesh and will only find peers using the + same value. +- **Check relay reachability.** Open discovery is a + subscription rather than one-shot queries — if the + WebSocket connection to the relays is failing repeatedly, + no adverts arrive. Look for relay-connection errors in + `sudo journalctl -u fips -n 200`. +- **`policy: open` typo.** YAML accepts and ignores unknown + values silently. If `fipsctl show status` (or the daemon's + startup log) shows `policy: configured_only`, the YAML + didn't parse the new value — re-check spelling and + indentation. + +If too many peers are appearing and you want to dial down: + +- **Lower `open_discovery_max_pending`.** Default 64; setting + it to e.g. 16 caps in-flight connection attempts. Adverts + beyond that wait in a queue. +- **Use a custom `app`.** Move to a private namespace where + only nodes you're coordinating with publish. +- **Use the peer ACL.** See + [../reference/security.md](../reference/security.md) for + the ACL format if you want explicit allow/deny rules. + +## What's next + +- **Reach services on other mesh nodes.** + [reach-mesh-services](reach-mesh-services.md) drives `nc`, + `traceroute6`, `curl`, and `ssh` at peers by `.fips` name — + any of the open-discovered peers in your list, or any node + you reach through them. + +- **Host a service of your own.** + [host-a-service](host-a-service.md) brings up an HTTP server + addressable as `.fips`, bound to `fips0` so the + exposure is mesh-only, behind the mesh firewall. + +- [ground-up-mesh](ground-up-mesh.md) — Bring up two devices on + a shared physical link (Ethernet, WiFi, or Bluetooth) with no + pre-existing IP infrastructure. The second deployment mode of + FIPS, a parallel to the overlay-on-internet path the + Nostr-discovery tutorials covered. + +For the operator-style scenario reference covering all five +shapes of Nostr discovery side-by-side: + +- [../how-to/enable-nostr-discovery.md](../how-to/enable-nostr-discovery.md) + § Capability 3 — open discovery configuration knobs. + +For the wire format and discovery design: + +- [../reference/nostr-events.md](../reference/nostr-events.md) + — Kind 37195 advert format and the `app` namespace tag. +- [../design/fips-nostr-discovery.md](../design/fips-nostr-discovery.md) + — discovery runtime design, security and threat model. diff --git a/docs/tutorials/persistent-identity.md b/docs/tutorials/persistent-identity.md new file mode 100644 index 0000000..2d11951 --- /dev/null +++ b/docs/tutorials/persistent-identity.md @@ -0,0 +1,304 @@ +# Make Your Node's Identity Persistent + +After completing +[join-the-test-mesh](join-the-test-mesh.md), your daemon is +connected to the public test mesh — but its identity is +ephemeral. Every restart generates a fresh Nostr keypair, so the +npub the rest of the world would use to reach you changes every +time. This tutorial walks through pinning your node to a stable +keypair, locating it on disk, and protecting it. + +The whole exercise should take about ten minutes. + +## What you'll build + +```text + ┌─────────────────────────────────────────┐ + │ /etc/fips/fips.yaml │ + │ node: │ + │ identity: │ + │ persistent: true ← this flag │ + └────────────────┬────────────────────────┘ + │ daemon reads on start + ▼ + ┌─────────────────────────────────────────┐ + │ /etc/fips/fips.key 0600 root:root │ + │ /etc/fips/fips.pub 0644 root:root │ + └─────────────────────────────────────────┘ + your stable nsec / npub +``` + +After this tutorial your node will have: + +- A keypair on disk that the daemon reuses across restarts. +- An npub you can hand to other operators so they can add you to + their `peers:` list once and have the entry keep working. +- A clear understanding of which file holds the secret and how + to keep it that way. + +## Why a stable identity matters + +In FIPS your Nostr keypair *is* your node's identity in the most +literal sense. Several things derive from it: + +- Your `fd97:...` mesh address — derived from the public key. +- Your `.fips` DNS name — the npub itself with `.fips` + appended. +- Every authenticated connection — Noise IK at the mesh layer, + XK at the session layer, both prove you hold the matching + secret key. + +> **A keypair, briefly.** Nostr identities are secp256k1 +> keypairs. The private half is the `nsec` (a bech32-encoded +> secret key); the public half is the `npub` (a bech32-encoded +> public key). The daemon needs the `nsec` to sign messages and +> complete handshakes; the rest of the world only sees the +> `npub` and uses it as your address. + +The daemon supports two ways of holding that keypair: + +> **Ephemeral vs. persistent.** +> +> - *Ephemeral* (the default): the daemon mints a brand-new +> keypair every time it starts, kept only in memory. No +> long-term secret is ever written to disk; nothing on your +> machine ties one run to the next; the npub your daemon +> presents to the network is fresh on every restart. This is +> the safe-by-default posture — your node has no persistent +> identity unless you explicitly ask for one. +> - *Persistent*: the daemon reads (or, on first start, +> generates and writes) a keypair stored at +> `/etc/fips/fips.key`. The npub stays the same across +> restarts, reboots, and reinstalls as long as that file is +> preserved. You take on the cost of protecting an on-disk +> secret in exchange for being addressable by a stable name. + +Persistent identity is a deliberate trade. You give up the +ephemeral default's privacy posture — once your npub is stable, +every connection your node makes is correlatable across time — +and you take on a real secret-management responsibility. In +return you get two things you can't get any other way: + +1. **Other operators can reference you by npub.** In + [join-the-test-mesh](join-the-test-mesh.md) you added + `test-us01` to your `peers:` list by its npub. That entry + only works because `test-us01`'s npub doesn't change. If + anyone is going to reach you the same way, your npub has to + be just as stable. +2. **Services on your node get a fixed address.** The + [host-a-service](host-a-service.md) tutorial walks through + running an HTTP server addressable as + `.fips`. Clients reach the service by that + name; if your npub changes on every restart, every + client's address book breaks. + +Both of these are good reasons. Neither is automatic — if your +node is purely a *client*, reaching out to others without +hosting anything itself, you may legitimately want to stay on +the ephemeral default. The rest of this tutorial assumes you've +decided you want a stable identity. + +## Step 1: Note your current ephemeral npub + +Before changing anything, capture the npub the daemon is using +right now so you can compare against it after the switch. + +```sh +sudo fipsctl show status | grep '"npub"' +``` + +You'll see one line like: + +```text +"npub": "npub1abc...xyz" +``` + +Make a note of it. We expect this to change. + +## Step 2: Enable persistent identity in the config + +Open `/etc/fips/fips.yaml` and find the `node:` block. The +shipped default has the relevant fragment commented out; make it +look like this: + +```yaml +node: + identity: + persistent: true +``` + +Save the file. That is the only configuration change. + +The daemon's behavior on the next restart: + +- If `/etc/fips/fips.key` already exists, load it and use that + identity. +- If it does not exist, generate a fresh keypair, write it to + `/etc/fips/fips.{key,pub}` with the correct file modes, and + use that. + +## Step 3: Restart the daemon + +```sh +sudo systemctl restart fips +sudo systemctl status fips +``` + +Status should show `active (running)` within a couple of +seconds. Confirm the new identity is in use: + +```sh +sudo fipsctl show status | grep '"npub"' +``` + +The npub should be **different** from the one in Step 1 — the +daemon discarded the old in-memory ephemeral keypair and minted +a new one which it has now persisted to disk. From here forward +this is *your* npub. + +## Step 4: Locate the keypair on disk + +The daemon wrote two files: + +```sh +sudo ls -l /etc/fips/fips.key /etc/fips/fips.pub +``` + +Expect: + +```text +-rw------- 1 root root ... fips.key +-rw-r--r-- 1 root root ... fips.pub +``` + +The public file is safe to share — it is your address: + +```sh +sudo cat /etc/fips/fips.pub +``` + +This must match the `npub` reported by `sudo fipsctl show +status`. Hand this string to anyone you want to be reachable +from; they paste it into their own `peers:` block as the `npub:` +field. + +The private file is the secret. **Do not** `cat` or paste its +contents anywhere — there is no reason to see it, and any line +of shell history or screen capture that contains it has captured +the secret. + +## Step 5: Verify it survives a restart + +Restart once more to confirm the daemon is reading `fips.key` +rather than re-generating it: + +```sh +sudo systemctl restart fips +sudo fipsctl show status | grep '"npub"' +``` + +The npub should match Step 3 exactly. If it does not, the daemon +was unable to read `fips.key` (most likely a permission problem) +— see [Troubleshooting](#troubleshooting). + +## Step 6: Protect the nsec + +`fips.key` is the only thing standing between you and someone +else impersonating your node. The daemon ships it with the right +permissions; the operator's job is to keep them that way. + +What that means in practice: + +- **Do not loosen the file mode.** `0600` (read/write for owner + only — here `root`) is correct; `chmod 0644` to "fix" a + permission error puts the secret on display to every account + on the host. +- **Do not commit it to source control.** If you maintain + configuration in a Git repo, exclude `fips.key`; if you use + ansible-vault or a similar mechanism, encrypt it. `fips.pub` + may be checked in freely. +- **Do not paste it into chat or email.** Operators sometimes + share config snippets to demonstrate a setup; redact the + contents of `fips.key` to a placeholder before doing so. +- **Back it up the way you back up an SSH host key.** Treat the + file (or its contents) the same way you would treat + `/etc/ssh/ssh_host_ed25519_key`: encrypted, offline, available + to recover the *same* identity if the host disappears. + +There is no in-protocol "key change" message in FIPS. If +`fips.key` is lost, the npub is lost — your node will come back +up with a new identity and every downstream reference to the +old one will be stale. + +## What you've learned + +- **Identity = keypair.** Every FIPS node is a Nostr keypair; + the npub is its address, the nsec is its credential. +- **The flag.** `node.identity.persistent: true` in + `/etc/fips/fips.yaml` is the difference between ephemeral and + persistent identity. +- **Where it lives.** `/etc/fips/fips.key` and + `/etc/fips/fips.pub`, mode `0600` and `0644`, owned + `root:root`. +- **What to share.** `fips.pub` is public; `fips.key` is not. +- **What it buys you.** A npub other operators can add to their + `peers:` list once, and that addresses the services your node + will eventually run. + +## Troubleshooting + +If the post-restart npub does not match `fips.pub`: + +- **Check file permissions.** + `sudo ls -l /etc/fips/fips.key`. If the mode is not `0600` or + the owner is not `root:root`, the daemon may have refused to + read it. Restore with + `sudo chmod 0600 /etc/fips/fips.key && sudo chown root:root + /etc/fips/fips.key`. +- **Check the journal.** `sudo journalctl -u fips -n 100` after + the restart will show one of: + - `Loaded persistent identity from key file path=...` — good. + - `Generated persistent identity, saved to key file ...` — + also good, but only expected on the first start after the + flag flip. + - `Using ephemeral identity (new keypair each start)` — the + config flag was not picked up; re-check the indentation of + the `persistent: true` line. + +If you see `Generated persistent identity...` on every start, +the file is being written but not read on subsequent starts; +this is almost always the same permission/path issue. + +## What's next + +- **Resolve peer addresses via Nostr.** + [resolve-peers-via-nostr](resolve-peers-via-nostr.md) walks + through the smallest useful step toward Nostr-mediated + discovery: keep your peer entry, drop its hard-coded address, + and let the daemon look up the current endpoint from public + Nostr relays. The first of three tutorials covering Nostr + discovery; advertising your own node and open ambient + discovery come next. + +- **Reach services on other mesh nodes.** + [reach-mesh-services](reach-mesh-services.md) drives `nc`, + `traceroute6`, `curl`, and `ssh` at peers by `.fips` name and + shows that the FIPS data plane is just IPv6 from an + application's point of view. + +- **Host a service of your own.** + [host-a-service](host-a-service.md) walks through bringing up + an HTTP server addressable as `.fips`, bound to + `fips0` so the exposure is mesh-only, behind the mesh + firewall. + +For the alternative provisioning paths — minting a keypair with +`fipsctl keygen` before the daemon ever starts, or importing an +existing Nostr `nsec` — and the key-rotation procedure, see the +operator-style recipe at +[../how-to/persistent-identity.md](../how-to/persistent-identity.md). + +For the full identity model: + +- [../design/fips-architecture.md](../design/fips-architecture.md) + — how npubs become `NodeAddr`s and IPv6 ULAs. diff --git a/docs/tutorials/reach-mesh-services.md b/docs/tutorials/reach-mesh-services.md new file mode 100644 index 0000000..54c61ce --- /dev/null +++ b/docs/tutorials/reach-mesh-services.md @@ -0,0 +1,253 @@ +# Reach Services on Other Mesh Nodes + +In [join-the-test-mesh](join-the-test-mesh.md) you used `ping6` +to reach `test-us01` and `test-us02` by their `.fips` names. +This tutorial generalizes that to any IPv6-capable tool you +already use — `nc`, `traceroute6`, `curl`, `ssh`, `scp`, +anything — and gets you comfortable with the daemon's IPv6 +adapter, which makes the FIPS mesh look like an ordinary IPv6 +network to applications that already know how to use IPv6. + +The whole exercise should take about ten minutes. + +## What you'll do + +You'll ping a mesh node (recap), attempt a TCP connection to it +with `nc`, and trace the packet path with `traceroute6` — all +by hostname. By the end you will have driven three different +IPv6 tools at a mesh address and seen each one work the same +way it works on the regular internet. + +> **An IPv6 adapter for a non-IPv6 mesh.** The FIPS network +> itself routes blobs of data between npub-addressed nodes; +> on its own it has nothing to do with IPv6. The daemon +> includes an *IPv6 adapter* that presents the mesh as an +> ordinary IPv6 interface (`fips0`), so existing IP software +> works without modification. The kernel routes packets to +> it, applications open IPv6 sockets through it, and the +> adapter handles encapsulating each packet and routing it +> through the mesh to the matching adapter on the other side. +> Any tool that speaks IPv6 works unchanged. + +The IPv6 adapter is currently the main way operators use the +FIPS network, which is why most of the new-user progression is +about it. Native applications can use the mesh without going +through IPv6 at all, but that is out of scope for this tutorial. + +## Addressing a mesh node + +Throughout this tutorial — and any time you reach across the +mesh — use a node's `.fips` hostname directly. There are two +forms: + +- **`.fips`** — the canonical form. Every node has one, + always. This is the long bech32 npub with `.fips` appended. +- **`.fips`** — the convenience form, *if* you (or + the package) have an entry for the node in + `/etc/fips/hosts`. The installer ships entries for the + public test mesh, so `test-us01.fips` works on a fresh + install. + +These are real hostnames as far as your kernel is concerned. +Pass them to any IPv6-capable tool — `ping6`, `nc`, `curl`, +`ssh`, `traceroute6`, anything — the same way you would pass a +hostname on the public internet. There is no separate +"resolve to address first" step you ever need to perform; if +the tool takes a hostname, it accepts a `.fips` hostname. + +> **Where the address comes from.** Every FIPS node's mesh +> address is the SHA-256 of its public key, truncated to the +> bottom 64 bits and prepended with `fd97:`. Names of the form +> `.fips` and any shortname mapped in `/etc/fips/hosts` +> are aliases for that address. The daemon's local DNS +> responder hands the answer back to your kernel without ever +> talking to a remote DNS server. + +## Step 1: Ping a mesh node (recap) + +```sh +ping6 -c 4 test-us01.fips +``` + +You did this in [join-the-test-mesh](join-the-test-mesh.md). +Four replies, RTT in the tens of milliseconds (depending on +where you are relative to `test-us01`). Nothing new — but it +confirms the mesh data plane is healthy before you try +anything else. + +## Step 2: Attempt a TCP connection + +`ping6` proves ICMPv6 reaches the destination. To prove TCP +reaches it, use `nc` (netcat) to attempt a connection to a port. +Pick any port — whether it has a service listening or not, the +attempt proves the data plane carries your TCP segments +end-to-end: + +```sh +nc -6 -vz test-us01.fips 22 2>&1 +``` + +You will see one of two outcomes: + +```text +Connection to test-us01.fips 22 port [tcp/ssh] succeeded! +``` + +or: + +```text +nc: connect to test-us01.fips port 22 (tcp) failed: Connection refused +``` + +Both are good. The first means a service is listening on that +port and accepted your TCP handshake. The second means your TCP +SYN reached the remote node's kernel, which sent back a TCP RST +because no service was bound — and that RST traveled all the way +back through the mesh to your `nc` process. + +> **What a `Connection refused` proves.** A connection-refused +> response is *not* a network failure. It means the destination +> host is alive and reachable, the TCP stack on the far end +> processed your SYN, and the reply made it home. Compare with +> what you would get if the address were unreachable: +> `Network is unreachable` or a timeout. Either of the two +> outcomes above demonstrates a working end-to-end TCP path. + +If the port you tried happens to have a service, attach `-` +instead of `-z` and you can read the banner directly: + +```sh +nc -6 -v test-us01.fips 22 +``` + +The remote node's SSH banner, if any, will print on the next +line. Type `Ctrl-C` to disconnect — you have not authenticated, +just banner-grabbed. + +If `nc` is not installed, the same demonstration works with +`curl` against TCP/80: + +```sh +curl -6 -v --connect-timeout 5 http://test-us01.fips/ 2>&1 | head +``` + +The TCP connection result is in the first few lines of `curl`'s +verbose output. The HTTP response code is irrelevant — what +matters is whether the connection itself succeeded. + +## Step 3: Trace the path + +`traceroute6` shows the IPv6 hops between you and a +destination: + +```sh +traceroute6 -n test-us02.fips +``` + +You will see exactly one line — `test-us02`'s mesh address. +That is the only IPv6 hop between your `fips0` and +`test-us02`'s `fips0`, even though at the FIPS-mesh layer +your packet is being forwarded through your peer `test-us01` +on the way to `test-us02`. The mesh-layer forwarding is +invisible to `traceroute6` because it lives below the IPv6 +adapter. + +> **Two layers, two ideas of "hop".** The FIPS mesh routes +> blobs between npub-addressed nodes and can pass through +> several intermediate peers — your packet to `test-us02` is +> handed off to `test-us01` first. The IPv6 adapter, sitting +> on top of that, presents every reachable mesh node as a +> direct IPv6 neighbor: one hop, on a flat fabric. From +> `traceroute6`'s perspective the multi-hop FIPS path is +> hidden — it sees only the source and destination IPv6 +> adapters. To see what's happening at the mesh layer, see +> [ipv6-adapter-walkthrough](ipv6-adapter-walkthrough.md), +> which traces one `ssh` request from DNS query to far-side +> TUN with `fipstop` and `fipsctl` running alongside. + +If `traceroute6` is not installed, `mtr` and other IPv6 path +tools produce the same single-hop result. The single-hop +behavior is a property of the IPv6 adapter, not of the tool. + +## What you've learned + +You have driven three IPv6 tools at mesh nodes you reach over +the mesh, all by `.fips` hostname, and they all worked the same +way they work everywhere else: + +- **Addressing.** `.fips` is the canonical hostname for + any node; `.fips` is the convenience form when + `/etc/fips/hosts` has an entry. Use these in any tool that + takes an IPv6 hostname — there is no separate resolution + step you ever need to perform. +- **Reachability.** `ping6` confirms the remote node's `fips0` + answers ICMPv6 echo from your `fips0`. +- **TCP.** `nc` confirms TCP segments traverse the mesh and the + far side responds (whether with a banner, a refusal, or a + service of its own). +- **Path.** `traceroute6` shows exactly one IPv6 hop to any + reachable mesh node, because the multi-hop FIPS-mesh-layer + forwarding lives below the IPv6 adapter and is invisible + to IPv6 tooling. + +The conceptual takeaway is the one in the callout at the top: +the daemon's IPv6 adapter takes care of presenting the FIPS +mesh as ordinary IPv6 to every tool you already know. To +consume any service hosted on any mesh node — SSH, HTTP, file +transfer, custom protocols — you use the IPv6 client you +would use anywhere else. The hostname looks unusual +(`.fips`), but the API surface is unchanged. + +## Troubleshooting + +If a tool reports "Network is unreachable" or hangs: + +- **Confirm the link is healthy.** + `sudo fipsctl show peers` should show `test-us01` with active + connectivity. If the link to your direct peer is down, nothing + past it is reachable. +- **Confirm `fips0` is up.** `ip -6 addr show fips0` should show + one `fd97:...` address. If `fips0` is missing, the daemon did + not bring up the TUN — verify the daemon is running with the + privileges it needs. The default is to run as root; if you + dropped privileges per + [../how-to/run-as-unprivileged-user.md](../how-to/run-as-unprivileged-user.md), + re-check that the `setcap` and systemd override survived your + last package upgrade. +- **Confirm the name resolves.** If `ping6 test-us01.fips` + fails with `unknown host` or `Name or service not known`, + the system resolver is not consulting the daemon's `.fips` + responder. The installer wires this up automatically; the + "Reaching mesh nodes by name" section of + [../getting-started.md](../getting-started.md) describes + what the wiring looks like and how to confirm it. + +If `nc` or `curl` reports a timeout (rather than a refusal or +success), the destination node is unreachable from your +daemon — possible mesh-routing transient. Try again, or ping +first: if `ping6` succeeds but TCP times out, it is the +specific port being filtered on the destination, not a path +problem. + +## What's next + +- [host-a-service](host-a-service.md) — Bring up a small HTTP + server on your node, bind it to `fips0` so it is mesh-only, + and confirm another mesh node (or your own machine) can + reach it through the same data plane you just exercised. + Covers bind-interface choice and the mesh firewall. + +For "what's actually in those packets": + +- [../design/fips-architecture.md](../design/fips-architecture.md) + — the protocol stack and the two-layer encryption model. +- [../design/fips-mesh-layer.md](../design/fips-mesh-layer.md) — + Noise IK link encryption, hop-by-hop forwarding. +- [../design/fips-session-layer.md](../design/fips-session-layer.md) + — end-to-end Noise XK between source and destination. + +For the trace-it-yourself version of the path you just +exercised, see +[ipv6-adapter-walkthrough](ipv6-adapter-walkthrough.md), which +walks one `ssh` from DNS query through session setup to the +far-side TUN with `fipstop` and `fipsctl` running alongside. diff --git a/docs/tutorials/resolve-peers-via-nostr.md b/docs/tutorials/resolve-peers-via-nostr.md new file mode 100644 index 0000000..3b887aa --- /dev/null +++ b/docs/tutorials/resolve-peers-via-nostr.md @@ -0,0 +1,300 @@ +# Resolve Peer Addresses via Nostr + +After +[persistent-identity](persistent-identity.md), your daemon has +a stable npub and is peered with `test-us01` over a hard-coded +UDP address (`test-us01.fips.network:2121`). That static +address works fine until `test-us01` moves to a new IP, swaps +ports, or starts publishing additional endpoints you'd want to +reach. The npub is stable; the set of network endpoints behind +it may not be. + +This tutorial shows the smallest useful step toward Nostr- +mediated discovery: keep the peer entry but drop its address, +let your daemon ask public Nostr relays for the peer's current +endpoint, and verify the link still works. You will not be +publishing anything yourself yet — this is the consume-only +case. + +The whole exercise should take about ten minutes. + +## What you'll build + +```text + ┌──────────────────────────┐ + │ Nostr relays │ + │ relay.damus.io │ + │ nos.lol │ + │ offchain.pub │ + └────────────▲─────────────┘ + "what's test-us01's │ signed advert + current address?" │ (Kind 37195) + │ from test-us01 + ┌───────────────────────┐ │ + │ your fips daemon │ ─────────┘ + │ peers: │ + │ - test-us01 npub │ ─── dial resolved UDP ──▶ test-us01 + │ via_nostr: true │ + └───────────────────────┘ +``` + +You'll change two things in `/etc/fips/fips.yaml`: + +- Add a `node.discovery.nostr` block that turns the consume- + side of Nostr discovery on. +- Edit the existing `test-us01` peer entry to drop its hard- + coded `addresses:` block and add `via_nostr: true`. + +After restart, the daemon will fetch `test-us01`'s current +advert from the relays, use the endpoint listed there, and +peer normally. + +## How Nostr discovery resolves an address + +Every FIPS daemon with `node.discovery.nostr.advertise: true` +publishes a signed Nostr event (Kind 37195) listing the +transport endpoints it is willing to accept connections on. +The event is signed by the daemon's secret key, so anyone +who has the corresponding npub can verify the advert really +came from that node. + +`test-us01` runs with `advertise: true`. Its current advert is +visible to any Nostr client. + +> **Identity is stable; endpoints are not.** A peer's npub is +> a long-lived identifier — it is who they are. Their UDP +> address, port, or transport choice is metadata that may +> change. Nostr discovery lets you bind your peer entry to the +> npub and lets the relay tell your daemon the current +> endpoint at dial time. + +There are two halves to this — consuming adverts (looking up +peers by npub) and publishing adverts (being lookup-able). This +tutorial covers only the consume half. + +> **Consume vs. publish.** This tutorial enables only the +> *consume* side: your daemon queries relays to resolve peers +> by npub. It does not publish an advert of its own — others +> still cannot find you by your npub yet. The next tutorial +> (`advertise-your-node`) handles the publish side. + +## Step 1: Confirm your starting state + +You should currently have: + +- A persistent npub from + [persistent-identity](persistent-identity.md). Confirm: + + ```sh + sudo fipsctl show status | grep '"npub"' + ``` + +- A working static peering with `test-us01`. Confirm: + + ```sh + sudo fipsctl show peers + ``` + + Expect `test-us01` listed with `connectivity` active and a + `transport_addr` of roughly `217.77.8.91:2121`. + +If either of those isn't true, finish the previous two +tutorials first; the Nostr discovery layer is built on top of +that working state. + +## Step 2: Enable the consume side of Nostr discovery + +Open `/etc/fips/fips.yaml` and add a `discovery` block under +`node:`: + +```yaml +node: + identity: + persistent: true + discovery: + nostr: + enabled: true + advertise: false +``` + +Two knobs, one job each: + +- `enabled: true` turns on the Nostr discovery runtime — the + daemon connects to a default relay set + (`wss://relay.damus.io`, `wss://nos.lol`, + `wss://offchain.pub`) and is now able to query and consume + adverts. +- `advertise: false` keeps the publish side off. Your daemon + will not publish an advert of its own at this stage. (This + is the default, but it is good practice to make the choice + explicit while you're learning.) + +## Step 3: Switch the peer entry to `via_nostr` + +Find the `peers:` block you added during +[join-the-test-mesh](join-the-test-mesh.md) and change it from +this: + +```yaml +peers: + - npub: "npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98" + alias: "test-us01" + addresses: + - transport: udp + addr: "test-us01.fips.network:2121" + connect_policy: auto_connect +``` + +to this: + +```yaml +peers: + - npub: "npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98" + alias: "test-us01" + via_nostr: true + connect_policy: auto_connect +``` + +What changed: the `addresses:` list is gone, replaced by +`via_nostr: true`. The npub stays — it is what the daemon +matches against the advert publisher's pubkey. + +Save the file. + +## Step 4: Restart the daemon + +```sh +sudo systemctl restart fips +sudo systemctl status fips +``` + +The status output should show `active (running)` within a +couple of seconds. The Nostr discovery runtime starts alongside +the rest of the daemon, fetches `test-us01`'s advert from the +default relays, and uses the endpoint listed there to dial. + +The resolution itself happens at debug-log level, so you will +not see it in the default-level journal. The user-facing way to +confirm everything worked is `fipsctl show peers` in the next +step. (To watch the resolution in the journal, run the daemon +manually with `RUST_LOG=fips::discovery::nostr=debug`; not +necessary for this tutorial.) + +## Step 5: Verify the resolved endpoint + +```sh +sudo fipsctl show peers +``` + +`test-us01` should appear with `connectivity` active and a +`transport_addr` reflecting the address that was resolved from +the advert — `217.77.8.91:2121` at time of writing. That field +is the strong signal: nothing in your config gave the daemon +that IP, yet there it is. + +You can confirm independently that the address came from the +advert. The advert is a public Nostr event — anyone can fetch +it. With the `nak` Nostr CLI installed: + +```sh +nak req -k 37195 -d "fips-overlay-v1" \ + -a 06f11c31227938ab98ba982280d2826f66a063f9efe7e342e81d6a76c677d7ed \ + --limit 1 wss://relay.damus.io +``` + +(That hex pubkey is the same identity as +`npub1qmc3...zel98` — Nostr filters take hex.) The `content` +field of the returned event lists the `endpoints` array; one +of its entries should match what `fipsctl show peers` is +using. That is what your daemon just did, signed and verified +by the Nostr layer. + +## Step 6: Confirm reachability still works + +```sh +ping6 -c 4 test-us01.fips +``` + +Expect four replies, exactly as in +[join-the-test-mesh](join-the-test-mesh.md) (which used the +full npub form). Nothing about the data plane has changed; +only the way you discovered the endpoint to dial. + +## What you've learned + +- **Adverts are signed.** Every Nostr discovery advert is + signed by the publisher's secret key, so the address you + resolved through a public relay is trustworthy in the same + sense the peer's npub is. +- **`via_nostr` replaces a static address.** A peer entry + with no `addresses:` block and `via_nostr: true` directs + the daemon to look the endpoint up at dial time. +- **The relay set is small and public.** Three default + relays today; the daemon round-robins queries across them. + No central FIPS infrastructure is involved. +- **Static and Nostr can mix.** You replaced the static + address with `via_nostr` here, but you could have kept both + — when both are present, static endpoints are tried first + and Nostr-resolved endpoints are appended as a fallback. + Useful when you want a fast-path direct dial but a + resilient fallback. + +## Troubleshooting + +If the link does not come up: + +- **No advert on the relays.** If the peer's daemon is + offline or has `advertise: false`, no advert exists for + your daemon to consume. Verify with `nak` (Step 5) — if the + query returns nothing, that is the problem and it is on the + peer's side. Re-add the static `addresses:` entry as a + fallback while you wait for the peer to come back up. +- **Relay reachability.** `Connected to relay` lines should + appear for at least one of the three default relays. If + none do, your network may be filtering outbound WebSocket + traffic or DNS for those hostnames. Check the journal for + TLS/DNS errors. +- **Stale cache.** The daemon caches resolved endpoints + briefly. If a peer's advert changes mid-session and you + hit a stale entry, restart the daemon to force a fresh + query. +- **Persistent identity not on.** If the journal shows + `Using ephemeral identity (new keypair each start)`, the + daemon falls back to ephemeral and the consume-side may + not behave as expected. Re-check + [persistent-identity](persistent-identity.md) Step 2. + +## What's next + +- **Advertise your own node.** + [advertise-your-node](advertise-your-node.md) publishes your + daemon's UDP endpoint on Nostr so other operators can add + you to their `peers:` list with `via_nostr: true` and reach + you the way you just reached `test-us01`. Includes a section + on `udp:nat` NAT traversal for symmetric-NAT networks. + +- **Discover peers with no prior configuration.** + [open-discovery](open-discovery.md) switches your daemon to + `policy: open` so the ambient namespace itself populates + your peer list — no static `peers:` entries required (the + static ones can stay too; the two mechanisms run in + parallel). + +For the operator-style scenario reference covering all five +shapes of Nostr discovery side-by-side (consume-only, +publish-direct, publish-Tor, NAT traversal, open): + +- [../how-to/enable-nostr-discovery.md](../how-to/enable-nostr-discovery.md) + — five scenarios with minimal YAML fragments. + +For the design and security model: + +- [../design/fips-nostr-discovery.md](../design/fips-nostr-discovery.md) + — discovery runtime architecture, advert format, threat + model. + +For the wire-format details: + +- [../reference/nostr-events.md](../reference/nostr-events.md) + — Kind 37195 advert format, Kind 21059 traversal signaling, + Kind 10050 inbox-relay list. diff --git a/packaging/common/hosts b/packaging/common/hosts index 2c1c17f..da26d35 100644 --- a/packaging/common/hosts +++ b/packaging/common/hosts @@ -15,8 +15,18 @@ # Peer aliases from fips.yaml are also resolved automatically. # Entries here take precedence over peer aliases on conflict. # -# Examples: -# vps-tx npub10yffd020a4ag8zcy75f9pruq3rnghvvhd5hphl9s62zgp35s560qrksp9u -# vps-chi npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98 -# # Changes are picked up automatically on the next DNS query. + +# --------------------------------------------------------------------------- +# Public test mesh — operated by the FIPS project for new-user on-ramps and +# integration testing. Accept inbound peering from arbitrary npubs without +# coordination. Safe to remove if you are not joining the public test mesh. +# --------------------------------------------------------------------------- +test-us01 npub1qmc3cvfz0yu2hx96nq3gp55zdan2qclealn7xshgr448d3nh6lks7zel98 +test-us02 npub10yffd020a4ag8zcy75f9pruq3rnghvvhd5hphl9s62zgp35s560qrksp9u +test-us03 npub136yqae6na688fs75g95ppps3lxe07fvxefj77938zf47uhm6074sxw8ctm +test-us03-next npub15m6c4ghuegx4pcde6tra8f7smn8vfv2wundyxwhkjynuerkrzmgsy09sh3 +test-us04 npub1gd7ye2qp2lphhzx75fynnjzaxx4dqanddecet0wtt5ss5ek8h9ps62wdkf +test-de01 npub1260n42s06vzc7796w0fh3ny7zcpw6tlk4gq3940gmfrzl5c9pv2s3657q8 +test-es01 npub17lpmzulpc98d8ff727k6e98atxn3phzupzsqqwe54ytduym747ws4tw5zm +test-uk01 npub1u0z26dc4qeneu5rvwvmpfhtwh3522ed6rlgxr9jarrfnjrc6ew4qxjysrs diff --git a/packaging/openwrt-ipk/files/etc/fips/fips.yaml b/packaging/openwrt-ipk/files/etc/fips/fips.yaml index 301c35f..7d6f2d3 100644 --- a/packaging/openwrt-ipk/files/etc/fips/fips.yaml +++ b/packaging/openwrt-ipk/files/etc/fips/fips.yaml @@ -1,7 +1,7 @@ # FIPS Node Configuration # # Edit this file before starting the FIPS daemon. -# Full reference: https://github.com/jmcorgan/fips/blob/master/docs/design/fips-configuration.md +# Full reference: https://github.com/jmcorgan/fips/blob/master/docs/reference/configuration.md # # After editing, restart the daemon: # /etc/init.d/fips restart