diff --git a/README.md b/README.md index b1ecfe8..b751761 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,28 @@ sudo ./target/release/fips -c fips.yaml See [docs/design/fips-configuration.md](docs/design/fips-configuration.md) for the full configuration reference. +### Inspect + +While a node is running, use `fipsctl` to inspect its state: + +``` +fipsctl show status # Node status overview +fipsctl show peers # Authenticated peers +fipsctl show links # Active links +fipsctl show tree # Spanning tree state +fipsctl show sessions # End-to-end sessions +fipsctl show bloom # Bloom filter state +fipsctl show mmp # MMP metrics summary +fipsctl show cache # Coordinate cache stats +fipsctl show connections # Pending handshake connections +fipsctl show transports # Transport instances +fipsctl show routing # Routing table summary +``` + +`fipsctl` communicates with the node via a Unix domain control socket +(enabled by default). All queries are read-only. Use `-s ` to +override the socket path. + ### Multi-node Testing See [testing/](testing/) for Docker-based integration test harnesses including diff --git a/docs/design/README.md b/docs/design/README.md index a821c55..c4a2796 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -31,15 +31,13 @@ specific topics. | Document | Description | | -------- | ----------- | -| [fips-spanning-tree.md](fips-spanning-tree.md) | Spanning tree algorithms: root election, parent selection, coordinates | +| [fips-spanning-tree.md](fips-spanning-tree.md) | Spanning tree algorithms: root discovery, parent selection, coordinates | | [fips-bloom-filters.md](fips-bloom-filters.md) | Bloom filter math: FPR analysis, size classes, split-horizon | ### Implementation | Document | Description | | -------- | ----------- | -| [fips-software-architecture.md](fips-software-architecture.md) | Stable architectural decisions guiding the codebase | -| [fips-state-machines.md](fips-state-machines.md) | Phase-based state machine pattern (Rust enum-of-structs) | | [fips-configuration.md](fips-configuration.md) | YAML configuration reference | ### Supplemental @@ -54,16 +52,16 @@ specific topics. fips-intro.md │ ┌──────────────┼──────────────┐ - ▼ ▼ ▼ - fips-transport-layer fips-mesh- fips-software- - │ operation architecture + ▼ ▼ │ + fips-transport-layer fips-mesh- │ + │ operation │ ▼ │ │ - fips-mesh-layer ◄────────┤ ▼ - │ │ fips-state-machines - ▼ │ - fips-session-layer ├──► fips-spanning-tree - │ │ │ - ▼ └──► fips-bloom-filters + fips-mesh-layer ◄────────┤ │ + │ │ │ + ▼ ├──► fips-spanning-tree + fips-session-layer │ │ + │ └──► fips-bloom-filters + ▼ fips-ipv6-adapter fips-wire-formats (referenced by all layer docs) diff --git a/docs/design/diagrams/ancestry-entry.svg b/docs/design/diagrams/ancestry-entry.svg new file mode 100644 index 0000000..1798d94 --- /dev/null +++ b/docs/design/diagrams/ancestry-entry.svg @@ -0,0 +1,28 @@ + + + + + + AncestryEntry (32 bytes) + + + 0–15 + + + node_addr + 16 bytes — routing identifier + + + 16–23 + + + sequence + 8 bytes LE + + + 24–31 + + + timestamp + 8 bytes LE — Unix seconds + diff --git a/docs/design/diagrams/common-prefix.svg b/docs/design/diagrams/common-prefix.svg new file mode 100644 index 0000000..52c9c47 --- /dev/null +++ b/docs/design/diagrams/common-prefix.svg @@ -0,0 +1,39 @@ + + + + + + FMP/FSP Common Prefix (4 bytes) + + + 0 + 4 + 8 + 16 + 32 + + + + ver + 4 bits + + + + phase + 4 bits + + + + flags + 1 byte + + + + payload_len + 2 bytes LE + + + byte 0 + byte 1 + bytes 2-3 + diff --git a/docs/design/diagrams/coords-required.svg b/docs/design/diagrams/coords-required.svg new file mode 100644 index 0000000..c9af0b7 --- /dev/null +++ b/docs/design/diagrams/coords-required.svg @@ -0,0 +1,34 @@ + + + + + + CoordsRequired (0x20) — 34 bytes payload + + + 0–1 + + + msg_type + 0x20 + + + flags (reserved) + + + 2–17 + + + dest_addr + 16 bytes — NodeAddr we couldn't route to + + + 18–33 + + + reporter + 16 bytes — NodeAddr of reporting router + + + plaintext error signal (U flag set) · total on wire: 38 bytes (4 prefix + 34 payload) + diff --git a/docs/design/diagrams/disconnect.svg b/docs/design/diagrams/disconnect.svg new file mode 100644 index 0000000..8272d82 --- /dev/null +++ b/docs/design/diagrams/disconnect.svg @@ -0,0 +1,18 @@ + + + + + + Disconnect (0x50) — 2 bytes + + + 0–1 + + + msg_type + 0x50 + + + reason + 1 byte + diff --git a/docs/design/diagrams/dynamics-convergence.svg b/docs/design/diagrams/dynamics-convergence.svg new file mode 100644 index 0000000..0ece708 --- /dev/null +++ b/docs/design/diagrams/dynamics-convergence.svg @@ -0,0 +1,100 @@ + + + + + + Three-Node Convergence + + + + + + + + + + T0 — Isolated + + ★ root + ★ root + ★ root + + + A + + + B + + + C + + node_addr: A < B < C + + + + + T1 — Links form + + ★ root + ★ root + ★ root + + + + + + + + A + + + B + + + C + + + + + T2 — B re-parents + + ★ root + ★ root + + + + + + + + A + + + B + + + C + + + + + T3 — Converged + + ★ root + + + + + + + + A + + + B + + + C + + A ← B ← C + diff --git a/docs/design/diagrams/dynamics-ex1-physical.svg b/docs/design/diagrams/dynamics-ex1-physical.svg new file mode 100644 index 0000000..d4eff0e --- /dev/null +++ b/docs/design/diagrams/dynamics-ex1-physical.svg @@ -0,0 +1,38 @@ + + + + + Office Network — Physical Topology + node_addr: A < C < B < E < D + + + + + + + + + + + + + + + A + + + + B + + + + D + + + + C + + + + E + diff --git a/docs/design/diagrams/dynamics-ex1-tree.svg b/docs/design/diagrams/dynamics-ex1-tree.svg new file mode 100644 index 0000000..5d58654 --- /dev/null +++ b/docs/design/diagrams/dynamics-ex1-tree.svg @@ -0,0 +1,43 @@ + + + + + Office Network — Converged Tree + + + + + + + + + + + + + + + + + + + + A + ★ root + + + + B + + + + C + + + + D + + + + E + diff --git a/docs/design/diagrams/dynamics-ex2-physical.svg b/docs/design/diagrams/dynamics-ex2-physical.svg new file mode 100644 index 0000000..fb064b8 --- /dev/null +++ b/docs/design/diagrams/dynamics-ex2-physical.svg @@ -0,0 +1,48 @@ + + + + + Mixed Network — Physical Topology + node_addr: B < A < D < E < C + + + + + + + + + + + + + + + + fiber + fiber + fiber + DSL 1Mbps + radio + 9600bps + + + + A + + + + B + + + + C + + + + D + + + + E + diff --git a/docs/design/diagrams/dynamics-ex2-tree.svg b/docs/design/diagrams/dynamics-ex2-tree.svg new file mode 100644 index 0000000..8afc302 --- /dev/null +++ b/docs/design/diagrams/dynamics-ex2-tree.svg @@ -0,0 +1,60 @@ + + + + + Mixed Network — Cost-Weighted Tree + + + + + + + + + + + + + + + + + + + + + eff 1.01 + eff 1.01 + eff 2.01 + eff 2.25 + + + + B + ★ root + + + + A + + + + D + + + + E + + + + C + diff --git a/docs/design/diagrams/dynamics-ex3-partition.svg b/docs/design/diagrams/dynamics-ex3-partition.svg new file mode 100644 index 0000000..2f521f2 --- /dev/null +++ b/docs/design/diagrams/dynamics-ex3-partition.svg @@ -0,0 +1,58 @@ + + + + + Network Partition — Independent Trees + + + + + + Partition 1 + Partition 2 + + + + + + + + + + + + + + + + WAN failed + + + + A + ★ root + + + + B + + + + C + + + + E + ★ root + + + + F + + + + G + diff --git a/docs/design/diagrams/dynamics-ex3-topology.svg b/docs/design/diagrams/dynamics-ex3-topology.svg new file mode 100644 index 0000000..3b598b2 --- /dev/null +++ b/docs/design/diagrams/dynamics-ex3-topology.svg @@ -0,0 +1,56 @@ + + + + + Two-Site Network — Physical Topology + node_addr: A < E < B < F < C < G + + + + Site 1 + + + + Site 2 + + + + + + + + + + + + + + WAN + + + + A + + + + B + + + + C + + + + E + + + + F + + + + G + diff --git a/docs/design/diagrams/dynamics-link-addition.svg b/docs/design/diagrams/dynamics-link-addition.svg new file mode 100644 index 0000000..b37ff96 --- /dev/null +++ b/docs/design/diagrams/dynamics-link-addition.svg @@ -0,0 +1,93 @@ + + + + + + Link Addition — D Re-parents to A + + + Before + After + + + → + + + + + + + + ★ root + + + depth 0 + depth 1 + depth 2 + depth 3 + + + + + + + + + A + + + + B + + + + C + + + + D + + + + + + + ★ root + + + depth 0 + depth 1 + depth 2 + depth 3 + + + + D + + + + + + + + + + + + A + + + + B + + + + D + + + + C + + + new link + diff --git a/docs/design/diagrams/dynamics-link-removal.svg b/docs/design/diagrams/dynamics-link-removal.svg new file mode 100644 index 0000000..a55f3a8 --- /dev/null +++ b/docs/design/diagrams/dynamics-link-removal.svg @@ -0,0 +1,76 @@ + + + + + + Link Removal — B↔C Link Fails + + + + + + Before + + + ★ root + + A + + + + + + + B + + + + + + + D + + + + + + + + + + + C + + + After + + + ★ root + + A + + + + + + + B + + + + + + + D + + + ★ root + + C + + + (isolated) + + + + diff --git a/docs/design/diagrams/dynamics-node-join-steps.svg b/docs/design/diagrams/dynamics-node-join-steps.svg new file mode 100644 index 0000000..000a849 --- /dev/null +++ b/docs/design/diagrams/dynamics-node-join-steps.svg @@ -0,0 +1,140 @@ + + + + + + Node B Joins via D — State Progression + + + T1 — Link established + T2–T3 — B re-parents to D + T4–T5 — Filter propagation + + + + + + + + + + + + ★ root + + + + + + + + + new link + + + + A + + + + C + + + + D + + + + E + + + + B + + + + + + + + ★ root + + + + + + + + + + + + A + + + + C + + + + D + + + + E + + + + B + + + parent=D, seq=2 + + + + + + + + ★ root + + + + + + + + + + + + A + + + + C + + + + D + + + + E + + + + B + + + + + + + + + FilterAnnounce + + + B reachable via D + diff --git a/docs/design/diagrams/dynamics-node-join.svg b/docs/design/diagrams/dynamics-node-join.svg new file mode 100644 index 0000000..72a6a17 --- /dev/null +++ b/docs/design/diagrams/dynamics-node-join.svg @@ -0,0 +1,82 @@ + + + + + + Node Joining — B Connects to D + + + Before + After + + + → + + + + + + + + ★ root + + A + + + + + + + + + + C + + + + D + + + + E + + + + B + + + + + + + + ★ root + + A + + + + + + + + + + C + + + + D + + + + E + + + + + + + + B + diff --git a/docs/design/diagrams/dynamics-partition.svg b/docs/design/diagrams/dynamics-partition.svg new file mode 100644 index 0000000..7bff8fe --- /dev/null +++ b/docs/design/diagrams/dynamics-partition.svg @@ -0,0 +1,95 @@ + + + + + + Network Partition — Link C↔D Fails + + + + + + Before + + + ★ root + + A + + + + + + + B + + + + + + + C + + + + + + + + + + + D + + + + + + + E + + + After + + + Partition 1 + Partition 2 + + + + + + + + ★ root + + A + + + + + + + B + + + + + + + C + + + + + ★ root + + D + + + + + + + E + diff --git a/docs/design/diagrams/established-complete-frame.svg b/docs/design/diagrams/established-complete-frame.svg new file mode 100644 index 0000000..fdbdb6c --- /dev/null +++ b/docs/design/diagrams/established-complete-frame.svg @@ -0,0 +1,91 @@ + + + + + + Complete Encrypted Frame (37 + body bytes) + + + 0 + 4 + 8 + 16 + 32 + + + 0–3 + + + ver + 4 bits + + + phase + 4 bits + + + flags + 1 byte + + + payload_len + 2 bytes LE + + + 4–7 + + + receiver_idx + 4 bytes LE + + + 8–11 + + + counter / lower 4 bytes LE + + + 12–15 + + + counter / upper 4 bytes LE + + + AAD + + + + + + + ── ciphertext (encrypted with ChaCha20-Poly1305) ── + + + 16–19 + + + timestamp + 4 bytes LE + + + 20 + + + msg_type + 1 byte + + + message body ... + + + ... + + + message body (variable length) + + + +16 + + + Poly1305 AEAD tag (16 bytes) + diff --git a/docs/design/diagrams/established-inner-header.svg b/docs/design/diagrams/established-inner-header.svg new file mode 100644 index 0000000..4293563 --- /dev/null +++ b/docs/design/diagrams/established-inner-header.svg @@ -0,0 +1,31 @@ + + + + + + Established Frame Inner Header (5 bytes, plaintext after decrypt) + + + 0 + 8 + 16 + 32 + + + 0–3 + + + timestamp + 4 bytes LE + + + 4 + + + msg_type + 1 byte + + + + message body ... + diff --git a/docs/design/diagrams/established-outer-header.svg b/docs/design/diagrams/established-outer-header.svg new file mode 100644 index 0000000..377b247 --- /dev/null +++ b/docs/design/diagrams/established-outer-header.svg @@ -0,0 +1,52 @@ + + + + + + Established Frame Outer Header (16 bytes, AEAD AAD) + + + 0 + 4 + 8 + 16 + 32 + + + 0–3 + + + ver + 4 bits + + + phase + 4 bits + + + flags + 1 byte + + + payload_len + 2 bytes LE + + + 4–7 + + + receiver_idx + 4 bytes LE + + + 8–11 + + + counter / lower 4 bytes LE + + + 12–15 + + + counter / upper 4 bytes LE + diff --git a/docs/design/diagrams/filter-announce.svg b/docs/design/diagrams/filter-announce.svg new file mode 100644 index 0000000..711b166 --- /dev/null +++ b/docs/design/diagrams/filter-announce.svg @@ -0,0 +1,43 @@ + + + + + + FilterAnnounce (0x20) — 11 + filter bytes + + + 0–3 + + + msg_type + 0x20 + + + sequence + + + 4–8 + + + 8 bytes LE — monotonic counter + + + 9–10 + + + hash_count + 1 byte + + + size_class + 1 byte + + + 11– + + + filter_bits (variable, 512 << size_class bytes) + + + v1 payload: 1,035 bytes (11 header + 1,024 filter) + diff --git a/docs/design/fips-architecture-overview.svg b/docs/design/diagrams/fips-architecture-overview.svg similarity index 100% rename from docs/design/fips-architecture-overview.svg rename to docs/design/diagrams/fips-architecture-overview.svg diff --git a/docs/design/fips-bloom-propagation.svg b/docs/design/diagrams/fips-bloom-propagation.svg similarity index 100% rename from docs/design/fips-bloom-propagation.svg rename to docs/design/diagrams/fips-bloom-propagation.svg diff --git a/docs/design/fips-coordinate-discovery.svg b/docs/design/diagrams/fips-coordinate-discovery.svg similarity index 100% rename from docs/design/fips-coordinate-discovery.svg rename to docs/design/diagrams/fips-coordinate-discovery.svg diff --git a/docs/design/fips-identity-derivation.svg b/docs/design/diagrams/fips-identity-derivation.svg similarity index 99% rename from docs/design/fips-identity-derivation.svg rename to docs/design/diagrams/fips-identity-derivation.svg index 6749b57..05ac257 100644 --- a/docs/design/fips-identity-derivation.svg +++ b/docs/design/diagrams/fips-identity-derivation.svg @@ -95,7 +95,7 @@ - add fd::/8 prefix + add fd00::/8 prefix diff --git a/docs/design/fips-mesh-topology.svg b/docs/design/diagrams/fips-mesh-topology.svg similarity index 100% rename from docs/design/fips-mesh-topology.svg rename to docs/design/diagrams/fips-mesh-topology.svg diff --git a/docs/design/fips-node-architecture.svg b/docs/design/diagrams/fips-node-architecture.svg similarity index 99% rename from docs/design/fips-node-architecture.svg rename to docs/design/diagrams/fips-node-architecture.svg index e5b794e..61c72be 100644 --- a/docs/design/fips-node-architecture.svg +++ b/docs/design/diagrams/fips-node-architecture.svg @@ -40,7 +40,7 @@ IPv6 TUN Adapter fips0 + DNS (.fips) - npub → fd::/8 IPv6 + npub → fd00::/8 IPv6 FIPS-aware apps diff --git a/docs/design/fips-osi-mapping.svg b/docs/design/diagrams/fips-osi-mapping.svg similarity index 100% rename from docs/design/fips-osi-mapping.svg rename to docs/design/diagrams/fips-osi-mapping.svg diff --git a/docs/design/fips-protocol-stack.svg b/docs/design/diagrams/fips-protocol-stack.svg similarity index 100% rename from docs/design/fips-protocol-stack.svg rename to docs/design/diagrams/fips-protocol-stack.svg diff --git a/docs/design/fips-routing-decision.svg b/docs/design/diagrams/fips-routing-decision.svg similarity index 100% rename from docs/design/fips-routing-decision.svg rename to docs/design/diagrams/fips-routing-decision.svg diff --git a/docs/design/diagrams/fsp-complete-message.svg b/docs/design/diagrams/fsp-complete-message.svg new file mode 100644 index 0000000..27543ed --- /dev/null +++ b/docs/design/diagrams/fsp-complete-message.svg @@ -0,0 +1,97 @@ + + + + + + FSP Complete Encrypted Message (34 + body bytes) + + + 0 + 4 + 8 + 16 + 32 + + + 0–3 + + + ver + 4 bits + + + phase + 4 bits + + + flags + 1 byte + + + payload_len + 2 bytes LE + + + 4–7 + + + counter / lower 4 bytes LE + + + 8–11 + + + counter / upper 4 bytes LE + + + AAD + + + + + + opt + + + cleartext coordinates (if CP flag set, variable length) + + + + ── ciphertext (encrypted with ChaCha20-Poly1305) ── + + + +0–3 + + + timestamp + 4 bytes LE + + + +4 + + + msg_type + 1 byte + + + inner_flags + 1 byte + + + body ... + + + ... + + + message body (variable length) + + + +16 + + + Poly1305 AEAD tag (16 bytes) + + + overhead: 34 bytes (12 header + 6 inner + 16 tag) · coords add variable length + diff --git a/docs/design/diagrams/handshake-flow.svg b/docs/design/diagrams/handshake-flow.svg new file mode 100644 index 0000000..3c44c49 --- /dev/null +++ b/docs/design/diagrams/handshake-flow.svg @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + FMP Handshake Flow (Noise IK) + + + Initiator + Responder + + + + + + + generates sender_idx + generates ephemeral keypair + + + + + [0x01|flags=0|len] | sender_idx | noise_msg1 + phase 0x1 — 114 bytes + + + validates msg1 + learns initiator's static key + generates sender_idx + generates ephemeral keypair + + + + + + [0x02|flags=0|len] | sender_idx | receiver_idx | noise_msg2 + phase 0x2 — 69 bytes + + + validates msg2 + derives session keys + + + + HANDSHAKE COMPLETE + + + + first encrypted frame: + + + [0x00|flags|len] | receiver_idx | counter=0 | ciphertext+tag + phase 0x0 — established frame + + + Both parties hold identical symmetric keys. Epoch exchange enables restart detection. + diff --git a/docs/design/diagrams/lookup-request.svg b/docs/design/diagrams/lookup-request.svg new file mode 100644 index 0000000..afb9811 --- /dev/null +++ b/docs/design/diagrams/lookup-request.svg @@ -0,0 +1,72 @@ + + + + + + LookupRequest (0x30) — 303 + 16n bytes + + + 0–3 + + + msg_type + 0x30 + + + request_id + + + 4–8 + + + 8 bytes LE — unique random identifier + + + 9–24 + + + target + 16 bytes — NodeAddr being sought + + + 25–40 + + + origin + 16 bytes — requester's NodeAddr + + + 41–43 + + + ttl + 1 byte + + + min_mtu + 2 bytes LE + + + 44–45 + + + origin_coords_cnt + + + 46– + + + origin_coords × n (16 bytes each — NodeAddr only) + + + +1+256 + + + hash_cnt + + + visited_bits (256 bytes) + + + total: 303 + (n × 16) bytes, where n = origin depth + 1 + diff --git a/docs/design/diagrams/lookup-response.svg b/docs/design/diagrams/lookup-response.svg new file mode 100644 index 0000000..71ddbfc --- /dev/null +++ b/docs/design/diagrams/lookup-response.svg @@ -0,0 +1,56 @@ + + + + + + LookupResponse (0x31) — 93 + 16n bytes + + + 0–3 + + + msg_type + 0x31 + + + request_id + + + 4–8 + + + 8 bytes LE — echoes the request's ID + + + 9–24 + + + target + 16 bytes — NodeAddr that was found + + + 25–28 + + + path_mtu + 2 bytes LE + + + target_coords_cnt + 2 bytes LE + + + 29– + + + target_coords × n (16 bytes each — NodeAddr only) + + + +64 + + + Schnorr proof (64 bytes) + + + total: 93 + (n × 16) bytes · proof signs (request_id || target || target_coords) + diff --git a/docs/design/diagrams/mtu-exceeded.svg b/docs/design/diagrams/mtu-exceeded.svg new file mode 100644 index 0000000..28de0a2 --- /dev/null +++ b/docs/design/diagrams/mtu-exceeded.svg @@ -0,0 +1,41 @@ + + + + + + MtuExceeded (0x22) — 36 bytes payload + + + 0–1 + + + msg_type + 0x22 + + + flags (reserved) + + + 2–17 + + + dest_addr + 16 bytes — destination being forwarded to + + + 18–33 + + + reporter + 16 bytes — NodeAddr of reporting router + + + 34–35 + + + mtu + 2 bytes LE — bottleneck MTU + + + plaintext error signal (U flag set) · total on wire: 40 bytes (4 prefix + 36 payload) + diff --git a/docs/design/diagrams/noise-ik-msg1.svg b/docs/design/diagrams/noise-ik-msg1.svg new file mode 100644 index 0000000..5d2458a --- /dev/null +++ b/docs/design/diagrams/noise-ik-msg1.svg @@ -0,0 +1,67 @@ + + + + + + Noise IK Message 1 — phase 0x1 (114 bytes) + + + 0–3 + + + ver + 4 bits + + + phase + 4 bits + + + flags + 1 byte + + + payload_len + 2 bytes LE + + + 4–7 + + + sender_idx + 4 bytes LE + + + Noise IK msg1 (106 bytes) + + + + + + 8–40 + + + ephemeral_pubkey + + + 33 bytes (compressed secp256k1) + + + 41–89 + + + encrypted_static + + + 49 bytes (static key 33 + AEAD tag 16) + + + 90–113 + + + encrypted_epoch + 24 bytes (epoch 8 + AEAD tag 16) + + + total: 114 bytes · payload_len = 110 · pattern: e, es, s, ss + diff --git a/docs/design/diagrams/noise-ik-msg2.svg b/docs/design/diagrams/noise-ik-msg2.svg new file mode 100644 index 0000000..4878f8b --- /dev/null +++ b/docs/design/diagrams/noise-ik-msg2.svg @@ -0,0 +1,65 @@ + + + + + + Noise IK Message 2 — phase 0x2 (69 bytes) + + + 0–3 + + + ver + 4 bits + + + phase + 4 bits + + + flags + 1 byte + + + payload_len + 2 bytes LE + + + 4–7 + + + sender_idx + 4 bytes LE + + + 8–11 + + + receiver_idx + 4 bytes LE + + + Noise IK msg2 (57 bytes) + + + + + + 12–44 + + + ephemeral_pubkey + + + 33 bytes (compressed secp256k1) + + + 45–68 + + + encrypted_epoch + 24 bytes (epoch 8 + AEAD tag 16) + + + total: 69 bytes · payload_len = 65 · pattern: e, ee, se + diff --git a/docs/design/diagrams/path-broken.svg b/docs/design/diagrams/path-broken.svg new file mode 100644 index 0000000..3a80723 --- /dev/null +++ b/docs/design/diagrams/path-broken.svg @@ -0,0 +1,43 @@ + + + + + + PathBroken (0x21) — 36 + 16n bytes payload + + + 0–1 + + + msg_type + 0x21 + + + flags (reserved) + + + 2–17 + + + dest_addr + 16 bytes — unreachable NodeAddr + + + 18–33 + + + reporter + 16 bytes — NodeAddr of reporting router + + + 34–35 + + + last_coords_count + + + 36– + + + last_known_coords × n (16 bytes each — stale coords) + diff --git a/docs/design/diagrams/path-mtu-notification.svg b/docs/design/diagrams/path-mtu-notification.svg new file mode 100644 index 0000000..c5abe67 --- /dev/null +++ b/docs/design/diagrams/path-mtu-notification.svg @@ -0,0 +1,17 @@ + + + + + + PathMtuNotification (0x13) — 2 bytes body + + + 0–1 + + + path_mtu + 2 bytes LE — minimum MTU observed along forward path + + + body only — carried inside FSP encrypted message (prefix + inner header + AEAD tag) + diff --git a/docs/design/diagrams/receiver-report.svg b/docs/design/diagrams/receiver-report.svg new file mode 100644 index 0000000..9b5bc82 --- /dev/null +++ b/docs/design/diagrams/receiver-report.svg @@ -0,0 +1,106 @@ + + + + + + ReceiverReport (0x02) — 68 bytes + + + 0–3 + + + msg_type + 0x02 + + + reserved (3 bytes zero) + + + 4–11 + + + highest_counter + 8 bytes LE + + + 12–19 + + + cumulative_packets_recv + 8 bytes LE + + + 20–27 + + + cumulative_bytes_recv + 8 bytes LE + + + 28–33 + + + timestamp_echo + 4 bytes LE + + + dwell_time + 2 bytes LE ms + + + 34–39 + + + max_burst + 2 bytes LE + + + mean_burst + 2 bytes u8.8 + + + reserved + + + 40–47 + + + jitter + 4 bytes LE µs + + + ecn_ce_count + 4 bytes LE + + + 48–55 + + + owd_trend + 4 bytes i32 LE µs/s + + + burst_loss_count + 4 bytes LE + + + 56–63 + + + reorder_count + 4 bytes LE cumulative + + + interval_pkts_recv + 4 bytes LE + + + 64–67 + + + interval_bytes_recv + 4 bytes LE + + + total: 68 bytes + diff --git a/docs/design/diagrams/sender-report.svg b/docs/design/diagrams/sender-report.svg new file mode 100644 index 0000000..9fb5942 --- /dev/null +++ b/docs/design/diagrams/sender-report.svg @@ -0,0 +1,66 @@ + + + + + + SenderReport (0x01) — 48 bytes + + + 0–3 + + + msg_type + 0x01 + + + reserved (3 bytes zero) + + + 4–11 + + + interval_start_counter + 8 bytes LE + + + 12–19 + + + interval_end_counter + 8 bytes LE + + + 20–27 + + + interval_start_ts + 4 bytes LE + + + interval_end_ts + 4 bytes LE + + + 28–31 + + + interval_bytes_sent + 4 bytes LE + + + 32–39 + + + cumulative_packets_sent + 8 bytes LE + + + 40–47 + + + cumulative_bytes_sent + 8 bytes LE + + + total: 48 bytes + diff --git a/docs/design/diagrams/session-ack.svg b/docs/design/diagrams/session-ack.svg new file mode 100644 index 0000000..699006c --- /dev/null +++ b/docs/design/diagrams/session-ack.svg @@ -0,0 +1,47 @@ + + + + + + SessionAck (phase 0x2) — Noise XK msg2 + + + 0–3 + + + FSP common prefix + ver=0, phase=0x2, flags=0, payload_len + + + 4 + + + flags + 1 byte — reserved + + + 5– + + + src_coords_count(2) + src_coords(16 × n) — acknowledger + + + ... + + + dest_coords_count(2) + dest_coords(16 × m) — initiator + + + ... + + + hs_len + 2 bytes LE + + + handshake_payload + Noise XK msg2 (57 bytes — ephemeral + epoch) + + + typical ~190 bytes (depth-dependent, carries both endpoints' coords) + diff --git a/docs/design/diagrams/session-datagram.svg b/docs/design/diagrams/session-datagram.svg new file mode 100644 index 0000000..4473d98 --- /dev/null +++ b/docs/design/diagrams/session-datagram.svg @@ -0,0 +1,46 @@ + + + + + + SessionDatagram (0x00) — 36 + payload bytes + + + 0–3 + + + msg_type + 0x00 + + + ttl + 1 byte + + + path_mtu + 2 bytes LE + + + 4–19 + + + src_addr + 16 bytes — source NodeAddr + + + 20–35 + + + dest_addr + 16 bytes — destination NodeAddr + + + 36– + + + payload (variable — session-layer message) + + + fixed header: 36 bytes · payload opaque to transit nodes + ttl decremented and path_mtu min'd at each forwarding hop + diff --git a/docs/design/diagrams/session-msg3.svg b/docs/design/diagrams/session-msg3.svg new file mode 100644 index 0000000..4db2df2 --- /dev/null +++ b/docs/design/diagrams/session-msg3.svg @@ -0,0 +1,35 @@ + + + + + + SessionMsg3 (phase 0x3) — Noise XK msg3 + + + 0–3 + + + FSP common prefix + ver=0, phase=0x3, flags=0, payload_len + + + 4–6 + + + flags + 1 byte + + + handshake_len + 2 bytes LE + + + 7– + + + handshake_payload + Noise XK msg3 (73 bytes — encrypted static + encrypted epoch) + + + ~80 bytes · no coordinates (both endpoints already have them) + diff --git a/docs/design/diagrams/session-setup.svg b/docs/design/diagrams/session-setup.svg new file mode 100644 index 0000000..e054f2f --- /dev/null +++ b/docs/design/diagrams/session-setup.svg @@ -0,0 +1,47 @@ + + + + + + SessionSetup (phase 0x1) — Noise XK msg1 + + + 0–3 + + + FSP common prefix + ver=0, phase=0x1, flags=0, payload_len + + + 4 + + + flags + 1 byte — bit 0: REQUEST_ACK, bit 1: BIDIRECTIONAL + + + 5– + + + src_coords_count(2) + src_coords(16 × n) + + + ... + + + dest_coords_count(2) + dest_coords(16 × m) + + + ... + + + hs_len + 2 bytes LE + + + handshake_payload + Noise XK msg1 (33 bytes — ephemeral key) + + + typical ~170 bytes (depth-dependent) + diff --git a/docs/design/diagrams/session-warmup-fsm.svg b/docs/design/diagrams/session-warmup-fsm.svg new file mode 100644 index 0000000..4d1f4f5 --- /dev/null +++ b/docs/design/diagrams/session-warmup-fsm.svg @@ -0,0 +1,64 @@ + + + + + + Coordinate Warmup State Machine + + + + + + WARMUP + + + send coords via CP + or CoordsWarmup + + + + + + + N packets sent + + + + + + MINIMAL + + + no coords attached + + + + + + + CoordsRequired + or PathBroken + + + + + + SEND CoordsWarmup (0x14) + + + rate-limited + + + + + + + counter reset + + + + session established + + + also on LookupResponse + diff --git a/docs/design/diagrams/tree-announce.svg b/docs/design/diagrams/tree-announce.svg new file mode 100644 index 0000000..0932d7d --- /dev/null +++ b/docs/design/diagrams/tree-announce.svg @@ -0,0 +1,59 @@ + + + + + + TreeAnnounce (0x10) — 100 + 32n bytes + + + 0–3 + + + msg_type + 0x10 + + + version + 0x01 + + + sequence + + + 4–9 + + + 8 bytes LE — monotonic counter + + + 10–17 + + + timestamp + 8 bytes LE — Unix seconds + + + 18–33 + + + parent + 16 bytes — NodeAddr (self = root) + + + 34–35 + + + ancestry_count + + + 36– + + + AncestryEntry × n (32 bytes each) + + + +64 + + + Schnorr signature (64 bytes) + diff --git a/docs/design/fips-bloom-filters.md b/docs/design/fips-bloom-filters.md index 585ad79..fda6565 100644 --- a/docs/design/fips-bloom-filters.md +++ b/docs/design/fips-bloom-filters.md @@ -6,12 +6,31 @@ selection. It is a supporting reference — for how bloom filters fit into the overall routing system, see [fips-mesh-operation.md](fips-mesh-operation.md). +## What Is a Bloom Filter? + +A bloom filter is a space-efficient probabilistic data structure that +tests whether an element is a member of a set. It uses a bit array and +multiple hash functions to represent a set compactly, with one key +tradeoff: + +- **No false negatives** — if the filter says "not present," the + element is definitely not in the set +- **Possible false positives** — if the filter says "present," the + element is *probably* in the set, but might not be + +This makes bloom filters well-suited for routing: a definitive "no" +eliminates a peer from consideration, while a "maybe" simply means the +peer is worth checking. The false positive rate is controlled by the +filter size and number of hash functions relative to the number of +entries. + ## Purpose Each node maintains bloom filters summarizing which destinations are -reachable through each of its peers. Bloom filters provide **candidate -selection**: they narrow which peers are worth considering when forwarding -a packet to a given destination. The actual forwarding decision is made by +reachable through each of its peers. When forwarding a packet, a node +checks its peers' filters to identify potential routing paths — typically +up or down the spanning tree, unless a mesh peer provides a shortcut +directly to the destination. The actual forwarding decision is made by tree coordinate distance ranking. Bloom filters answer a single question: "can peer P possibly reach @@ -23,55 +42,85 @@ destination D?" The answer is either "no" (definitive) or "maybe" | Parameter | Value | Rationale | | --------- | ----- | --------- | | Size | 1 KB (8,192 bits) | Balance between accuracy and bandwidth | -| Hash functions (k) | 5 | Compromise between optimal k=7 for 800 entries and accommodating up to ~1,600 entries | +| Hash functions (k) | 5 | Optimal at ~1,200 entries; good compromise for 800–1,600 range | | Size class | 1 (v1 mandated) | `512 << 1 = 1024` bytes | ### Why k = 5 -The optimal number of hash functions depends on the expected occupancy: +The optimal number of hash functions is k_opt = (m/n) × ln(2). For +m = 8,192 bits: -- At 800 entries (typical for moderate-degree nodes), optimal k ≈ 7 -- At 1,600 entries (hub nodes), optimal k ≈ 4 +| n (entries) | k_opt | Rounded | +| ----------- | ----- | ------- | +| 800 | 7.1 | 7 | +| 1,200 | 4.7 | **5** | +| 1,600 | 3.5 | 4 | -k = 5 is a practical compromise that provides reasonable false positive -rates across the expected range of occupancy. +k = 5 is optimal at n ≈ 1,200 and a practical compromise across the +800–1,600 range. The FPR penalty versus the per-n optimal k is under +1 percentage point throughout this range. ## False Positive Rate (FPR) Analysis The false positive rate for a bloom filter with m bits, k hash functions, -and n entries is approximately: +and n entries is: ```text -FPR ≈ (1 - e^(-kn/m))^k +FPR = (1 - e^(-kn/m))^k ``` With m = 8,192 bits and k = 5: -| Node Degree | Expected Entries | FPR | Impact | -| ----------- | ---------------- | --- | ------ | -| 5 (IoT) | 100–200 | ~0.02% | Negligible | -| 8 (typical) | 250–400 | ~0.3% | Negligible | -| 12 (well-connected) | 500–800 | ~2.4% | Minor — occasional unnecessary discovery | -| 20+ (hub) | 1,200–1,800 | 7.5–15% | Elevated — more false positive candidates | +| n (entries) | Fill % | FPR | Impact | +| ----------- | ------ | --- | ------ | +| 200 | 11.5% | 0.002% | Negligible | +| 400 | 21.7% | 0.048% | Negligible | +| 800 | 38.6% | 0.86% | Negligible | +| 1,200 | 51.9% | 3.8% | Minor — occasional unnecessary candidate | +| 1,600 | 62.3% | 9.4% | Elevated | +| 2,400 | 78.3% | 27% | Poor — filter losing discrimination | +| 3,350 | 87% | 50% | Useless for candidate selection | -At moderate network sizes, filters are highly accurate. At larger scales -(~1M nodes), hub nodes with many peers see elevated FPR. False positives -cause unnecessary candidate evaluation (and potentially unnecessary -discovery attempts) but do not affect routing correctness — the tree -distance calculation makes the actual forwarding decision. +### Filter Occupancy Model + +Filter entries are determined by **network size and tree position**, not +node degree. Under tree-only merge with split-horizon: + +- **Upward filter** (to parent): contains the node's subtree — small + for deep nodes, at most N for the root +- **Downward filter** (to a child): contains the complement of the + child's subtree — approximately N × (b-1)/b entries for branching + factor b + +The worst-case filter is the downward direction from the root's child, +containing roughly N × (b-1)/b entries. Upward filters remain compact +even in large networks. + +For a tree with branching factor b ≈ 5: + +| Network size (N) | Worst-case n | FPR | Assessment | +| ----------------- | ------------ | --- | ---------- | +| 500 | 400 | 0.048% | Negligible | +| 1,000 | 800 | 0.86% | Good | +| 2,000 | 1,600 | 9.4% | Acceptable | +| 3,000 | 2,400 | 27% | Poor | +| 5,000 | 4,000 | 63% | Filter nearly useless | + +Upward filters remain excellent: a depth-2 node in a 10,000-node tree +has a subtree of ~400 entries (FPR 0.048%). ### Saturation Behavior -A bloom filter with 8,192 bits and k = 5 saturates (FPR approaches 100%) -around 3,000–4,000 entries. Beyond this, every query returns "maybe" and the -filter provides no candidate selection value. +At 50% FPR (n ≈ 3,350), the filter provides no better than coin-flip +discrimination and is effectively useless for candidate selection. +Full saturation (FPR > 99%) occurs around n ≈ 10,000. Tree-only merge propagation mitigates saturation by limiting each filter's content to tree-relevant entries. A node's outgoing filter to its parent contains only its subtree; its outgoing filter to a child contains the complement. Neither filter contains entries from mesh shortcuts' transitive -information, keeping filter occupancy proportional to the node's position -in the tree rather than the total mesh connectivity. +information. This keeps upward filters compact regardless of network size, +but downward filters grow proportionally with N. ## Per-Peer Filter Model @@ -179,38 +228,10 @@ property of the data structure). Entries are expired through: removed, and outbound filters are recomputed - **Filter replacement**: Each FilterAnnounce completely replaces the previous filter for that peer -- **Implicit timeout**: If no FilterAnnounce is received from a peer within - a threshold period, the peer's filter may be considered stale (tied to - link liveness detection) - -## Size Classes and Folding - -### Size Class Table - -| size_class | Bytes | Bits | Status | -| ---------- | ----- | ---- | ------ | -| 0 | 512 | 4,096 | Reserved | -| 1 | 1,024 | 8,192 | **v1 (MUST use)** | -| 2 | 2,048 | 16,384 | Reserved | -| 3 | 4,096 | 32,768 | Reserved | - -v1 nodes MUST use size_class = 1 and MUST reject FilterAnnounce messages -with any other size_class. - -### Folding (Forward Compatibility) - -Larger filters can be **folded** to smaller sizes by OR-ing the two halves -together. A 2 KB filter folds to 1 KB by OR-ing the upper and lower -halves. This preserves the "maybe present" property (no false negatives -introduced) but increases the false positive rate. - -Folding enables future protocol versions where hub nodes maintain larger -filters (lower FPR) while constrained nodes fold down to the mandated size. -A node receiving a larger filter folds it locally before use if needed. - -The hash function design supports folding: membership tests at a smaller -size use `hash(item, i) % smaller_bit_count`, which maps to the same bit -positions that folding produces. +- **Implicit timeout**: If a peer becomes unresponsive, the MMP link + liveness detector eventually declares the link dead and removes the + peer, which triggers filter cleanup as a side effect of peer removal. + There is no independent filter staleness timer. ## Membership Test @@ -244,30 +265,75 @@ overhead. See [fips-wire-formats.md](fips-wire-formats.md) for the complete wire format reference. -## Scale Considerations +## Scale and Size Classes -### Small Networks (< 1,000 nodes) +### v1 Scale Limits -Filters are very accurate (FPR < 1% for typical node degrees). Candidate -selection effectively identifies the correct forwarding peer on the first -try for most destinations. +Coordinate-based tree distance checking ensures correct routing decisions +at all network sizes — bloom filters are an optimization that narrows the +set of peers considered, not a correctness requirement. As filters +saturate, routing still works; it just evaluates more candidates per hop. -### Medium Networks (1,000–100,000 nodes) +With the v1 mandatory 1 KB filter (size_class 1): -Filters remain accurate for typical nodes. Hub nodes (20+ peers) may see -elevated FPR but the tree distance ranking correctly selects the best -candidate regardless. +- **Small networks (< 1,000 nodes)**: Both upward and downward filters + are highly accurate (worst-case FPR < 1%). Filters effectively narrow + candidates to the correct forwarding peer on the first try. -### Large Networks (> 100,000 nodes) +- **Medium networks (1,000–2,000 nodes)**: Upward filters remain + excellent. Downward filters approach 10% FPR at the worst case + (near-root nodes). The additional false positives add minor overhead + to candidate evaluation but do not affect routing correctness. -Hub nodes approach filter saturation. The filter still provides value -(some peers can be definitively excluded) but the candidate set grows -larger. The tree distance calculation becomes the primary routing -discriminator. +- **Large networks (> 2,000 nodes)**: Downward filters lose most of + their discrimination value. At N ≈ 5,000, downward filters for + near-root nodes have ~63% FPR, providing little benefit over evaluating + all peers. Upward filters remain compact and accurate at any scale. + Routing correctness is unaffected — greedy tree distance still selects + the right next hop — but the efficiency gain from bloom filter + pre-selection diminishes. -Future mitigation: hub nodes could use larger filters (size_class 2 or 3) -while constrained nodes fold to size_class 1. This requires protocol -negotiation not present in v1. +### Size Class Table + +| size_class | Bytes | Bits | Status | +| ---------- | ----- | ---- | ------ | +| 0 | 512 | 4,096 | Reserved | +| 1 | 1,024 | 8,192 | **v1 (MUST use)** | +| 2 | 2,048 | 16,384 | Reserved | +| 3 | 4,096 | 32,768 | Reserved | + +FMP v1 mandates size_class = 1. Nodes MUST use size_class = 1 and MUST +reject FilterAnnounce messages with any other size_class. The size_class +field is reserved in the wire format to support future protocol versions +with larger default filter sizes. + +### Scaling Strategy + +The 1 KB filter becomes a practical limitation beyond ~2,000 nodes. The +size class mechanism provides the path forward: future FMP versions may +use larger default filters (size_class 2 or 3) to support larger networks +while remaining compatible with constrained nodes through folding. +Size_class 2 (2 KB, 16,384 bits) would roughly double the practical +network size limit. + +The envisioned approach is that hub nodes near the root — which carry the +largest downward filters — would use larger size classes, while leaf nodes +and resource-constrained nodes continue with smaller filters. A node +receiving a filter larger than its own size class folds it down locally. +The mechanism by which heterogeneous filter sizes propagate through the +tree is a future design direction not specified in v1. See +[IDEA-0043](../../ideas/IDEA-0043-heterogeneous-filter-propagation.md). + +### Folding + +Larger filters can be **folded** to smaller sizes by OR-ing the two halves +together. A 2 KB filter folds to 1 KB by OR-ing the upper and lower +halves. This preserves the "maybe present" property (no false negatives +introduced) but increases the false positive rate. + +The hash function design supports folding: membership tests at a smaller +size use `hash(item, i) % smaller_bit_count`, which maps to the same bit +positions that folding produces. ## Implementation Status diff --git a/docs/design/fips-configuration.md b/docs/design/fips-configuration.md index 6dcfe61..02e6f3a 100644 --- a/docs/design/fips-configuration.md +++ b/docs/design/fips-configuration.md @@ -57,9 +57,8 @@ peers: # Static peer list | `node.control.socket_path` | string | *(auto)* | Socket file path. Default: `$XDG_RUNTIME_DIR/fips/control.sock` if `XDG_RUNTIME_DIR` is set, otherwise `/tmp/fips-control.sock` | The control socket provides read-only access to node state via the -`fipsctl` command-line tool. See -[fips-software-architecture.md](fips-software-architecture.md) for the -protocol and command list. +`fipsctl` command-line tool. See the project +[README](../../README.md#inspect) for the command list. All tunable protocol parameters live under `node.*`, organized as sysctl-style dotted paths. The top-level sections (`tun`, `dns`, `transports`, `peers`) @@ -285,7 +284,7 @@ dns: transports: udp: bind_addr: "0.0.0.0:4000" - mtu: 1197 + mtu: 1472 peers: - npub: "npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le" diff --git a/docs/design/fips-intro.md b/docs/design/fips-intro.md index 50bc326..e4ff9a4 100644 --- a/docs/design/fips-intro.md +++ b/docs/design/fips-intro.md @@ -108,14 +108,14 @@ and applications know nothing about which physical media carry their traffic. This separation means new transports, protocol features, and application interfaces can be added independently. -![Protocol Stack](fips-protocol-stack.svg) +![Protocol Stack](diagrams/fips-protocol-stack.svg) ### Mapping to Traditional Networking Readers familiar with the OSI model or TCP/IP networking may find it helpful to see how FIPS concepts relate to traditional layers: -![OSI Mapping](fips-osi-mapping.svg) +![OSI Mapping](diagrams/fips-osi-mapping.svg) Note that FMP spans what would traditionally be separate link and network layers. This is intentional — in a self-organizing mesh, the same layer that @@ -168,7 +168,7 @@ Two application interfaces sit at the top of the stack: a native datagram API addressed by npub, and an IPv6 TUN adapter that maps npubs to `fd00::/8` addresses so unmodified IP applications can use the network transparently. -![Node Architecture](fips-node-architecture.svg) +![Node Architecture](diagrams/fips-node-architecture.svg) The mesh routes application traffic across heterogeneous transports transparently. A packet may traverse WiFi, Ethernet, UDP/IP, and Tor links @@ -177,7 +177,7 @@ which transports are involved. Each hop is independently encrypted at the link layer, while a single end-to-end session protects the payload across the entire path. -![Architecture Overview](fips-architecture-overview.svg) +![Architecture Overview](diagrams/fips-architecture-overview.svg) --- @@ -198,7 +198,7 @@ same keypair. ### FIPS Identity Handling -![Identity Derivation](fips-identity-derivation.svg) +![Identity Derivation](diagrams/fips-identity-derivation.svg) The pubkey is the node's cryptographic identity, used in Noise IK handshakes for both link and session encryption. It is never exposed beyond the @@ -299,7 +299,7 @@ complementary mechanisms provide the information each node needs. ### Spanning Tree: The Coordinate System -![Mesh Topology](fips-mesh-topology.svg) +![Mesh Topology](diagrams/fips-mesh-topology.svg) Nodes self-organize into a spanning tree through gossip — each node exchanges announcements with its direct peers and independently selects a @@ -363,7 +363,7 @@ the full network. See [fips-bloom-filters.md](fips-bloom-filters.md) for filter parameters and mathematical properties. -![Bloom filter propagation on a spanning tree](fips-bloom-propagation.svg) +![Bloom filter propagation on a spanning tree](diagrams/fips-bloom-propagation.svg) The outbound filter for peer Q merges this node's identity with tree peer inbound filters except Q's (split-horizon exclusion). This creates @@ -405,7 +405,7 @@ These are cached at each node after being learned through discovery (LookupRequest/LookupResponse) or session establishment (SessionSetup). The coordinate cache is the critical piece that enables efficient forwarding. -![Per-hop routing decision flowchart](fips-routing-decision.svg) +![Per-hop routing decision flowchart](diagrams/fips-routing-decision.svg) ### Coordinate Caching and Discovery @@ -419,7 +419,7 @@ Session establishment (SessionSetup) also carries coordinates, warming transit node caches along the path so that data packets can be forwarded without individual discovery at each hop. -![Coordinate discovery and cache warming sequence](fips-coordinate-discovery.svg) +![Coordinate discovery and cache warming sequence](diagrams/fips-coordinate-discovery.svg) ### Error Recovery @@ -783,8 +783,6 @@ of self-sovereign identity systems. No novel cryptography is introduced. | Document | Description | | -------- | ----------- | -| [fips-software-architecture.md](fips-software-architecture.md) | Stable architectural decisions guiding the codebase | -| [fips-state-machines.md](fips-state-machines.md) | Phase-based state machine pattern (Rust) | | [fips-configuration.md](fips-configuration.md) | YAML configuration reference | ### External References diff --git a/docs/design/fips-ipv6-adapter.md b/docs/design/fips-ipv6-adapter.md index 79f4b8b..66351d3 100644 --- a/docs/design/fips-ipv6-adapter.md +++ b/docs/design/fips-ipv6-adapter.md @@ -10,7 +10,8 @@ can communicate over the mesh without modification. The adapter bridges two worlds: IPv6 applications that address destinations by IP address, and the FIPS mesh that addresses destinations by public key (npub). The adapter handles the translation: DNS resolution from npub to -`fd::/8` address, identity cache management so FIPS can route IPv6 packets, +`fd00::/8` ULA (Unique Local Address) address, identity cache management so +FIPS can route IPv6 packets, MTU enforcement so packets fit through the mesh, and the TUN interface that connects to the kernel's IPv6 stack. @@ -21,7 +22,7 @@ native FIPS datagram API, addressing destinations directly by npub. ### The Problem -IPv6 addresses in the `fd::/8` range are derived from public keys via a +IPv6 addresses in the `fd00::/8` range are derived from public keys via a one-way hash (SHA-256). Given only an IPv6 address, the public key cannot be recovered — and without the public key, FIPS cannot compute the node_addr needed for routing. @@ -35,7 +36,7 @@ DNS resolution serves as the "routing intent" signal. When an application resolves `npub1xxx...xxx.fips`, the FIPS DNS service: 1. Extracts the npub from the `.fips` domain name -2. Derives the `fd::/8` IPv6 address from the public key +2. Derives the `fd00::/8` IPv6 address from the public key 3. Primes the identity cache with the mapping (IPv6 address prefix ↔ NodeAddr ↔ PublicKey) 4. Returns the IPv6 address to the application @@ -54,7 +55,7 @@ for address derivation. ### Traffic Without Prior DNS Lookup -A packet may arrive at the TUN for an `fd::/8` destination without a prior DNS +A packet may arrive at the TUN for an `fd00::/8` destination without a prior DNS lookup — cached address, manual configuration, etc. Since the address derivation is one-way, the npub cannot be recovered from the address alone. @@ -82,24 +83,37 @@ SHA-256 → node_addr (16 bytes, truncated) fd + node_addr[0..15] → IPv6 address (16 bytes) ``` -The `fd` prefix ensures no collision with addresses in use on the underlying -transport network. These are overlay identifiers — they appear in the TUN -interface for application compatibility but are not routable on the underlying -transport. +The `fd` prefix places FIPS addresses in the IPv6 Unique Local Address (ULA) +space defined by RFC 4193. ULAs are the IPv6 equivalent of RFC 1918 private +addresses (10.x, 172.16.x, 192.168.x) — they are reserved for local use and +are not routable over the public Internet. This means FIPS overlay addresses +cannot conflict with native IPv6 traffic that may be present on the same host +or network, and they will not leak beyond the local system even if routing is +misconfigured. These are overlay identifiers — they appear in the TUN +interface for application compatibility but have no meaning outside the FIPS +mesh. ## Identity Cache -The identity cache maps the FIPS address prefix (15 bytes — the IPv6 address -minus the `fd` prefix) to `(NodeAddr, PublicKey)`. This cache is needed only -when using the IPv6 adapter; the native FIPS API provides the public key -directly. +The derivation from public key to NodeAddr and IPv6 address is one-way +(SHA-256 truncation). Given a destination IPv6 address from an outbound packet +on the TUN interface, the adapter cannot recover the public key or NodeAddr +needed for FIPS routing. The identity cache provides the reverse lookup: +it maps the FIPS address prefix (15 bytes — the IPv6 address minus the `fd` +prefix) back to `(NodeAddr, PublicKey)`, allowing the adapter to route +IPv6 traffic into the mesh. This cache is needed only when using the IPv6 +adapter; the native FIPS API provides the public key directly. ### Eviction Policy The mapping is deterministic (derived from the public key) and never becomes stale. The cache uses **LRU-only eviction** bounded by a configurable size (default 10K entries). There is no TTL — entries are evicted only when the -cache is full and space is needed for a new entry. +cache is full and space is needed for a new entry. LRU-only eviction is +necessary because there is no other way for the FIPS router to recover the +routing identity from an IPv6 address, and IPv6 traffic for a destination may +arrive an arbitrarily long time after the DNS resolution that populated the +cache entry. ### Relationship to DNS TTL @@ -111,9 +125,17 @@ entry hasn't been evicted by memory pressure. ## MTU Enforcement -FIPS encapsulation adds overhead to every packet. The adapter must ensure -that IPv6 packets from applications fit within the FIPS encapsulation budget -after all layers of wrapping. +FIPS does not provide fragmentation or reassembly at the session or mesh +protocol layers — every datagram must fit in a single transport-layer packet. +Some transports may perform fragmentation and reassembly internally (e.g., BLE +L2CAP) and can advertise a larger virtual MTU than the physical medium +supports, but this is transparent to FIPS. The mesh layer provides two +facilities to manage MTU across heterogeneous paths: route discovery can +constrain results to paths that support a required minimum MTU, and transit +nodes that cannot forward an oversized datagram send an MtuExceeded error +signal back to the source. The adapter must ensure that IPv6 packets from +applications fit within the FIPS encapsulation budget after all layers of +wrapping. ### Encapsulation Overhead @@ -156,8 +178,9 @@ transport path MTU for the IPv6 adapter is therefore: ``` Transports with smaller MTUs (radio at ~250 bytes, serial at 256 bytes) cannot -support the IPv6 adapter — applications on those transports must use the -native FIPS datagram API. +support the IPv6 adapter without some form of internal fragmentation and +reassembly. Otherwise, applications on those transports must use the native +FIPS datagram API. ### ICMP Packet Too Big @@ -206,10 +229,10 @@ defining feature. ### Architecture ```text -Applications (sockets using fd::/8 addresses) +Applications (sockets using fd00::/8 addresses) │ ▼ -Kernel IPv6 Stack (routing: fd::/8 → fips0) +Kernel IPv6 Stack (routing: fd00::/8 → fips0) │ ▼ TUN Device (fips0) @@ -222,7 +245,7 @@ TUN Device (fips0) The TUN reader receives raw IPv6 packets from applications and processes them: 1. Validate IPv6 header -2. Extract destination `fd::/8` address +2. Extract destination `fd00::/8` address 3. Look up identity cache — miss returns ICMPv6 Destination Unreachable 4. Retrieve NodeAddr and PublicKey from cache 5. Look up or establish FSP session @@ -247,7 +270,7 @@ handle. The Linux kernel routing table processes rules in priority order: 1. **Local table**: Intercepts traffic to addresses assigned to this machine -2. **Main table**: Routes `fd::/8` to the TUN device +2. **Main table**: Routes `fd00::/8` to the TUN device This means every packet arriving at the TUN reader is guaranteed to be for a *remote* FIPS destination. No "is this for me?" check is needed on the read @@ -283,10 +306,10 @@ TUN device creation requires `CAP_NET_ADMIN`. Options: | TCP MSS clamping (SYN + SYN-ACK) | **Implemented** | | DNS service (.fips domain) | **Implemented** | | Per-destination route MTU (netlink) | Planned | -| Transit MTU error signal | Planned | +| Transit MTU error signal | **Implemented** | | Path MTU tracking (SessionDatagram field) | **Implemented** | | Path MTU notification (end-to-end echo) | **Implemented** | -| Endpoint fragmentation/reassembly | Future direction | +| Endpoint fragmentation/reassembly | Transport drivers | ## Design Considerations @@ -318,11 +341,6 @@ plaintext. Fragmenting encrypted datagrams would require either exposing plaintext structure to transit nodes (unacceptable) or reassembly before decryption (opens attack surface). -Endpoint-only fragmentation (fragment before session encryption, reassemble -after decryption) is a future direction that avoids these objections. Each -fragment would be independently encrypted and look like a normal data packet -to transit nodes. - ## References - [fips-intro.md](fips-intro.md) — Protocol overview and architecture diff --git a/docs/design/fips-mesh-layer.md b/docs/design/fips-mesh-layer.md index 4dcc13f..c68ad03 100644 --- a/docs/design/fips-mesh-layer.md +++ b/docs/design/fips-mesh-layer.md @@ -280,8 +280,8 @@ authenticated peer regardless of what transport address the packet arrived from. FMP updates the peer's current address to the packet's source address, and subsequent outbound packets use the updated address. -This allows peers to change transport addresses (IP:port for UDP, connection -handle for TCP) without session interruption. The mechanism is: +This allows peers to change transport addresses (e.g., IP:port for UDP) +without session interruption. The mechanism is: 1. Packet arrives from a different address than expected 2. Receiver index lookup finds the session diff --git a/docs/design/fips-mesh-operation.md b/docs/design/fips-mesh-operation.md index 7ca60b1..395d0d7 100644 --- a/docs/design/fips-mesh-operation.md +++ b/docs/design/fips-mesh-operation.md @@ -23,8 +23,8 @@ Two complementary mechanisms provide the information each node needs: ancestry path from itself to the root. These coordinates enable distance calculations between any two nodes without global topology knowledge. - **Bloom filters** summarize which destinations are reachable through each - peer. They provide candidate selection — narrowing which peers are worth - considering for forwarding a given destination. + peer. Because they propagate along tree edges, they encode directional + reachability — which subtree contains a given destination. Together, they enable a routing decision process that is local, efficient, and self-healing. @@ -48,7 +48,7 @@ enable: Nodes self-organize into a spanning tree through distributed parent selection: -1. **Root election**: The node with the smallest node_addr becomes the root. +1. **Root discovery**: The node with the smallest node_addr becomes the root. No election protocol — this is a consequence of each node independently preferring lower-addressed roots. 2. **Parent selection**: Each node selects a single parent from among its @@ -75,10 +75,76 @@ Changes cascade through the tree: TreeAnnounce propagation is rate-limited at 500ms minimum interval per peer. A tree of depth D reconverges in roughly D×0.5s to D×1.0s. +### How the Tree Adapts to Link Quality + +The initial tree forms based on hop count alone — all links default to a +cost of 1.0 before measurements are available. As the Metrics Measurement +Protocol (MMP) accumulates bidirectional delivery ratios and round-trip +time estimates, each node computes a per-link cost: + +```text +link_cost = ETX × (1.0 + SRTT_ms / 100.0) +``` + +ETX (Expected Transmission Count) captures loss — a perfect link has +ETX = 1.0, while 10% loss in each direction yields ETX ≈ 1.23. The SRTT +term weights latency so that a low-loss but high-latency link (e.g., a +satellite hop) costs more than a low-loss, low-latency link. + +Parent selection uses **effective depth** rather than raw hop count: + +```text +effective_depth = peer.depth + link_cost_to_peer +``` + +This allows a node to trade a shorter but lossy path for a longer but +higher-quality one. A node two hops from the root over clean links +(effective depth ≈ 3.0) is preferred over a node one hop away over a +degraded link (effective depth ≈ 4.5). + +Parent reselection is triggered by three paths: + +1. **TreeAnnounce**: When a peer announces a new tree position, the node + re-evaluates using current link costs +2. **Periodic re-evaluation**: Every 60s (configurable), the node + re-evaluates its parent choice using the latest MMP metrics, catching + gradual link degradation that doesn't trigger TreeAnnounce +3. **Parent loss**: When the current parent is removed, the node + immediately selects the best alternative + +To prevent oscillation from metric noise, parent switches are subject to +**hysteresis**: a candidate must offer an effective depth at least 20% +better than the current parent to trigger a switch. A **hold-down period** +(default 30s) suppresses non-mandatory re-evaluation after a switch, +allowing MMP metrics to stabilize on the new link before reconsidering. + +### Flap Dampening + +Unstable links that repeatedly connect and disconnect can cause cascading +tree reconvergence. The spanning tree uses flap dampening with hysteresis +and hold-down periods to suppress rapid parent oscillation. Links that flap +above a configurable threshold are temporarily penalized, preventing them +from being selected as parent until the link stabilizes. + +### Link Liveness + +Each node sends a dedicated **Heartbeat** message (0x51, 1 byte, no +payload) to every peer at a fixed interval (default 10s). Any +authenticated encrypted frame — heartbeat, MMP report, TreeAnnounce, +data packet — resets the peer's liveness timer. On an idle link with no +application data or topology changes, the heartbeat is the only traffic +that keeps the link alive. + +Peers that are silent for a configurable dead timeout (default 30s) are +considered dead and removed from the peer table. With the default 10s +heartbeat interval, a peer must miss three consecutive heartbeats before +removal. This triggers tree reconvergence and bloom filter recomputation +for the affected subtree. + ### Partition Handling -If the network partitions, each segment independently elects its own root -(the smallest node_addr in the segment) and reconverges. When segments +If the network partitions, each segment independently rediscovers its own +root (the smallest node_addr in the segment) and reconverges. When segments rejoin, nodes discover the globally-smallest root through TreeAnnounce exchange and reconverge to a single tree. @@ -94,9 +160,10 @@ Each node maintains a bloom filter per peer, answering: "can peer P possibly reach destination D?" The answer is either "no" (definitive) or "maybe" (probabilistic — false positives are possible). -This is **candidate selection**, not routing. Bloom filters identify which -peers are worth considering for a destination, but the actual forwarding -decision uses tree coordinate distance to rank those candidates. +Because filters propagate along tree edges with split-horizon exclusion, +they encode directional reachability: a bloom hit on a tree peer reliably +indicates which subtree contains the destination. When multiple peers match, +tree coordinate distance ranks them. ### How Filters Propagate @@ -135,9 +202,9 @@ Updates are rate-limited at 500ms to prevent storms during topology changes. At moderate network sizes, bloom filters are highly accurate. At larger scales (~1M nodes), hub nodes with many peers may see elevated false positive -rates (7–15% for nodes with 20+ peers). False positives cause unnecessary -discovery attempts but do not affect routing correctness — the tree distance -calculation makes the actual forwarding decision. +rates (7–15% for nodes with 20+ peers). False positives may cause a packet +to be forwarded toward the wrong subtree, but the self-distance check at +each hop prevents loops and the packet falls through to greedy tree routing. See [fips-bloom-filters.md](fips-bloom-filters.md) for filter parameters, FPR calculations, and size class folding. @@ -155,8 +222,8 @@ priority chain. This is the core routing algorithm. 2. **Direct peer** — The destination is an authenticated neighbor. Forward directly. No coordinates or bloom filters needed. -3. **Bloom-guided candidate selection** — One or more peers' bloom filters - contain the destination. Select the best candidate by composite key: +3. **Bloom-guided routing** — One or more peers' bloom filters contain the + destination. Select the best peer by composite key: `(link_cost, tree_distance, node_addr)`. This requires the destination's tree coordinates to be in the local coordinate cache. @@ -189,10 +256,12 @@ through to greedy tree routing (functional but suboptimal). When bloom filters identify multiple candidate peers, they are ranked by a composite key: -1. **link_cost** — Per-link quality metric. ETX is computed from bidirectional - delivery ratios in MMP metrics and is used in cost-based parent selection - via `effective_depth = depth + link_cost`, but is not yet wired into - `find_next_hop()` candidate ranking. +1. **link_cost** — Per-link quality metric derived from ETX (Expected + Transmission Count), computed from bidirectional delivery ratios in MMP + metrics. In practice this is an uncommon tie-breaker: most forwarding + decisions are resolved by tree distance alone, and link_cost only + differentiates candidates when multiple peers offer the same tree distance + to the destination. 2. **tree_distance** — Coordinate-based distance to destination through this peer 3. **node_addr** — Deterministic tie-breaker @@ -285,9 +354,10 @@ to a visited filter (preventing loops on a single path), and forwards to all peers not in the visited filter. Bloom filters may help direct the flood toward likely candidates. -**Deduplication**: Nodes maintain a short-lived request_id dedup cache to -drop convergent duplicates (the same request arriving via different paths). -This is a protocol requirement, not an optimization. +**Deduplication**: Nodes maintain a short-lived request_id dedup cache +(default 10s window) to drop convergent duplicates (the same request +arriving via different paths). This is a protocol requirement, not an +optimization. ### LookupResponse @@ -528,10 +598,12 @@ When traffic resumes: 3. Coordinates: discovery may be needed if cache has expired 4. SessionSetup re-warms transit caches on the new path -## Leaf-Only Operation *(future direction)* +## Leaf-Only Operation *(under development)* -Leaf-only operation is a planned optimization for resource-constrained nodes -(sensors, battery-powered devices). Not currently implemented. +Leaf-only operation is an optimization for resource-constrained nodes +(sensors, battery-powered devices). The core infrastructure exists (config +flag, node constructor, bloom filter support) but is not yet enabled in +normal operation. ### Concept @@ -558,7 +630,7 @@ The upstream peer: Even as a leaf-only node, it still: -- Maintains its own Noise IK link session with the upstream peer +- Maintains its own Noise IK link session with the upstream peer (FMP layer) - Can establish end-to-end FSP sessions with arbitrary destinations - Has its own identity (npub, node_addr) @@ -621,9 +693,12 @@ recovery). | LookupResponse proof verification | **Implemented** | | Discovery reverse-path routing | **Implemented** | | Error signal rate limiting | **Implemented** | -| Leaf-only operation | Future direction | +| Flap dampening (hysteresis + hold-down) | **Implemented** | +| Link liveness (dead timeout) | **Implemented** | +| Discovery request deduplication | **Implemented** | +| Leaf-only operation | Under development | | Link cost in parent selection (ETX) | **Implemented** | -| Link cost in candidate ranking | Future direction | +| Link cost in candidate ranking | **Implemented** | | Discovery path accumulation | Future direction | ## References diff --git a/docs/design/fips-session-layer.md b/docs/design/fips-session-layer.md index 87a83fa..ca4dd5f 100644 --- a/docs/design/fips-session-layer.md +++ b/docs/design/fips-session-layer.md @@ -38,7 +38,7 @@ encrypts the payload, and routes through FMP. No DNS involvement. ### IPv6 Adapter -Unmodified IPv6 applications use a TUN device with `fd::/8` routing. A local +Unmodified IPv6 applications use a TUN device with `fd00::/8` routing. A local DNS service maps npub → IPv6 address and primes the identity cache. Packets arriving at the TUN are translated to FIPS datagrams and routed through FSP. @@ -111,10 +111,10 @@ node, FMP delivers it to FSP for session-layer processing. Sessions are established on demand when the first datagram needs to be sent to a destination with no existing session. -FSP uses Noise XK for session key agreement. The initiator knows the -destination's npub (required for XK's pre-message `s` token); the -responder learns the initiator's identity from msg3 (not msg1, unlike -IK at the link layer). This provides stronger initiator identity hiding +FSP uses Noise XK for session key agreement (Noise Protocol Framework; +Perrin 2018). The initiator knows the destination's npub (required for +XK's pre-message `s` token); the responder learns the initiator's +identity from msg3 (not msg1, unlike IK at the link layer). This provides stronger initiator identity hiding — the initiator's static key is encrypted under the established shared secret rather than under only the responder's static key. @@ -133,7 +133,8 @@ and SessionMsg3: Each side's epoch (an 8-byte random value generated at startup) is exchanged encrypted in msg2 and msg3. On subsequent handshakes, an epoch mismatch indicates the peer has restarted, triggering session -re-establishment. +re-establishment — similar to IKEv2's INITIAL_CONTACT notification for +peer restart detection (RFC 7296 §2.4). Packets that trigger session establishment are queued (with bounded buffer) and transmitted after the session is established. @@ -154,14 +155,15 @@ the forward path (e.g., after tree reconvergence). ### Simultaneous Initiation When both nodes attempt to establish a session simultaneously ("crossing -hellos"), a deterministic tie-breaker resolves the conflict: +hellos"), a deterministic tie-breaker resolves the conflict, mirroring +IKEv2's simultaneous initiation resolution (RFC 7296 §2.8): - If `local_node_addr < remote_node_addr`: Continue as initiator, ignore incoming setup - If `local_node_addr > remote_node_addr`: Abort own initiation, switch to responder role -This ensures exactly one handshake completes. +This lowest-address-wins approach ensures exactly one handshake completes. ### Data Transfer @@ -194,7 +196,9 @@ The fresh SessionSetup re-warms the caches, maintaining routing continuity. ### Session Independence from Transport Sessions exist above the routing layer and are bound to npub identities, not -transport addresses or routing paths. A session survives: +transport addresses or routing paths — following WireGuard's approach +(Donenfeld 2017) of binding cryptographic sessions to identity keys rather +than network addresses. A session survives: - Transport failover (UDP → Ethernet → back to UDP) - Route changes (different intermediate hops) @@ -237,7 +241,9 @@ operations) rather than under only the responder's static key. | Hash | SHA-256 | Nostr-native | | Key derivation | HKDF-SHA256 | Standard Noise KDF | -These choices prioritize compatibility with the Nostr cryptographic stack. +These choices prioritize compatibility with the Nostr cryptographic stack — +secp256k1 + ChaCha20-Poly1305 + SHA-256 aligns with the NIP-44 encrypted +messaging standard. ### secp256k1 Parity Normalization @@ -271,14 +277,16 @@ authentication for every packet, matching WireGuard and Lightning's approach. ### Forward Secrecy -Ephemeral keys in the Noise handshake provide forward secrecy. Compromise of -static keys (nsec) does not reveal past session keys, because session keys are -derived in part from ephemeral-ephemeral DH (`ee`), and ephemeral keys are -discarded after the handshake. +Ephemeral keys in the Noise handshake provide forward secrecy — a standard +property of Noise handshake patterns that include ephemeral key exchange. +Compromise of static keys (nsec) does not reveal past session keys, because +session keys are derived in part from ephemeral-ephemeral DH (`ee`), and +ephemeral keys are discarded after the handshake. ## Replay Protection -FSP uses explicit 8-byte counters on the wire for replay protection. Each side +FSP uses explicit 8-byte counters on the wire for replay protection, adapted +from the DTLS anti-replay mechanism (Rescorla & Modadugu, RFC 6347). Each side maintains a monotonically increasing send counter, included in the 12-byte cleartext header of every encrypted message. The receiver maintains a sliding window (2048-entry bitmap) tracking which counters have been seen. @@ -287,8 +295,9 @@ This design is critical for operation over unreliable transports. Under UDP packet loss or reordering, implicit nonce counters (where the receiver increments on each decrypt attempt) would desynchronize permanently — a failed `decrypt()` increments the nonce, and the desync grows with each lost packet. -Explicit counters allow the receiver to decrypt any packet independently, -regardless of what packets were lost or reordered. +Explicit counters with a sliding bitmap window — the standard DTLS approach — +allow the receiver to decrypt any packet independently, regardless of what +packets were lost or reordered. The same `ReplayWindow` and `decrypt_with_replay_check()` implementation is used at both the link and session layers. @@ -302,7 +311,9 @@ entry expires, it cannot forward data packets (which carry only addresses, not coordinates) and sends a CoordsRequired error. FSP uses a hybrid warmup strategy combining proactive piggybacking with -reactive standalone messages to keep transit caches populated: +reactive standalone messages to keep transit caches populated (Yggdrasil +uses a similar approach of embedding coordinates in session traffic to +warm transit node caches): ### Proactive Warmup Phase @@ -329,15 +340,12 @@ Transit nodes extract coordinates via the existing `try_warm_coord_cache()` code path with zero changes to the transit forwarding logic. CoordsWarmup is indistinguishable from any other CP-flagged message at the transit layer. -Wire format: - -```text -FSP header (12 bytes, AAD): ver=0, phase=0, flags=CP, counter, payload_len -Cleartext coords: src_coords + dst_coords (same encoding as CP flag) -AEAD ciphertext: inner_header(6) + Poly1305 tag(16) = 22 bytes - -Total FSP payload: 12 + coords + 22 -``` +On the wire, a CoordsWarmup message consists of the standard 12-byte FSP +header (used as AEAD AAD) with ver=0, phase=0, flags=CP, followed by +cleartext source and destination coordinates (same encoding as any +CP-flagged packet), followed by 22 bytes of AEAD ciphertext (6-byte inner +header and 16-byte Poly1305 tag). The total FSP payload is 12 bytes of +header, the coordinate data, and 22 bytes of ciphertext. ### Steady State @@ -368,30 +376,11 @@ signal generation (100ms per destination). ### Warmup State Machine -```text - ┌──────────────┐ - │ WARMUP │ ◄── Send first N packets with coords (CP or CoordsWarmup) - └──────┬───────┘ - │ N packets sent without CoordsRequired - ▼ - ┌──────────────┐ - │ MINIMAL │ ◄── Send packets without coords - └──────┬───────┘ - │ CoordsRequired or PathBroken received - ▼ - ┌──────────────────────────┐ - │ SEND CoordsWarmup (0x14) │ ◄── Immediate standalone warmup (rate-limited) - └──────────┬───────────────┘ - │ - ▼ - ┌──────────────┐ - │ WARMUP │ ◄── Counter reset, piggyback coords again - └──────────────┘ -``` +![Coordinate warmup state machine](diagrams/session-warmup-fsm.svg) ## Identity Cache -The identity cache maps FIPS address prefix (15 bytes, the `fd::/8` IPv6 +The identity cache maps FIPS address prefix (15 bytes, the `fd00::/8` IPv6 address minus the `fd` prefix) to `(NodeAddr, PublicKey)`. This cache is needed only when using the IPv6 adapter — the native FIPS API provides the public key directly. @@ -482,7 +471,8 @@ bandwidth cost: clamped to [500ms, 10s] with a cold-start interval of 1s ### Path MTU Tracking PathMtuNotification (message type 0x13) provides end-to-end path MTU -feedback: +feedback, adapting RFC 1191 Path MTU Discovery for overlay networks — the +transit-node `min()` propagation replaces ICMP Packet Too Big: 1. The source sets `path_mtu` in each SessionDatagram envelope to its outbound link MTU. @@ -500,7 +490,8 @@ feedback: ### Send Failure Backoff When a session MMP report cannot be delivered (destination unreachable, no -route), the sender applies exponential backoff to the probe interval: +route), the sender applies exponential backoff to the probe interval (a +standard distributed systems pattern for transient failure handling): - Each consecutive failure doubles the interval: 2x, 4x, 8x, 16x, 32x - Backoff caps at 32x the base interval (5 consecutive failures) @@ -553,6 +544,8 @@ MMP session metrics session=npub1tdwa...84le rtt=4.3ms loss=0.6% jitter=0.2ms go ## References +### FIPS Internal Documentation + - [fips-intro.md](fips-intro.md) — Protocol overview and architecture - [fips-mesh-layer.md](fips-mesh-layer.md) — FMP specification (below FSP) - [fips-ipv6-adapter.md](fips-ipv6-adapter.md) — IPv6 adaptation layer (above FSP) @@ -560,3 +553,29 @@ MMP session metrics session=npub1tdwa...84le rtt=4.3ms loss=0.6% jitter=0.2ms go error recovery - [fips-wire-formats.md](fips-wire-formats.md) — Wire format reference for all session message types + +### External References + +- Perrin, T. ["The Noise Protocol Framework"](https://noiseprotocol.org/noise.html). + Revision 34, 2018. *Framework for building crypto protocols using Diffie-Hellman + key agreement and AEAD ciphers. FSP uses the XK handshake pattern.* + +- Donenfeld, J.A. ["WireGuard: Next Generation Kernel Network Tunnel"](https://www.wireguard.com/papers/wireguard.pdf). + NDSS 2017. *Transport-independent cryptographic sessions bound to identity keys + rather than network addresses; AEAD-only authentication model.* + +- Rescorla, E., Modadugu, N. [RFC 6347](https://datatracker.ietf.org/doc/html/rfc6347): + "Datagram Transport Layer Security Version 1.2". 2012. *Explicit sequence numbers + with sliding bitmap window for replay protection over unreliable transports.* + +- Kaufman, C., Hoffman, P., Nir, Y., Eronen, P., Kivinen, T. + [RFC 7296](https://datatracker.ietf.org/doc/html/rfc7296): + "Internet Key Exchange Protocol Version 2 (IKEv2)". 2014. *Simultaneous + initiation resolution (§2.8) and INITIAL_CONTACT peer restart detection (§2.4).* + +- Mogul, J., Deering, S. [RFC 1191](https://datatracker.ietf.org/doc/html/rfc1191): + "Path MTU Discovery". 1990. *End-to-end path MTU discovery; FSP adapts this for + overlay networks using transit-node min() propagation.* + +- [Yggdrasil Network](https://yggdrasil-network.github.io/). *Coordinate-based + overlay routing with session traffic used to warm transit node coordinate caches.* diff --git a/docs/design/fips-software-architecture.md b/docs/design/fips-software-architecture.md deleted file mode 100644 index a3510f7..0000000 --- a/docs/design/fips-software-architecture.md +++ /dev/null @@ -1,365 +0,0 @@ -# FIPS Software Architecture - -This document describes the stable architectural decisions that guide the -FIPS codebase — the "why" behind the code's shape. It covers design -principles and patterns that are expected to remain stable as the -implementation evolves. For protocol behavior and wire formats, see the -protocol layer documents. - -## Ownership and Entity Hierarchy - -A FIPS node owns transports, which produce links, which authenticate into -peers: - -```text -Node -├── Transports (HashMap) -│ └── Each transport instance manages one communication medium -├── Links (HashMap) -│ └── Each link is a connection to a remote endpoint over a transport -├── Peers (HashMap) -│ └── Each peer is an authenticated remote FIPS node -├── TreeState — local view of the spanning tree -├── CoordCache — destination coordinates for routing -├── Sessions — end-to-end FSP sessions (HashMap by NodeAddr) -└── Identity — this node's cryptographic identity -``` - -**Key ownership rules**: - -- A transport exists for the lifetime of the node (configured at startup) -- A link is created when connecting to a remote endpoint and destroyed when - the connection terminates -- A peer is created when a link successfully authenticates (Noise IK - handshake) and destroyed when the link goes down -- Links and peers have a one-to-one mapping with coupled lifecycles — - peer teardown implies link teardown - -## Event-Driven Execution Model - -The node uses an async select loop as its main event loop, multiplexing -events from all sources into a single processing stream: - -- **Transport events**: Inbound datagrams from all transports arrive via a - shared mpsc channel -- **Timer events**: Periodic and one-shot timers for keepalive, stale peer - detection, cache expiry, handshake timeouts -- **TUN events**: Outbound IPv6 packets from local applications -- **Control events**: Identity registrations from DNS, shutdown signals - -Within the select loop, events are dispatched to focused handler functions -organized by concern (handshake processing, gossip handling, forwarding, -session management, timeout handling). Each handler operates on the node's -state directly — there is no separate message-passing between internal -components. - -**Why a single select loop**: FIPS protocol operations frequently need to -read and modify multiple pieces of state (e.g., forwarding a packet reads -the coordinate cache, peer ancestry, and bloom filters simultaneously). -A single-threaded event loop avoids the complexity of locking and provides -deterministic ordering of state changes. - -**Exceptions**: The TUN reader and writer run in separate blocking threads -because TUN I/O is blocking (kernel file descriptor). They communicate with -the main event loop via channels. - -## Phase-Based State Machine Pattern - -FIPS entities use a Rust enum-of-structs pattern for state machines where -each phase carries only the data relevant to that phase: - -```rust -enum PeerSlot { - Connecting(PeerConnection), // handshake in progress - Active(ActivePeer), // authenticated, participating -} -``` - -Each variant holds a different struct with phase-appropriate fields. The -`PeerConnection` struct carries handshake state; `ActivePeer` carries tree -position, bloom filters, and link statistics. Transitioning between phases -consumes the old struct and produces the new one, making it impossible to -access handshake state after authentication is complete. - -This pattern enforces at the type level that code handling an authenticated -peer cannot accidentally reference handshake state, and vice versa. - -See [fips-state-machines.md](fips-state-machines.md) for a detailed -treatment of this pattern. - -## Two-Layer Encryption Rationale - -FIPS uses independent Noise encryption at two layers (IK at FMP, XK at FSP): - -| Layer | Scope | What It Protects | -| ----- | ----- | ---------------- | -| FMP (mesh) | Hop-by-hop | All traffic on each peer link | -| FSP (session) | End-to-end | Application payload between endpoints | - -**Why two layers instead of one**: - -- **Link encryption** protects all traffic from passive observers on the - underlying transport — including routing metadata (TreeAnnounce, bloom - filters, discovery messages) that would otherwise be visible -- **Session encryption** protects application payloads from intermediate - routing nodes, which must decrypt link encryption to read routing headers -- Both layers always apply. For adjacent peers, traffic is encrypted twice. - This eliminates special cases ("local peer" vs. "remote destination") and - means topology changes (a direct peer becomes multi-hop) don't affect - sessions. - -**Why different patterns**: FMP uses Noise IK (2-message) because link peers -are configured and directly connected — the single round trip is preferred. -FSP uses Noise XK (3-message) because session traffic traverses untrusted -intermediate nodes — the extra round trip buys stronger initiator identity -hiding. Both layers reuse the same cryptographic stack (secp256k1, -ChaCha20-Poly1305, SHA-256), minimizing cryptographic dependencies. - -## Identity Model - -FIPS uses three related but distinct identifiers at different layers: - -```text -keypair (secp256k1) - │ - ├── pubkey (32 bytes) — the endpoint identity, used in Noise handshakes - │ - ├── node_addr = SHA-256(pubkey)[0..16] — routing identifier, visible to - │ transit nodes, cannot be reversed to pubkey - │ - └── IPv6 address = fd + node_addr[0..15] — overlay address for IPv6 - applications -``` - -**Privacy property**: Transit nodes see only node_addrs in packet headers. -They can forward traffic without knowing the Nostr identities of the -endpoints. An observer can verify "does this node_addr belong to pubkey X?" -but cannot enumerate communicating identities from traffic alone. - -**Self-sovereign**: Nodes generate their own identities without coordination. -The identity system uses Nostr keypairs (secp256k1), so existing npub/nsec -pairs work directly. - -## Protocol Self-Healing Design - -FIPS control protocols are designed for eventual consistency, tolerating -packet loss without acknowledgment/retry machinery: - -| Protocol | Self-Healing Property | -| -------- | --------------------- | -| TreeAnnounce | Full state with monotonic sequence; lost announcement recovered on next send | -| FilterAnnounce | Full filter replacement with sequence; stale filter recovered on next update | -| LookupRequest | Timeout-based retry at application level | -| SessionSetup | Timeout-based retry; lost setup triggers re-establishment on first data failure | -| CoordsRequired/PathBroken | Rate-limited, best-effort; lost error recovered by session idle timeout | - -**Why no ack/retry**: FIPS operates over unreliable transports (primarily -UDP). Adding reliability to control messages would require per-message state, -retransmission timers, and acknowledgment tracking — complexity that gossip -protocols avoid by sending full state periodically. A lost TreeAnnounce is -simply replaced by the next one, which carries the same or newer state. - -## Metrics Measurement Protocol - -MMP is instantiated at two independent layers, each with its own -configuration and state: - -- **Link layer**: One `MmpPeerState` per `ActivePeer`. Measures per-hop - quality using the FMP counter and timestamp fields that already exist on - every encrypted frame. No additional message overhead beyond periodic - SenderReport/ReceiverReport exchanges. - -- **Session layer**: One `MmpSessionState` per established `SessionEntry`. - Measures end-to-end quality using the FSP counter and timestamp fields. - Reports are encrypted and forwarded through every transit link. - -Both instantiations use identical algorithms (SRTT, loss, jitter, dual EWMA, -OWD trend) but are configured independently via `node.mmp.*` and -`node.session_mmp.*`. This allows operators to run Full mode on links (low -overhead, single hop) while using Lightweight mode for sessions (reduces -bandwidth cost on transit links). - -### Peer Display Names - -The node maintains a `peer_aliases` map (`HashMap`) populated -from the `peers[].alias` field in configuration. All log output uses -`peer_display_name()` to show human-readable names (e.g., "node-b") instead -of truncated public keys, improving operator experience. - -### Buffer Sizing Chain - -Under high forwarding load, back-pressure propagates through: - -1. **UDP socket receive buffer** (`transports.udp.recv_buf_size`, default 2 MB) — - kernel-level buffer for incoming datagrams. -2. **Packet channel** (`node.buffers.packet_channel`, default 1024) — async - channel from transport receive loop to the node's RX event loop. -3. **Processing** — decryption, routing decision, forwarding. - -If the packet channel fills (RX loop can't keep up), the transport receive -loop blocks, and the kernel receive buffer absorbs bursts. If the kernel -buffer also fills, incoming datagrams are silently dropped -(`RcvbufErrors` in `/proc/net/snmp`). The 2 MB default socket buffer was -chosen to handle ~85 MB/s forwarding throughput without kernel drops. - -## Bounded State Principle - -FIPS nodes maintain state proportional to O(P × D), where P is the number -of direct peers and D is the tree depth — not O(N) where N is the network -size. - -What each node stores: - -| State | Size | Scope | -| ----- | ---- | ----- | -| Peer ancestry (TreeAnnounce) | P × D entries | Direct peers only | -| Bloom filters | P × 1 KB | One per peer | -| Coordinate cache | Configurable (50K default) | Destinations actively routed | -| Identity cache | Configurable (10K default) | IPv6 adapter only | -| Sessions | Configurable (10K default) | Active end-to-end sessions | - -A node does not know about nodes in distant parts of the network. It knows -its direct peers, their tree positions, and the destinations it has recently -routed traffic to. This scales naturally: adding nodes to the network does -not increase the per-node state of existing nodes (except for a slight -increase in bloom filter occupancy). - -## Transport Opacity - -Transport addresses are opaque byte vectors above FMP. The transport layer -interprets them (e.g., UDP parses "ip:port" strings); all layers above treat -them as handles passed back to the transport for sending. - -**Architectural boundary**: Adding a new transport type (e.g., BLE) requires -implementing the transport trait and potentially a new `TransportHandle` -variant. No changes to FMP, FSP, or any routing logic. The transport trait -defines the interface: - -- `send(addr, data)` — send a datagram -- `mtu()` — maximum datagram size -- `start()` / `stop()` — lifecycle -- `discover()` — optional endpoint discovery - -Inbound datagrams are pushed via a shared channel, aggregating all transports -into a single event stream for the main loop. - -## Cache Architecture - -### Unified Coordinate Cache - -The coordinate cache maps `NodeAddr → TreeCoordinate`. It was originally two -separate caches (session-populated and discovery-populated) but was merged -into a single cache because both stored the same type of data and the -distinction was conceptual, not functional. - -Key properties: - -- **TTL-based expiration** (300s default) with **refresh on use** — active - routing resets the TTL, keeping hot entries alive -- **LRU eviction** when full — least recently used entries are evicted first -- **Flush on parent change** — when the local node's tree parent changes, - the entire cache is flushed because the node's own coordinates have - changed, making cached distance calculations potentially invalid - -### Identity Cache (LRU-Only) - -The identity cache maps FIPS address prefix → (NodeAddr, PublicKey). The -mapping is deterministic (derived from public key) and never becomes stale, -so there is no TTL — only LRU eviction bounded by a configurable size. - -This cache is needed only by the IPv6 adapter. The native FIPS API provides -the public key directly. - -### Timer Ordering - -Cache and session timers are ordered to ensure correct lifecycle behavior: - -```text -Session idle timeout (90s) < Coordinate cache TTL (300s) ≤ DNS TTL (300s) -``` - -When traffic stops, the session tears down first (90s). When traffic -resumes, a fresh SessionSetup re-warms transit caches that are still within -their TTL (300s). This ordering prevents the case where a session outlives -its transit cache entries, which would cause routing failures. - -## Receive Path Design - -Transports use a channel-push model rather than a poll/receive method. Each -transport takes a sender handle (`PacketTx`) at construction and spawns an -internal receive loop that pushes inbound datagrams onto the channel. The -node's main select loop reads from the corresponding receiver. - -**Why push, not poll**: Async Rust cannot express async methods on trait -objects (the `Transport` trait is synchronous). The channel-push model works -around this limitation: the concrete transport implementation (e.g., -`UdpTransport`) spawns its own async receive task and pushes to a channel, -while the trait surface remains synchronous for `send()`, `mtu()`, etc. - -The `TransportHandle` enum provides async dispatch for methods that need it -(like `send_async()`) without requiring dyn dispatch. - -## Control Socket - -FIPS exposes a read-only observability interface through a Unix domain -socket. The control socket provides runtime visibility into node state -without affecting protocol operation. - -### Architecture - -The control socket is a `UnixListener` integrated into the node's async -runtime. It accepts connections, reads a single JSON request per -connection, dispatches to query handlers that read (but never modify) -node state, and returns a JSON response. - -- **Max request size**: 4,096 bytes -- **I/O timeout**: 5 seconds per connection -- **Socket cleanup**: Automatically removes stale sockets on startup - -### fipsctl Binary - -The `fipsctl` command-line tool communicates with the control socket: - -```text -fipsctl [OPTIONS] - -OPTIONS: - -s, --socket Control socket path override - -COMMANDS: - show status Node status overview - show peers Authenticated peers - show links Active links - show tree Spanning tree state - show sessions End-to-end sessions - show bloom Bloom filter state - show mmp MMP metrics summary - show cache Coordinate cache stats - show connections Pending handshake connections - show transports Transport instances - show routing Routing table summary -``` - -### Protocol - -Request and response are line-delimited JSON: - -```json -{"command":"show_status"} -{"status":"ok","data":{...}} -``` - -Error responses use `{"status":"error","message":"..."}`. - -All queries are read-only — the control socket cannot modify node state, -trigger actions, or shut down the node. - -## References - -- [fips-intro.md](fips-intro.md) — Protocol overview -- [fips-mesh-layer.md](fips-mesh-layer.md) — FMP specification -- [fips-session-layer.md](fips-session-layer.md) — FSP specification -- [fips-state-machines.md](fips-state-machines.md) — Phase-based state - machine pattern -- [fips-configuration.md](fips-configuration.md) — YAML configuration - reference diff --git a/docs/design/fips-spanning-tree.md b/docs/design/fips-spanning-tree.md index 2e3bd0d..72d68f9 100644 --- a/docs/design/fips-spanning-tree.md +++ b/docs/design/fips-spanning-tree.md @@ -6,9 +6,29 @@ for readers who want to understand the tree internals — for how the spanning tree fits into the overall mesh operation, see [fips-mesh-operation.md](fips-mesh-operation.md). +## What Is a Spanning Tree? + +A spanning tree is a subset of the links in a mesh network that: + +- **Reaches every node** — no node is disconnected +- **Contains no cycles** — there is exactly one path between any two nodes +- **Has a single root** — one distinguished node from which all paths descend +- **Assigns every non-root node exactly one parent** — creating a + hierarchy from leaves to root + +Because a tree has no cycles, any node's position can be described by +its path to the root. This path serves as a coordinate in a virtual +address space, enabling distance calculations and routing decisions +without global topology knowledge. + +There is nothing special about the root node other than providing the +center of the coordinate system. Being root implies no additional +processing, routing, or operational burden — the root runs the same +protocol as every other node. + ## Purpose -The spanning tree gives every node in the mesh a **coordinate** — its +The FIPS spanning tree gives every node in the mesh a **coordinate** — its ancestry path from itself to the root. These coordinates enable: - Distance calculation between any two nodes without global topology @@ -16,24 +36,31 @@ ancestry path from itself to the root. These coordinates enable: - Greedy routing where each hop reduces distance to the destination - Loop-free forwarding guaranteed by strictly-decreasing distance -## Root Election +## Root Discovery The root is the node with the **lexicographically smallest node_addr** among all reachable nodes. There is no election protocol, no voting, no -negotiation. Each node independently evaluates the TreeAnnounce messages -from its peers and selects the minimum root. +negotiation — each node independently discovers the same root by +evaluating the TreeAnnounce messages from its peers and selecting the +minimum root. When a node first joins the network with no peers, it is its own root. As it connects to peers and receives their TreeAnnounce messages, it discovers smaller node_addrs and converges to the global root. -If the network partitions, each segment independently elects its own root +If the network partitions, each segment independently discovers its own root (the smallest node_addr in that segment). When segments rejoin, all nodes discover the globally-smallest root through TreeAnnounce exchange and reconverge to a single tree. ## Parent Selection +Parent selection and reselection is the primary means by which the mesh +self-organizes into an efficient routing structure. By having each node +choose the parent with the best measured link performance, packet +routing up and down the tree follows the best available path that +reduces distance to the destination. + Each node selects a single parent from among its direct peers. Parent selection uses cost-weighted depth to balance tree depth against link quality. @@ -79,9 +106,9 @@ immediate parent reselection: delaying legitimate reconvergence. Mandatory switches (parent loss, root change) bypass dampening. - **Local-only metrics**: Link costs use only locally measured MMP data - (ETX and SRTT). No cumulative path costs are propagated and no wire - format changes are required. This avoids the trust problems inherent - in self-reported cost metrics in a permissionless network. + (ETX and SRTT). No cumulative path costs are propagated, avoiding + the trust problems inherent in self-reported cost metrics in a + permissionless network. ### After Parent Change @@ -126,7 +153,7 @@ tree_distance(a, b): Example: If A has coordinates `[A, X, Y, Root]` and B has coordinates `[B, Z, Y, Root]`, the common suffix is `[Y, Root]` (length 2). Distance = -(3 - 2) + (3 - 2) = 2 hops. +(4 - 2) + (4 - 2) = 4 hops. The self-distance check in greedy routing uses this calculation: a packet is forwarded to a peer only if the peer is strictly closer to the destination @@ -193,10 +220,8 @@ for stronger verification. ### Timestamp - Type: u64, Unix seconds -- Used for stale detection, not versioning -- A root declaration is considered stale after `ROOT_TIMEOUT` (60 minutes) - without refresh. In practice, heartbeat cascading handles root departure - for the common case (see spanning-tree-dynamics.md §11) +- Advisory only — not used in any decision logic, as there is no way to + verify its accuracy from peers ## Reconvergence @@ -214,7 +239,7 @@ When a node fails (link timeout or disconnect): When the network partitions: 1. Nodes in each segment lose peers across the partition boundary -2. If the root was in the other segment, affected nodes elect a new segment +2. If the root was in the other segment, affected nodes discover a new segment root (smallest node_addr in their segment) 3. Each segment reconverges independently @@ -262,14 +287,12 @@ Example: In a 1000-node network with depth 10 and 5 peers, a node stores | FLAP_WINDOW_SECS | 60s | Sliding window for counting parent switches | | FLAP_DAMPENING_SECS | 120s | Extended hold-down duration when flap threshold exceeded | | ANNOUNCE_MIN_INTERVAL | 500ms | Minimum between announcements to same peer | -| ROOT_TIMEOUT | 60 min | Root declaration considered stale (not yet enforced; heartbeat cascading covers common case) | -| TREE_ENTRY_TTL | 5–10 min | Individual entry expiration | ## Implementation Status | Feature | Status | | ------- | ------ | -| Root election (smallest node_addr) | **Implemented** | +| Root discovery (smallest node_addr) | **Implemented** | | Cost-based parent selection with hysteresis | **Implemented** | | Hold-down timer after parent change | **Implemented** | | Periodic cost-based parent re-evaluation | **Implemented** | @@ -280,8 +303,6 @@ Example: In a 1000-node network with depth 10 and 5 peers, a node stores | Rate limiting (500ms per peer) | **Implemented** | | Coord cache flush on parent change | **Implemented** | | Flap dampening (extended hold-down on rapid switches) | **Implemented** | -| Root timeout enforcement | Planned | -| Tree entry TTL enforcement | Planned | | Per-ancestry-entry signatures | Future direction | ## References diff --git a/docs/design/fips-state-machines.md b/docs/design/fips-state-machines.md deleted file mode 100644 index f4f98ec..0000000 --- a/docs/design/fips-state-machines.md +++ /dev/null @@ -1,381 +0,0 @@ -# FIPS State Machine Design - -This document describes the phase-based state machine pattern used throughout -FIPS, where different lifecycle phases are represented by distinct struct types -wrapped in an enum rather than a single struct with a state field. - -## Pattern Overview - -### Traditional Approach (Single Struct + State Enum) - -```rust -enum PeerState { - Connecting, - Authenticating, - Active, - Disconnected, -} - -struct Peer { - identity: PeerIdentity, - state: PeerState, - // Fields needed by ALL states - ephemeral_key: Option, // Only used during auth - session_keys: Option, // Only valid when Active - tree_coords: Option, // Only valid when Active - // ... -} - -impl Peer { - fn handle_packet(&mut self, packet: &[u8]) { - match self.state { - PeerState::Connecting => { /* must check we're not Active */ } - PeerState::Active => { /* must check session_keys.is_some() */ } - // ... - } - } -} -``` - -**Problems:** - -- Fields that only apply to certain states are `Option` or uninitialized -- Methods must check state before operating (runtime errors possible) -- Auth-phase secrets (ephemeral keys) persist in memory after auth completes -- Single struct grows to accommodate all phases - -### Phase-Based Approach (Enum of Structs) - -```rust -/// What the Node stores per peer slot -enum PeerSlot { - Connecting(Box), - Active(Box), -} - -/// Handles authentication handshake only -struct PeerConnection { - link_id: LinkId, - direction: LinkDirection, - // Handshake-specific state - handshake_state: HandshakeState, - expected_identity: Option, - noise_handshake: Option, - noise_session: Option, - // Timing - started_at: u64, - last_activity: u64, -} - -/// Fully authenticated peer -struct ActivePeer { - identity: PeerIdentity, - link_id: LinkId, - connectivity: ConnectivityState, - noise_session: Option, - // Routing state - declaration: Option, - ancestry: Option, - inbound_filter: Option, - last_seen: u64, -} -``` - -**Benefits:** - -- Each struct only contains fields relevant to that phase -- Methods can't be called in wrong state (compile-time safety) -- Noise handshake state automatically dropped when `PeerConnection` → `ActivePeer` -- Each phase struct is smaller, simpler, independently testable - -## Transition Pattern - -In FIPS, connections and peers are stored in separate maps rather than a -unified `PeerSlot` map (though `PeerSlot` exists for cases where either phase -is needed). PeerConnection uses `&mut self` methods for handshake steps, and -the Node orchestrates promotion when the handshake completes: - -```rust -impl PeerConnection { - /// Start handshake as initiator, returns msg1 to send. - fn start_handshake(&mut self, our_keypair: Keypair, current_time_ms: u64) - -> Result, NoiseError>; - - /// Process incoming msg1 (responder), returns msg2 to send. - fn receive_handshake_init(&mut self, our_keypair: Keypair, message: &[u8], - current_time_ms: u64) -> Result, NoiseError>; - - /// Complete handshake by processing msg2 (initiator). - fn complete_handshake(&mut self, message: &[u8], current_time_ms: u64) - -> Result<(), NoiseError>; - - /// Take the completed NoiseSession for use in ActivePeer. - fn take_session(&mut self) -> Option; -} - -/// Result of promoting a connection to active peer. -enum PromotionResult { - /// New peer created successfully. - Promoted(NodeAddr), - /// Cross-connection detected, this connection lost tie-breaker. - CrossConnectionLost { winner_link_id: LinkId }, - /// Cross-connection detected, this connection won, old one replaced. - CrossConnectionWon { loser_link_id: LinkId, node_addr: NodeAddr }, -} -``` - -The Node promotes a completed connection by moving data between maps: - -```rust -// In the Node's handshake completion handler: -// 1. Remove from connections map -let conn = self.connections.remove(&link_id).unwrap(); - -// 2. Extract session and identity from completed connection -let session = conn.take_session().unwrap(); -let identity = conn.expected_identity().unwrap().clone(); - -// 3. Create active peer with the session -let peer = ActivePeer::with_session( - identity, link_id, current_time_ms, - session, our_index, their_index, - transport_id, source_addr, conn.link_stats().clone(), -); - -// 4. Insert into peers map (checking for cross-connections) -self.peers.insert(*peer.node_addr(), peer); -``` - -## Timeout Handling - -Each phase struct tracks its own timing using Unix milliseconds (`u64`). -PeerConnection provides a simple boolean timeout check: - -```rust -impl PeerConnection { - /// Check if the connection has timed out. - fn is_timed_out(&self, current_time_ms: u64, timeout_ms: u64) -> bool { - self.idle_time(current_time_ms) > timeout_ms - } - - /// Time since last activity. - fn idle_time(&self, current_time_ms: u64) -> u64 { - current_time_ms.saturating_sub(self.last_activity) - } -} -``` - -The Node's event loop periodically scans the connections map for timeouts: - -```rust -// In the Node's periodic maintenance: -let now_ms = current_time_ms(); -let mut timed_out = vec![]; - -for (link_id, conn) in &self.connections { - if conn.is_timed_out(now_ms, HANDSHAKE_TIMEOUT_MS) { - timed_out.push(*link_id); - } -} - -for link_id in timed_out { - self.connections.remove(&link_id); - // Clean up link, free index, etc. -} -``` - -## Application in FIPS - -### Peer Lifecycle - -```text -PeerSlot::Connecting(PeerConnection) - │ - │ Noise IK handshake (2 messages) - ▼ -PeerSlot::Active(ActivePeer) - │ - │ Disconnect message (0x50) / link failure / timeout - ▼ -[removed from peers map, index freed, link cleaned up] -``` - -**PeerConnection** contains: - -- Noise IK handshake state (`noise_handshake`, `handshake_state`) -- Expected identity (known for outbound, learned for inbound) -- Direction (`LinkDirection::Inbound` vs `LinkDirection::Outbound`) -- Session index tracking (`our_index`, `their_index`) -- Timing (`started_at`, `last_activity` in Unix milliseconds) - -**ActivePeer** contains: - -- `NoiseSession` (symmetric keys for encrypt/decrypt) -- `ConnectivityState` (Connected, Stale, Reconnecting, Disconnected) -- Tree position (`declaration`, `ancestry`) -- Bloom filter (`inbound_filter`, with sequence tracking) -- Session indices and transport address (for wire protocol dispatch) -- Statistics (`last_seen`, `link_stats`, `authenticated_at`) - -### End-to-End Session Lifecycle (FSP) - -```text -EndToEndState::Initiating(HandshakeState) - │ - │ Noise XK msg1 sent (SessionSetup) - │ Awaiting msg2 (SessionAck) - │ - │ msg2 received, processed - │ msg3 sent (SessionMsg3) - ▼ -EndToEndState::Established(NoiseSession) -``` - -```text -[msg1 received from remote initiator] - │ - │ Noise XK msg1 processed, msg2 sent (SessionAck) - ▼ -EndToEndState::AwaitingMsg3(HandshakeState) - │ - │ msg3 received, processed - ▼ -EndToEndState::Established(NoiseSession) -``` - -**EndToEndState** is an enum with three variants: - -- **Initiating(HandshakeState)** — We initiated the session, sent - SessionSetup with XK msg1, awaiting SessionAck (msg2) -- **AwaitingMsg3(HandshakeState)** — We are the XK responder, processed - msg1, sent msg2, awaiting msg3 from initiator -- **Established(NoiseSession)** — Handshake complete, symmetric session - keys derived, ready for application data - -The three-message flow (XK pattern) means the responder has an intermediate -`AwaitingMsg3` state that does not exist in the link-layer IK handshake -(which completes in two messages). The initiator transitions directly from -Initiating to Established upon processing msg2 and sending msg3. - -Each side's epoch (8-byte random startup value) is exchanged encrypted in -msg2 and msg3. Epoch mismatch on subsequent handshakes indicates peer -restart. - -### Link Lifecycle (Connection-Oriented Transports) - -Links use a simple state enum rather than phase-based structs, since link -state transitions are straightforward: - -```rust -enum LinkState { - Connecting, // Connection in progress (connection-oriented only) - Connected, // Ready for traffic - Disconnected, // Was connected, now gone - Failed, // Connection attempt failed -} - -struct Link { - link_id: LinkId, - transport_id: TransportId, - remote_addr: TransportAddr, - direction: LinkDirection, - state: LinkState, - stats: LinkStats, - // ... -} -``` - -For connectionless transports (UDP), links are immediately `Connected` - -no `Connecting` phase needed. A phase-based approach (separate structs for -connecting vs established) would be valuable for transports with complex -connection setup like Tor circuit building. - -### Node Lifecycle - -```rust -enum NodeState { - Created, - Starting, - Running, - Stopping, - Stopped, -} -``` - -The Node uses a simple state enum because startup/shutdown are brief and -don't need complex per-phase logic. A phase-based approach (separate structs -per phase) would be useful if startup involved multi-step async operations -with retries. - -### Transport Lifecycle - -```rust -enum TransportState { - Configured, - Starting, - Up, - Down, - Failed, -} -``` - -Like `NodeState`, this uses a simple state enum because transport startup is -straightforward. A phase-based approach would be valuable for transports with -complex initialization (e.g., Tor bootstrap with multi-step circuit building). - -## When to Use This Pattern - -**Use phase-based structs when:** - -- Different phases have different fields (auth secrets vs session keys) -- Phase-specific logic is complex enough to benefit from isolation -- Security-sensitive data should be dropped after phase completion -- You want compile-time enforcement of valid operations per phase - -**Use simple state enum when:** - -- All phases share the same fields -- Phase transitions are simple (just flip a flag) -- The struct is small and phase logic is trivial - -## Lookup Tables - -Rather than a unified `PeerSlot` map, the Node stores connections and peers -in separate maps optimized for their phase: - -```rust -struct Node { - // Handshake phase: indexed by LinkId (identity not yet known) - connections: HashMap, - - // Active phase: indexed by NodeAddr (verified identity) - peers: HashMap, - - // Reverse lookup: (transport_id, remote_addr) → LinkId - addr_to_link: HashMap, - - // Wire protocol dispatch: (transport_id, our_index) → NodeAddr - peers_by_index: HashMap<(TransportId, u32), NodeAddr>, -} -``` - -For inbound connections from unknown addresses: - -1. Receive Noise IK msg1 → decrypt to extract sender's static key (identity) -2. Create new PeerConnection with discovered identity -3. Add to `connections` (by LinkId) and `addr_to_link` -4. After handshake completes, promote to `peers` (indexed by NodeAddr) and - `peers_by_index` (for session index dispatch) - -## Summary - -The phase-based state machine pattern provides: - -1. **Type safety** - Can't call auth methods on active peer -2. **Memory efficiency** - Phase-specific data dropped on transition -3. **Clarity** - Each struct is focused and comprehensible -4. **Security** - Handshake state (ephemeral keys) dropped after auth -5. **Testability** - Each phase testable in isolation - -The cost is slightly more complex transition handling in the event loop, -but this is offset by simpler per-phase logic. diff --git a/docs/design/fips-transport-layer.md b/docs/design/fips-transport-layer.md index 7db8d75..4343917 100644 --- a/docs/design/fips-transport-layer.md +++ b/docs/design/fips-transport-layer.md @@ -16,7 +16,7 @@ to the FIPS Mesh Protocol (FMP) above. The transport layer deals exclusively in **transport addresses** — IP:port tuples, MAC addresses, .onion identifiers, radio device addresses. These are opaque to every layer above FMP. The mapping from transport address to FIPS -identity happens at the link layer after the Noise IK handshake completes. +identity happens at the link layer after the Noise IK link handshake completes. The word "peer" belongs to the link layer and above; the transport layer knows only about remote endpoints identified by transport addresses. @@ -69,7 +69,7 @@ forwarding and LookupResponse transit annotation. For connection-oriented transports, manage the underlying connection: TCP handshake, Tor circuit establishment, Bluetooth pairing. FMP cannot begin -the Noise IK handshake until the transport-layer connection is established. +the Noise IK link handshake until the transport-layer connection is established. Connectionless transports (UDP, raw Ethernet) skip this — datagrams can flow immediately to any reachable address. @@ -133,7 +133,7 @@ and duplication at the routing layer. **Connection model**: Connectionless transports (UDP, raw Ethernet) allow immediate datagram exchange. Connection-oriented transports (TCP, Tor, BLE) -require connection setup before FMP can begin the Noise IK handshake, +require connection setup before FMP can begin the Noise IK link handshake, adding startup latency. **Stream vs. datagram**: Datagram transports have natural packet boundaries. @@ -422,5 +422,3 @@ quality difference is significant. Link cost is not yet used in - [fips-intro.md](fips-intro.md) — Protocol overview and layer architecture - [fips-mesh-layer.md](fips-mesh-layer.md) — FMP specification (the layer above) - [fips-wire-formats.md](fips-wire-formats.md) — Transport framing details -- [fips-software-architecture.md](fips-software-architecture.md) — Transport - trait implementation details diff --git a/docs/design/fips-wire-formats.md b/docs/design/fips-wire-formats.md index 1cbba6a..db22c2b 100644 --- a/docs/design/fips-wire-formats.md +++ b/docs/design/fips-wire-formats.md @@ -17,18 +17,11 @@ how application data is wrapped through each layer. ## Transport Framing -### UDP - -FIPS packets are carried directly in UDP datagrams. No additional framing is -needed — each UDP datagram contains exactly one FIPS link-layer packet. - -### Stream Transports *(future direction)* - -TCP, WebSocket, and Tor transports provide a byte stream, not datagrams. The -common prefix `payload_len` field provides integrated stream framing — the -receiver reads the 4-byte common prefix, then reads exactly the number of -bytes indicated by `payload_len` (plus any phase-specific header and AEAD -tag). No separate length prefix is needed. +Datagram-oriented transports (UDP, raw Ethernet, radio) preserve natural +packet boundaries and require no additional framing. Stream-oriented +transports (TCP, WebSocket, Tor) must delineate FIPS packets within the +byte stream; the common prefix `payload_len` field provides this +framing directly. ## Link-Layer Formats @@ -38,12 +31,7 @@ length. ### Common Prefix (4 bytes) -```text -┌──────────────────────┬───────────┬───────────────┐ -│ ver(4) + phase(4) │ flags │ payload_len │ -│ 1 byte │ 1 byte │ 2 bytes LE │ -└──────────────────────┴───────────┴───────────────┘ -``` +![Common Prefix](diagrams/common-prefix.svg) | Field | Size | Description | | ----- | ---- | ----------- | @@ -78,12 +66,7 @@ encrypted link-layer message. **Outer header** (16 bytes, used as AEAD AAD): -```text -┌──────────────────────┬───────────┬───────────────┬──────────────┬──────────┐ -│ ver(4) + phase(4) │ flags │ payload_len │ receiver_idx │ counter │ -│ 1 byte │ 1 byte │ 2 bytes LE │ 4 bytes LE │ 8 bytes LE│ -└──────────────────────┴───────────┴───────────────┴──────────────┴──────────┘ -``` +![Established frame outer header](diagrams/established-outer-header.svg) | Field | Size | Description | | ----- | ---- | ----------- | @@ -96,12 +79,7 @@ ChaCha20-Poly1305 AEAD construction. **Encrypted inner header** (5 bytes, first bytes of plaintext): -```text -┌───────────────┬──────────┐ -│ timestamp │ msg_type │ -│ 4 bytes LE │ 1 byte │ -└───────────────┴──────────┘ -``` +![Established frame inner header](diagrams/established-inner-header.svg) | Field | Size | Description | | ----- | ---- | ----------- | @@ -113,15 +91,7 @@ the 1-byte message type and message-specific fields. **Complete encrypted frame**: -```text -┌──────────────────────────────────────┬───────────────────────────┐ -│ outer header (16 bytes, used as AAD) │ ciphertext + AEAD tag │ -│ │ (inner_hdr + body) + 16 │ -└──────────────────────────────────────┴───────────────────────────┘ - -Total overhead: 37 bytes (16 outer + 5 inner + 16 AEAD tag) -Minimum frame: 37 bytes (empty body) -``` +![Complete encrypted frame](diagrams/established-complete-frame.svg) ### Message Type Table @@ -141,14 +111,7 @@ Minimum frame: 37 bytes (empty body) Handshake initiation from connecting party. -```text -┌──────────────────────┬─────────────┬─────────────────────────────────────────┐ -│ common prefix │ sender_idx │ Noise IK message 1 │ -│ 4 bytes │ 4 bytes LE │ 106 bytes │ -└──────────────────────┴─────────────┴─────────────────────────────────────────┘ - -Total: 114 bytes -``` +![Noise IK message 1](diagrams/noise-ik-msg1.svg) Common prefix: ver=0, phase=0x1, flags=0, payload_len=110 (4 + 106). @@ -172,14 +135,7 @@ Noise pattern: `-> e, es, s, ss` with epoch payload Handshake response from responder. -```text -┌──────────────────────┬─────────────┬──────────────┬──────────────────────────┐ -│ common prefix │ sender_idx │ receiver_idx │ Noise IK message 2 │ -│ 4 bytes │ 4 bytes LE │ 4 bytes LE │ 57 bytes │ -└──────────────────────┴─────────────┴──────────────┴──────────────────────────┘ - -Total: 69 bytes -``` +![Noise IK message 2](diagrams/noise-ik-msg2.svg) Common prefix: ver=0, phase=0x2, flags=0, payload_len=65 (4 + 4 + 57). @@ -215,32 +171,7 @@ Each party in a link session maintains two indices: ### Handshake Flow -```text -Initiator Responder ---------- --------- -generates sender_idx -generates ephemeral keypair - - [0x01|flags=0|len] | sender_idx | noise_msg1 - ------------------------------------------------> - - validates msg1 - learns initiator's static key - generates sender_idx - generates ephemeral keypair - - [0x02|flags=0|len] | sender_idx | receiver_idx | noise_msg2 - <------------------------------------------------ - -validates msg2 -derives session keys - -=============== HANDSHAKE COMPLETE =============== - -First encrypted frame: - [0x00|flags|len] | receiver_idx | counter=0 | ciphertext+tag - ------------------------------------------------> -``` +![Handshake flow](diagrams/handshake-flow.svg) ## Link-Layer Message Types @@ -253,6 +184,8 @@ message-specific fields. Spanning tree state announcement, exchanged between direct peers only. +![TreeAnnounce](diagrams/tree-announce.svg) + | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | | 0 | msg_type | 1 byte | 0x10 | @@ -266,6 +199,8 @@ Spanning tree state announcement, exchanged between direct peers only. **AncestryEntry** (32 bytes): +![AncestryEntry](diagrams/ancestry-entry.svg) + | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | | 0 | node_addr | 16 bytes | Node's routing identifier | @@ -286,6 +221,8 @@ includes self) Bloom filter reachability update, exchanged between direct peers only. +![FilterAnnounce](diagrams/filter-announce.svg) + | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | | 0 | msg_type | 1 byte | 0x20 | @@ -310,6 +247,8 @@ With link overhead: 1,072 bytes. Coordinate discovery request, flooded through the mesh. +![LookupRequest](diagrams/lookup-request.svg) + | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | | 0 | msg_type | 1 byte | 0x30 | @@ -335,6 +274,8 @@ Coordinate discovery request, flooded through the mesh. Coordinate discovery response, greedy-routed back to requester. +![LookupResponse](diagrams/lookup-response.svg) + | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | | 0 | msg_type | 1 byte | 0x31 | @@ -367,6 +308,8 @@ The source verifies the proof upon receipt. Encapsulated session-layer payload for multi-hop forwarding. +![SessionDatagram](diagrams/session-datagram.svg) + | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | | 0 | msg_type | 1 byte | 0x00 | @@ -389,6 +332,8 @@ independently of link encryption. Orderly link teardown with reason code. +![Disconnect](diagrams/disconnect.svg) + | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | | 0 | msg_type | 1 byte | 0x50 | @@ -412,6 +357,8 @@ Orderly link teardown with reason code. Sent by the frame sender to provide interval-based transmission statistics. +![SenderReport](diagrams/sender-report.svg) + | Offset | Field | Size | Encoding | | ------ | ----- | ---- | -------- | | 0 | msg_type | 1 | `0x01` | @@ -430,6 +377,8 @@ Sent by the frame sender to provide interval-based transmission statistics. Sent by the frame receiver to provide loss, jitter, and timing feedback. +![ReceiverReport](diagrams/receiver-report.svg) + | Offset | Field | Size | Encoding | | ------ | ----- | ---- | -------- | | 0 | msg_type | 1 | `0x02` | @@ -524,12 +473,7 @@ body. **Complete encrypted message**: -```text -┌─────────────────────────────────┬─────────────────┬───────────────────────────┐ -│ header (12 bytes, used as AAD) │ [coords if CP] │ ciphertext + AEAD tag │ -│ │ │ (inner_hdr + body) + 16 │ -└─────────────────────────────────┴─────────────────┴───────────────────────────┘ -``` +![FSP complete encrypted message](diagrams/fsp-complete-message.svg) ### FSP Session Message Types @@ -558,8 +502,16 @@ happens at the session level based on the FSP message type. Establishes a session and warms transit coordinate caches. Contains the first message of the Noise XK handshake (ephemeral key only — the initiator's static identity is not revealed until msg3). + +SessionSetup, SessionAck, and SessionMsg3 are identified by the **phase** +field in the FSP common prefix (0x1, 0x2, 0x3), not by a message type +byte. The `msg_type` field in the encrypted inner header applies only to +established-phase (0x0) messages. + Encoded with FSP prefix: ver=0, phase=0x1, flags=0, payload_len. +![SessionSetup](diagrams/session-setup.svg) + **Body** (after 4-byte FSP prefix): | Offset | Field | Size | Description | @@ -578,6 +530,8 @@ Second message of the Noise XK handshake. The responder sends its ephemeral key and encrypted epoch. Encoded with FSP prefix: ver=0, phase=0x2, flags=0, payload_len. +![SessionAck](diagrams/session-ack.svg) + **Body** (after 4-byte FSP prefix): | Offset | Field | Size | Description | @@ -597,6 +551,8 @@ its encrypted static identity and epoch. After msg3, both parties derive identical symmetric session keys and the session is established. Encoded with FSP prefix: ver=0, phase=0x3, flags=0, payload_len. +![SessionMsg3](diagrams/session-msg3.svg) + **Body** (after 4-byte FSP prefix): | Offset | Field | Size | Description | @@ -625,6 +581,8 @@ The body after the inner header is delivered directly to the TUN interface. Sent by the destination to report the observed forward-path MTU. +![PathMtuNotification](diagrams/path-mtu-notification.svg) + | Offset | Field | Size | Encoding | | ------ | ----- | ---- | -------- | | 0 | path_mtu | 2 | u16 LE — minimum MTU observed along the forward path | @@ -662,50 +620,61 @@ on the wire. ### CoordsRequired (0x20) Plaintext error signal — transit node lacks coordinates for destination. -Encoded with FSP prefix: ver=0, phase=0x0, U flag set, payload_len. +Identified by the FSP phase field, not a separate message type byte +(same as SessionSetup/SessionAck/SessionMsg3). +Encoded with FSP prefix: ver=0, phase=0x0, U flag set, payload_len=34. -**Body** (after 4-byte FSP prefix + 1-byte msg_type): +![CoordsRequired](diagrams/coords-required.svg) + +**Payload** (after 4-byte FSP prefix): | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | -| 0 | flags | 1 byte | Reserved | -| 1 | dest_addr | 16 bytes | NodeAddr we couldn't route to | -| 17 | reporter | 16 bytes | NodeAddr of reporting router | +| 0 | msg_type | 1 byte | 0x20 | +| 1 | flags | 1 byte | Reserved | +| 2 | dest_addr | 16 bytes | NodeAddr we couldn't route to | +| 18 | reporter | 16 bytes | NodeAddr of reporting router | -**Body size**: 33 bytes. Total with prefix + msg_type: 38 bytes. +**Payload size**: 34 bytes. **Total on wire**: 38 bytes (4 prefix + 34 payload). ### PathBroken (0x21) Plaintext error signal — greedy routing reached a dead end. Encoded with FSP prefix: ver=0, phase=0x0, U flag set, payload_len. -**Body** (after 4-byte FSP prefix + 1-byte msg_type): +![PathBroken](diagrams/path-broken.svg) + +**Payload** (after 4-byte FSP prefix): | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | -| 0 | flags | 1 byte | Reserved | -| 1 | dest_addr | 16 bytes | Unreachable NodeAddr | -| 17 | reporter | 16 bytes | NodeAddr of reporting router | -| 33 | last_coords_count | 2 bytes LE | Number of stale coordinate entries | -| 35 | last_known_coords | 16 x n bytes | Stale coordinates that failed | +| 0 | msg_type | 1 byte | 0x21 | +| 1 | flags | 1 byte | Reserved | +| 2 | dest_addr | 16 bytes | Unreachable NodeAddr | +| 18 | reporter | 16 bytes | NodeAddr of reporting router | +| 34 | last_coords_count | 2 bytes LE | Number of stale coordinate entries | +| 36 | last_known_coords | 16 x n bytes | Stale coordinates that failed | ### MtuExceeded (0x22) Plaintext error signal — forwarded packet exceeds the next-hop link MTU. Sent by a transit router back to the source when a SessionDatagram cannot be forwarded because its size exceeds the outgoing link's MTU. -Encoded with FSP prefix: ver=0, phase=0x0, U flag set, payload_len. +Encoded with FSP prefix: ver=0, phase=0x0, U flag set, payload_len=36. -**Body** (after 4-byte FSP prefix + 1-byte msg_type): +![MtuExceeded](diagrams/mtu-exceeded.svg) + +**Payload** (after 4-byte FSP prefix): | Offset | Field | Size | Description | | ------ | ----- | ---- | ----------- | -| 0 | flags | 1 byte | Reserved | -| 1 | dest_addr | 16 bytes | NodeAddr of the destination being forwarded to | -| 17 | reporter | 16 bytes | NodeAddr of the router that detected the MTU violation | -| 33 | mtu | 2 bytes LE | Bottleneck MTU at the reporting router | +| 0 | msg_type | 1 byte | 0x22 | +| 1 | flags | 1 byte | Reserved | +| 2 | dest_addr | 16 bytes | NodeAddr of the destination being forwarded to | +| 18 | reporter | 16 bytes | NodeAddr of the router that detected the MTU violation | +| 34 | mtu | 2 bytes LE | Bottleneck MTU at the reporting router | -**Body size**: 35 bytes. Total with prefix + msg_type: 40 bytes. +**Payload size**: 36 bytes. **Total on wire**: 40 bytes (4 prefix + 36 payload). The source uses the reported MTU to adjust its session-layer path MTU estimate. MtuExceeded is the reactive complement to the proactive diff --git a/docs/design/spanning-tree-dynamics.md b/docs/design/spanning-tree-dynamics.md index 44700a1..1cb4933 100644 --- a/docs/design/spanning-tree-dynamics.md +++ b/docs/design/spanning-tree-dynamics.md @@ -10,8 +10,6 @@ For spanning tree algorithms and data structures, see [fips-spanning-tree.md](fips-spanning-tree.md). For how the spanning tree fits into mesh routing, see [fips-mesh-operation.md](fips-mesh-operation.md). -The protocol is based on Yggdrasil v0.5's CRDT gossip design. - ## Contents 1. [Core Concepts](#1-core-concepts) @@ -32,9 +30,11 @@ The protocol is based on Yggdrasil v0.5's CRDT gossip design. ### The CRDT Approach -The spanning tree is maintained as a distributed soft-state CRDT-Set. Each node -makes independent local decisions about parent selection, gossips these decisions -to peers, and the system converges to a consistent structure without coordination. +The spanning tree is maintained as a distributed soft-state CRDT-Set (see +Shapiro et al., "Conflict-free Replicated Data Types"). Each node makes +independent local decisions about parent selection, gossips these decisions +to peers, and the system converges to a consistent structure without +coordination. Key properties: @@ -53,24 +53,34 @@ A node's TreeState contains only: 2. **Direct peer declarations** - each peer's parent selection 3. **Ancestry of peers** - the chain from each peer up to root -This is **O(P × D)** entries where P is peer count and D is tree depth—not O(N) -where N is network size. A node does *not* know about: +This is **O(P × D)** entries where P is peer count and D is tree depth — not +O(N) where N is network size. This bounded-state approach follows +Yggdrasil/Ironwood's design, where each node knows only its own ancestry and +direct peer information, in contrast to classical STP (IEEE 802.1D) where all +bridges participate in a global election. A node does *not* know about: - Other subtrees branching off its ancestors - Siblings of ancestors - Nodes in distant parts of the network -This bounded state is sufficient to compute the node's own tree coordinates and -distances to any node whose coordinates it learns (via lookup responses). +This bounded state is sufficient to compute the node's own tree coordinates — +the path from the node to root, used as an address for greedy routing — and +distances to any node whose coordinates it learns (via lookup responses). The +theoretical foundation for this approach is Kleinberg's proof that every +connected graph has a greedy embedding in hyperbolic space (2007), with +practical embedding via spanning trees explored by Cvetkovski and Crovella +(2009). See [References](#references) for the full citations. **Example**: In a 1000-node network with tree depth 10, a node with 5 peers maintains roughly 50 TreeState entries, not 1000. -### Root Election +### Root Discovery The root is deterministic: the node with the lexicographically smallest node_addr -among all reachable nodes. No explicit election protocol exists—each node -independently derives the same answer from its local TreeState. +among all reachable nodes. No explicit election protocol exists — each node +independently derives the same answer from its local TreeState. This approach +derives from Yggdrasil's spanning tree design, which itself echoes IEEE 802.1D +STP's bridge ID concept but without the explicit BPDU exchange. --- @@ -80,26 +90,20 @@ When a node starts with no peers, it bootstraps as a single-node network. ### Step-by-Step: Isolated Startup -``` -Time T0: Node A starts -├── Generates or loads keypair (npub_A, nsec_A) -├── Computes node_addr_A = SHA-256(npub_A) -├── Initializes empty TreeState -├── Sets parent = self (A is its own root) -├── Sets sequence = 1 -└── Records timestamp = now +**T0: Node A starts.** -State after T0: - TreeState_A = { (A, parent=A, seq=1, ts=T0) } - Root_A = A - Coordinate_A = [A] -``` +- Generates or loads keypair `(npub_A, nsec_A)` +- Computes `node_addr_A = SHA-256(npub_A)` +- Initializes empty TreeState +- Sets `parent = self` (A is its own root), `sequence = 1` +- Records current timestamp -At this point, node A is a fully functional single-node FIPS network. It can: +After T0, A's TreeState contains a single entry `(A, parent=A, seq=1)`, +its root is A, and its coordinate is `[A]`. -- Accept incoming peer connections -- Route packets to itself -- Respond to lookups for its own address +At this point, node A is a fully functional single-node FIPS network. It +can accept incoming peer connections, route packets to itself, and respond +to lookups for its own address. ### What Triggers State Changes @@ -114,94 +118,50 @@ While isolated, A's state only changes on: When a new node connects to an existing network, a sequence of gossip exchanges integrates it into the spanning tree. -### Step-by-Step: Node B Joins via Node A +### Step-by-Step: Node B Joins via Node D -**Initial state**: Network has nodes A (root), C, D, E. Node B is new. +**Initial state**: Network has nodes A (root), C, D, E in an established +tree. Node B is new and isolated — its TreeState contains only its own +entry `(B, parent=B, seq=1)` and it considers itself root. -``` -Existing tree structure: - A (root, smallest node_addr) - /|\ - C D E +![Node Joining — B Connects to D](diagrams/dynamics-node-join.svg) -B's initial state (before connecting): - TreeState_B = { (B, parent=B, seq=1) } - Root_B = B -``` +The following steps trace B's integration into the tree: -**T1: B establishes link to D** +![Node B Joins via D — State Progression](diagrams/dynamics-node-join-steps.svg) -``` -Link established B ←→ D +**T1: Link established.** B and D establish a peer link. Both sides +immediately exchange TreeAnnounce messages. B sends its self-rooted +declaration `(parent=B, seq=1)` with ancestry `[B]`. D sends its +declaration `(parent=A, seq=47)` with ancestry `[D, A]`. -Immediate actions: -├── B sends TreeAnnounce to D: -│ └── Contains: B's declaration (parent=B, seq=1), B's ancestry (just B) -│ -└── D sends TreeAnnounce to B: - └── Contains: D's declaration (parent=A, seq=47), D's ancestry [D, A] -``` +**T2: B processes D's announcement.** B verifies D's outer declaration +signature and accepts A's ancestry entry on transitive trust through D +(in v1, only the sender's outer signature is verified). B merges both +entries into its TreeState, discovers that `node_addr_A < node_addr_B`, +and adopts A as root. With D as the only peer offering a path to A, B +selects D as parent. -**T2: B processes D's announcement** +**T3: B updates its declaration.** B increments its sequence number to 2, +sets `parent=D`, signs the new declaration, and computes its tree +coordinate `[B, D, A]`. B is now part of the spanning tree at depth 2. -``` -B receives D's TreeAnnounce: -├── Verifies signature on D's parent declaration -├── Verifies signature on A's self-declaration (from ancestry) -├── Merges into TreeState_B: -│ └── TreeState_B = { (B, parent=B, seq=1), (D, parent=A, seq=47), (A, parent=A, seq=203) } -│ -├── Evaluates root: -│ └── Compares node_addr_A vs node_addr_B -│ └── If A < B: Root_B = A (A has smaller node_addr) -│ -└── Evaluates parent selection: - └── Only peer is D - └── D has path to new root A - └── B selects D as parent -``` +**T4: B announces to D.** B sends a TreeAnnounce containing its updated +declaration `(parent=D, seq=2)` with ancestry `[B, D, A]`. D merges B's +entry into its own TreeState. D's coordinate `[D, A]` is unchanged — B's +arrival adds a child but does not affect D's path to root. -**T3: B updates its declaration** +**T5: Filter propagation.** D recomputes its outbound bloom filters (now +including B) and sends FilterAnnounce to all peers — parent A and any +mesh peers. A receives D's updated filter and now knows that B is +probably reachable through D (probabilistically — bloom filters have +no false negatives but possible false positives). Filter updates +propagate transitively through tree edges toward root. -``` -B's state change: -├── parent_B = D (was: B) -├── sequence_B = 2 (incremented) -├── timestamp_B = T3 -└── Signs new declaration - -TreeState_B = { (B, parent=D, seq=2), (D, parent=A, seq=47), (A, parent=A, seq=203) } -Root_B = A -Coordinate_B = [B, D, A] -``` - -**T4: B announces to D** - -``` -B sends TreeAnnounce to D: -└── Contains: B's new declaration (parent=D, seq=2), ancestry [B, D, A] - -D receives and merges: -├── TreeState_D now includes B's entry -├── D's coordinate unchanged: [D, A] -└── D can now route to B -``` - -**T5: D updates its bloom filter**: - -``` -D adds B's node_addr to its bloom filter -D sends FilterAnnounce to parent A - -A merges D's bloom filter with its view of D's subtree -A now knows "B is reachable through D" (probabilistically) - -This bloom filter update propagates toward root. -``` - -**Important**: D does NOT include B's declaration in TreeAnnounce to A. Tree -gossip only includes the sender's ancestry (path to root), not children. Most -nodes never learn B's declaration—they learn B is *reachable* via bloom filters. +**Key point**: D does *not* include B's declaration in TreeAnnounce to A. +Tree gossip carries only the sender's ancestry (path to root), not +children. Most nodes never learn B's declaration directly — they learn B +is reachable via bloom filter propagation through the tree. ### Convergence Time @@ -214,12 +174,18 @@ The propagation time is O(tree depth), not O(network size). In the example: - B's coordinates are known immediately (B computes from D's ancestry) - B's reachability propagates via bloom filter: D → A (1 hop to root) -- Any node wanting to reach B does a bloom filter lookup for candidate selection +- Any node wanting to reach B checks bloom filters to identify routing paths - Total: 1-2 gossip rounds for B to be locatable +Note: The use of bloom filters for reachability identification is a FIPS +addition. Yggdrasil uses DHT-based lookup flooding to discover coordinates; +FIPS replaces this with bloom filter summaries that propagate through tree +edges, providing O(1) per-peer reachability checks. See +[fips-bloom-filters.md](fips-bloom-filters.md) for details. + Note: Nodes A, C, E never add B to their TreeState. They can still route to B -by using bloom filter lookup for candidate selection to get B's coordinates, -then coordinate-based greedy routing. +by checking bloom filters to identify which peer can reach B, obtaining B's +coordinates via lookup, then using coordinate-based greedy routing. --- @@ -237,80 +203,59 @@ each node only knows its own ancestry and peers. Convergence means: When multiple isolated nodes connect simultaneously, the network must: -1. Elect a single root (determined by smallest node_addr) +1. Discover a single root (determined by smallest node_addr) 2. Form a loop-free tree structure 3. Propagate ancestry information along peer links **Example: Three nodes connect simultaneously** -``` -T0: Nodes A, B, C start isolated - Each is its own root - node_addr ordering: A < B < C +![Three-Node Convergence](diagrams/dynamics-convergence.svg) -T1: Links form: A ←→ B, B ←→ C +Three nodes A, B, C start isolated, each self-rooted (`node_addr: +A < B < C`, so A will be the global root). -T2: Gossip round 1 - A sends to B: (A, parent=A) - B sends to A: (B, parent=B) - B sends to C: (B, parent=B) - C sends to B: (C, parent=C) +**T0 — Isolated.** Each node considers itself root with `seq=1`. -T3: Processing round 1 - B learns A < B, adopts A as root, selects A as parent - C learns B exists (but B still claims self as root) +**T1 — Links form.** Links A–B and B–C are established. All nodes +exchange TreeAnnounce messages with their peers. A sends `(parent=A, +seq=1)` to B; B sends `(parent=B)` to A and C; C sends `(parent=C)` +to B. At this instant, all three still believe they are root. -T4: Gossip round 2 - B sends to A: (B, parent=A) — B has re-parented - B sends to C: (B, parent=A), ancestry includes A +**T2 — B re-parents.** B receives A's announcement, discovers +`node_addr_A < node_addr_B`, adopts A as root, and selects A as parent. +C receives B's announcement but B still claimed self as root at the time +it was sent — C has no reason to change yet. B now sends its updated +declaration `(parent=A)` with ancestry `[B, A]` to both peers. This +carries A's information transitively to C. -T5: Processing round 2 - C learns A (via B's ancestry), A < C - C adopts A as root, selects B as parent - -T6: Gossip round 3 - C sends to B: (C, parent=B) - -T7: Converged state - Root = A - Tree: A ← B ← C -``` +**T3 — Converged.** C receives B's updated announcement, discovers A +through B's ancestry, determines `node_addr_A < node_addr_C`, adopts A +as root, and selects B as parent. After C announces its new parent to B, +the tree is stable: A ← B ← C. Root information propagated through two +gossip rounds — matching the tree depth of 2. ### Convergence Properties -**Consistency guarantee**: After gossip quiesces: +The three-node example illustrates the general properties described in +§1. Convergence required two gossip rounds — one per level of tree +depth — with no coordination between nodes. Each node made independent +local decisions (root comparison, parent selection) and the CRDT merge +rule (highest sequence number wins) ensured that all pairwise views +converged to the same result. -- All nodes agree on the identity of the root -- Each node has a stable parent selection -- Peered nodes have identical views of their shared ancestry (the CRDT property) -- Any two nodes can compute accurate distance via their coordinates +In general, convergence time is bounded by `depth × gossip_interval`. +Parallel gossip on multiple links typically achieves convergence faster +than this worst case, since nodes at different depths process +announcements concurrently. The three-node walkthrough shows this: +B and C process announcements from different gossip rounds in parallel, +and the tree stabilizes as soon as C's re-parent announcement reaches B. -Nodes do *not* have global knowledge—a leaf node knows nothing about distant -subtrees. But any node can locate any other node via Bloom-guided candidate -selection and then route using tree coordinate distance. - -**Convergence time**: Bounded by tree depth × gossip interval. For a tree of -depth D with gossip interval G: - -- Worst case: D × G for root information to propagate to deepest leaf -- Typical case: Faster due to parallel gossip on multiple links - -**No coordination required**: Convergence emerges from: - -- Deterministic root election (smallest node_addr) -- Deterministic merge rules (highest sequence wins) -- Eventually consistent gossip - -### Partial Convergence States - -During convergence, the network may temporarily have: - -- **Multiple roots**: Different partitions with different root beliefs -- **Inconsistent coordinates**: Nodes computing distances from stale state -- **Routing failures**: Coordinate-based greedy routing may fail until coordinates stabilize - -These are transient. The protocol guarantees eventual convergence, not instant -consistency. +During convergence, the network may transiently exhibit multiple roots +(each partition with its own root belief), inconsistent coordinates, and +routing failures. These are resolved as gossip propagates — the protocol +guarantees eventual convergence, not instant consistency (following the +epidemic dissemination model; see Kermarrec, "Gossiping in Distributed +Systems"). --- @@ -329,53 +274,71 @@ Adding a link can: **Example: Better path discovered** -``` -Before: A ← B ← C ← D (linear chain, A is root) - D's coordinate: [D, C, B, A], depth 3 +![Link Addition — D Re-parents to A](diagrams/dynamics-link-addition.svg) -New link: A ←→ D established +**Initial state.** Nodes A, B, C, and D form a linear chain with A as root. +B is A's child at depth 1, C is B's child at depth 2, and D is C's child at +depth 3 with coordinate `[D, C, B, A]`. Every packet D sends toward root +traverses three hops. -D receives A's announcement directly: -├── A's ancestry: [A] (depth 0) -├── D evaluates: going through A gives depth 1 vs current depth 3 -├── If improvement > stability threshold: -│ └── D re-parents to A -│ └── D's new coordinate: [D, A], depth 1 +**New link established.** A direct link between A and D comes up. Both sides +immediately exchange TreeAnnounce messages. D receives A's announcement +carrying ancestry `[A]` at depth 0 — a direct path to the root that D has +never seen before. -After: A is root - ├── B (depth 1) - │ └── C (depth 2) - └── D (depth 1) -``` +**D evaluates parent.** D compares the new path through A (depth 1, one hop) +against its current path through C (depth 3, three hops). The depth +improvement of 2 far exceeds the hysteresis threshold (see [§8](#8-parent-selection)), +so the switch is not suppressed. + +**D re-parents to A.** D selects A as its new parent, increments its sequence +number, and computes its new coordinate `[D, A]` at depth 1. D sends a +TreeAnnounce to all peers — A (new parent), C (former parent), and any mesh +peers. + +**Tree settles.** The resulting tree has A as root with two children: B at +depth 1 (unchanged) and D at depth 1 (formerly depth 3). C remains at +depth 2 under B — it was B's child before the link addition and is unaffected +by D's re-parenting. D's path to root shortened from three hops to one. ### Link Removal Removing a link can: 1. **Remove parent** → must find new parent -2. **Partition the network** → separate root election +2. **Partition the network** → separate root discovery 3. **Remove non-parent peer** → minimal impact **Example: Parent link fails** -``` -Before: A ← B ← C, B ← D - C's parent is B +![Link Removal — B↔C Link Fails](diagrams/dynamics-link-removal.svg) -Link B ←→ C fails: -├── C detects link failure (see Section 7) -├── C's TreeState still contains B's entry (hasn't expired) -├── C has no peers with path to A -├── C becomes its own root temporarily -│ -└── If C has other peers: - └── C may discover path to A through them - └── C re-parents to best available peer +**Initial state.** Nodes A, B, C, and D form a tree with A as root. B is +A's child at depth 1, and both C and D are children of B at depth 2. C's +parent is B, with coordinate `[C, B, A]`. -└── If C is truly isolated: - └── C remains its own root - └── C is now a separate single-node network -``` +**Link failure.** The link between B and C fails. C detects the failure +through the heartbeat timeout mechanism described in +[§7](#7-link-failure-detection). At this point C's TreeState still +contains B's entry — it has not yet expired — but the underlying transport +link is gone. + +**C loses its path to root.** C examines its remaining peers and finds none +with a valid path to root A. With no alternative parent available, C has no +choice but to become its own root temporarily — it increments its sequence +number and begins announcing itself as root of a single-node tree. + +**Two possible outcomes.** If C has other peers not shown in this example, +it may receive a TreeAnnounce carrying a path to A through a different +branch of the mesh. In that case C re-parents to the best available peer +and rejoins the original tree. If C is truly isolated with no remaining +peers, it stays as its own root and operates as an independent single-node +network. + +**D is unaffected.** Node D's parent is B, not C, so the B–C link failure +does not disrupt D's path to root. D continues operating at depth 2 with +coordinate `[D, B, A]` and is unaware of C's situation unless it was also +peered with C. ### Reconvergence Dynamics @@ -383,6 +346,9 @@ Link B ←→ C fails: the improvement exceeds cost-based hysteresis (`parent_hysteresis`, default 0.2 = 20% improvement required). A hold-down timer (`hold_down_secs`, default 30s) further suppresses non-mandatory re-evaluation after a switch. +Hysteresis and hold-down timers are well-established techniques in routing +protocol design (used in OSPF, BGP, and IS-IS); FIPS adapts these to the +specific context of tree-coordinate routing with local-only link metrics. See §8 for details. **Sequence number advancement**: Each parent change increments the sequence @@ -403,17 +369,19 @@ Network partitions create isolated segments that must operate independently. A partition occurs when there's no path between two sets of nodes: -``` -Before: - A ← B ← C ← D ← E - (A is root) +![Network Partition — Link C↔D Fails](diagrams/dynamics-partition.svg) -Link C ←→ D fails: +**Initial state.** Nodes A, B, C, D, and E form a linear chain with A as +root. B is A's child at depth 1, C is B's child at depth 2, D is C's +child at depth 3, and E is D's child at depth 4. -After: - Partition 1: A ← B ← C - Partition 2: D ← E (or E ← D, depending on node_addrs) -``` +**Link failure.** The link between C and D fails. Because C and D are +the only connection between the two halves of the chain, no alternative +path exists — the network splits into two partitions. Partition 1 +contains nodes A, B, and C with the original root A still reachable. +Partition 2 contains D and E, which must rediscover its new root: the node with +the smaller `node_addr` between D and E becomes root of the isolated +fragment. ### Partition Detection @@ -422,63 +390,51 @@ Nodes detect they're partitioned when: 1. **Parent unreachable**: Direct link to parent fails 2. **Root unreachable**: No peer has path to current root -**Detection via gossip staleness** (not currently implemented — see -[Known Limitations](#known-limitations)): - -In principle, nodes would also detect partitions through root entry staleness: -if no fresh root announcements arrive within a timeout, the root is presumed -departed. This requires tracking root entry timestamps and enforcing expiration, -which is not yet implemented. Currently, only direct parent loss (case 1) and -absence of any peer with a path to root (case 2) trigger partition detection. - ### Independent Operation -Each partition operates as an independent network: +Each partition operates as an independent network. -``` -Partition 1 (nodes A, B, C): -├── Root = A (unchanged, A still reachable) -├── Tree structure unchanged -└── Routing works within partition +**Partition 1 (nodes A, B, C).** From this partition's perspective, +nothing has changed except that D's TreeState entries eventually expire. +Root A is still directly reachable by B, and C's path through B to A +remains intact. The tree structure is unchanged and routing within the +partition continues normally. -Partition 2 (nodes D, E): -├── Previous root A is unreachable -├── D and E exchange announcements -├── New root = min(node_addr_D, node_addr_E) -├── Tree forms between D and E -└── Routing works within partition -``` +**Partition 2 (nodes D, E).** D detects that its parent C is unreachable +and that no remaining peer offers a path to A. D becomes its own root +temporarily. D and E then exchange TreeAnnounce messages and converge on a +new root — whichever of D or E has the smaller `node_addr`. A two-node +tree forms between them and routing within the partition works as expected, +though neither node can reach A, B, or C. ### Partition Healing -When connectivity is restored: +When connectivity is restored, the two partitions merge through normal +gossip exchange. -``` -Link C ←→ D restored: +**T1:** The link between C and D is re-established. Both sides immediately +exchange TreeAnnounce messages. C sends its declaration with `root=A` and +ancestry `[C, B, A]`. D sends its declaration with `root=D` (assuming +`node_addr_D < node_addr_E`) and ancestry `[D]`. -T1: C and D exchange TreeAnnounce - C sends: root=A, ancestry [C, B, A] - D sends: root=D (assuming D < E), ancestry [D] +**T2:** D processes C's announcement and learns about node A for the first +time since the partition. Because `node_addr_A < node_addr_D`, D adopts A +as the new root and selects C as parent — C is the only peer offering a +path to A. -T2: D processes C's announcement - D learns about A - If A < D: D adopts A as new root - D selects C as parent (path to A) +**T3:** D updates its declaration and announces to E. E receives D's new +ancestry containing A, learns that a smaller root exists, and re-evaluates +its own parent selection accordingly. -T3: D announces to E - E learns about A through D's new ancestry - E re-evaluates and re-parents if needed - -T4: Merged network - Single root (A) - All nodes reachable via unified tree structure - (Each node still only knows its own ancestry, not global topology) -``` +**T4:** The network has merged back into a single tree with root A. All +five nodes are reachable via the unified tree structure. As always, each +node knows only its own ancestry and direct peer information — no node has +a global view of the topology. ### Root Stability Across Partitions A key design consideration: the root should be stable to minimize reconvergence. -If partition 2 elected a "temporary" root with a large node_addr, healing is cheap— +If partition 2 discovered a "temporary" root with a large node_addr, healing is cheap— that root immediately defers to the global root. If by chance partition 2's root has a smaller node_addr than partition 1's root, @@ -492,51 +448,43 @@ Detecting failed links is critical for timely reconvergence. ### Detection Mechanisms -**Traffic-based detection** (Yggdrasil v0.5 approach): +**MMP heartbeat-based detection** (following the general approach of +heartbeat-based failure detectors; see Das et al., "SWIM" for the +theoretical framework): -``` -On sending data to peer: - set read_deadline = now + peer_timeout +**Heartbeat sending.** Every `heartbeat_interval` (default 10 seconds), if +no frame has been sent to a peer recently, the node sends a Heartbeat +message (link-layer `msg_type` `0x51`, no payload). This ensures that even +an idle link generates periodic traffic for liveness detection. -On receiving data from peer: - clear read_deadline +**Tick-based expiry.** On every tick, the node checks each peer's +`last_recv_time`. If `now - peer.last_recv_time` exceeds +`link_dead_timeout` (default 30 seconds), the link is declared dead. The +peer is removed from the active peer set and, if the dead peer was the +current parent, reconvergence is triggered immediately. -On deadline expiration: - mark link as failed - remove peer from active peers - trigger reconvergence if peer was parent -``` - -This avoids dedicated keepalive traffic—normal protocol messages serve as -implicit heartbeats. - -**Explicit keepalive** (for idle links): - -``` -If no traffic sent to peer in keepalive_interval: - send Dummy message (type 0x00) - expect acknowledgment within peer_timeout -``` +Any successfully decrypted frame — data, gossip, MMP report, or +heartbeat — updates the peer's `last_recv_time`. The heartbeat serves +as an explicit keepalive when the link is idle. Under normal traffic, +application data and protocol messages provide implicit liveness +indication. ### Failure Response -When a link failure is detected: +When a link failure is detected, the node first removes the peer from its +active peer set. What happens next depends on the failed peer's role. -``` -link_failed(peer): - remove peer from active_peers +**Parent failure (critical).** If the failed peer was the current parent, +the node has lost its path to root. It calls `select_new_parent()` to find +another peer with a valid root path. If no valid parent is available, the +node becomes its own root — the same bootstrap state as +[§2](#2-single-node-startup). In either case, the node announces its +updated declaration to all remaining peers. - if peer == current_parent: - // Critical: lost path to root - select_new_parent() - if no_valid_parent_available: - become_own_root() - announce_to_all_peers() - else: - // Non-critical: lost a potential route - // TreeState entries for peer will expire naturally - // May trigger parent re-evaluation if peer was better path -``` +**Non-parent failure.** If the failed peer was not the current parent, the +impact is less severe. The peer's TreeState entries are removed +immediately. The node may re-evaluate parent selection if the lost peer had +been offering a better path, but the current path to root remains intact. ### Timing Considerations @@ -545,35 +493,25 @@ link_failed(peer): - Short timeout: Quick failure detection, but transient issues cause flapping - Long timeout: Stable under jitter, but slow to respond to real failures -**Typical values** (see [fips-spanning-tree.md](fips-spanning-tree.md) for -current FIPS-specific parameters): - -``` -peer_timeout: 10-30 seconds -keepalive_interval: peer_timeout / 3 -gossip_interval: on topology change (no periodic refresh) -tree_entry_ttl: not currently enforced (see Known Limitations) -``` +**FIPS parameters** (see [fips-spanning-tree.md](fips-spanning-tree.md) +and [fips-mesh-layer.md](fips-mesh-layer.md) for complete reference): +`heartbeat_interval_secs` is 10 (send heartbeat if link idle), +`link_dead_timeout_secs` is 30 (declare link dead after no traffic), and +gossip is event-driven on topology change with no periodic refresh. ### Asymmetric Failures -Links may fail asymmetrically (A can send to B, but not receive): +Links may fail asymmetrically — for example, A can send frames to B but +B's frames never reach A. In this scenario, B detects the failure first: +it receives no frames from A within `link_dead_timeout` and marks the link +dead. A, however, is still receiving B's traffic and does not yet know +anything is wrong. -``` -A → B: working -B → A: failed - -B detects: no responses from A, marks link failed -A doesn't detect: still receiving from B - -Resolution: -├── B stops sending to A -├── A eventually times out waiting for B's traffic -├── Both sides converge to "link failed" state -``` - -The protocol handles this through bidirectional timeout—both sides must -see traffic to consider the link alive. +**Resolution through bidirectional timeout.** Once B declares the link +dead, it stops sending to A. A then stops receiving B's traffic, and after +its own `link_dead_timeout` expires, A also marks the link dead. Both sides +converge to the same "link failed" state, though B detects it up to +`link_dead_timeout` seconds before A does. --- @@ -585,62 +523,56 @@ Parent selection determines tree structure and routing efficiency. The implementation uses cost-weighted depth to balance tree depth against link quality. Each candidate parent is evaluated by its **effective depth** — the -tree depth plus a local link cost penalty derived from MMP metrics. +tree depth plus a local link cost penalty derived from MMP metrics. This +cost-aware parent selection is a FIPS addition — Yggdrasil selects parents +purely by tree depth without link quality consideration. **Algorithm** (`TreeState::evaluate_parent()` in `tree/state.rs`): -``` -evaluate_parent(peer_costs: HashMap): - // 1. Find smallest root reachable through any peer - smallest_root = min(peer.root for peer in peers_with_coords) +1. **Find the smallest reachable root.** The function examines all peers that + have advertised coordinates and identifies the numerically smallest root + address among them. If the local node is itself that smallest root and is + already acting as root, no change is needed and the function returns early. - if self == smallest_root and is_root: - return None // Already root, no change +2. **Compute effective depth for each candidate.** For every peer whose + announced root matches the smallest root, the algorithm calculates + `effective_depth = peer.depth + link_cost`, where `link_cost` comes from + `peer_costs` (MMP-derived) or defaults to 1.0 when metrics have not yet + converged. The best candidate is the peer with the lowest effective depth, + with ties broken by numerically smallest `NodeAddr`. If the best candidate + is already the current parent, no switch is needed. - // 2. Compute effective depth for each candidate - for each peer with peer.root == smallest_root: - link_cost = peer_costs.get(peer) or 1.0 // default optimistic - peer.effective_depth = peer.depth + link_cost +3. **Check for mandatory switches.** Two conditions bypass all stability + mechanisms and trigger an immediate parent change: the current parent is no + longer reachable (link lost), or a strictly better root has been discovered. + These cases cannot wait for hold-down or hysteresis because the current tree + position is already invalid or suboptimal at the root level. - best_peer = min(candidates, key=effective_depth, tiebreak=node_addr) +4. **Hold-down check.** For non-mandatory switches, the algorithm checks + whether enough time has elapsed since the last parent change. If + `last_parent_switch + hold_down_secs` is still in the future, the switch is + suppressed. This prevents rapid oscillation when multiple candidates compete. - if best_peer == current_parent: - return None // Already using best +5. **Hysteresis check.** Even after hold-down expires, a same-root switch + requires significant improvement. The algorithm computes the current + parent's effective depth and compares it against the best candidate's. The + switch proceeds only if `best_effective_depth < current_effective_depth * + (1.0 - parent_hysteresis)`, requiring a 20% improvement by default. + Otherwise the current parent is retained, favoring stability over marginal + gains. - // 3. Mandatory switches — bypass hysteresis and hold-down - if current_parent not in peers: - return best_peer // Parent lost - if current_root != smallest_root: - return best_peer // Better root found - - // 4. Hold-down check — suppress non-mandatory switches - if last_parent_switch + hold_down > now: - return None // Too soon after last switch - - // 5. Hysteresis — require significant improvement - current_parent_eff = current_parent.depth + peer_costs.get(current_parent) - if best_eff_depth < current_parent_eff * (1.0 - parent_hysteresis): - return best_peer - - return None // Not enough improvement -``` - -**Parameters**: - -``` -parent_hysteresis = 0.2 // 20% improvement required for same-root switch -hold_down_secs = 30 // Suppress re-evaluation after parent switch -reeval_interval_secs = 60 // Periodic re-evaluation independent of TreeAnnounce -``` +**Parameters.** The three tuning parameters are `parent_hysteresis = 0.2` (20% +improvement required for a same-root switch), `hold_down_secs = 30` (suppress +re-evaluation after a parent switch), and `reeval_interval_secs = 60` (periodic +re-evaluation independent of TreeAnnounce traffic). **Link cost formula** (`ActivePeer::link_cost()` in `peer/active.rs`): +`link_cost = etx * (1.0 + srtt_ms / 100.0)` -``` -link_cost = etx * (1.0 + srtt_ms / 100.0) -``` - -Where ETX (Expected Transmission Count) comes from bidirectional MMP delivery -ratios and SRTT (Smoothed Round-Trip Time) from MMP timestamp-echo. When MMP +Where ETX (Expected Transmission Count, from De Couto et al., "A +High-Throughput Path Metric for Multi-Hop Wireless Routing", 2003) comes from +bidirectional MMP delivery ratios and SRTT (Smoothed Round-Trip Time) from MMP +timestamp-echo. When MMP metrics have not yet converged, `link_cost` defaults to 1.0, preserving depth-only behavior as a graceful fallback. @@ -660,8 +592,9 @@ with current MMP link costs, independent of TreeAnnounce traffic. ### Design Rationale: Local-Only Cost Metrics -The original design considered cumulative path costs (OSPF-style, where each -hop adds its link cost and the total is advertised in TreeAnnounce). This +The original design considered cumulative path costs (OSPF-style — see +RFC 2328 — where each hop adds its link cost and the total is advertised +in TreeAnnounce). This approach was rejected for three independent reasons: 1. **Unverifiable self-reporting**: In a permissionless network, a node can @@ -720,63 +653,32 @@ Once converged, what does the network look like and how does it behave? ### Steady State Gossip Pattern -``` -Normal operation (no topology changes): - -Root A: No periodic announcements — announces only if topology changes - └── Root does not refresh its timestamp periodically - -Each node: Sends TreeAnnounce only when its own state changes - └── Parent change, new peer, or peer departure - -Typical gossip per node in steady state: -├── No periodic sends — tree gossip is entirely change-driven -├── Announce on parent selection change -├── Announce on peer link up/down -└── Zero gossip traffic when topology is stable -``` +**Normal operation.** During normal operation with no topology changes, the root +does not send periodic announcements or refresh its timestamp — it announces +only when its own state changes. Every other node behaves the same way, sending +a TreeAnnounce only on parent selection change or peer link up/down. Tree gossip +is entirely change-driven: when the topology is stable, gossip traffic drops to +zero. ### Expected Steady State Properties -**Gossip volume**: +**Gossip volume.** Each topology change event produces an update of roughly 100 +bytes for the node's own declaration, plus a variable delta for changed +ancestors, giving a total ranging from O(100 bytes) to O(depth * 100 bytes). In +steady state with no topology changes, gossip traffic is zero — there are no +periodic refreshes. Traffic resumes only when links change or nodes join and +depart, and remains negligible compared to application traffic. -``` -Per topology change event: -├── Own declaration update: ~100 bytes -├── Delta of changed ancestors: varies -└── Total: O(100 bytes) to O(depth * 100 bytes) +**Memory usage.** Each node's `TreeState` stores its own entry (~100 bytes), +direct peer entries (~100 bytes each), and ancestry entries (~100 bytes each, +O(depth) per peer), giving a total of `O(peers * depth * 100)` bytes. For a +typical node with 5 peers at depth 10, this works out to roughly 5 KB of tree +state. -In steady state (no topology changes): -├── Zero gossip traffic — no periodic refreshes -├── Traffic resumes only when links change or nodes join/depart -└── Negligible compared to application traffic -``` - -**Memory usage**: - -``` -Per node TreeState: -├── Own entry: ~100 bytes -├── Direct peers: ~100 bytes each -├── Ancestry entries: ~100 bytes each, O(depth) per peer -└── Total: O(peers * depth * 100 bytes) - -For node with 5 peers, depth 10: -└── ~5 KB of tree state -``` - -**CPU usage**: - -``` -Per gossip message received: -├── Signature verification: O(ancestry_length) -├── TreeState merge: O(ancestry_length) -├── Parent re-evaluation: O(peers) -└── Total: O(peers + depth) per message - -In steady state with infrequent updates: -└── Negligible CPU overhead -``` +**CPU usage.** Processing each received gossip message involves signature +verification at O(ancestry_length), TreeState merge at O(ancestry_length), and +parent re-evaluation at O(peers), for a total cost of O(peers + depth) per +message. In steady state with infrequent updates, CPU overhead is negligible. ### Monitoring Steady State @@ -803,40 +705,31 @@ Warning signs of instability: **Scenario**: Five nodes (A-E) in an office. A is the router with internet, B-E are workstations. All connected via ethernet switch. -``` -Physical topology (full mesh via switch): - A ──── B - │╲ ╱│ - │ ╲ ╱ │ - │ ╳ │ - │ ╱ ╲ │ - │╱ ╲│ - D ──── C ──── E +![Office Network — Physical Topology](diagrams/dynamics-ex1-physical.svg) -node_addr ordering: A < C < B < E < D -``` +**Physical topology.** All five nodes connect through a shared ethernet +switch, giving A, B, C, and D full-mesh connectivity with direct links +between every pair. E connects only to C. The `node_addr` ordering is +A < C < B < E < D, making A the root candidate. **Tree formation**: -``` -T0: All nodes start, each is own root +![Office Network — Converged Tree](diagrams/dynamics-ex1-tree.svg) -T1: Links established (all pairs discover each other) +**T0: Bootstrap.** All nodes start independently, each declaring itself +as root with depth 0. -T2: Gossip exchange - Nodes learn about A through peer announcements - B, C, D select A as parent (direct link) - E learns about A via peers' ancestry +**T1: Link establishment.** Peer links come up across the switch. All +pairs within the full-mesh subset (A, B, C, D) discover each other; +E discovers C. -T3: Converged tree (assuming equal link costs): - A (root) - /│\ - B C D - │ - E +**T2: Gossip exchange.** Nodes learn about A through peer +TreeAnnounce messages. B, C, and D each have a direct link to A and +select it as parent. E learns about A via C's ancestry. - E selects C as parent (or any direct peer with path to A) -``` +**T3: Converged tree.** Assuming equal link costs, A is root with +children B, C, and D at depth 1. E selects C as parent (or any direct +peer with a path to A) and sits at depth 2. **Steady state**: @@ -847,187 +740,137 @@ T3: Converged tree (assuming equal link costs): **Link failure scenario**: -``` -Link A ←→ C fails: +**T1: Failure detected.** The link between A and C fails. C detects +the loss when no traffic arrives from A and the deadline expires. C's +TreeState still has A as root (not expired), and C has peers B, D, and +E through the switch's full-mesh connectivity. -T1: C detects (no traffic from A, deadline expires) - C's current TreeState still has A as root (not expired) - C has peers B, D, E (assuming full connectivity) +**T2: Parent re-selection.** C evaluates its remaining peers for a +path to A. Both B and D have direct links to A, so C selects one of +them as its new parent based on cost. -T2: C queries peers for path to A - B and D both have direct path to A - C selects B or D as new parent (based on cost) +**T3: Announcement propagation.** C announces its new parent to all +peers. E receives the update; its path to root now goes through +C → B → A (or C → D → A depending on C's selection). -T3: C announces new parent to all peers - E receives, E's path to root now goes C → B → A (or C → D → A) - -T4: Reconverged tree (if C selected B): - A (root) - /│ - B D - │ - C - │ - E -``` +**T4: Reconverged tree.** If C selected B, the tree becomes: A is root +with children B and D at depth 1, C is a child of B at depth 2, and E +is a child of C at depth 3. ### Example 2: Mesh Network with Constrained Links **Scenario**: Rural network with mixed connectivity. Some high-bandwidth internet links, some low-bandwidth radio links. -``` -Physical topology: - A ═══════ B (═══ = fiber, 1 Gbps) - │ ║ - │(radio) ║(fiber) - │ 9600bps ║ - │ ║ - C ─────── D ═══════ E - (DSL) (fiber) - 1 Mbps +![Mixed Network — Physical Topology](diagrams/dynamics-ex2-physical.svg) -node_addr ordering: B < A < D < E < C -``` +**Physical topology.** Five nodes with heterogeneous links. A and B +connect via fiber (1 Gbps), as do B–D and D–E. C connects to D over +DSL (1 Mbps) and to A over a 9600 bps radio link. The `node_addr` +ordering is B < A < D < E < C, making B the root candidate. **Local link costs** (using `link_cost = etx * (1.0 + srtt_ms / 100.0)`): -``` -Assumed MMP measurements after convergence: -A ═ B: fiber, 1ms RTT, 0% loss → link_cost = 1.0 * (1 + 1/100) ≈ 1.01 -B ═ D: fiber, 1ms RTT, 0% loss → link_cost = 1.0 * (1 + 1/100) ≈ 1.01 -D ═ E: fiber, 1ms RTT, 0% loss → link_cost = 1.0 * (1 + 1/100) ≈ 1.01 -C — D: DSL, 20ms RTT, 2% loss → link_cost = 1.04 * (1 + 20/100) ≈ 1.25 -A ~ C: radio, 500ms RTT, 5% loss→ link_cost = 1.11 * (1 + 500/100) ≈ 6.66 -``` +After MMP measurements converge, the three fiber links (A–B, B–D, +D–E) all measure 1 ms RTT with 0% loss, yielding `link_cost = +1.0 * (1 + 1/100)` ≈ 1.01 each. The DSL link C–D measures 20 ms RTT +with 2% loss, giving `link_cost = 1.04 * (1 + 20/100)` ≈ 1.25. The +radio link A–C measures 500 ms RTT with 5% loss, giving `link_cost = +1.11 * (1 + 500/100)` ≈ 6.66. **Tree formation with effective depth**: -``` -Root = B (smallest node_addr, depth 0) +![Mixed Network — Cost-Weighted Tree](diagrams/dynamics-ex2-tree.svg) -Parent selection (each node evaluates effective_depth = peer.depth + link_cost): -├── A: peers are B (depth 0, cost 1.01 → eff 1.01), C (depth ?, cost 6.66) -│ └── Selects B (lowest effective depth) -│ -├── D: peers are B (depth 0, cost 1.01 → eff 1.01), -│ C (depth ?, cost 1.25), E (depth ?, cost 1.01) -│ └── Selects B (direct, lowest effective depth) -│ -├── E: peer is D (depth 1, cost 1.01 → eff 2.01) -│ └── Selects D (only candidate) -│ -└── C: peers are A (depth 1, cost 6.66 → eff 7.66), - D (depth 1, cost 1.25 → eff 2.25) - └── Selects D (eff 2.25 vs 7.66 — much lower) +**Root selection.** B has the smallest `node_addr` and becomes root at +depth 0. Each node evaluates `effective_depth = peer.depth + link_cost` +for its candidates. -Resulting tree: - B (root, depth 0) - / \ - A D (both depth 1) - |\ - E C (both depth 2) -``` +**A** sees B at depth 0 with cost 1.01 (effective 1.01) and C whose +depth is not yet resolved. A selects B as parent. + +**D** sees B at depth 0 with cost 1.01 (effective 1.01), plus C and E +whose depths are not yet resolved. D selects B as parent. + +**E** has only D as a peer. D is at depth 1 with cost 1.01 (effective +2.01). E selects D as its only candidate. + +**C** sees A at depth 1 with cost 6.66 (effective 7.66) and D at +depth 1 with cost 1.25 (effective 2.25). C selects D — the DSL link +is far cheaper than the radio link. + +**Resulting tree.** B is root at depth 0 with children A and D at +depth 1. Under D, both E and C sit at depth 2. **Note**: C chooses D despite both being at depth 1 — the DSL link to D (eff 2.25) far beats the radio link to A (eff 7.66). With local-only costs, each node evaluates only its direct link quality, not cumulative path cost. -**Radio link failure**: +**Radio link failure.** If the A–C radio link fails, there is no tree +impact since C's parent is D, not A. C loses a potential backup path +but the current tree is unchanged. -``` -If A ~ C radio fails: -└── No tree impact (C's parent is D, not A) -└── C loses a potential backup path, but current tree unchanged - -If D — C DSL fails: -├── C loses parent (mandatory switch — bypasses hysteresis and hold-down) -├── C's only remaining peer is A (radio) -├── C selects A as parent (eff depth = 1 + 6.66 = 7.66) -└── Tree reconverges with C as child of A at depth 2 -``` +**DSL link failure.** If the D–C DSL link fails, C loses its parent. +This is a mandatory switch that bypasses hysteresis and hold-down. C's +only remaining peer is A (via radio), so C selects A as parent with an +effective depth of 1 + 6.66 = 7.66. The tree reconverges with C as a +child of A at depth 2. ### Example 3: Network Partition and Healing **Scenario**: Two office sites connected by a single WAN link. -``` -Site 1: WAN link Site 2: -A ─── B ─────────────────────────── E ─── F - │ │ - C G +![Two-Site Network — Physical Topology](diagrams/dynamics-ex3-topology.svg) -node_addr ordering: A < E < B < F < C < G -``` +**Physical topology.** Site 1 contains nodes A, B, and C. A connects +to B, and B connects to C. Site 2 contains nodes E, F, and G. E +connects to F, and F connects to G. A single WAN link bridges the two +sites between B and E. The `node_addr` ordering is +A < E < B < F < C < G. -**Normal operation**: - -``` -Root = A (global smallest) -Tree: - A - └── B - ├── C - └── E (via WAN) - └── F - └── G -``` +**Normal operation.** A has the globally smallest `node_addr` and +serves as root. B is A's child at depth 1. Under B, C sits at depth 2 +and E (via the WAN link) also at depth 2. F is E's child at depth 3, +and G is F's child at depth 4. **Partition (WAN fails)**: -``` -T1: B ←→ E link fails - B detects: E unreachable - E detects: B unreachable +![Network Partition — Independent Trees](diagrams/dynamics-ex3-partition.svg) -T2: Site 1 state: - Root = A (still reachable) - Tree unchanged for A, B, C - E's entry in B's TreeState expires +**T1: WAN link fails.** The B–E link goes down. B detects that E is +unreachable; E detects that B is unreachable. -T3: Site 2 state: - E loses path to A - E evaluates remaining peers: F - F has no path to A either - E compares: node_addr_E < node_addr_F - E becomes new root for Site 2 +**T2: Site 1 unaffected.** A is still reachable, so the tree for A, +B, and C remains unchanged. E's entry in B's TreeState expires and is +removed. -T4: Site 2 reconverges: - E (root) - └── F - └── G +**T3: Site 2 elects a new root.** E loses its path to A. It evaluates +its remaining peer F, but F has no path to A either. Since +`node_addr_E < node_addr_F`, E becomes the new root for Site 2. -Network is now two separate trees with roots A and E. -``` +**T4: Site 2 reconverges.** E is root at depth 0, F is E's child at +depth 1, and G is F's child at depth 2. The network now operates as +two separate trees with roots A and E. **Partition heals**: -``` -T5: WAN link restored - B ←→ E exchange announcements +**T5: WAN link restored.** The B–E link comes back up. B and E +immediately exchange TreeAnnounce messages. -T6: E receives B's announcement: - B's ancestry: [A, B] - E learns: A exists, node_addr_A < node_addr_E - E adopts A as root - E selects B as parent +**T6: Root discovery.** E receives B's announcement carrying ancestry +`[A, B]`. E learns that A exists and `node_addr_A < node_addr_E`, so +A is the superior root. E adopts A as root and selects B as parent. -T7: E announces to F: - E's new ancestry: [E, B, A] - F learns about A - F re-parents (E is still valid parent, now with path to A) +**T7: Propagation to F.** E announces its new state to F with +ancestry `[E, B, A]`. F learns about A and re-parents — E is still a +valid parent, now with a path to A. -T8: F announces to G: - Similar propagation +**T8: Propagation to G.** F announces to G, and G similarly adopts A +as root through the updated ancestry chain. -T9: Merged network: - A (root) - └── B - ├── C - └── E - └── F - └── G -``` +**T9: Merged network.** The tree is whole again: A is root, B is A's +child at depth 1, C and E are B's children at depth 2, F is E's child +at depth 3, and G is F's child at depth 4. **Convergence time**: 4 gossip rounds (depth of Site 2's subtree is 3, plus initial exchange). @@ -1036,65 +879,22 @@ initial exchange). ## Known Limitations -The following limitations exist in the current implementation relative to the -design described in this document. They are documented here to guide future -work. +The following limitations exist in the current implementation. They are +documented here to guide future work. -### Root Timeout Not Enforced +### No Root Staleness Detection -The design specifies a 60-minute root timeout (§6 partition detection) after -which nodes should treat the root as departed and re-elect. The current -implementation does not track root entry timestamps or perform staleness -checks. +The implementation has no mechanism to detect that the root has become +unreachable without direct parent loss. Partition detection relies entirely +on link-level failure detection cascading through the tree. -**Mitigation**: Heartbeat cascading significantly reduces the practical -impact. When the root disappears, its direct children detect the parent loss -(keepalive timeout), re-elect, and announce new coordinates. This cascades -down the tree — each level's children detect their parent's changed state and -re-evaluate. For the common case of root departure, the tree reconverges -without an explicit root timeout. +**Mitigation**: MMP heartbeat cascading handles the common case. When the +root disappears, its direct children detect the link dead timeout, +rediscover the root, and announce new coordinates. This cascades down the +tree — each level's children detect their parent's changed state and +re-evaluate. The tree reconverges without an explicit root staleness check. -**Remaining gap**: If an intermediate node maintains a link to the root but -that link silently stops forwarding (no keepalive failure), nodes below it -would retain stale root state. This is an unusual failure mode — most link -failures are detected by keepalive timeouts. An explicit root timeout would -provide defense-in-depth for this edge case. - -### Known Limitation: No TTL on Tree Entries - -The design specifies a 5-10 minute TTL on tree entries (§8 timing -parameters). Peer entries in `TreeState` are never expired; they persist until -explicitly removed by peer disconnection. - -**Impact**: Stale ancestry information from departed nodes remains in -`TreeState`, potentially affecting coordinate computation. In practice, this -is partially mitigated by parent loss handling, but entries for non-parent -peers that depart without a graceful disconnect will linger. - -**Required fix**: Add a `last_seen` timestamp to peer entries in `TreeState`. -In `check_tree_state()`, expire entries older than `tree_entry_ttl`. When -entries expire, re-evaluate parent selection if the expired entry was the -current parent. - -### Known Limitation: No Partition Detection - -The design describes partition detection via gossip staleness (§6) where -nodes detect isolation when root announcements stop arriving and the root -entry eventually expires, triggering independent partition operation. - -**Impact**: Without root timeout enforcement (see above), partitioned nodes -cannot detect that they've lost connectivity to the root. They continue with -stale coordinates rather than forming an independent partition with a local -root. This affects only the case where the path to root is broken at some -intermediate point — direct parent loss is handled correctly. - -**Required fix**: Depends on root timeout implementation. Once root timeout -is enforced, partition detection follows naturally: a node whose root entry -expires and has no peer with a fresher root declaration is partitioned. It -becomes its own root and announces, allowing the partition to converge -independently. - -### Known Limitation: Remaining Stability Gaps +### Stability Mechanisms The primary stability mechanisms are implemented: @@ -1110,32 +910,11 @@ The primary stability mechanisms are implemented: switches (parent loss, root change) bypass dampening. The flap counter resets when the window expires naturally. -Remaining gaps: - -- No sequence number advancement rate limiting -- No minimum stable state duration before re-announcing - -**Impact**: These mechanisms compose to bound announcement traffic even -under rapid link flapping. The hold-down timer limits the rate of parent -switches (at most one non-mandatory switch per 30s), flap dampening -catches pathological patterns that persist beyond the hold-down window, -and per-peer rate limiting (500ms) bounds announcement frequency. - -### Known Limitation: Integration Test Gaps - -Unit tests for `TreeState` and `TreeCoordinate` are comprehensive, and basic -integration tests verify TreeAnnounce exchange and parent ancestry -propagation. However, the following failure scenarios lack test coverage: - -- Root node failure and network-wide re-election -- Network partition formation and independent operation -- Partition healing and root convergence -- Stale entry cleanup (depends on TTL implementation) -- Parent flapping under rapid topology changes (flap dampening is - implemented; integration test coverage is the remaining gap) - -These tests are blocked on or related to the limitations above and should be -added as each limitation is resolved. +These mechanisms compose to bound announcement traffic even under rapid link +flapping. The hold-down timer limits the rate of parent switches (at most +one non-mandatory switch per 30s), flap dampening catches pathological +patterns that persist beyond the hold-down window, and per-peer rate +limiting (500ms) bounds announcement frequency. --- @@ -1144,23 +923,48 @@ added as each limitation is resolved. The gossip-based spanning tree protocol achieves distributed coordination through: -1. **Deterministic root election** - Smallest node_addr, no negotiation needed +1. **Deterministic root discovery** - Smallest node_addr, no negotiation needed 2. **Cost-aware parent selection** - Each node independently chooses lowest effective depth to root using local link metrics -3. **CRDT merge semantics** - Conflicts resolved by sequence number, then timestamp +3. **CRDT merge semantics** - Conflicts resolved by sequence number (higher wins) 4. **Bounded state** - O(peers × depth) entries per node, not O(network size) 5. **Depth-proportional convergence** - Scales with tree height, not node count -6. **Traffic-based failure detection** - No dedicated keepalive overhead +6. **MMP-based failure detection** - Heartbeat keepalives with link dead timeout 7. **Stability thresholds** - Hysteresis, hold-down, and flap dampening prevent flapping on similar-cost paths Each node maintains only its own ancestry and direct peer information—not global -topology. Reachability to arbitrary destinations is provided by Bloom-guided -candidate selection (bloom filters propagating up the tree), with coordinate -discovery via lookup protocol and coordinate-based greedy routing for forwarding. +topology. Reachability to arbitrary destinations is identified by bloom filter +checks (filters propagating through tree edges), with coordinate discovery via +lookup protocol and coordinate-based greedy routing for forwarding. The protocol handles partitions gracefully (independent operation), heals automatically when connectivity returns, and adapts to heterogeneous link costs to form efficient tree structures. +### Prior Art and FIPS Contributions + +The protocol builds on established foundations and adds several new elements: + +**Derived from prior work**: + +- Spanning tree coordinate routing (Yggdrasil/Ironwood, building on Kleinberg + 2007 and Cvetkovski/Crovella 2009) +- Deterministic root discovery via smallest identifier (Yggdrasil; echoes + IEEE 802.1D STP bridge ID selection) +- CRDT-based distributed state (Shapiro et al. 2011) +- Gossip dissemination (epidemic model; Kermarrec 2007) +- Heartbeat-based failure detection (SWIM; Das et al. 2002) +- ETX link metric (De Couto et al. 2003) +- Hysteresis and hold-down for route stability (OSPF, BGP, IS-IS) + +**FIPS additions**: + +- Cost-aware parent selection using local-only link metrics (effective depth = + tree depth + link cost), replacing Yggdrasil's depth-only selection +- Combined ETX + SRTT link cost formula with MMP-measured components +- Flap dampening with mandatory switch bypass +- Announcement suppression for transient state changes +- Tree-only bloom filter merge with split-horizon exclusion + --- ## References @@ -1207,6 +1011,24 @@ costs to form efficient tree structures. ACM Journal of Experimental Algorithmics 2020. *Achieved 100% success ratio with 6% stretch on Internet graph.* +#### Link Metrics + +- De Couto, D., Aguayo, D., Bicket, J., Morris, R. + "A High-Throughput Path Metric for Multi-Hop Wireless Routing". + MobiCom 2003. *Introduced ETX (Expected Transmission Count) as a link + quality metric for wireless mesh networks.* + +#### Routing Protocol Stability + +- IEEE 802.1D. "IEEE Standard for Local and Metropolitan Area + Networks: Media Access Control (MAC) Bridges". *Spanning Tree + Protocol (STP) — root election via bridge ID, BPDU exchange.* + +- Moy, J. [RFC 2328](https://datatracker.ietf.org/doc/html/rfc2328): + "OSPF Version 2". 1998. *Link-state routing with cumulative path + costs and SPF computation. FIPS's local-only cost approach is + contrasted with OSPF's cumulative model in §8.* + #### Distributed Systems Primitives - Shapiro, M., Preguiça, N., Baquero, C., Zawirski, M. diff --git a/src/bloom/mod.rs b/src/bloom/mod.rs index d50532c..663bd99 100644 --- a/src/bloom/mod.rs +++ b/src/bloom/mod.rs @@ -8,7 +8,7 @@ //! ## v1 Parameters //! //! - Size: 1 KB (8,192 bits) - sized for actual ~400-800 entry occupancy -//! - Hash functions: k=5 - optimal for 800-1,600 entries at 1KB +//! - Hash functions: k=5 - optimal at ~1,200 entries, good for 800-1,600 //! - Bandwidth: 1 KB/announce (75% reduction from original 4KB design) //! //! These parameters are right-sized for typical network occupancy of @@ -24,7 +24,7 @@ pub use state::BloomState; /// Default filter size in bits (1KB = 8,192 bits). /// -/// Sized for ~800-1,600 entries with <5% FPR at typical occupancy (~400 entries). +/// Sized for ~800-1,600 entries. FPR ~0.05% at 400 entries, ~0.9% at 800. /// This is v1 protocol default (size_class=1). pub const DEFAULT_FILTER_SIZE_BITS: usize = 8192; @@ -33,8 +33,8 @@ pub const DEFAULT_FILTER_SIZE_BYTES: usize = DEFAULT_FILTER_SIZE_BITS / 8; /// Default number of hash functions. /// -/// k=5 is optimal for 800-1,600 entries at 1KB filter size. -/// At 400 entries: FPR ~0.3%. At 800 entries: FPR ~2.4%. +/// k=5 is optimal at ~1,200 entries and a good compromise for 800-1,600. +/// At 400 entries: FPR ~0.05%. At 800 entries: FPR ~0.9%. pub const DEFAULT_HASH_COUNT: u8 = 5; /// Size class for v1 protocol (1 KB filters).