mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-31 03:56:15 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Arjen
|
081855e8e2 | ||
|
Arjen
|
3d81f4b1f5 | ||
|
Arjen
|
0450b11d5f | ||
|
Arjen
|
4afa27f056 | ||
|
Arjen
|
0aa50d245c | ||
|
Arjen
|
abb0701048 | ||
|
Arjen
|
5c02d33ad8 | ||
|
Arjen
|
5083a09223 | ||
|
Arjen
|
16ead248d0 | ||
|
Arjen
|
9e59659073 | ||
|
Arjen
|
1611282047 | ||
|
Arjen
|
786758e8c3 | ||
|
Arjen
|
0cf035a0e1 |
@@ -5,23 +5,6 @@ junit = { path = "junit.xml" }
|
||||
# occasional msg1 under burst load even with the per-edge repair loop.
|
||||
# Allow a retry rather than failing the whole CI run on a single
|
||||
# dropped packet.
|
||||
#
|
||||
# Deliberately scoped to [profile.ci] and NOT applied locally, which makes
|
||||
# the two gates disagree: the hosted runner retries a flaky test twice, the
|
||||
# local sweep fails on the first failure. The asymmetry is intended and this
|
||||
# is the record of why, since an undocumented one is indistinguishable from
|
||||
# an oversight.
|
||||
#
|
||||
# It is here for shared-runner packet loss, a property of the hosted
|
||||
# environment and not of the code. Applying it locally would suppress a real
|
||||
# local flake, and a failure that only reproduces under load is a robustness
|
||||
# bug to fix rather than to retry past. Keeping the local sweep strict is
|
||||
# what makes it the sharper of the two gates.
|
||||
#
|
||||
# The cost, stated rather than hidden: a test that fails once and passes on
|
||||
# retry is reported green here with no separate signal, so a genuine
|
||||
# intermittent failure can be absorbed. If that starts mattering, the fix is
|
||||
# to surface retried-but-passed tests, not to drop the retries.
|
||||
retries = 2
|
||||
|
||||
[test-groups]
|
||||
|
||||
+196
-88
@@ -38,12 +38,12 @@ env:
|
||||
# unreliable on GitHub-hosted runners.
|
||||
# tor-directory — same; live Tor dependency.
|
||||
#
|
||||
# The two runners express the same work in different matrix shapes, and the
|
||||
# parity guard compares through that shape rather than around it: chaos legs
|
||||
# are compared per scenario (and per flag) via their `scenario:` field,
|
||||
# deb-install legs per distro. The one leg still compared at leg granularity
|
||||
# is dns-resolver — a single leg here, running all of its scenarios
|
||||
# internally, exactly as the local suite does.
|
||||
# Granularity-only differences (same coverage, different matrix shape —
|
||||
# NOT a divergence):
|
||||
# deb-install — split here into per-distro legs (debian12/debian13/
|
||||
# ubuntu22/ubuntu24/ubuntu26) for parallelism; local runs the
|
||||
# same distro set in one suite.
|
||||
# dns-resolver — single leg here; runs all scenarios (same as local).
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -52,29 +52,6 @@ env:
|
||||
# Builds on Linux x86_64, Linux aarch64, and macOS.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
jobs:
|
||||
ci-parity:
|
||||
name: CI parity
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install Python deps
|
||||
run: pip3 install --quiet pyyaml
|
||||
- name: Check local and GitHub runners cover the same work
|
||||
run: bash testing/check-ci-parity.sh
|
||||
- name: Check test log matchers against the strings src/ emits
|
||||
run: python3 testing/check-log-strings.py
|
||||
- name: Check no tested function's exit status is a log call's
|
||||
run: python3 testing/check-trailing-log.py
|
||||
- name: Check nothing resolves the shared mutable test image
|
||||
run: bash testing/check-image-scoping.sh
|
||||
# Hermetic: synthetic ping functions, no containers, ~45s. Lives beside
|
||||
# the other two so both runners gate on it identically — putting it in
|
||||
# only one would create exactly the drift check-ci-parity.sh exists to
|
||||
# catch, and it is invisible to that checker either way since it is not
|
||||
# a matrix suite.
|
||||
- name: Run convergence-gate unit tests
|
||||
run: bash testing/lib/wait-converge-test.sh
|
||||
|
||||
fmt:
|
||||
name: Format check
|
||||
runs-on: ubuntu-latest
|
||||
@@ -110,59 +87,6 @@ jobs:
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-
|
||||
- run: cargo clippy --all-targets --all-features -- -D warnings
|
||||
# An optional feature means two source trees, and --all-features lints
|
||||
# only one of them. The default build is what ships, so lint it
|
||||
# explicitly: without this stage, code that compiles only with
|
||||
# `profiling` enabled would pass CI while breaking every release build.
|
||||
# Mirrored in testing/ci-local.sh — check-ci-parity.sh compares
|
||||
# integration suites only and will not catch a stage added to one runner
|
||||
# and not the other.
|
||||
- name: Clippy (default features)
|
||||
run: cargo clippy --all-targets -- -D warnings
|
||||
- name: Build with the tick-body profiler enabled
|
||||
run: cargo build --workspace --features profiling
|
||||
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
# Android cross-check
|
||||
#
|
||||
# FIPS runs on Android as an embedded library — the host app owns the TUN
|
||||
# (an Android VpnService), so there are no daemon binaries to package, unlike
|
||||
# the desktop targets. This job only cross-compiles the library for the
|
||||
# android target to guard the android-only cfg paths (and the `not(android)`
|
||||
# exclusions) from silently bit-rotting; nothing else in CI compiles them.
|
||||
# cargo-ndk wires the NDK toolchain, which is required even for a check
|
||||
# because `ring` compiles C at build time.
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
android-check:
|
||||
name: Android cross-check (aarch64)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install Rust toolchain (+ Android target)
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
target: aarch64-linux-android
|
||||
components: clippy
|
||||
cache: false
|
||||
rustflags: ''
|
||||
- name: Cache Cargo registry + build
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-cargo-android-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-
|
||||
- name: Install cargo-ndk
|
||||
uses: taiki-e/install-action@v2
|
||||
with:
|
||||
tool: cargo-ndk
|
||||
- name: Clippy the library for Android
|
||||
run: |
|
||||
export ANDROID_NDK_HOME="${ANDROID_NDK_HOME:-$ANDROID_NDK_LATEST_HOME}"
|
||||
cargo ndk -t arm64-v8a clippy --lib -- -D warnings
|
||||
|
||||
build:
|
||||
name: Build (${{ matrix.os }})
|
||||
@@ -304,12 +228,6 @@ jobs:
|
||||
check_name: Unit Tests Summary
|
||||
fail_on_failure: false
|
||||
|
||||
# The `profiling` feature adds a module, a recorder and a writer thread
|
||||
# that the default-feature run above never compiles, so its own tests do
|
||||
# not execute there. Mirrored in testing/ci-local.sh.
|
||||
- name: Run library tests with the tick-body profiler enabled
|
||||
run: cargo test --lib --features profiling
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Job 2b – Unit tests (macOS)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -433,6 +351,22 @@ jobs:
|
||||
- suite: static-chain
|
||||
type: static
|
||||
topology: chain
|
||||
# ── Rekey integration test ──────────────────────────────────────────
|
||||
- suite: rekey
|
||||
type: rekey
|
||||
topology: rekey
|
||||
- suite: rekey-accept-off
|
||||
type: rekey-accept-off
|
||||
topology: rekey-accept-off
|
||||
- suite: rekey-outbound-only
|
||||
type: rekey-outbound-only
|
||||
topology: rekey-outbound-only
|
||||
# ── Inbound max_peers admission-cap test ───────────────────────
|
||||
- suite: admission-cap
|
||||
type: admission-cap
|
||||
topology: mesh
|
||||
- suite: acl-allowlist
|
||||
type: acl-allowlist
|
||||
# ── Firewall baseline (fips0 nftables default-deny) ────────────
|
||||
- suite: firewall
|
||||
type: firewall
|
||||
@@ -441,6 +375,9 @@ jobs:
|
||||
type: gateway
|
||||
topology: gateway
|
||||
# ── Chaos / stochastic scenarios ───────────────────────────────────
|
||||
- suite: chaos-smoke-10
|
||||
type: chaos
|
||||
scenario: smoke-10
|
||||
- suite: churn-mixed-10
|
||||
type: chaos
|
||||
scenario: churn-mixed
|
||||
@@ -454,9 +391,30 @@ jobs:
|
||||
- suite: tcp-mesh
|
||||
type: chaos
|
||||
scenario: tcp-mesh
|
||||
- suite: bottleneck-parent
|
||||
type: chaos
|
||||
scenario: bottleneck-parent
|
||||
- suite: cost-avoidance
|
||||
type: chaos
|
||||
scenario: cost-avoidance
|
||||
- suite: cost-reeval
|
||||
type: chaos
|
||||
scenario: cost-reeval
|
||||
- suite: cost-stability
|
||||
type: chaos
|
||||
scenario: cost-stability
|
||||
- suite: depth-vs-cost
|
||||
type: chaos
|
||||
scenario: depth-vs-cost
|
||||
- suite: mixed-technology
|
||||
type: chaos
|
||||
scenario: mixed-technology
|
||||
- suite: congestion-stress
|
||||
type: chaos
|
||||
scenario: congestion-stress
|
||||
- suite: bloom-storm
|
||||
type: chaos
|
||||
scenario: bloom-storm
|
||||
# ── Sidecar deployment ──────────────────────────────────────────
|
||||
- suite: sidecar
|
||||
type: sidecar
|
||||
@@ -569,6 +527,120 @@ jobs:
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile ${{ matrix.topology }} down --volumes --remove-orphans
|
||||
|
||||
# ── Rekey integration test ──────────────────────────────────────────────
|
||||
- name: Generate and inject configs (rekey)
|
||||
if: matrix.type == 'rekey'
|
||||
run: |
|
||||
bash testing/static/scripts/generate-configs.sh rekey
|
||||
bash testing/static/scripts/rekey-test.sh inject-config
|
||||
|
||||
- name: Start containers (rekey)
|
||||
if: matrix.type == 'rekey'
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey up -d
|
||||
|
||||
- name: Run rekey test
|
||||
if: matrix.type == 'rekey'
|
||||
run: bash testing/static/scripts/rekey-test.sh
|
||||
|
||||
- name: Collect logs on failure (rekey)
|
||||
if: matrix.type == 'rekey' && failure()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey logs --no-color
|
||||
|
||||
- name: Stop containers (rekey)
|
||||
if: matrix.type == 'rekey' && always()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey down --volumes --remove-orphans
|
||||
|
||||
# ── Rekey + accept_connections=false variant ──────────────────────────
|
||||
- name: Generate and inject configs (rekey-accept-off)
|
||||
if: matrix.type == 'rekey-accept-off'
|
||||
env:
|
||||
REKEY_TOPOLOGY: rekey-accept-off
|
||||
REKEY_ACCEPT_OFF_NODES: b
|
||||
run: |
|
||||
bash testing/static/scripts/generate-configs.sh rekey-accept-off
|
||||
bash testing/static/scripts/rekey-test.sh inject-config
|
||||
|
||||
- name: Start containers (rekey-accept-off)
|
||||
if: matrix.type == 'rekey-accept-off'
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey-accept-off up -d
|
||||
|
||||
- name: Run rekey test (accept-off variant)
|
||||
if: matrix.type == 'rekey-accept-off'
|
||||
env:
|
||||
REKEY_TOPOLOGY: rekey-accept-off
|
||||
REKEY_ACCEPT_OFF_NODES: b
|
||||
run: bash testing/static/scripts/rekey-test.sh
|
||||
|
||||
- name: Collect logs on failure (rekey-accept-off)
|
||||
if: matrix.type == 'rekey-accept-off' && failure()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey-accept-off logs --no-color | tail -300
|
||||
|
||||
- name: Stop containers (rekey-accept-off)
|
||||
if: matrix.type == 'rekey-accept-off' && always()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey-accept-off down --volumes --remove-orphans
|
||||
|
||||
# ── Rekey + udp.outbound_only=true variant ─────────────────────────────
|
||||
- name: Generate and inject configs (rekey-outbound-only)
|
||||
if: matrix.type == 'rekey-outbound-only'
|
||||
env:
|
||||
REKEY_TOPOLOGY: rekey-outbound-only
|
||||
REKEY_OUTBOUND_ONLY_NODES: b
|
||||
run: |
|
||||
bash testing/static/scripts/generate-configs.sh rekey-outbound-only
|
||||
bash testing/static/scripts/rekey-test.sh inject-config
|
||||
|
||||
- name: Start containers (rekey-outbound-only)
|
||||
if: matrix.type == 'rekey-outbound-only'
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey-outbound-only up -d
|
||||
|
||||
- name: Run rekey test (outbound-only variant)
|
||||
if: matrix.type == 'rekey-outbound-only'
|
||||
env:
|
||||
REKEY_TOPOLOGY: rekey-outbound-only
|
||||
REKEY_OUTBOUND_ONLY_NODES: b
|
||||
run: bash testing/static/scripts/rekey-test.sh
|
||||
|
||||
- name: Collect logs on failure (rekey-outbound-only)
|
||||
if: matrix.type == 'rekey-outbound-only' && failure()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey-outbound-only logs --no-color | tail -300
|
||||
|
||||
- name: Stop containers (rekey-outbound-only)
|
||||
if: matrix.type == 'rekey-outbound-only' && always()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey-outbound-only down --volumes --remove-orphans
|
||||
|
||||
# ── ACL allowlist integration test ─────────────────────────────────────
|
||||
- name: Run ACL allowlist integration test
|
||||
if: matrix.type == 'acl-allowlist'
|
||||
run: bash testing/acl-allowlist/test.sh --skip-build --keep-up
|
||||
|
||||
- name: Collect logs on failure (acl-allowlist)
|
||||
if: matrix.type == 'acl-allowlist' && failure()
|
||||
run: |
|
||||
docker compose -f testing/acl-allowlist/docker-compose.yml logs --no-color
|
||||
|
||||
- name: Stop containers (acl-allowlist)
|
||||
if: matrix.type == 'acl-allowlist' && always()
|
||||
run: |
|
||||
docker compose -f testing/acl-allowlist/docker-compose.yml down --volumes --remove-orphans
|
||||
|
||||
# ── Firewall baseline integration test ─────────────────────────────────
|
||||
- name: Run firewall baseline integration test
|
||||
if: matrix.type == 'firewall'
|
||||
@@ -699,6 +771,42 @@ jobs:
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile gateway down --volumes --remove-orphans
|
||||
|
||||
# ── Inbound max_peers admission-cap integration test ────────────────
|
||||
# Lowers node.max_peers on one mesh node and asserts the inbound cap
|
||||
# holds under sustained retry pressure: denied peers keep retrying but
|
||||
# are never promoted to an active session. The admission-cap-test.sh
|
||||
# assertions are tailored per link-layer handshake variant; the leg
|
||||
# itself is uniform. Static-style harness on the shared mesh profile.
|
||||
- name: Generate configs (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/generate-configs.sh mesh
|
||||
|
||||
- name: Inject admission-cap config (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/admission-cap-test.sh inject-config
|
||||
|
||||
- name: Start containers (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh up -d
|
||||
|
||||
- name: Run admission-cap test
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/admission-cap-test.sh
|
||||
|
||||
- name: Collect logs on failure (admission-cap)
|
||||
if: matrix.type == 'admission-cap' && failure()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh logs --no-color | tail -300
|
||||
|
||||
- name: Stop containers (admission-cap)
|
||||
if: matrix.type == 'admission-cap' && always()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh down --volumes --remove-orphans
|
||||
|
||||
# ── Real-deb install integration ────────────────────────────────────
|
||||
# The deb-install harness builds its own .deb from source in a
|
||||
# cargo-deb builder image; the pre-built Linux binary from the
|
||||
|
||||
@@ -285,8 +285,6 @@ jobs:
|
||||
"$FILES_DIR/etc/fips/firewall.sh"
|
||||
"$FILES_DIR/etc/hotplug.d/net/99-fips"
|
||||
"$FILES_DIR/etc/uci-defaults/90-fips-setup"
|
||||
"$FILES_DIR/usr/bin/fips-mesh-setup"
|
||||
"$FILES_DIR/usr/bin/fips-ap-setup"
|
||||
)
|
||||
fail=0
|
||||
for f in "${TARGETS[@]}"; do
|
||||
@@ -406,8 +404,6 @@ jobs:
|
||||
./usr/bin/fipsctl
|
||||
./usr/bin/fipstop
|
||||
./usr/bin/fips-gateway
|
||||
./usr/bin/fips-mesh-setup
|
||||
./usr/bin/fips-ap-setup
|
||||
./etc/init.d/fips
|
||||
./etc/init.d/fips-gateway
|
||||
./etc/fips/fips.yaml
|
||||
@@ -721,7 +717,6 @@ jobs:
|
||||
|
||||
for path in \
|
||||
usr/bin/fips usr/bin/fipsctl usr/bin/fipstop usr/bin/fips-gateway \
|
||||
usr/bin/fips-mesh-setup usr/bin/fips-ap-setup \
|
||||
etc/init.d/fips etc/init.d/fips-gateway \
|
||||
etc/fips/fips.yaml etc/fips/firewall.sh etc/dnsmasq.d/fips.conf \
|
||||
etc/sysctl.d/fips-gateway.conf etc/sysctl.d/fips-bridge.conf \
|
||||
|
||||
@@ -33,11 +33,6 @@ __pycache__/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
|
||||
# Per-run build contexts created by testing/ci-local.sh. Its teardown normally
|
||||
# removes them, but the CI worker's SIGKILL runs no trap, so one can survive a
|
||||
# preempted run; ci-cleanup.sh sweeps the survivors.
|
||||
/testing/docker-*/
|
||||
|
||||
# Runtime artifacts from running fips in-tree during local testing.
|
||||
# Root-anchored so legitimately-tracked fips.yaml under packaging/ and
|
||||
# examples/ stays included.
|
||||
|
||||
-143
@@ -9,153 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
|
||||
- An optional tick-body profiler behind the new `profiling` Cargo feature,
|
||||
**off by default**. When enabled, `fipsctl profile tick on [--dir PATH]` /
|
||||
`off` / `status` starts and stops a capture at runtime with no restart. Each
|
||||
capture writes one tab-separated file (default `/var/log/fips`, capped at
|
||||
32 MB) carrying, per ten-second interval, the exact count, max and total for
|
||||
every step of the rx-loop tick arm, the whole-tick span, and gauges for ticks,
|
||||
peer count, the gap between successive tick-arm entries and the resulting
|
||||
arm-starvation delay. With the feature off the instrumentation macro is a pure
|
||||
pass-through, so a default build contains no timing code on the tick path.
|
||||
`LogsDirectory=fips` was added to the packaged systemd units so the capture
|
||||
directory is created and cleaned up declaratively.
|
||||
|
||||
- `packaging/debian/build-deb.sh --features <list>` builds the `.deb` with a
|
||||
Cargo feature list, which is how an instrumented package is produced for a
|
||||
measurement run. The auto-derived dev Version gains a matching `+<features>`
|
||||
marker, so a feature build and a default build of the same commit are no
|
||||
longer indistinguishable: without it the two carry byte-identical versions,
|
||||
an install of one over the other is an apt no-op, and the running node offers
|
||||
no way to tell which one it has. The marker sorts above the unmarked build, so
|
||||
installing a feature build is an upgrade and reverting to the default build is
|
||||
a downgrade — revert with `dpkg -i` rather than `apt install`. `--features` is
|
||||
refused together with `--no-build`, which would stamp the marker onto binaries
|
||||
the features never reached.
|
||||
|
||||
### Changed
|
||||
|
||||
- The Ethernet transport's per-interface `discovery` flag was renamed to
|
||||
`listen` (`transports.ethernet.*`) to match the symmetric `announce`
|
||||
(transmit) / `listen` (receive) neighbor-beacon vocabulary. The old
|
||||
`discovery:` key is still accepted via a serde alias, so deployed configs
|
||||
continue to load unchanged; `Config::to_yaml()` re-emits it under the
|
||||
canonical `listen:` name. Update your `fips.yaml` to `listen:`.
|
||||
|
||||
- The mesh-lookup control-metrics family is now emitted under the key
|
||||
`lookup` in `fipsctl stats metrics` and `show routing`. The former key
|
||||
`discovery` is still emitted as a deprecated alias carrying identical
|
||||
counters; update dashboards and alerts to read `lookup`.
|
||||
- The overloaded `node.discovery.*` config table was split into
|
||||
`node.lookup.*` (mesh-lookup scalars: `ttl`, `attempt_timeouts_secs`,
|
||||
`recent_expiry_secs`, `backoff_base_secs`, `backoff_max_secs`,
|
||||
`forward_min_interval_secs`) and `node.rendezvous.*` (peer rendezvous:
|
||||
`nostr.*`, `lan.*`). A deployed `node.discovery:` block still loads and is
|
||||
folded into the new tables with a one-time deprecation warning; migrate your
|
||||
`fips.yaml` to the new keys.
|
||||
|
||||
- `SessionDatagram::decrement_ttl` and `SessionDatagram::can_forward` now match
|
||||
the forwarder's IP hop-limit semantics: `decrement_ttl` decrements first and
|
||||
reports false when the result is zero, and `can_forward` is true only at a
|
||||
TTL of 2 or more.
|
||||
|
||||
### Deprecated
|
||||
|
||||
- The `discovery` metric-family key (control-socket JSON). It is dual-emitted
|
||||
alongside the new `lookup` key during a migration window and will be removed.
|
||||
Migrate dashboards/alerts from `discovery.*` to `lookup.*`.
|
||||
- The `node.discovery.*` config table. Its keys were split into `node.lookup.*`
|
||||
(mesh-lookup) and `node.rendezvous.*` (peer rendezvous). A legacy
|
||||
`node.discovery:` block still applies for now with a deprecation warning and
|
||||
will be removed; migrate to `node.lookup.*` / `node.rendezvous.*`.
|
||||
- The Ethernet `transports.ethernet.discovery` flag, renamed to
|
||||
`transports.ethernet.listen`. The old key is still accepted via a serde
|
||||
alias and will be removed at the v2 cutover; migrate to `listen`.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Nostr NAT traversal no longer breaks after the host suspends. The traversal
|
||||
clock cached a Unix timestamp once at startup and advanced it with a
|
||||
monotonic `Instant`, which does not tick while a machine is asleep, so after
|
||||
a suspend the daemon's idea of the time trailed real time by the suspend
|
||||
duration for the rest of the process lifetime. Every NIP-40 expiration it
|
||||
computed was therefore published already in the past: relays dropped the
|
||||
offers as expired, the initiator logged a signal timeout waiting for an
|
||||
answer, and traversal stayed broken until the daemon was restarted. The
|
||||
clock now reads the wall clock on every call. This is not macOS-specific,
|
||||
though a laptop that sleeps is where it is easiest to hit; any host that
|
||||
suspends or hibernates was affected. Reported in
|
||||
[#128](https://github.com/jmcorgan/fips/issues/128).
|
||||
|
||||
- `SessionDatagram` hop-limit handling now follows IP semantics. Delivery to
|
||||
the addressed node is no longer TTL-gated, and a forwarder decrements before
|
||||
deciding rather than after, so a datagram that would leave with a TTL of zero
|
||||
is dropped instead of transmitted. Previously the TTL check ran ahead of the
|
||||
local-delivery test, so a datagram addressed to this node that arrived with
|
||||
TTL 0 was dropped, and a forwarder receiving a transit datagram at TTL 1
|
||||
transmitted it at TTL 0 for the next hop to discard, wasting one transmission
|
||||
per expiring datagram. The reachable radius is unchanged, because the two
|
||||
behaviors compensated exactly: a path of `h` links still delivers for any
|
||||
source TTL of `h` or more. During a rolling upgrade, an unupgraded forwarder
|
||||
feeding an upgraded destination delivers one hop further than either version
|
||||
does on its own; no version mix delivers less far. The `TtlExhausted` reject
|
||||
counter now charges at the node that makes the decision rather than at the
|
||||
hop after it.
|
||||
|
||||
## [0.4.1] - 2026-07-19
|
||||
|
||||
### Changed
|
||||
|
||||
- `node.bloom.max_inbound_fpr` default raised from `0.10` to `0.20`. The
|
||||
cap rejects inbound `FilterAnnounce` whose FPR (`fill^k`) exceeds it. On
|
||||
the fixed 1 KB / k=5 filter, `0.10` corresponds to fill 0.631 (~1,630
|
||||
reachable entries), and the busiest nodes' aggregates had again begun to
|
||||
reach it as the mesh grew. `0.20` (fill 0.7248, ~2,114 entries) restores
|
||||
headroom without materially weakening the antipoison gate: a saturated or
|
||||
poisoned filter is ~100% FPR and still rejected. This is the second raise
|
||||
of this cap in two releases; the fixed 1 KB filter is the underlying
|
||||
constraint, and the structural remedy is the v2 filter work rather than a
|
||||
further raise. A node running this default accepts announcements that a
|
||||
v0.4.0 node drops, so during a rolling upgrade the two versions can
|
||||
disagree about mesh size.
|
||||
- Bloom filter probing computes its SHA-256 digest once per operation
|
||||
rather than once per hash function. All k indices were already derived
|
||||
from a single digest, but the digest was recomputed inside the
|
||||
per-function loop, so every insert and membership test hashed the same
|
||||
bytes `hash_count` times (5x at the default). Output is bit-for-bit
|
||||
identical; this is the hottest path in packet forwarding and mesh-size
|
||||
estimation.
|
||||
- Identity operations reuse one shared `secp256k1` context instead of
|
||||
constructing a fresh one at every sign, verify, and key-derive site.
|
||||
Each construction allocated a context and ran randomization and blinding
|
||||
table setup. Behavior is unchanged: the same API calls are made, only the
|
||||
context lifetime differs, and the shared context still performs the
|
||||
standard construction-time blinding.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Spanning tree: the coordinate cache is now invalidated when the parent
|
||||
link is lost through peer removal. That path reparents or self-roots the
|
||||
node but omitted the invalidation every other position-change path
|
||||
performs, so cached entries for downstream destinations kept the node's
|
||||
now-stale coordinate prefix. Because routing access refreshes an entry's
|
||||
TTL, an actively routed stale entry never self-expired and was corrected
|
||||
only by a fresh insert.
|
||||
- Discovery: applying a `LookupResponse` now keeps the tighter of the
|
||||
cached and received `path_mtu` rather than overwriting unconditionally.
|
||||
A looser estimate arriving in a later response could clobber a tighter
|
||||
value already learned from a reactive `MtuExceeded` or
|
||||
`PathMtuNotification`, loosening a clamp that had been correctly
|
||||
tightened.
|
||||
|
||||
### Removed
|
||||
|
||||
- The `parent_switched` spanning-tree metric counter. It was incremented on
|
||||
the line immediately before `parent_switches` at every site and never
|
||||
independently, so the two were always identical. `parent_switches`
|
||||
remains as the sole counter. Consumers reading `parent_switched` from the
|
||||
control socket or `fipstop` should use `parent_switches`.
|
||||
|
||||
## [0.4.0] - 2026-06-27
|
||||
|
||||
### Added
|
||||
|
||||
Generated
-1
@@ -1087,7 +1087,6 @@ dependencies = [
|
||||
"hex",
|
||||
"hkdf",
|
||||
"libc",
|
||||
"libm",
|
||||
"mdns-sd",
|
||||
"nostr",
|
||||
"nostr-sdk",
|
||||
|
||||
-14
@@ -11,21 +11,12 @@ readme = "README.md"
|
||||
keywords = ["mesh", "p2p", "decentralized", "overlay-network", "nostr"]
|
||||
categories = ["network-programming", "command-line-utilities", "cryptography"]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Tick-body profiler (`src/instr`). Off by default: enabling it edits 26 call
|
||||
# sites in the rx loop's hot tick arm, and only a compile-time gate makes the
|
||||
# default build's neutrality a property of the generated code rather than of a
|
||||
# runtime check. Build with `--features profiling` for a measurement run.
|
||||
profiling = []
|
||||
|
||||
[dependencies]
|
||||
ratatui = "0.30"
|
||||
secp256k1 = { version = "0.30", features = ["rand", "global-context"] }
|
||||
sha2 = "0.10"
|
||||
hkdf = "0.12"
|
||||
ring = "0.17"
|
||||
libm = "0.2"
|
||||
rand = "0.10.1"
|
||||
crossbeam-channel = "0.5"
|
||||
thiserror = "2.0"
|
||||
@@ -117,8 +108,3 @@ path = "src/bin/fips-gateway.rs"
|
||||
[[bin]]
|
||||
name = "fipstop"
|
||||
path = "src/bin/fipstop/main.rs"
|
||||
|
||||
[[bench]]
|
||||
name = "routing_next_hop"
|
||||
path = "benches/routing_next_hop.rs"
|
||||
harness = false
|
||||
|
||||
@@ -124,7 +124,7 @@ platform.
|
||||
| Ethernet | ✅ | ✅ | ❌ | ❌ | ✅ |
|
||||
| Tor | ✅ | ✅ | ✅ | ❌ | ✅ |
|
||||
| Nym | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| BLE | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| BLE | ✅ | ❌ | ❌ | ✅ | ❌ |
|
||||
|
||||
On Linux, a source build requires `libclang` — the LAN gateway's
|
||||
nftables bindings are generated by `bindgen` at build time, which
|
||||
@@ -213,7 +213,7 @@ testing/ Docker-based integration test harnesses + chaos simulation
|
||||
## Status & roadmap
|
||||
|
||||
FIPS is at **v0.5.0-dev** on the `master` branch.
|
||||
[v0.4.1](https://github.com/jmcorgan/fips/releases/tag/v0.4.1) has
|
||||
[v0.4.0](https://github.com/jmcorgan/fips/releases/tag/v0.4.0) has
|
||||
shipped; this development line continues the testing-and-polishing
|
||||
track toward v0.5.0. The core protocol works end-to-end over
|
||||
UDP, TCP, Ethernet, Tor, Nym, and Bluetooth on a global, public test
|
||||
|
||||
+284
-109
@@ -1,134 +1,302 @@
|
||||
# FIPS v0.4.1
|
||||
# FIPS v0.4.0
|
||||
|
||||
**Released**: 2026-07-19
|
||||
**Released**: 2026-06-21 (provisional)
|
||||
|
||||
v0.4.1 is a maintenance release on the v0.4.x line. It raises the default
|
||||
antipoison cap on inbound bloom filter announcements, removes a redundant
|
||||
spanning-tree metric counter, fixes two convergence and path-MTU bugs, and
|
||||
cuts per-packet CPU in the bloom and identity paths. There is no wire
|
||||
format change and no new feature surface.
|
||||
v0.4.0 is the throughput-and-observability release on the v0.3.x wire
|
||||
format. It adds two new ways for nodes to find and reach each other (the
|
||||
Nym mixnet transport and opt-in mDNS LAN discovery), overhauls the data
|
||||
plane for higher single-node throughput and lower per-packet CPU, moves
|
||||
the entire operator read surface off the data-plane hot path so
|
||||
observability stays responsive under load, ships a reworked `fipstop`
|
||||
TUI, and hardens FMP and FSP rekey to be hitless under packet loss in
|
||||
both directions. It also folds in the accumulated mesh-convergence,
|
||||
admission-control, and packaging fixes from the maintenance line.
|
||||
|
||||
v0.4.1 is wire-compatible with v0.4.0. Nodes can be upgraded one at a time
|
||||
with no coordinated restart, though one behavior change below is worth
|
||||
reading before you start a rolling upgrade.
|
||||
v0.4.0 is wire-compatible with v0.3.0. Mixed meshes interoperate; there
|
||||
is no flag-day upgrade. A deployed v0.3.0 node and an upgraded v0.4.0
|
||||
node peer, rekey, and route normally, so you can roll the upgrade out
|
||||
across a mesh in any order.
|
||||
|
||||
## At a glance
|
||||
|
||||
- `node.bloom.max_inbound_fpr` default moves from `0.10` to `0.20`.
|
||||
- The `parent_switched` metric counter is gone. Use `parent_switches`.
|
||||
- Spanning tree no longer serves stale coordinates after a parent link is
|
||||
lost through peer removal.
|
||||
- Discovery no longer loosens a path MTU clamp it had correctly tightened.
|
||||
- Bloom probing and identity operations do measurably less work per call,
|
||||
with identical results.
|
||||
- New outbound Nym mixnet transport with a single-container demo and a
|
||||
new mixnet-relay example.
|
||||
- Opt-in mDNS / DNS-SD discovery on the local link.
|
||||
- Data-plane overhaul: off-task encrypt and decrypt worker pools, GSO,
|
||||
connected-UDP send path, copy-avoidance on receive, batched macOS
|
||||
receive.
|
||||
- The full `show_*` read surface now serves off the receive loop, so
|
||||
`fipsctl` and `fipstop` stay responsive on loaded nodes; a new
|
||||
counter-only `show_metrics` query enables a Prometheus scraper at no
|
||||
hot-path cost.
|
||||
- Reworked `fipstop` TUI on a machine-verified render-snapshot base.
|
||||
- Rekey is now hitless under loss and reordering in both directions.
|
||||
- New packaging targets: an OpenWrt `.apk` for OpenWrt 25+ and a Nix
|
||||
flake for reproducible from-source builds on Nix/NixOS.
|
||||
- Six route-class transit counters partition forwarded traffic by its
|
||||
tree relationship to the next hop, visible via `show_routing` and
|
||||
`show_status`.
|
||||
|
||||
## What's new
|
||||
|
||||
### Nym mixnet transport
|
||||
|
||||
FIPS can now peer over the [Nym](https://nymtech.net/) mixnet for
|
||||
metadata-resistant connectivity. The new `transports.nym` transport
|
||||
makes outbound connections through a `nym-socks5-client` SOCKS5 proxy
|
||||
that you run alongside the daemon (for example as a service running
|
||||
alongside the fips daemon, or as a sidecar container). The transport
|
||||
waits at startup for the nym-socks5-client to become ready before giving
|
||||
up.
|
||||
|
||||
This is a privacy and anonymity deployment mode chosen for its own
|
||||
properties. It mixes your FIPS traffic into the Nym cover-traffic
|
||||
network so that link-level observers cannot correlate which mesh peers
|
||||
are talking. A new `examples/sidecar-nostr-mixnet-relay/` demonstrates a
|
||||
FIPS-reachable Nostr relay peered across the mixnet end to end, and a
|
||||
single-container demo ships with the transport.
|
||||
|
||||
Enable it by adding a `transports.nym` instance and pointing it at your
|
||||
running nym-socks5-client. See the transports reference for the field
|
||||
set.
|
||||
|
||||
### mDNS LAN discovery
|
||||
|
||||
Nodes on a shared local link can now find each other with zero address
|
||||
configuration. The opt-in `node.discovery.lan` path runs an mDNS /
|
||||
DNS-SD responder and browser: each node advertises a FIPS service record
|
||||
on the link and adopts the peers it discovers. This complements the
|
||||
existing Nostr-mediated overlay discovery for the common case where the
|
||||
peers are simply on the same LAN.
|
||||
|
||||
Turn it on with `node.discovery.lan.enabled: true`. `service_type` and
|
||||
`scope` tune the advertised service record and which interfaces
|
||||
participate. Discovery on the local link needs no relay and no STUN.
|
||||
|
||||
### Data-plane throughput overhaul
|
||||
|
||||
The receive and send paths were reworked for higher single-node
|
||||
throughput and lower per-packet CPU, building on the v0.3.0
|
||||
crypto-backend swap:
|
||||
|
||||
- **Off-task encrypt and decrypt.** Per-peer encrypt and decrypt now run
|
||||
on dedicated worker tasks rather than inline on the receive loop, so a
|
||||
single busy peer no longer serializes the whole node's crypto.
|
||||
- **GSO and connected-UDP send.** The Linux send path uses generic
|
||||
segmentation offload and a connected-UDP socket where available,
|
||||
cutting syscall overhead on bulk flows.
|
||||
- **Copy-avoidance on receive.** The receive hot path avoids buffer
|
||||
copies it previously made per packet.
|
||||
- **Batched macOS receive.** macOS gains a `recvmsg_x` batched receive,
|
||||
mirroring the Linux `recvmmsg` batching from v0.3.0.
|
||||
- **Shared immutable-state context and an atomic metric registry.**
|
||||
Immutable per-node state moved into a single shared context, and
|
||||
counters live in an atomic metric registry that the new `show_metrics`
|
||||
query reads without touching the hot path.
|
||||
|
||||
These are all internal to the data plane and require no operator action.
|
||||
|
||||
### Observability off the hot path
|
||||
|
||||
Every read-only control query now renders from a snapshot published once
|
||||
per tick into a lock-free `ArcSwap`, served from the control accept task
|
||||
instead of round-tripping the data-plane receive loop. This covers
|
||||
`show_status`, `show_stats_*`, `show_peers`, `show_sessions`,
|
||||
`show_links`, `show_connections`, `show_transports`, `show_mmp`,
|
||||
`show_tree`, `show_bloom`, `show_cache`, `show_routing`,
|
||||
`show_identity_cache`, `show_acl`, `show_listening_sockets`, and the new
|
||||
`show_metrics`. Only the mutating `connect` and `disconnect` commands
|
||||
still reach the loop.
|
||||
|
||||
The practical effect: on a loaded node where the receive loop was busy,
|
||||
`fipsctl` and `fipstop` queries previously stalled or timed out (the
|
||||
five-second query pattern operators saw). They now answer promptly
|
||||
regardless of data-plane load. Per-entity snapshots reuse unchanged rows
|
||||
by pointer, so the per-tick publish cost stays bounded as peer and
|
||||
session counts grow.
|
||||
|
||||
A new **`show_metrics`** query (surfaced as `fipsctl stats metrics`)
|
||||
returns a counter-only snapshot of every metric family. It is the
|
||||
enabler for a Prometheus scraper that pulls node counters at no hot-path
|
||||
cost.
|
||||
|
||||
Six **route-class transit counters** partition transit-forwarded packets
|
||||
by their tree relationship to the chosen next hop — tree-up, tree-down,
|
||||
tree-down-cross, cross-link descend, cross-link ascend, and direct-peer
|
||||
— and the six classes sum to `forwarded_packets`. They surface through
|
||||
`show_routing` and `show_status`, and the `fipstop` routing tab is
|
||||
reorganized so its two columns separate own/endpoint traffic from
|
||||
forwarded/transit traffic with the tree-down-cross line visually flagged.
|
||||
|
||||
### Reworked fipstop TUI
|
||||
|
||||
`fipstop` gets a rendering, navigation, and read-surface overhaul on a
|
||||
machine-verified base: a render-snapshot harness asserts the exact text
|
||||
grid and per-cell style of every view against canned control-socket
|
||||
output. New daemon-resolved fields surface through the snapshots,
|
||||
including effective persistence, root and is-root state, a
|
||||
per-transport-type peer-count map, per-peer effective depth, the root
|
||||
npub, and the last-sent uptree filter fill ratio with the subtree size
|
||||
estimate.
|
||||
|
||||
A separate fix clears a garbled-screen problem on startup and stray
|
||||
bytes on quit, most visible over SSH and inside tmux: startup now forces
|
||||
a full repaint before the first draw, and quit stops and joins the
|
||||
stdin-poll thread before restoring the terminal, so post-raw-mode
|
||||
keystrokes no longer echo onto the restored screen.
|
||||
|
||||
### Rekey reliability
|
||||
|
||||
FMP and FSP session rekey are now hitless under packet loss and
|
||||
reordering in both directions:
|
||||
|
||||
- Inbound frames are authenticated against the pending session before
|
||||
the K-bit cutover promotes it, so a spoofed or stale frame cannot
|
||||
derail a rekey in progress.
|
||||
- Rekey message-1 retransmission is bounded, and the link-dead heartbeat
|
||||
is rekey-aware so an in-flight rekey is not mistaken for a dead link.
|
||||
- FSP session rekey holds connectivity across the rekey window under
|
||||
loss and reordering.
|
||||
- Dual-initiation races (both peers starting a rekey at once on a
|
||||
high-latency link) are desynchronized with symmetric jitter so the two
|
||||
sides converge on one session rather than fighting.
|
||||
- An exhausted retransmission-budget abort, an expected and self-limiting
|
||||
outcome on lossy or high-latency links, is logged at debug rather than
|
||||
warn.
|
||||
|
||||
The net operator takeaway: rekey completes cleanly without dropping
|
||||
traffic, even on lossy or high-latency links, and the log no longer
|
||||
cries wolf when a rekey gives up and retries.
|
||||
|
||||
### New packaging targets
|
||||
|
||||
- **OpenWrt `.apk`.** A new `.apk` package targets OpenWrt 25+, where
|
||||
apk-tools is the mandatory package manager; the existing `.ipk`
|
||||
continues to cover OpenWrt 24.x and earlier. It is built SDK-free,
|
||||
reusing the `.ipk` cross-compile and installed-filesystem payload, and
|
||||
releases publish `.apk` artifacts and checksums alongside `.ipk`. Like
|
||||
the `.ipk`, the package is unsigned and installed with
|
||||
`apk add --allow-untrusted`.
|
||||
- **Nix flake.** A `flake.nix` at the project root builds all four
|
||||
binaries (`fips`, `fipsctl`, `fips-gateway`, `fipstop`) from source on
|
||||
Nix/NixOS, pinning the exact toolchain and wiring the native build
|
||||
dependencies so no host setup is needed beyond Nix with flakes
|
||||
enabled. It exposes `nix build`, `nix run`, a `nix develop` dev shell,
|
||||
and `nix flake check`, with `flake.lock` committed for reproducibility.
|
||||
|
||||
## Behavior changes worth flagging
|
||||
|
||||
### The inbound filter FPR cap default doubles again
|
||||
These affect operators on upgrade.
|
||||
|
||||
`node.bloom.max_inbound_fpr` goes from `0.10` to `0.20`. The cap rejects
|
||||
inbound `FilterAnnounce` frames whose advertised false positive rate
|
||||
exceeds it. On the fixed 1 KB, k=5 filter, `0.10` corresponds to a fill of
|
||||
0.631 and roughly 1,630 reachable entries, and the busiest nodes'
|
||||
aggregates had started reaching that ceiling as the mesh grew. `0.20`
|
||||
corresponds to a fill of 0.7248 and roughly 2,114 entries.
|
||||
|
||||
Be aware that this is the second time in two releases that this default
|
||||
has doubled, for the same reason both times. That is worth stating plainly
|
||||
rather than repeating the previous release's framing: raising the cap buys
|
||||
headroom, it does not fix anything. The real constraint is the fixed 1 KB
|
||||
filter size, which is a protocol constant. The structural remedy is the v2
|
||||
filter work, where filter capacity scales with the mesh instead of being
|
||||
pinned. This release is an interim step to keep legitimate aggregates from
|
||||
being rejected until that lands. It is not the start of a pattern of
|
||||
raising the cap once per release, and if you are sizing capacity planning
|
||||
around this number, plan against the v2 work rather than against a third
|
||||
raise.
|
||||
|
||||
The antipoison property the cap exists for is preserved. A saturated or
|
||||
deliberately poisoned filter still presents an FPR near 100% and is still
|
||||
rejected.
|
||||
|
||||
**This matters during a rolling upgrade.** A v0.4.1 node accepts a
|
||||
`FilterAnnounce` with a derived FPR between 0.10 and 0.20; a v0.4.0 node
|
||||
drops the same frame, and the drop is silent on the wire with no NACK. The
|
||||
cap also gates the mesh size estimator, which declines to produce a value
|
||||
when any contributing filter is over the cap. So while a mesh is partly
|
||||
upgraded, upgraded and not-yet-upgraded nodes can legitimately report
|
||||
different mesh sizes, or one can report a size while the other reports
|
||||
unknown. This resolves once every node is on v0.4.1. If you want to avoid
|
||||
the window entirely, set `node.bloom.max_inbound_fpr: 0.10` explicitly in
|
||||
your config before upgrading and remove it after the last node is done.
|
||||
|
||||
### The `parent_switched` counter is removed
|
||||
|
||||
`parent_switched` was incremented on the line immediately before
|
||||
`parent_switches` at every site and never independently, so the two
|
||||
counters always held the same value. `parent_switched` is now gone from
|
||||
the tree metrics, the control socket snapshot, and the `fipstop` tree
|
||||
view. `parent_switches` remains and is unchanged.
|
||||
|
||||
If you scrape the control socket, or have dashboards or alerts referencing
|
||||
`parent_switched`, point them at `parent_switches`. Anything still asking
|
||||
for `parent_switched` will find nothing rather than a zero.
|
||||
- **Bloom filter antipoison cap raised.** `node.bloom.max_inbound_fpr`
|
||||
moves from 0.05 to 0.10, accepting filters with a higher derived
|
||||
false-positive rate before rejecting them. This reduces spurious
|
||||
filter rejections on larger meshes while keeping the antipoison
|
||||
protection in place.
|
||||
- **TCP inbound cap honors `max_connections`.** The TCP inbound accept
|
||||
ceiling now resolves from explicit per-transport
|
||||
`max_inbound_connections`, then node-wide
|
||||
`node.limits.max_connections`, then the built-in default of 256.
|
||||
Previously the TCP inbound ceiling was hardwired to 256 and ignored
|
||||
`max_connections`, so raising it had no effect on inbound TCP.
|
||||
- **Static host aliases hot-reload.** `/etc/fips/hosts` now reloads on
|
||||
mtime change once per tick rather than only at startup, so display
|
||||
names in `fipsctl` and `fipstop` reflect edits without a daemon
|
||||
restart. The peer ACL reloads through the same lock-free snapshot
|
||||
mechanism.
|
||||
- **Quieter logs on busy public-mesh nodes.** Routine per-peer
|
||||
connection-lifecycle and capacity-cap events, no-route session-datagram
|
||||
drops, and exhausted rekey-budget aborts are demoted to debug, so
|
||||
genuinely notable info and warn lines are no longer drowned out.
|
||||
- **More visible drops.** Receive-path silent rejections now flow
|
||||
through typed reject-reason counters, and discovery counts requests
|
||||
dropped when the dedup cache is full (`req_dedup_cache_full`, visible
|
||||
via `show_routing`). Drops that were previously silent are now
|
||||
countable.
|
||||
- **Tor connect-refused accounting.** The Tor transport increments its
|
||||
`connect_refused` statistic (the "Refused" line in `fipstop`) on an
|
||||
actively-refused SOCKS5 connect, instead of recording every connect
|
||||
failure as a generic SOCKS5 error.
|
||||
|
||||
## Notable bug fixes
|
||||
|
||||
### Stale coordinates after losing a parent through peer removal
|
||||
The CHANGELOG has the exhaustive list. This is the operator-relevant
|
||||
subset of fixes for behavior that shipped in v0.3.0.
|
||||
|
||||
When a node's parent link dropped via peer removal, the node correctly
|
||||
reparented or self-rooted, but skipped the coordinate cache invalidation
|
||||
that every other position-change path performs. Cached entries for
|
||||
downstream destinations kept the node's old coordinate prefix. This did
|
||||
not self-correct the way a stale cache entry normally would: routing
|
||||
access refreshes an entry's TTL, so an entry that was actively being
|
||||
routed through never expired, and was only fixed by an unrelated fresh
|
||||
insert. Both invalidation classes now run on this path, matching the
|
||||
loop-detection branch.
|
||||
|
||||
### Discovery could loosen a tightened path MTU clamp
|
||||
|
||||
An originator handling a `LookupResponse` overwrote its cached path MTU
|
||||
unconditionally. If a reactive `MtuExceeded` or `PathMtuNotification` had
|
||||
already taught it a tighter value, a later, looser discovery estimate
|
||||
would clobber that and re-loosen the clamp, risking a return to dropped
|
||||
oversized packets. The cached and received values are now compared and the
|
||||
tighter one is kept.
|
||||
- **Symmetric peer teardown on manual disconnect.** A manual
|
||||
`fipsctl disconnect` now sends the peer a scoped Disconnect so both
|
||||
ends tear down and re-handshake cleanly. Previously a manual
|
||||
disconnect tore down only the local side, leaving the peer with a
|
||||
stale session that was never re-adopted as a child and whose bloom
|
||||
filter was never re-recorded.
|
||||
- **Gateway holds long-lived and DNS-cached mappings.** `fips-gateway`
|
||||
no longer drops a virtual-IP mapping while traffic is still flowing.
|
||||
The mapping TTL clock previously advanced only on DNS re-query, so a
|
||||
busy long-lived or DNS-cached client could have its mapping reclaimed
|
||||
mid-flow. The tick now refreshes the mapping whenever conntrack reports
|
||||
active sessions and recovers a draining mapping to active when traffic
|
||||
resumes; only genuinely idle mappings drain.
|
||||
- **Accurate mesh-size estimate under filter overlap.** The mesh-size
|
||||
estimator now estimates the cardinality of the OR-union of self plus
|
||||
every connected peer's inbound filter, instead of summing per-filter
|
||||
cardinalities of tree peers. Summing assumed the filters were disjoint,
|
||||
so a stale or oversized parent filter or a routing loop inflated the
|
||||
reported mesh size and a tree rebalance flapped the count. OR-union
|
||||
deduplicates overlap, equals the old result in the disjoint case, and
|
||||
removes the estimate's dependence on tree-declaration cache freshness.
|
||||
- **Single-uplink node reattaches within a round-trip.** A node with one
|
||||
tree peer, which has periodic parent re-evaluation disabled, was left
|
||||
self-rooted and unreachable if its one-shot attaching TreeAnnounce was
|
||||
lost, until the next periodic re-broadcast. Tree-position exchange is
|
||||
now self-healing on the receive path: a node that hears an announce
|
||||
advertising a strictly worse root echoes its own declaration back,
|
||||
provoking the better-rooted peer to re-push its real position
|
||||
immediately.
|
||||
- **macOS self-connections work end to end (#117).** Traffic a macOS
|
||||
node sends to its own `<npub>.fips` address is now delivered locally
|
||||
for full TCP/UDP, not just `ping6`. The point-to-point `utun` egresses
|
||||
self-addressed packets into the daemon with an unfinished transport
|
||||
checksum (macOS offloads it on the `lo0` loopback route), so
|
||||
re-injecting them verbatim made the local stack drop every segment the
|
||||
MSS-clamp rewrite did not happen to fix and self-connections
|
||||
half-opened and hung. The hairpin path now recomputes the TCP/UDP
|
||||
checksum before re-injection. Linux was unaffected.
|
||||
|
||||
## Upgrade notes
|
||||
|
||||
This is a drop-in upgrade from v0.4.0 with no wire format change, no
|
||||
config migration, and no coordinated restart. Upgrade nodes in whatever
|
||||
order you like.
|
||||
Operator-actionable items moving from v0.3.0 to v0.4.0:
|
||||
|
||||
Two things to do rather than assume:
|
||||
- **Wire-compatible, no flag day.** v0.4.0 peers with v0.3.0. Upgrade
|
||||
nodes in any order. During a rolling upgrade you may see some log lines
|
||||
on the upgraded side as it interacts with not-yet-upgraded peers;
|
||||
behavior is correct, log noise only.
|
||||
- **Bloom antipoison cap default changed.** `node.bloom.max_inbound_fpr`
|
||||
now defaults to 0.10 (was 0.05). If you set this explicitly, review
|
||||
whether you still want the old value.
|
||||
- **New optional config surfaces.** `transports.nym` (outbound Nym
|
||||
mixnet) and `node.discovery.lan` (mDNS LAN discovery) are both opt-in
|
||||
and off by default. Adding them is the only way to turn the new paths
|
||||
on.
|
||||
- **TCP inbound cap.** If you relied on the old hardwired 256 inbound-TCP
|
||||
ceiling, note it now honors `max_inbound_connections` then
|
||||
`node.limits.max_connections` then 256.
|
||||
- **New observability query.** `fipsctl stats metrics` (the
|
||||
`show_metrics` control query) returns a counter-only snapshot suitable
|
||||
for a scraper.
|
||||
|
||||
1. If you monitor `parent_switched`, move to `parent_switches` before
|
||||
upgrading, or your dashboards will go blank rather than error.
|
||||
2. During the rolling window, expect upgraded and not-yet-upgraded nodes
|
||||
to potentially disagree about mesh size, per the FPR cap section above.
|
||||
This is expected and self-resolves. Do not chase it as a bug unless it
|
||||
persists after every node reports `0.4.1`.
|
||||
|
||||
If you have pinned `node.bloom.max_inbound_fpr` explicitly in your config,
|
||||
your setting is honored and nothing changes for you. The change only
|
||||
affects nodes taking the default.
|
||||
|
||||
Downgrading to v0.4.0 is supported and needs no special handling.
|
||||
|
||||
## Getting v0.4.1
|
||||
## Getting v0.4.0
|
||||
|
||||
- **Linux x86_64 / aarch64**: `.deb` and tarball at the
|
||||
[v0.4.1 release page](https://github.com/jmcorgan/fips/releases/tag/v0.4.1).
|
||||
[v0.4.0 release page](https://github.com/jmcorgan/fips/releases/tag/v0.4.0).
|
||||
- **Arch Linux**: `fips` from the AUR.
|
||||
- **macOS**: `.pkg` at the v0.4.1 release page.
|
||||
- **Windows**: ZIP at the v0.4.1 release page.
|
||||
- **macOS**: `.pkg` at the v0.4.0 release page.
|
||||
- **Windows**: ZIP at the v0.4.0 release page.
|
||||
- **OpenWrt**: `.ipk` (OpenWrt 24.x and earlier) or `.apk` (OpenWrt 25+)
|
||||
at the v0.4.1 release page.
|
||||
- **From source**: `cargo build --release` from a checkout of the v0.4.1
|
||||
at the v0.4.0 release page.
|
||||
- **From source**: `cargo build --release` from a checkout of the v0.4.0
|
||||
tag (Rust 1.94.1 per `rust-toolchain.toml`; `libclang-dev` is a
|
||||
required Linux build prerequisite).
|
||||
- **Nix / NixOS**: `nix build .#fips` from a checkout of the v0.4.1 tag
|
||||
- **Nix / NixOS**: `nix build .#fips` from a checkout of the v0.4.0 tag
|
||||
builds the binaries from source with the pinned toolchain and no manual
|
||||
prerequisites (see the Nix section of `packaging/README.md`).
|
||||
|
||||
@@ -141,6 +309,13 @@ The full per-commit changelog lives in
|
||||
Thanks to everyone who contributed code, packaging work, bug reports, or
|
||||
reviews to this release.
|
||||
|
||||
- [@jcorgan](https://github.com/jmcorgan): release shepherd, spanning-tree
|
||||
and discovery fixes, bloom and identity performance work, antipoison cap
|
||||
change, and testing.
|
||||
- [@jcorgan](https://github.com/jmcorgan): release shepherd, high-level
|
||||
design, control read plane, rekey hardening, admission, bug fixes,
|
||||
testing, packaging, PR coordination, and issue resolution.
|
||||
- [@mmalmi](https://github.com/mmalmi): opt-in mDNS LAN discovery and
|
||||
data-plane performance work.
|
||||
- [@Origami74](https://github.com/Origami74): macOS packaging and
|
||||
website coordination.
|
||||
- [@dskvr](https://github.com/dskvr): AUR packaging.
|
||||
- [@oleksky](https://github.com/oleksky): Nym mixnet transport and the
|
||||
single-container mixnet demo.
|
||||
|
||||
@@ -1,365 +0,0 @@
|
||||
//! Micro-benchmark quantifying the per-forwarded-packet heap-allocation cost
|
||||
//! of the routing next-hop candidate-assembly path.
|
||||
//!
|
||||
//! `find_next_hop` runs once per forwarded data packet. Its sans-IO core
|
||||
//! assembles a `Vec<Candidate>` by enumerating every peer through the
|
||||
//! `RoutingView` seam: `peer_addrs()` materializes a `Vec<NodeAddr>` of all
|
||||
//! peers, the survivors are snapshotted (each cloning its `TreeCoordinate`),
|
||||
//! and the result is collected into a second `Vec`. This bench measures that
|
||||
//! per-call allocation against a fused zero-alloc reference that iterates the
|
||||
//! peer map directly and borrows coordinates instead of cloning.
|
||||
//!
|
||||
//! Visibility caveat: the production `routing_candidates` / `select_best_candidate`
|
||||
//! / `RoutingView` / `Candidate` are `pub(crate)` (src/proto/routing/core.rs)
|
||||
//! and are not re-exported at the crate root, so an external bench crate cannot
|
||||
//! name them. Rather than change production visibility, this file reproduces
|
||||
//! that path verbatim over the real public `NodeAddr` / `TreeCoordinate` /
|
||||
//! `CoordEntry` / `BloomFilter` types with the same iterator chain and the same
|
||||
//! `HashMap`-backed view the shell uses (src/node/mod.rs NodeRoutingView). The
|
||||
//! allocation behavior is therefore identical to production by construction;
|
||||
//! only the symbol identity differs.
|
||||
|
||||
use std::alloc::{GlobalAlloc, Layout, System};
|
||||
use std::collections::HashMap;
|
||||
use std::hint::black_box;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
|
||||
use fips::{BloomFilter, NodeAddr, TreeCoordinate};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Counting global allocator: bumps a process-global counter on every heap
|
||||
// allocation operation (alloc / alloc_zeroed / realloc). Sampled tightly and
|
||||
// single-threaded in `report_allocs` so no unrelated allocations are captured.
|
||||
// ---------------------------------------------------------------------------
|
||||
struct CountingAlloc;
|
||||
|
||||
static ALLOCS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
unsafe impl GlobalAlloc for CountingAlloc {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
ALLOCS.fetch_add(1, Ordering::Relaxed);
|
||||
unsafe { System.alloc(layout) }
|
||||
}
|
||||
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
|
||||
unsafe { System.dealloc(ptr, layout) }
|
||||
}
|
||||
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
|
||||
ALLOCS.fetch_add(1, Ordering::Relaxed);
|
||||
unsafe { System.alloc_zeroed(layout) }
|
||||
}
|
||||
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
|
||||
ALLOCS.fetch_add(1, Ordering::Relaxed);
|
||||
unsafe { System.realloc(ptr, layout, new_size) }
|
||||
}
|
||||
}
|
||||
|
||||
#[global_allocator]
|
||||
static GLOBAL: CountingAlloc = CountingAlloc;
|
||||
|
||||
const PEER_COUNTS: [usize; 4] = [8, 32, 128, 256];
|
||||
/// Fraction of peers whose bloom filter reports the destination reachable.
|
||||
const REACH_NUMERATOR: usize = 1;
|
||||
const REACH_DENOMINATOR: usize = 2;
|
||||
/// Tree depth for synthetic coordinates (self..root), a realistic mesh depth.
|
||||
const COORD_DEPTH: usize = 8;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reproduction of the pub(crate) routing seam (src/proto/routing/core.rs).
|
||||
// ---------------------------------------------------------------------------
|
||||
trait RoutingView {
|
||||
fn peer_addrs(&self) -> Vec<NodeAddr>;
|
||||
fn peer_may_reach(&self, peer: &NodeAddr, dest: &NodeAddr) -> bool;
|
||||
fn peer_can_send(&self, peer: &NodeAddr) -> bool;
|
||||
fn peer_link_cost(&self, peer: &NodeAddr) -> f64;
|
||||
fn peer_coords(&self, peer: &NodeAddr) -> Option<TreeCoordinate>;
|
||||
}
|
||||
|
||||
struct Candidate {
|
||||
addr: NodeAddr,
|
||||
can_send: bool,
|
||||
link_cost: f64,
|
||||
coords: Option<TreeCoordinate>,
|
||||
}
|
||||
|
||||
/// Verbatim from `routing::routing_candidates` (core.rs). Allocates the
|
||||
/// `peer_addrs` Vec, clones each survivor's coords, and collects into a Vec.
|
||||
fn routing_candidates(rv: &impl RoutingView, dest: &NodeAddr) -> Vec<Candidate> {
|
||||
rv.peer_addrs()
|
||||
.into_iter()
|
||||
.filter(|peer| rv.peer_may_reach(peer, dest))
|
||||
.map(|peer| Candidate {
|
||||
can_send: rv.peer_can_send(&peer),
|
||||
link_cost: rv.peer_link_cost(&peer),
|
||||
coords: rv.peer_coords(&peer),
|
||||
addr: peer,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Verbatim from `routing::select_best_candidate` (core.rs). Pure, no alloc.
|
||||
fn select_best_candidate(
|
||||
candidates: &[Candidate],
|
||||
dest_coords: &TreeCoordinate,
|
||||
my_coords: &TreeCoordinate,
|
||||
) -> Option<NodeAddr> {
|
||||
let my_distance = my_coords.distance_to(dest_coords);
|
||||
let mut best: Option<(&Candidate, f64, usize)> = None;
|
||||
for candidate in candidates {
|
||||
if !candidate.can_send {
|
||||
continue;
|
||||
}
|
||||
let cost = candidate.link_cost;
|
||||
let dist = candidate
|
||||
.coords
|
||||
.as_ref()
|
||||
.map(|pc| pc.distance_to(dest_coords))
|
||||
.unwrap_or(usize::MAX);
|
||||
if dist >= my_distance {
|
||||
continue;
|
||||
}
|
||||
let dominated = match &best {
|
||||
None => true,
|
||||
Some((_, best_cost, best_dist)) => {
|
||||
cost < *best_cost
|
||||
|| (cost == *best_cost && dist < *best_dist)
|
||||
|| (cost == *best_cost
|
||||
&& dist == *best_dist
|
||||
&& candidate.addr < best.as_ref().unwrap().0.addr)
|
||||
}
|
||||
};
|
||||
if dominated {
|
||||
best = Some((candidate, cost, dist));
|
||||
}
|
||||
}
|
||||
best.map(|(candidate, _, _)| candidate.addr)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bench-local view, HashMap-backed exactly like src/node/mod.rs NodeRoutingView.
|
||||
// ---------------------------------------------------------------------------
|
||||
struct BenchPeer {
|
||||
bloom: BloomFilter,
|
||||
can_send: bool,
|
||||
link_cost: f64,
|
||||
}
|
||||
|
||||
struct BenchView {
|
||||
peers: HashMap<NodeAddr, BenchPeer>,
|
||||
coords: HashMap<NodeAddr, TreeCoordinate>,
|
||||
}
|
||||
|
||||
impl RoutingView for BenchView {
|
||||
fn peer_addrs(&self) -> Vec<NodeAddr> {
|
||||
self.peers.keys().copied().collect()
|
||||
}
|
||||
fn peer_may_reach(&self, peer: &NodeAddr, dest: &NodeAddr) -> bool {
|
||||
self.peers.get(peer).is_some_and(|p| p.bloom.contains(dest))
|
||||
}
|
||||
fn peer_can_send(&self, peer: &NodeAddr) -> bool {
|
||||
self.peers.get(peer).is_some_and(|p| p.can_send)
|
||||
}
|
||||
fn peer_link_cost(&self, peer: &NodeAddr) -> f64 {
|
||||
self.peers.get(peer).map_or(f64::INFINITY, |p| p.link_cost)
|
||||
}
|
||||
fn peer_coords(&self, peer: &NodeAddr) -> Option<TreeCoordinate> {
|
||||
self.coords.get(peer).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
/// Zero-alloc reference: what an iterator/visitor seam would do. Iterates the
|
||||
/// peer map directly, fuses the may_reach + can_send filters, borrows coords
|
||||
/// instead of cloning, and tracks the best hop inline. No Vec, no coord clone.
|
||||
fn resolve_next_hop_zeroalloc(
|
||||
view: &BenchView,
|
||||
dest: &NodeAddr,
|
||||
dest_coords: &TreeCoordinate,
|
||||
my_coords: &TreeCoordinate,
|
||||
) -> Option<NodeAddr> {
|
||||
let my_distance = my_coords.distance_to(dest_coords);
|
||||
let mut best: Option<(NodeAddr, f64, usize)> = None;
|
||||
for (addr, peer) in &view.peers {
|
||||
if !peer.bloom.contains(dest) {
|
||||
continue;
|
||||
}
|
||||
if !peer.can_send {
|
||||
continue;
|
||||
}
|
||||
let cost = peer.link_cost;
|
||||
let dist = view
|
||||
.coords
|
||||
.get(addr)
|
||||
.map(|pc| pc.distance_to(dest_coords))
|
||||
.unwrap_or(usize::MAX);
|
||||
if dist >= my_distance {
|
||||
continue;
|
||||
}
|
||||
let dominated = match &best {
|
||||
None => true,
|
||||
Some((best_addr, best_cost, best_dist)) => {
|
||||
cost < *best_cost
|
||||
|| (cost == *best_cost && dist < *best_dist)
|
||||
|| (cost == *best_cost && dist == *best_dist && *addr < *best_addr)
|
||||
}
|
||||
};
|
||||
if dominated {
|
||||
best = Some((*addr, cost, dist));
|
||||
}
|
||||
}
|
||||
best.map(|(addr, _, _)| addr)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario construction.
|
||||
// ---------------------------------------------------------------------------
|
||||
fn addr(tag: u8, i: u16) -> NodeAddr {
|
||||
let mut b = [0u8; 16];
|
||||
b[0] = tag;
|
||||
b[1..3].copy_from_slice(&i.to_le_bytes());
|
||||
NodeAddr::from_bytes(b)
|
||||
}
|
||||
|
||||
/// A depth-`COORD_DEPTH` coordinate whose leaf is `leaf`, sharing a fixed
|
||||
/// interior path and root with `shared_tag`. Peers built with the dest's
|
||||
/// shared_tag sit close to the destination (distance 2); a distinct shared_tag
|
||||
/// sits far (near the root), modeling our own position.
|
||||
fn coord(leaf: NodeAddr, shared_tag: u8) -> TreeCoordinate {
|
||||
let mut path = Vec::with_capacity(COORD_DEPTH);
|
||||
path.push(leaf);
|
||||
for level in 1..(COORD_DEPTH - 1) {
|
||||
path.push(addr(shared_tag, level as u16));
|
||||
}
|
||||
path.push(addr(9, 0)); // common root
|
||||
TreeCoordinate::from_addrs(path).expect("valid coord path")
|
||||
}
|
||||
|
||||
struct Scenario {
|
||||
view: BenchView,
|
||||
dest: NodeAddr,
|
||||
dest_coords: TreeCoordinate,
|
||||
my_coords: TreeCoordinate,
|
||||
}
|
||||
|
||||
impl Scenario {
|
||||
fn new(n: usize) -> Self {
|
||||
let dest = addr(2, 0);
|
||||
// Destination path uses interior tag 4; peers reuse tag 4 so survivors
|
||||
// are close to the destination. Our own coords use tag 5 (far).
|
||||
let dest_coords = coord(dest, 4);
|
||||
let my_coords = coord(addr(6, 0), 5);
|
||||
|
||||
let mut peers = HashMap::new();
|
||||
let mut coords = HashMap::new();
|
||||
for i in 0..n {
|
||||
let paddr = addr(1, i as u16);
|
||||
let mut bloom = BloomFilter::new();
|
||||
// Realistic fill: a handful of unrelated reachable addrs.
|
||||
for f in 0..4u16 {
|
||||
bloom.insert(&addr(7, i as u16 * 4 + f));
|
||||
}
|
||||
// A controlled fraction advertise the destination as reachable.
|
||||
if (i % REACH_DENOMINATOR) < REACH_NUMERATOR {
|
||||
bloom.insert(&dest);
|
||||
}
|
||||
peers.insert(
|
||||
paddr,
|
||||
BenchPeer {
|
||||
bloom,
|
||||
can_send: true,
|
||||
link_cost: 1.0 + (i as f64) * 0.01,
|
||||
},
|
||||
);
|
||||
// Peers share the destination's interior path (tag 4) → close.
|
||||
coords.insert(paddr, coord(paddr, 4));
|
||||
}
|
||||
|
||||
Self {
|
||||
view: BenchView { peers, coords },
|
||||
dest,
|
||||
dest_coords,
|
||||
my_coords,
|
||||
}
|
||||
}
|
||||
|
||||
fn survivors(&self) -> usize {
|
||||
self.view
|
||||
.peers
|
||||
.values()
|
||||
.filter(|p| p.bloom.contains(&self.dest))
|
||||
.count()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Allocation-per-call report (printed once, before criterion timing).
|
||||
// ---------------------------------------------------------------------------
|
||||
fn count_allocs<T>(iters: usize, mut f: impl FnMut() -> T) -> f64 {
|
||||
for _ in 0..8 {
|
||||
black_box(f());
|
||||
}
|
||||
let start = ALLOCS.load(Ordering::Relaxed);
|
||||
for _ in 0..iters {
|
||||
black_box(f());
|
||||
}
|
||||
let end = ALLOCS.load(Ordering::Relaxed);
|
||||
(end - start) as f64 / iters as f64
|
||||
}
|
||||
|
||||
fn report_allocs() {
|
||||
const ITERS: usize = 2000;
|
||||
println!("\n=== allocations per call (heap alloc ops: alloc+alloc_zeroed+realloc) ===");
|
||||
println!(
|
||||
"{:>6} {:>10} {:>16} {:>16}",
|
||||
"peers", "survivors", "current/call", "zero-alloc/call"
|
||||
);
|
||||
for &n in &PEER_COUNTS {
|
||||
let s = Scenario::new(n);
|
||||
let survivors = s.survivors();
|
||||
let current = count_allocs(ITERS, || {
|
||||
let cands = routing_candidates(&s.view, &s.dest);
|
||||
select_best_candidate(&cands, &s.dest_coords, &s.my_coords)
|
||||
});
|
||||
let zero = count_allocs(ITERS, || {
|
||||
resolve_next_hop_zeroalloc(&s.view, &s.dest, &s.dest_coords, &s.my_coords)
|
||||
});
|
||||
println!("{n:>6} {survivors:>10} {current:>16.2} {zero:>16.2}");
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
fn bench_next_hop(c: &mut Criterion) {
|
||||
report_allocs();
|
||||
|
||||
let mut group = c.benchmark_group("find_next_hop");
|
||||
for &n in &PEER_COUNTS {
|
||||
let scenario = Scenario::new(n);
|
||||
group.bench_with_input(BenchmarkId::new("current_alloc", n), &n, |b, _| {
|
||||
b.iter(|| {
|
||||
let cands = routing_candidates(&scenario.view, &scenario.dest);
|
||||
black_box(select_best_candidate(
|
||||
&cands,
|
||||
&scenario.dest_coords,
|
||||
&scenario.my_coords,
|
||||
))
|
||||
});
|
||||
});
|
||||
group.bench_with_input(BenchmarkId::new("zero_alloc_ref", n), &n, |b, _| {
|
||||
b.iter(|| {
|
||||
black_box(resolve_next_hop_zeroalloc(
|
||||
&scenario.view,
|
||||
&scenario.dest,
|
||||
&scenario.dest_coords,
|
||||
&scenario.my_coords,
|
||||
))
|
||||
});
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group! {
|
||||
name = benches;
|
||||
config = Criterion::default().sample_size(50);
|
||||
targets = bench_next_hop
|
||||
}
|
||||
criterion_main!(benches);
|
||||
@@ -41,9 +41,18 @@ fn main() {
|
||||
// satisfy libdbus-sys's pkg-config cross-compile requirement, and musl
|
||||
// router targets don't run BlueZ by default anyway.
|
||||
println!("cargo:rustc-check-cfg=cfg(bluer_available)");
|
||||
// `ble_available` gates the platform-agnostic BLE transport module (pool,
|
||||
// discovery, per-peer PSM, the generic `BleTransport`). The module compiles
|
||||
// on every platform that has — or will have — a concrete `BleIo` backend;
|
||||
// the backend is selected per-platform (BluerIo on linux-glibc, BluestIo on
|
||||
// macOS, AndroidIo on Android, else the in-memory MockBleIo fallback).
|
||||
println!("cargo:rustc-check-cfg=cfg(ble_available)");
|
||||
let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();
|
||||
let target_env = std::env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default();
|
||||
if target_os == "linux" && target_env != "musl" {
|
||||
println!("cargo:rustc-cfg=bluer_available");
|
||||
}
|
||||
if matches!(target_os.as_str(), "linux" | "macos" | "android") {
|
||||
println!("cargo:rustc-cfg=ble_available");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -360,14 +360,14 @@ control socket and `fipstop` dashboard. (See `compute_mesh_size()` in
|
||||
|
||||
The estimator refuses to produce a value when any contributing filter
|
||||
is above the antipoison FPR cap (`node.bloom.max_inbound_fpr`,
|
||||
default `0.20`); a partial aggregate would silently underestimate.
|
||||
default `0.10`); a partial aggregate would silently underestimate.
|
||||
Consumers handle the resulting `None` by displaying an "unknown"
|
||||
state rather than a misleading number.
|
||||
|
||||
## Antipoison: Inbound FPR Cap
|
||||
|
||||
Inbound `FilterAnnounce` payloads are checked against
|
||||
`node.bloom.max_inbound_fpr` (default `0.20`). Filters whose
|
||||
`node.bloom.max_inbound_fpr` (default `0.10`). Filters whose
|
||||
estimated false positive rate exceeds the cap are dropped silently
|
||||
(no NACK on the wire) — they would otherwise inflate downstream
|
||||
candidate evaluation cost without contributing useful discrimination.
|
||||
|
||||
@@ -477,7 +477,7 @@ The TXT record carries three keys (`src/discovery/lan/mod.rs:47-55`):
|
||||
|
||||
Once per node tick, the node drains browser events and acts on them in
|
||||
`poll_lan_discovery()` (`src/node/lifecycle.rs:907`, called from
|
||||
`src/node/dataplane/rx_loop.rs:266`). For each discovered peer it finds
|
||||
`src/node/handlers/rx_loop.rs:266`). For each discovered peer it finds
|
||||
a UDP transport whose family matches the peer address, parses the
|
||||
`npub` into a `PeerIdentity`, skips peers it is already connected to or
|
||||
currently connecting to, and otherwise initiates a connection.
|
||||
|
||||
@@ -262,7 +262,7 @@ UDP (1500 vs 1472 MTU).
|
||||
- **No IP dependency**: Operates below the IP layer. Nodes on the same
|
||||
Ethernet segment can communicate without IP addresses or routing
|
||||
infrastructure
|
||||
- **Broadcast neighbor detection**: Nodes discover each other via periodic beacon
|
||||
- **Broadcast discovery**: Nodes discover each other via periodic beacon
|
||||
broadcasts on the shared medium, with no static peer configuration required
|
||||
- **Higher MTU**: Standard Ethernet frames carry 1500 bytes of payload,
|
||||
yielding an effective FIPS MTU of 1499 after the frame type prefix
|
||||
@@ -293,7 +293,7 @@ socket.
|
||||
| Addressing | 6-byte MAC address |
|
||||
| Platform | Linux only (`CAP_NET_RAW` required) |
|
||||
|
||||
### Neighbor Beacons
|
||||
### Beacon Discovery
|
||||
|
||||
Ethernet nodes discover peers via broadcast beacons sent to
|
||||
ff:ff:ff:ff:ff:ff. Each beacon is a 34-byte frame containing the sender's
|
||||
@@ -301,7 +301,7 @@ x-only public key. Receiving nodes extract the MAC source address from the
|
||||
frame and the public key from the payload, then report the discovered peer
|
||||
to FMP.
|
||||
|
||||
Four configuration flags control neighbor behavior — `listen`
|
||||
Four configuration flags control discovery behavior — `discovery`
|
||||
(listen for beacons), `announce` (broadcast beacons), `auto_connect`
|
||||
(initiate handshakes to discovered peers), and `accept_connections`
|
||||
(accept inbound handshakes). The flag table and per-flag defaults
|
||||
@@ -310,13 +310,13 @@ under `transports.ethernet.*`.
|
||||
|
||||
A typical discoverable node sets `announce`, `auto_connect`, and
|
||||
`accept_connections` all true. A passive listener uses just
|
||||
`listen: true` to observe the network without announcing itself.
|
||||
`discovery: true` to observe the network without announcing itself.
|
||||
|
||||
### WiFi Compatibility
|
||||
|
||||
WiFi interfaces in infrastructure (managed) mode work transparently for
|
||||
unicast — the mac80211 subsystem handles frame translation between 802.11
|
||||
and 802.3. Broadcast neighbor detection is unreliable in managed mode because
|
||||
and 802.3. Broadcast beacon discovery is unreliable in managed mode because
|
||||
access points commonly isolate clients from each other's broadcast traffic.
|
||||
|
||||
Startup logging:
|
||||
@@ -895,11 +895,11 @@ transitions through `Starting` to `Up` (operational). `stop()` moves to
|
||||
| --------- | ------ | ----- |
|
||||
| UDP/IP | **Implemented** | Primary transport, AsyncFd/recvmsg, SO_RXQ_OVFL kernel drop detection |
|
||||
| TCP/IP | **Implemented** | FMP header-based framing, non-blocking connect, per-connection MSS MTU |
|
||||
| Ethernet | **Implemented** | AF_PACKET SOCK_DGRAM, EtherType 0x2121, neighbor beacons, Linux only |
|
||||
| Ethernet | **Implemented** | AF_PACKET SOCK_DGRAM, EtherType 0x2121, beacon discovery, Linux only |
|
||||
| WiFi | **Implemented** (via Ethernet transport, infrastructure mode) | mac80211 translates 802.11↔802.3; broadcast beacons unreliable through APs |
|
||||
| Tor | **Implemented** | Outbound SOCKS5, inbound via onion service, .onion and clearnet addressing |
|
||||
| Nym | **Implemented** | Outbound-only SOCKS5 through nym-socks5-client, mixnet anonymity, IP/hostname addressing |
|
||||
| BLE | **Implemented** (Linux/glibc only; experimental) | L2CAP CoC, ATT_MTU negotiation, per-link MTU; musl/macOS/Windows skip |
|
||||
| BLE | **Implemented** (Linux/glibc and Android; experimental) | L2CAP CoC, ATT_MTU negotiation, per-link MTU; Linux via BlueZ, Android via the embedder radio bridge; macOS/Windows/musl skip |
|
||||
| Radio | Future direction | Constrained MTU (51–222 bytes) |
|
||||
| Serial | Future direction | SLIP/COBS framing, point-to-point |
|
||||
|
||||
|
||||
@@ -25,6 +25,4 @@ X" to "X is done".
|
||||
| [persistent-identity.md](persistent-identity.md) | Provision a stable Nostr keypair so the node keeps the same npub across restarts |
|
||||
| [host-aliases.md](host-aliases.md) | Use shortnames (`test-us01.fips`, `my-laptop.fips`) instead of full npubs by editing `/etc/fips/hosts` or setting peer aliases |
|
||||
| [set-up-bluetooth-peer.md](set-up-bluetooth-peer.md) | Configure a Bluetooth Low Energy peer link |
|
||||
| [set-up-80211s-mesh-backhaul.md](set-up-80211s-mesh-backhaul.md) | Link OpenWrt FIPS routers over an open 802.11s radio backhaul (FIPS provides encryption, authentication, and routing) |
|
||||
| [set-up-open-access-ssid.md](set-up-open-access-ssid.md) | Broadcast the open `!FIPS` access SSID so phones and laptops roam onto the mesh (one ESS: save once, roam every FIPS router) |
|
||||
| [diagnose-mtu-issues.md](diagnose-mtu-issues.md) | Triage MTU-shaped failures and rule out their imposters (bufferbloat, transport saturation) |
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
# Set Up an 802.11s Mesh Backhaul (OpenWrt)
|
||||
|
||||
Link FIPS routers over radio — no cables, no APs, no shared
|
||||
infrastructure — by running the Ethernet transport on an open 802.11s
|
||||
mesh interface. The radio layer provides nothing but L2 frames to
|
||||
direct neighbors; FIPS provides everything else: encryption and
|
||||
authentication (Noise IK), peer discovery (Ethernet beacons), and
|
||||
routing (the spanning tree).
|
||||
|
||||
For the transport design, see
|
||||
[../design/fips-transport-layer.md](../design/fips-transport-layer.md).
|
||||
For all `transports.ethernet.*` configuration keys, see
|
||||
[../reference/configuration.md](../reference/configuration.md).
|
||||
|
||||
## Why open, why forwarding off
|
||||
|
||||
Two deliberate choices distinguish this from a stock 802.11s setup:
|
||||
|
||||
- **`encryption none`** — the mesh is open on purpose. Every FIPS peer
|
||||
link is already authenticated and encrypted by the Noise IK
|
||||
handshake, so SAE at L2 would duplicate that work, add a shared
|
||||
credential to provision across routers, and (on ath10k) force the
|
||||
firmware into its slower raw Tx/Rx mode. A stranger can form an
|
||||
802.11s peering with your router *and* a FIPS peer link on top of it —
|
||||
the same open model as mDNS and BLE discovery, where the advert is
|
||||
only a hint and the handshake authenticates each link (no
|
||||
impersonation, no MITM) rather than gating who may peer. Admission is
|
||||
open up to the daemon's max-peers cap. What you concede: any nearby
|
||||
radio can peer and reach the FIPS overlay surface; L2 metadata (MAC
|
||||
addresses, frame sizes) is visible in the air; a hostile radio can
|
||||
burn airtime — all inherent to an open radio link.
|
||||
- **`mesh_fwding 0`** — disables 802.11s's own HWMP routing so each
|
||||
mesh link is a plain neighbor link. FIPS is the routing layer; two
|
||||
routing layers would fight, and broadcast discovery beacons would
|
||||
flood the whole mesh instead of reaching direct neighbors only.
|
||||
|
||||
The interface is **not** bridged into `br-lan` — the FIPS Ethernet
|
||||
transport binds it directly.
|
||||
|
||||
## When to use
|
||||
|
||||
- Two or more OpenWrt FIPS routers within radio range of each other,
|
||||
where running cable is impractical.
|
||||
- You want the mesh segment to keep working with zero shared
|
||||
credentials or per-site configuration ("flash and drop in").
|
||||
|
||||
It is **not** for connecting phones or laptops — client devices
|
||||
cannot join an 802.11s mesh. They enter the mesh through a normal AP
|
||||
on the same router (see constraints below), or over BLE.
|
||||
|
||||
## Requirements
|
||||
|
||||
- OpenWrt 22.03+ with the FIPS package installed.
|
||||
- A radio whose driver supports mesh point interfaces. Check with:
|
||||
|
||||
```sh
|
||||
iw list | grep -A 10 "Supported interface modes" | grep "mesh point"
|
||||
```
|
||||
|
||||
The mainstream OpenWrt chips (ath9k, ath10k, mt76) all qualify.
|
||||
- Ideally a dual- or tri-band router, so one band can be dedicated to
|
||||
the backhaul (see constraints).
|
||||
|
||||
## Step 1 — create the mesh interface(s)
|
||||
|
||||
On **each** router, run the helper once per radio you want in the
|
||||
backhaul:
|
||||
|
||||
```sh
|
||||
fips-mesh-setup radio1
|
||||
```
|
||||
|
||||
This creates an open 802.11s interface with mesh ID `fips-mesh` and
|
||||
HWMP forwarding off, attaches it to an unmanaged netifd interface (no
|
||||
IP configuration — none is needed), uncomments the matching `meshN`
|
||||
transport entry in `/etc/fips/fips.yaml` (see Step 2), and reloads the
|
||||
radio. Interfaces are named by radio index: `radio0` → `fips-mesh0`,
|
||||
`radio1` → `fips-mesh1`. Pass a second argument to use a different
|
||||
mesh ID.
|
||||
|
||||
Note: the helper runs `wifi reload`, which re-applies the whole
|
||||
wireless config and so briefly drops every client AP on all radios for
|
||||
a few seconds. `fips-mesh-setup remove` reloads the same way. Expect
|
||||
the blip if clients are connected.
|
||||
|
||||
On dual-band routers, meshing **both** bands is worth it: 2.4 GHz
|
||||
reaches further at lower rates, 5 GHz carries more over shorter
|
||||
links. Note this is **failover, not multipath**: FIPS keeps one
|
||||
active link per peer, so traffic uses one band at a time — the other
|
||||
is a standby that re-establishes the peer if the active link dies
|
||||
(detection via keepalive timeout, so a cutover takes seconds, not
|
||||
milliseconds):
|
||||
|
||||
```sh
|
||||
fips-mesh-setup radio0
|
||||
fips-mesh-setup radio1
|
||||
```
|
||||
|
||||
**Pin the same channel on every backhaul router, per band.** Mesh
|
||||
points only peer on the same channel, and the mesh inherits whatever
|
||||
the radio is set to — with `channel 'auto'` (the default on many
|
||||
devices) each router picks its own and the mesh silently never forms.
|
||||
The script prints the radio's current band and channel and warns on
|
||||
`auto`:
|
||||
|
||||
```sh
|
||||
uci set wireless.radio1.channel='36'
|
||||
uci commit wireless && wifi reload
|
||||
```
|
||||
|
||||
Prefer a non-DFS channel (36–48 on 5 GHz): on DFS channels the radio
|
||||
must wait ~60 s in CAC before transmitting after every reload.
|
||||
|
||||
Equivalent manual UCI (per radio), if you prefer to see what it does:
|
||||
|
||||
```sh
|
||||
uci batch <<'EOF'
|
||||
set wireless.fips_mesh_radio1=wifi-iface
|
||||
set wireless.fips_mesh_radio1.device='radio1'
|
||||
set wireless.fips_mesh_radio1.mode='mesh'
|
||||
set wireless.fips_mesh_radio1.mesh_id='fips-mesh'
|
||||
set wireless.fips_mesh_radio1.encryption='none'
|
||||
set wireless.fips_mesh_radio1.mesh_fwding='0'
|
||||
set wireless.fips_mesh_radio1.ifname='fips-mesh1'
|
||||
set wireless.fips_mesh_radio1.network='fips_mesh_radio1'
|
||||
set network.fips_mesh_radio1=interface
|
||||
set network.fips_mesh_radio1.proto='none'
|
||||
EOF
|
||||
uci commit
|
||||
wifi reload
|
||||
```
|
||||
|
||||
## Step 2 — check the FIPS transport binding
|
||||
|
||||
The `fips.yaml` shipped in the OpenWrt package carries one transport
|
||||
entry per radio, but **commented out** — so a stock install that never
|
||||
runs this helper logs no per-boot "interface missing" warning.
|
||||
`fips-mesh-setup` uncommented the matching `meshN` entry in Step 1, so
|
||||
there is normally nothing to do here. If you maintain your own config
|
||||
(or ran the manual UCI above instead of the helper), make sure the
|
||||
entries are present and uncommented:
|
||||
|
||||
```yaml
|
||||
transports:
|
||||
ethernet:
|
||||
mesh0:
|
||||
interface: "fips-mesh0"
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
mesh1:
|
||||
interface: "fips-mesh1"
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
```
|
||||
|
||||
## Step 3 — restart the daemon (order matters)
|
||||
|
||||
```sh
|
||||
/etc/init.d/fips restart
|
||||
```
|
||||
|
||||
Restart fips **after** the mesh interface is up. A transport whose
|
||||
interface is missing at startup is logged and skipped, not retried —
|
||||
so if the daemon comes up before the radio, the mesh transport stays
|
||||
dead until the next restart. (An interface that *vanishes and
|
||||
returns* after startup is recovered automatically; only the missing-
|
||||
at-startup case needs this ordering.)
|
||||
|
||||
## Verify
|
||||
|
||||
L2 first — the 802.11s peering, with a second configured router in
|
||||
range:
|
||||
|
||||
```sh
|
||||
iw dev fips-mesh0 station dump
|
||||
```
|
||||
|
||||
You should see one station entry per neighbor router, with signal
|
||||
levels. No entries means a radio problem, not a FIPS problem — triage
|
||||
in this order:
|
||||
|
||||
1. **Channel mismatch** (the most common cause): compare
|
||||
`iw dev fips-mesh0 info` on both routers — mesh ID *and* channel
|
||||
must match exactly.
|
||||
2. **The mesh interface never joined** — `iw dev fips-meshX info`
|
||||
shows `type mesh point` but **no channel line**, and `station dump`
|
||||
is empty. Usual cause: a client (`sta`) interface on the same
|
||||
radio. A STA must follow its upstream AP's channel, the whole
|
||||
radio follows the STA, and a mesh pinned to a different channel
|
||||
silently stays down. Check for a STA sharing the radio
|
||||
(`iw dev`, look for `type managed` on the same phy), compare
|
||||
`iw dev <sta-iface> info | grep channel`, and re-pin the mesh
|
||||
channel to match — on every backhaul router.
|
||||
3. **Is the other router transmitting at all?**
|
||||
|
||||
```sh
|
||||
iw dev fips-mesh0 scan | grep -i -B4 "MESH ID"
|
||||
```
|
||||
|
||||
Its mesh ID visible → transmission works, peering is failing
|
||||
(mesh ID typo, or one side has encryption set). Nothing visible →
|
||||
check `wifi status` on the other router, remember the ~60 s DFS
|
||||
CAC wait, and confirm the country code is set
|
||||
(`uci get wireless.radio1.country`) — an unset regdomain can
|
||||
block channels entirely.
|
||||
4. `logread | grep -iE "mesh|fips-mesh0"` on both sides.
|
||||
|
||||
Then the FIPS layer on top:
|
||||
|
||||
```sh
|
||||
logread | grep -i beacon # beacons flowing on the new transport
|
||||
fipsctl show peers # neighbor authenticated and connected
|
||||
fipsctl show links # link on the 'ethernet' transport
|
||||
```
|
||||
|
||||
Discovery is automatic: each node beacons its pubkey every few
|
||||
seconds, and `auto_connect` initiates the Noise handshake on first
|
||||
sight.
|
||||
|
||||
## Constraints
|
||||
|
||||
- **Airtime is shared per radio.** All virtual interfaces on one
|
||||
radio (AP + mesh) share one channel, and multi-hop forwarding on a
|
||||
single radio roughly halves throughput per hop. On dual/tri-band
|
||||
hardware, dedicate one band to `fips-mesh0` and serve clients on
|
||||
the others.
|
||||
- **AP + mesh coexistence is driver-dependent.** It works on the
|
||||
mainstream chips (this is the standard Freifunk/Gluon setup), but
|
||||
check `iw list` under "valid interface combinations" for your
|
||||
hardware.
|
||||
- **Clients can't join.** Phones and laptops reach the mesh through
|
||||
the router's normal AP or via BLE — never through the 802.11s
|
||||
interface.
|
||||
- **Radio links are lossy.** A neighbor at the edge of range will
|
||||
form an 802.11s peering yet deliver a fraction of its frames.
|
||||
Expect link-quality effects that don't exist on wired Ethernet.
|
||||
- **A client (STA) uplink on the same radio owns the channel.** The
|
||||
STA must follow whatever channel its upstream AP uses; every other
|
||||
interface on that radio follows the STA. A mesh pinned to a
|
||||
different channel silently never joins, and it does **not** recover
|
||||
when the STA disconnects — a `wifi reload` (plus a fips restart) is
|
||||
needed. A *roaming* uplink (travel-router / hotspot-chasing setups)
|
||||
is fundamentally incompatible with a fixed-channel mesh on the same
|
||||
radio: dedicate the mesh to the radio the STA never uses, and treat
|
||||
any mesh sharing a STA radio as best-effort.
|
||||
@@ -44,6 +44,7 @@ crate accordingly).
|
||||
| -------- | -------------- |
|
||||
| Linux (glibc) | Supported. |
|
||||
| Linux (musl, OpenWrt) | Disabled at build time. |
|
||||
| Android | Supported (native Android BLE, via the embedder's radio bridge). |
|
||||
| macOS | Not supported. |
|
||||
| Windows | Not supported. |
|
||||
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
# Set Up the Open FIPS Access SSID (OpenWrt)
|
||||
|
||||
Give phones and laptops a way in: every FIPS router broadcasts the
|
||||
same open SSID — `!FIPS` — from its access radio. Same SSID + unique
|
||||
BSSIDs is one standard ESS, so a client saves the network once and
|
||||
roams between all FIPS routers natively, with no per-router setup and
|
||||
no shared credentials (the Freifunk model). The leading `!` sorts the
|
||||
network to the top of alphabetically ordered pickers (iOS, desktop
|
||||
OSes — Android sorts by signal strength) and is part of the name:
|
||||
SSIDs match byte-for-byte or not at all. The radio layer provides
|
||||
nothing but open L2 to the nearest router; FIPS provides everything
|
||||
else: encryption and authentication (Noise IK), discovery
|
||||
(mDNS/Ethernet beacons), and mobility (the overlay identity survives
|
||||
roaming, so no 802.11r or L2 tricks are needed).
|
||||
|
||||
This is the *access* layer — how clients reach FIPS routers. For the
|
||||
router-to-router *backhaul*, see
|
||||
[set-up-80211s-mesh-backhaul.md](set-up-80211s-mesh-backhaul.md).
|
||||
For all `transports.ethernet.*` configuration keys, see
|
||||
[../reference/configuration.md](../reference/configuration.md).
|
||||
|
||||
## Why open, why this addressing
|
||||
|
||||
Three deliberate choices distinguish this from a stock guest network:
|
||||
|
||||
- **`encryption none`** — the SSID is open on purpose, and it *must*
|
||||
be. Clients key a saved network on SSID **plus security type**: if
|
||||
one router used a PSK and another OWE, the same `FIPS` name would be
|
||||
three different saved networks and roaming would break. Open is the
|
||||
only security type that needs zero provisioning, and OWE is left out
|
||||
for now for exactly this uniformity reason (OWE-transition mode is
|
||||
inconsistent across client vendors). Every FIPS peer link is already
|
||||
authenticated and encrypted by the Noise IK handshake. A stranger
|
||||
can associate *and* form a FIPS peer link — that is the point of open
|
||||
access; the handshake authenticates each link (no impersonation, no
|
||||
MITM) but does not gate who may peer, and admission is open up to the
|
||||
daemon's max-peers cap. What confines a hostile peer is the isolated
|
||||
`fips_ap` zone (no path to br-lan or the WAN — see below), not the
|
||||
handshake. What you concede: any nearby device can reach the FIPS
|
||||
overlay surface (handshake, discovery, lookup, routing) and peer with
|
||||
the router; L2 metadata is visible in the air; a hostile radio can
|
||||
burn airtime — all inherent to an open radio link.
|
||||
- **DHCPv4 from a fixed subnet, plus IPv6 router advertisements.**
|
||||
dnsmasq leases IPv4 out of `10.21.<N>.0/24` (`N` = the radio index;
|
||||
the prefix echoes FIPS port 2121). The subnet is deliberately
|
||||
**identical on every router**: a roaming phone keeps its lease
|
||||
across routers, and dnsmasq's authoritative mode (the OpenWrt
|
||||
default, pinned by the helper) ACKs a renew the new router never
|
||||
issued. odhcpd additionally announces a ULA prefix (`fd..`-range)
|
||||
for stateless SLAAC; DHCPv6 stays off. FIPS itself only needs
|
||||
link-local + mDNS, but Android's provisioning check requires an RA
|
||||
or a DHCP offer and *disconnects* with neither, and plain laptops
|
||||
expect a real IPv4 address. Works with or without an upstream —
|
||||
nothing here depends on the WAN. The IPv6 side stays per-router and
|
||||
disposable; in all cases the FIPS overlay identity, not the IP, is
|
||||
the mobility anchor.
|
||||
- **Isolated interface** — its own network and firewall zone, with no
|
||||
path to `br-lan` and no forwarding to the WAN. Inbound traffic is
|
||||
rejected except DHCPv4, ICMPv6 (SLAAC itself), mDNS, and the FIPS
|
||||
transport ports; the raw-Ethernet transport (EtherType 0x2121) is
|
||||
not IP and never traverses the firewall. AP client isolation is on, so clients
|
||||
cannot reach each other at L2 — two FIPS phones on one router still
|
||||
reach each other through the router at the overlay layer.
|
||||
|
||||
## The "no internet" behavior (expected, one-time acceptance)
|
||||
|
||||
The network intentionally provides **no internet**. On first connect,
|
||||
a phone's validation probe fails and it asks whether to stay on a
|
||||
network without internet access — choose **stay connected** and
|
||||
**don't ask again**. That choice is stored per SSID, so accepting it
|
||||
once covers every FIPS router anywhere.
|
||||
|
||||
After that, the network is marked "connected, no internet"
|
||||
(unvalidated) and the phone keeps **cellular as its default route**
|
||||
while staying associated — normal apps never notice the FIPS network
|
||||
exists. FIPS apps bind their sockets to the Wi-Fi network explicitly,
|
||||
so mesh traffic flows over Wi-Fi while everything else uses cellular.
|
||||
|
||||
## When to use
|
||||
|
||||
- Any FIPS router that should serve phones and laptops directly, not
|
||||
just peer with other routers.
|
||||
- You want clients to roam between FIPS routers with zero per-router
|
||||
or per-site configuration.
|
||||
|
||||
It is the complement of the 802.11s backhaul: the backhaul links
|
||||
routers (clients cannot join it), the access SSID admits clients.
|
||||
Both can share a radio, at an airtime cost (see constraints).
|
||||
|
||||
## Requirements
|
||||
|
||||
- OpenWrt 22.03+ with the FIPS package installed (fw4; dnsmasq and
|
||||
odhcpd are part of the default images).
|
||||
- Any radio — AP mode needs no special driver support.
|
||||
|
||||
## Step 1 — create the access point(s)
|
||||
|
||||
On **each** router, run the helper once per radio that should serve
|
||||
clients:
|
||||
|
||||
```sh
|
||||
fips-ap-setup radio0
|
||||
```
|
||||
|
||||
This creates an open AP with SSID `!FIPS` and client isolation, an
|
||||
isolated network with `10.21.<N>.1/24` and a static ULA `/64`, a
|
||||
DHCPv4 + RA dhcp config (dnsmasq leases, SLAAC, no DHCPv6), and a
|
||||
locked-down `fips_ap` firewall zone — then reloads the radio. Interfaces are named by radio index: `radio0` →
|
||||
`fips-ap0`, `radio1` → `fips-ap1`. Pass a second argument to use a
|
||||
different SSID — but the SSID, like the security type, must be
|
||||
identical on **all** routers or clients will treat them as separate
|
||||
networks and stop roaming.
|
||||
|
||||
On dual-band routers, run it for both radios so clients can pick
|
||||
either band:
|
||||
|
||||
```sh
|
||||
fips-ap-setup radio0
|
||||
fips-ap-setup radio1
|
||||
```
|
||||
|
||||
**Channels are free per router.** Unlike the mesh backhaul, there is
|
||||
no same-channel constraint — clients scan when they roam — so leave
|
||||
each router on whatever channel suits its RF environment.
|
||||
|
||||
Equivalent manual UCI (per radio), if you prefer to see what it does
|
||||
(`fdxx:...` stands for a `/64` out of the router's ULA prefix):
|
||||
|
||||
```sh
|
||||
uci batch <<'EOF'
|
||||
set wireless.fips_ap_radio0=wifi-iface
|
||||
set wireless.fips_ap_radio0.device='radio0'
|
||||
set wireless.fips_ap_radio0.mode='ap'
|
||||
set wireless.fips_ap_radio0.ssid='!FIPS'
|
||||
set wireless.fips_ap_radio0.encryption='none'
|
||||
set wireless.fips_ap_radio0.isolate='1'
|
||||
set wireless.fips_ap_radio0.ifname='fips-ap0'
|
||||
set wireless.fips_ap_radio0.network='fips_ap_radio0'
|
||||
set network.fips_ap_radio0=interface
|
||||
set network.fips_ap_radio0.proto='static'
|
||||
set network.fips_ap_radio0.ipaddr='10.21.0.1'
|
||||
set network.fips_ap_radio0.netmask='255.255.255.0'
|
||||
set network.fips_ap_radio0.ip6addr='fdxx:xxxx:xxxx:fa00::1/64'
|
||||
set dhcp.fips_ap_radio0=dhcp
|
||||
set dhcp.fips_ap_radio0.interface='fips_ap_radio0'
|
||||
set dhcp.fips_ap_radio0.ra='server'
|
||||
set dhcp.fips_ap_radio0.ra_default='2'
|
||||
set dhcp.fips_ap_radio0.dhcpv6='disabled'
|
||||
set dhcp.fips_ap_radio0.dhcpv4='server'
|
||||
set dhcp.fips_ap_radio0.start='10'
|
||||
set dhcp.fips_ap_radio0.limit='200'
|
||||
EOF
|
||||
uci commit
|
||||
wifi reload
|
||||
```
|
||||
|
||||
plus the `fips_ap` firewall zone (input/forward REJECT, no
|
||||
forwardings, ACCEPT rules for DHCPv4/UDP 67, ICMPv6, UDP 5353/2121,
|
||||
TCP 8443).
|
||||
|
||||
## Step 2 — check the FIPS transport binding
|
||||
|
||||
The `fips.yaml` shipped in the OpenWrt package carries one transport
|
||||
entry per access interface, but **commented out** — so a stock install
|
||||
that never runs this helper logs no per-boot "interface missing"
|
||||
warning. `fips-ap-setup` uncommented the matching `apN` entry in Step 1,
|
||||
and also enabled `node.rendezvous.lan` (the daemon's mDNS/DNS-SD
|
||||
rendezvous — phone FIPS apps cannot see raw-Ethernet beacons, so mDNS
|
||||
is how they find the daemon; the switch is daemon-wide and stays on if
|
||||
you later remove the AP). So there is normally nothing to do here. If
|
||||
you maintain your own config (or ran the manual UCI above instead of
|
||||
the helper), make sure both are present and uncommented:
|
||||
|
||||
```yaml
|
||||
node:
|
||||
rendezvous:
|
||||
lan:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
```yaml
|
||||
transports:
|
||||
ethernet:
|
||||
ap0:
|
||||
interface: "fips-ap0"
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
ap1:
|
||||
interface: "fips-ap1"
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
```
|
||||
|
||||
## Step 3 — restart the daemon (order matters)
|
||||
|
||||
```sh
|
||||
/etc/init.d/fips restart
|
||||
```
|
||||
|
||||
Restart fips **after** the AP interface is up. A transport whose
|
||||
interface is missing at startup is logged and skipped, not retried —
|
||||
so if the daemon comes up before the radio, the access transport
|
||||
stays dead until the next restart. (An interface that *vanishes and
|
||||
returns* after startup is recovered automatically; only the missing-
|
||||
at-startup case needs this ordering.)
|
||||
|
||||
## Verify
|
||||
|
||||
L2 and addressing first, with a phone or laptop connected to `!FIPS`:
|
||||
|
||||
```sh
|
||||
iw dev fips-ap0 station dump # one entry per associated client
|
||||
ip addr show dev fips-ap0 # 10.21.0.1/24 and the fd..::1/64
|
||||
cat /tmp/dhcp.leases # one lease per connected client
|
||||
```
|
||||
|
||||
No station entries means a radio problem; an association that drops
|
||||
after ~30 s usually means the client never got an address — check
|
||||
`logread | grep -e dnsmasq -e odhcpd` and that the
|
||||
`dhcp.fips_ap_radio0` section survived
|
||||
(`uci show dhcp | grep fips_ap`).
|
||||
|
||||
Then the FIPS layer on top, for a client running FIPS:
|
||||
|
||||
```sh
|
||||
logread | grep -i beacon # beacons flowing on the new transport
|
||||
fipsctl show peers # client authenticated and connected
|
||||
```
|
||||
|
||||
On the phone itself: the network shows "connected, no internet" and
|
||||
stays associated — that is the designed steady state, not an error.
|
||||
|
||||
## Constraints
|
||||
|
||||
- **SSID and security type must be uniform across ALL routers.**
|
||||
One router with a PSK (or OWE) under the same name splits the ESS
|
||||
into different saved networks and silently breaks roaming. Never
|
||||
"harden" a single router.
|
||||
- **Airtime is shared per radio.** An access AP and a mesh backhaul
|
||||
on the same radio share one channel. On dual/tri-band hardware,
|
||||
dedicate a band to the backhaul and serve clients on the others.
|
||||
- **Strangers can associate and peer — by design.** Open access means
|
||||
any nearby device can complete the Noise handshake and become a FIPS
|
||||
peer (up to the max-peers cap); the handshake authenticates each link,
|
||||
it does not restrict who joins. They reach only the FIPS overlay
|
||||
surface — the isolated zone gives no path to br-lan or the WAN. Do not
|
||||
add forwardings to the `fips_ap` zone: that would turn the open SSID
|
||||
into a hotspot and hand the isolation away.
|
||||
- **Roaming is client-driven.** Clients decide when to hop BSSIDs
|
||||
(standard ESS behavior); the IPv4 lease survives the hop (same
|
||||
subnet everywhere), the SLAAC address renumbers, and FIPS sessions
|
||||
ride through because the overlay identity is the anchor. Expect a
|
||||
brief L2 gap during the hop, as on any ESS without 802.11r.
|
||||
- **The `10.21.<N>.0/24` convention must hold everywhere.** Lease
|
||||
survival depends on every router serving the same subnet from the
|
||||
same radio index — the helper guarantees this; don't hand-pick
|
||||
per-router subnets. Two routers can lease the same address to two
|
||||
different clients; after a roam the conflict is caught (dnsmasq
|
||||
NAKs a renew for an address in use) and the client re-DHCPs. If a
|
||||
laptop is *also* wired to a LAN that really uses `10.21.<N>.0/24`,
|
||||
its routing table will conflict — a corner case worth knowing, not
|
||||
designing around: the zone forwards nowhere, so the FIPS side never
|
||||
reaches beyond the router either way.
|
||||
@@ -115,58 +115,6 @@ Tell the daemon to drop a peer link.
|
||||
| -------- | ----------- |
|
||||
| `peer` | npub (bech32) or hostname from `/etc/fips/hosts`. |
|
||||
|
||||
### `profile tick <on|off|status>`
|
||||
|
||||
> **Reading the output.** Step durations are wall clock measured across `await`
|
||||
> points, not CPU time: a step that waits on I/O accrues that wait, and other
|
||||
> tasks may run inside the span. That is the intended measure for head-of-line
|
||||
> delay, and it means a large step is not necessarily an expensive one.
|
||||
> `arm_starvation` is measured directly as the entry time minus the deadline
|
||||
> the interval scheduled that tick for. It is not derived from
|
||||
> `tick_entry_gap`, which carries no starvation signal on its own: under a
|
||||
> steady delay every gap is exactly one tick period.
|
||||
|
||||
Start, stop and inspect a capture of the rx-loop tick body. **Present
|
||||
only when both `fipsctl` and the daemon are built with
|
||||
`--features profiling`**; the feature is off by default, so a stock
|
||||
package does not carry this subcommand and a stock daemon reports
|
||||
`profile_tick_*` as an unknown command.
|
||||
|
||||
| Subcommand | Control-socket command | Description |
|
||||
| ---------- | ---------------------- | ----------- |
|
||||
| `profile tick on` | `profile_tick_on` | Create the capture file and start recording. Fails if a capture is already running (naming the active file) or if the directory cannot be written. |
|
||||
| `profile tick off` | `profile_tick_off` | Stop the capture. The writer is woken immediately, drains once more and is joined, so the command returns promptly. Succeeds, reporting nothing active, when no capture is running. |
|
||||
| `profile tick status` | `profile_tick_status` | Report `idle`, `running`, `stopped_by_cap` or `stopped_by_error`, plus the active path, bytes written, flush interval and byte cap. |
|
||||
|
||||
`profile tick on` options:
|
||||
|
||||
| Flag | Argument | Default | Description |
|
||||
| ---- | -------- | ------- | ----------- |
|
||||
| `--dir` | directory path | `/var/log/fips` | Where to write the capture. Created if absent. Use it to profile a non-root `cargo run`, or on a platform whose log root differs. |
|
||||
|
||||
One file is written per capture, named `profile-<UTC timestamp>.tsv`.
|
||||
It opens with a `#`-prefixed header block (node npub, build version,
|
||||
platform, configured tick period, flush interval, byte cap, start
|
||||
time), then a tab-separated column header, then one row per measured
|
||||
step per flush interval:
|
||||
|
||||
```text
|
||||
ts_unix kind domain name count max total unit
|
||||
```
|
||||
|
||||
`kind` is `step` for a timed span and `gauge` for a sampled scalar, so
|
||||
a gauge value never lands under a duration column; `unit` names the
|
||||
unit of `max` and `total` for that row. Every step present in the build
|
||||
gets a row every interval, including zero-count rows. Gauges cover
|
||||
ticks per interval, peer count, the wall gap between successive
|
||||
tick-arm entries, and the arm-starvation delay, which is measured
|
||||
against the deadline the tick was scheduled for rather than derived
|
||||
from the gap.
|
||||
|
||||
A capture stops itself on reaching 32 MB, appending a `#` line saying
|
||||
so; `profile tick status` then reports `stopped_by_cap` until the next
|
||||
`on` or `off` clears it.
|
||||
|
||||
## Exit Codes
|
||||
|
||||
| Code | Meaning |
|
||||
|
||||
@@ -277,7 +277,7 @@ Controls tree construction and parent selection.
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `node.bloom.update_debounce_ms` | u64 | `500` | Debounce interval for filter update propagation |
|
||||
| `node.bloom.max_inbound_fpr` | f64 | `0.20` | Antipoison cap: reject inbound `FilterAnnounce` frames whose advertised false-positive rate exceeds this value. Valid range `(0.0, 1.0)`. The default `0.20` corresponds to fill 0.7248 at k=5 (≈2,114 entries on the 1 KB filter); a saturated/poisoned filter is still ~100% FPR and rejected |
|
||||
| `node.bloom.max_inbound_fpr` | f64 | `0.10` | Antipoison cap: reject inbound `FilterAnnounce` frames whose advertised false-positive rate exceeds this value. Valid range `(0.0, 1.0)`. The default `0.10` corresponds to fill 0.631 at k=5 (≈1,630 entries on the 1 KB filter); a saturated/poisoned filter is still ~100% FPR and rejected |
|
||||
|
||||
Bloom filter size (1 KB), hash count (5), and size classes are protocol
|
||||
constants and not configurable.
|
||||
@@ -436,7 +436,7 @@ Requires `CAP_NET_RAW` or running as root. Linux only.
|
||||
| `mtu` | u16 | *(auto)* | Override MTU. Default: interface MTU minus 3 (for frame type + length prefix) |
|
||||
| `recv_buf_size` | usize | `2097152` | Socket receive buffer size in bytes (2 MB) |
|
||||
| `send_buf_size` | usize | `2097152` | Socket send buffer size in bytes (2 MB) |
|
||||
| `listen` | bool | `true` | Listen for neighbor beacons from other nodes |
|
||||
| `discovery` | bool | `true` | Listen for discovery beacons from other nodes |
|
||||
| `announce` | bool | `false` | Broadcast announcement beacons on the LAN |
|
||||
| `auto_connect` | bool | `false` | Auto-connect to discovered peers |
|
||||
| `accept_connections` | bool | `false` | Accept incoming connection attempts from discovered peers |
|
||||
@@ -450,7 +450,7 @@ transports:
|
||||
ethernet:
|
||||
lan:
|
||||
interface: "eth0"
|
||||
listen: true
|
||||
discovery: true
|
||||
announce: true
|
||||
backbone:
|
||||
interface: "eth1"
|
||||
@@ -458,7 +458,7 @@ transports:
|
||||
```
|
||||
|
||||
Each named instance operates independently with its own socket and
|
||||
neighbor state. The instance name is used in log messages and the
|
||||
discovery state. The instance name is used in log messages and the
|
||||
`name()` method on the Transport trait.
|
||||
|
||||
### TCP (`transports.tcp.*`)
|
||||
@@ -840,7 +840,7 @@ peers:
|
||||
### Mixed UDP + Ethernet Example
|
||||
|
||||
A node bridging internet peers (UDP) and a local Ethernet segment with
|
||||
neighbor beacons:
|
||||
beacon discovery:
|
||||
|
||||
```yaml
|
||||
node:
|
||||
@@ -856,7 +856,7 @@ transports:
|
||||
mtu: 1472
|
||||
ethernet:
|
||||
interface: "eth0"
|
||||
listen: true
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
@@ -944,7 +944,7 @@ node:
|
||||
flap_dampening_secs: 120 # extended hold-down on flap
|
||||
bloom:
|
||||
update_debounce_ms: 500
|
||||
max_inbound_fpr: 0.20 # antipoison cap on inbound FilterAnnounce FPR
|
||||
max_inbound_fpr: 0.10 # antipoison cap on inbound FilterAnnounce FPR
|
||||
session:
|
||||
default_ttl: 64
|
||||
pending_packets_per_dest: 16
|
||||
@@ -999,7 +999,7 @@ transports:
|
||||
# mtu: null # null = interface MTU - 3 (typically 1497)
|
||||
# recv_buf_size: 2097152 # 2 MB
|
||||
# send_buf_size: 2097152 # 2 MB
|
||||
# listen: true # listen for beacons
|
||||
# discovery: true # listen for beacons
|
||||
# announce: false # broadcast beacons
|
||||
# auto_connect: false # connect to discovered peers
|
||||
# accept_connections: false # accept inbound handshakes
|
||||
|
||||
@@ -159,21 +159,6 @@ not reproduced here to avoid duplicating the source.
|
||||
Both commands run on the daemon's main task and may block briefly
|
||||
while the node mutates its state.
|
||||
|
||||
#### Profiler toggle (`--features profiling` builds only)
|
||||
|
||||
| Command | Params | Behaviour |
|
||||
| ------- | ------ | --------- |
|
||||
| `profile_tick_on` | `dir` (optional directory path; default `/var/log/fips`) | Creates the capture file, publishes its path, and starts the writer thread. `data`: `state`, `path`, `interval_secs`, `byte_cap`. Errors if a capture is already running (naming the active file) or the directory is unwritable. |
|
||||
| `profile_tick_off` | — | Stops the capture, drains once more, joins the writer. `data`: `state`, `stopped`, `stopped_by_cap`, `stopped_by_error`, `path`, `bytes`. |
|
||||
| `profile_tick_status` | — | `data`: `state` (`idle` / `running` / `stopped_by_cap` / `stopped_by_error`), `path`, `bytes`, `byte_cap`, `interval_secs`. |
|
||||
|
||||
Unlike `connect` and `disconnect`, these three are served in the
|
||||
control accept task rather than on the daemon's main task. All of their
|
||||
state is process statics and none of them needs `&mut Node`, so
|
||||
routing them through the main loop would only make the toggle queue
|
||||
behind the tick body it exists to measure. They are absent from a
|
||||
default build, where the daemon answers them as unknown commands.
|
||||
|
||||
## Gateway Command Catalog
|
||||
|
||||
`fips-gateway` exposes a separate control socket with its own command
|
||||
|
||||
@@ -209,7 +209,7 @@ for the metadata-privacy model and the rejection of onion routing.
|
||||
| --------- | --------------- | ------------ | ------ |
|
||||
| UDP | None until `bind_addr` set | `0.0.0.0:2121` typical | Operator sets `transports.udp.bind_addr` |
|
||||
| TCP | None until `bind_addr` set | None — outbound-only without bind | Operator sets `transports.tcp.bind_addr` |
|
||||
| Ethernet | Listens on configured interface (raw `AF_PACKET`) | EtherType 0x2121 on selected interface | Per-flag `listen`, `announce`, `auto_connect`, `accept_connections` |
|
||||
| Ethernet | Listens on configured interface (raw `AF_PACKET`) | EtherType 0x2121 on selected interface | Per-flag `discovery`, `announce`, `auto_connect`, `accept_connections` |
|
||||
| Tor | None until `directory_service` configured | `127.0.0.1:8443` (loopback only) | Operator sets `transports.tor.directory_service` and configures `HiddenServiceDir` in `torrc` |
|
||||
| BLE | Off by default | n/a | Operator enables `transports.ble.*` |
|
||||
| Nostr discovery | Off by default | n/a (relay client, not a listener) | Operator sets `node.discovery.nostr.enabled: true` |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# FIPS v0.4.0
|
||||
|
||||
**Released**: 2026-06-27
|
||||
**Released**: 2026-06-21 (provisional)
|
||||
|
||||
v0.4.0 is the throughput-and-observability release on the v0.3.x wire
|
||||
format. It adds two new ways for nodes to find and reach each other (the
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
# FIPS v0.4.1
|
||||
|
||||
**Released**: 2026-07-19
|
||||
|
||||
v0.4.1 is a maintenance release on the v0.4.x line. It raises the default
|
||||
antipoison cap on inbound bloom filter announcements, removes a redundant
|
||||
spanning-tree metric counter, fixes two convergence and path-MTU bugs, and
|
||||
cuts per-packet CPU in the bloom and identity paths. There is no wire
|
||||
format change and no new feature surface.
|
||||
|
||||
v0.4.1 is wire-compatible with v0.4.0. Nodes can be upgraded one at a time
|
||||
with no coordinated restart, though one behavior change below is worth
|
||||
reading before you start a rolling upgrade.
|
||||
|
||||
## At a glance
|
||||
|
||||
- `node.bloom.max_inbound_fpr` default moves from `0.10` to `0.20`.
|
||||
- The `parent_switched` metric counter is gone. Use `parent_switches`.
|
||||
- Spanning tree no longer serves stale coordinates after a parent link is
|
||||
lost through peer removal.
|
||||
- Discovery no longer loosens a path MTU clamp it had correctly tightened.
|
||||
- Bloom probing and identity operations do measurably less work per call,
|
||||
with identical results.
|
||||
|
||||
## Behavior changes worth flagging
|
||||
|
||||
### The inbound filter FPR cap default doubles again
|
||||
|
||||
`node.bloom.max_inbound_fpr` goes from `0.10` to `0.20`. The cap rejects
|
||||
inbound `FilterAnnounce` frames whose advertised false positive rate
|
||||
exceeds it. On the fixed 1 KB, k=5 filter, `0.10` corresponds to a fill of
|
||||
0.631 and roughly 1,630 reachable entries, and the busiest nodes'
|
||||
aggregates had started reaching that ceiling as the mesh grew. `0.20`
|
||||
corresponds to a fill of 0.7248 and roughly 2,114 entries.
|
||||
|
||||
Be aware that this is the second time in two releases that this default
|
||||
has doubled, for the same reason both times. That is worth stating plainly
|
||||
rather than repeating the previous release's framing: raising the cap buys
|
||||
headroom, it does not fix anything. The real constraint is the fixed 1 KB
|
||||
filter size, which is a protocol constant. The structural remedy is the v2
|
||||
filter work, where filter capacity scales with the mesh instead of being
|
||||
pinned. This release is an interim step to keep legitimate aggregates from
|
||||
being rejected until that lands. It is not the start of a pattern of
|
||||
raising the cap once per release, and if you are sizing capacity planning
|
||||
around this number, plan against the v2 work rather than against a third
|
||||
raise.
|
||||
|
||||
The antipoison property the cap exists for is preserved. A saturated or
|
||||
deliberately poisoned filter still presents an FPR near 100% and is still
|
||||
rejected.
|
||||
|
||||
**This matters during a rolling upgrade.** A v0.4.1 node accepts a
|
||||
`FilterAnnounce` with a derived FPR between 0.10 and 0.20; a v0.4.0 node
|
||||
drops the same frame, and the drop is silent on the wire with no NACK. The
|
||||
cap also gates the mesh size estimator, which declines to produce a value
|
||||
when any contributing filter is over the cap. So while a mesh is partly
|
||||
upgraded, upgraded and not-yet-upgraded nodes can legitimately report
|
||||
different mesh sizes, or one can report a size while the other reports
|
||||
unknown. This resolves once every node is on v0.4.1. If you want to avoid
|
||||
the window entirely, set `node.bloom.max_inbound_fpr: 0.10` explicitly in
|
||||
your config before upgrading and remove it after the last node is done.
|
||||
|
||||
### The `parent_switched` counter is removed
|
||||
|
||||
`parent_switched` was incremented on the line immediately before
|
||||
`parent_switches` at every site and never independently, so the two
|
||||
counters always held the same value. `parent_switched` is now gone from
|
||||
the tree metrics, the control socket snapshot, and the `fipstop` tree
|
||||
view. `parent_switches` remains and is unchanged.
|
||||
|
||||
If you scrape the control socket, or have dashboards or alerts referencing
|
||||
`parent_switched`, point them at `parent_switches`. Anything still asking
|
||||
for `parent_switched` will find nothing rather than a zero.
|
||||
|
||||
## Notable bug fixes
|
||||
|
||||
### Stale coordinates after losing a parent through peer removal
|
||||
|
||||
When a node's parent link dropped via peer removal, the node correctly
|
||||
reparented or self-rooted, but skipped the coordinate cache invalidation
|
||||
that every other position-change path performs. Cached entries for
|
||||
downstream destinations kept the node's old coordinate prefix. This did
|
||||
not self-correct the way a stale cache entry normally would: routing
|
||||
access refreshes an entry's TTL, so an entry that was actively being
|
||||
routed through never expired, and was only fixed by an unrelated fresh
|
||||
insert. Both invalidation classes now run on this path, matching the
|
||||
loop-detection branch.
|
||||
|
||||
### Discovery could loosen a tightened path MTU clamp
|
||||
|
||||
An originator handling a `LookupResponse` overwrote its cached path MTU
|
||||
unconditionally. If a reactive `MtuExceeded` or `PathMtuNotification` had
|
||||
already taught it a tighter value, a later, looser discovery estimate
|
||||
would clobber that and re-loosen the clamp, risking a return to dropped
|
||||
oversized packets. The cached and received values are now compared and the
|
||||
tighter one is kept.
|
||||
|
||||
## Upgrade notes
|
||||
|
||||
This is a drop-in upgrade from v0.4.0 with no wire format change, no
|
||||
config migration, and no coordinated restart. Upgrade nodes in whatever
|
||||
order you like.
|
||||
|
||||
Two things to do rather than assume:
|
||||
|
||||
1. If you monitor `parent_switched`, move to `parent_switches` before
|
||||
upgrading, or your dashboards will go blank rather than error.
|
||||
2. During the rolling window, expect upgraded and not-yet-upgraded nodes
|
||||
to potentially disagree about mesh size, per the FPR cap section above.
|
||||
This is expected and self-resolves. Do not chase it as a bug unless it
|
||||
persists after every node reports `0.4.1`.
|
||||
|
||||
If you have pinned `node.bloom.max_inbound_fpr` explicitly in your config,
|
||||
your setting is honored and nothing changes for you. The change only
|
||||
affects nodes taking the default.
|
||||
|
||||
Downgrading to v0.4.0 is supported and needs no special handling.
|
||||
|
||||
## Getting v0.4.1
|
||||
|
||||
- **Linux x86_64 / aarch64**: `.deb` and tarball at the
|
||||
[v0.4.1 release page](https://github.com/jmcorgan/fips/releases/tag/v0.4.1).
|
||||
- **Arch Linux**: `fips` from the AUR.
|
||||
- **macOS**: `.pkg` at the v0.4.1 release page.
|
||||
- **Windows**: ZIP at the v0.4.1 release page.
|
||||
- **OpenWrt**: `.ipk` (OpenWrt 24.x and earlier) or `.apk` (OpenWrt 25+)
|
||||
at the v0.4.1 release page.
|
||||
- **From source**: `cargo build --release` from a checkout of the v0.4.1
|
||||
tag (Rust 1.94.1 per `rust-toolchain.toml`; `libclang-dev` is a
|
||||
required Linux build prerequisite).
|
||||
- **Nix / NixOS**: `nix build .#fips` from a checkout of the v0.4.1 tag
|
||||
builds the binaries from source with the pinned toolchain and no manual
|
||||
prerequisites (see the Nix section of `packaging/README.md`).
|
||||
|
||||
The full per-commit changelog lives in
|
||||
[`CHANGELOG.md`](../../CHANGELOG.md). Issues and discussion at
|
||||
[github.com/jmcorgan/fips](https://github.com/jmcorgan/fips).
|
||||
|
||||
## Contributors
|
||||
|
||||
Thanks to everyone who contributed code, packaging work, bug reports, or
|
||||
reviews to this release.
|
||||
|
||||
- [@jcorgan](https://github.com/jmcorgan): release shepherd, spanning-tree
|
||||
and discovery fixes, bloom and identity performance work, antipoison cap
|
||||
change, and testing.
|
||||
@@ -71,7 +71,7 @@ supplies the rest:
|
||||
- **Addressing**: the `fips0` adapter takes an `fd97:...` ULA
|
||||
derived from the npub. No DHCP. No SLAAC. The address is
|
||||
cryptographically tied to the identity.
|
||||
- **Neighbor detection**: each daemon broadcasts a small beacon on the
|
||||
- **Discovery**: each daemon broadcasts a small beacon on the
|
||||
link advertising its npub; the other daemon's listener picks
|
||||
it up and dials in over the same link.
|
||||
- **Routing**: the FIPS mesh layer builds its own spanning tree
|
||||
@@ -177,7 +177,7 @@ different interface names — that is normal.
|
||||
|
||||
Edit `/etc/fips/fips.yaml` on **both** nodes. Under
|
||||
`transports:`, add an `ethernet:` block. The key settings are
|
||||
the four neighbor flags — both nodes must opt in to all four,
|
||||
the four discovery flags — both nodes must opt in to all four,
|
||||
and they default to off:
|
||||
|
||||
```yaml
|
||||
@@ -185,7 +185,7 @@ transports:
|
||||
ethernet:
|
||||
interface: "<eth>" # the name from Step 1
|
||||
announce: true # broadcast our beacon on the link
|
||||
listen: true # listen for beacons (default; shown for clarity)
|
||||
discovery: true # listen for beacons (default; shown for clarity)
|
||||
auto_connect: true # dial peers we discover
|
||||
accept_connections: true # accept dial-ins from peers we discover
|
||||
```
|
||||
@@ -194,7 +194,7 @@ Each flag does one thing:
|
||||
|
||||
- `announce: true` — emit a small beacon every
|
||||
`beacon_interval_secs` (default 30s) carrying our npub.
|
||||
- `listen: true` — listen for incoming beacons; populate a
|
||||
- `discovery: true` — listen for incoming beacons; populate a
|
||||
candidate-peer list keyed by source MAC and observed npub.
|
||||
- `auto_connect: true` — when we see a beacon from an npub
|
||||
we have not yet peered with, initiate the outbound Noise
|
||||
@@ -218,7 +218,7 @@ is "all four flags on both ends."
|
||||
> lan:
|
||||
> interface: "eth0"
|
||||
> announce: true
|
||||
> listen: true
|
||||
> discovery: true
|
||||
> auto_connect: true
|
||||
> accept_connections: true
|
||||
> dongle:
|
||||
@@ -227,7 +227,7 @@ is "all four flags on both ends."
|
||||
> # ...
|
||||
> ```
|
||||
>
|
||||
> Each named instance runs its own socket and neighbor state.
|
||||
> Each named instance runs its own socket and discovery state.
|
||||
> A single ground-up link only needs the flat form shown
|
||||
> first; named instances become useful when the same node
|
||||
> bridges multiple physical segments.
|
||||
@@ -390,7 +390,7 @@ What you do need on the AP side:
|
||||
networks and "secure" enterprise APs ship with it on.
|
||||
When client isolation is on, the AP refuses to forward
|
||||
station-to-station frames — the broadcast beacons never
|
||||
arrive at the other node, and neighbor detection fails silently.
|
||||
arrive at the other node, and discovery fails silently.
|
||||
If beacons aren't crossing, this is the first thing to
|
||||
check.
|
||||
|
||||
@@ -401,7 +401,7 @@ adapter name.
|
||||
### Bluetooth LE (experimental but works)
|
||||
|
||||
BLE is a separate transport (`transports.ble.*`) with its own
|
||||
neighbor-detection model — L2CAP advertisements rather than raw L2
|
||||
discovery model — L2CAP advertisements rather than raw L2
|
||||
broadcasts. The shape of the tutorial is the same (advertise +
|
||||
scan + auto-connect + accept), but the prerequisites are
|
||||
different: BlueZ, `bluetoothd`, an HCI adapter, and the
|
||||
@@ -424,7 +424,7 @@ Windows builds skip it.
|
||||
a radio link), `CAP_NET_RAW`, and a few config flags on each
|
||||
end are sufficient. The mesh supplies its own identity,
|
||||
addressing, discovery, and routing.
|
||||
- **Neighbor detection is a four-flag opt-in.** `announce`, `listen`,
|
||||
- **Discovery is a four-flag opt-in.** `announce`, `discovery`,
|
||||
`auto_connect`, and `accept_connections` each control one
|
||||
thing; both ends must agree before a link will form.
|
||||
- **The two modes coexist.** Overlay peers and ground-up peers
|
||||
|
||||
@@ -178,7 +178,7 @@ The resolution itself happens at debug-log level, so you will
|
||||
not see it in the default-level journal. The user-facing way to
|
||||
confirm everything worked is `fipsctl show peers` in the next
|
||||
step. (To watch the resolution in the journal, run the daemon
|
||||
manually with `RUST_LOG=fips::nostr=debug`; not
|
||||
manually with `RUST_LOG=fips::discovery::nostr=debug`; not
|
||||
necessary for this tutorial.)
|
||||
|
||||
## Step 5: Verify the resolved endpoint
|
||||
|
||||
@@ -10,21 +10,12 @@ node:
|
||||
#
|
||||
# Or set an explicit key (overrides persistent):
|
||||
# nsec: "nsec1..."
|
||||
# Mesh-lookup protocol (node.lookup.*): the overlay coordinate-lookup engine
|
||||
# (mesh address -> coordinates). Defaults shown; uncomment to override.
|
||||
# lookup:
|
||||
# ttl: 64
|
||||
# attempt_timeouts_secs: [1, 2, 4, 8]
|
||||
# recent_expiry_secs: 10
|
||||
# backoff_base_secs: 0
|
||||
# backoff_max_secs: 0
|
||||
# forward_min_interval_secs: 2
|
||||
rendezvous:
|
||||
# Optional Nostr-mediated overlay endpoint rendezvous.
|
||||
discovery:
|
||||
# Optional Nostr-mediated overlay endpoint discovery.
|
||||
# nostr:
|
||||
# enabled: true
|
||||
# policy: configured_only # disabled | configured_only | open
|
||||
# open_discovery_max_pending: 64 # caps queued open-rendezvous retries
|
||||
# open_discovery_max_pending: 64 # caps queued open-discovery retries
|
||||
# app: "fips-overlay-v1"
|
||||
# advertise: true
|
||||
# advert_relays:
|
||||
@@ -43,17 +34,17 @@ node:
|
||||
# - "stun:stun.cloudflare.com:3478"
|
||||
# - "stun:global.stun.twilio.com:3478"
|
||||
#
|
||||
# Optional mDNS-based LAN rendezvous for sub-second same-LAN pairing.
|
||||
# Optional mDNS-based LAN discovery for sub-second same-LAN pairing.
|
||||
# Opt-in (default false): default-off avoids a per-LAN identity
|
||||
# broadcast on nodes that have deliberately disabled other rendezvous
|
||||
# broadcast on nodes that have deliberately disabled other discovery
|
||||
# channels, and avoids any multicast surprise on upgrade. Requires an
|
||||
# operational UDP transport (the advertised port is the one peers dial).
|
||||
# lan:
|
||||
# enabled: false
|
||||
# # Optional application/network scope carried in the LAN-only TXT
|
||||
# # record. Browsers that set a scope ignore adverts for other scopes.
|
||||
# # Kept separate from the Nostr rendezvous `app` tag so relay-visible
|
||||
# # adverts can stay generic while LAN rendezvous stays per-private-network.
|
||||
# # Kept separate from the Nostr discovery `app` tag so relay-visible
|
||||
# # adverts can stay generic while LAN discovery stays per-private-network.
|
||||
# # scope: "lab-floor-3"
|
||||
# # Advanced: overrides the mDNS service type. Leave unset in normal
|
||||
# # use — only needed to run multiple isolated services on one
|
||||
@@ -98,7 +89,7 @@ transports:
|
||||
# Ethernet transport — uncomment and set your interface name.
|
||||
# ethernet:
|
||||
# interface: "eth0"
|
||||
# listen: true
|
||||
# discovery: true
|
||||
# announce: true
|
||||
# auto_connect: true
|
||||
# accept_connections: true
|
||||
@@ -156,5 +147,5 @@ peers: []
|
||||
# - transport: udp
|
||||
# addr: "test-us01.fips.network:2121" # IP or hostname (e.g., "peer.example.com:2121")
|
||||
# - transport: udp
|
||||
# addr: "nat" # Use node.rendezvous.nostr for Nostr/STUN hole punching
|
||||
# addr: "nat" # Use node.discovery.nostr for Nostr/STUN hole punching
|
||||
# connect_policy: auto_connect
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# Build a .deb package for FIPS using cargo-deb.
|
||||
#
|
||||
# Usage: ./build-deb.sh [--target <triple>] [--version <version>] [--no-build]
|
||||
# [--features <list>]
|
||||
#
|
||||
# Prerequisites: cargo-deb (install with: cargo install cargo-deb)
|
||||
# Output: deploy/fips_<version>_<arch>.deb
|
||||
@@ -20,9 +19,6 @@ Options:
|
||||
--target <triple> Rust target triple to build/package
|
||||
--version <version> Override Debian package version
|
||||
--no-build Package existing binaries without running cargo build
|
||||
--features <list> Cargo features to build with (comma-separated). Marks the
|
||||
auto-derived Version so the package is distinguishable
|
||||
from a default build of the same commit.
|
||||
-h, --help Show this help
|
||||
EOF
|
||||
}
|
||||
@@ -30,7 +26,6 @@ EOF
|
||||
TARGET_TRIPLE=""
|
||||
VERSION_OVERRIDE=""
|
||||
NO_BUILD=0
|
||||
FEATURES=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -46,10 +41,6 @@ while [[ $# -gt 0 ]]; do
|
||||
NO_BUILD=1
|
||||
shift
|
||||
;;
|
||||
--features)
|
||||
FEATURES="${2:?missing value for --features}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
@@ -62,16 +53,6 @@ while [[ $# -gt 0 ]]; do
|
||||
esac
|
||||
done
|
||||
|
||||
# A feature build that skips the build step would stamp a feature-marked Version
|
||||
# onto whatever binaries already sit in target/, which is the one outcome the
|
||||
# marking exists to prevent. Refuse rather than emit a package that misdescribes
|
||||
# itself.
|
||||
if [[ -n "${FEATURES}" && "${NO_BUILD}" -eq 1 ]]; then
|
||||
echo "--features cannot be combined with --no-build: the features would not" >&2
|
||||
echo "reach the binaries, but the Version would claim they had." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
# Ensure cargo-deb is available
|
||||
@@ -100,33 +81,13 @@ if [[ -z "${VERSION_OVERRIDE}" ]]; then
|
||||
if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then
|
||||
DIRTY_SUFFIX=".dirty"
|
||||
fi
|
||||
# A feature build of a given commit is a different package from the
|
||||
# default build of that same commit, but nothing else in this version
|
||||
# says so: the crate version, the date and the sha are all identical.
|
||||
# Without a marker the two are byte-identical versions, so installing
|
||||
# one over the other is an apt no-op (the very failure the per-commit
|
||||
# version above exists to prevent) and the node offers no way to tell
|
||||
# which one it is running. Underscores and commas are not legal in a
|
||||
# Debian version, so the feature list is folded to dots.
|
||||
FEATURE_SUFFIX=""
|
||||
if [[ -n "${FEATURES}" ]]; then
|
||||
FEATURE_SUFFIX="+$(printf '%s' "${FEATURES}" | tr -c 'a-zA-Z0-9.' '.')"
|
||||
fi
|
||||
# Debian Version: <upstream>~dev+git<YYYYMMDD>.<sha>[.dirty][+<features>]-1
|
||||
# Debian Version: <upstream>~dev+git<YYYYMMDD>.<sha>[.dirty]-1
|
||||
# The "~" makes every dev build sort BEFORE the eventual tagged
|
||||
# release; the date+sha makes consecutive dev builds compare as
|
||||
# different versions; the trailing "-1" is the Debian revision.
|
||||
# The feature suffix sorts ABOVE the unsuffixed build, so installing a
|
||||
# feature build is an upgrade and reverting to the default build is a
|
||||
# downgrade — which apt refuses without being told to, and `dpkg -i`
|
||||
# performs. Revert with `dpkg -i`, not `apt install`.
|
||||
VERSION_OVERRIDE="${BASE_VERSION}~dev+git${GIT_DATE}.${GIT_SHA}${DIRTY_SUFFIX}${FEATURE_SUFFIX}-1"
|
||||
VERSION_OVERRIDE="${BASE_VERSION}~dev+git${GIT_DATE}.${GIT_SHA}${DIRTY_SUFFIX}-1"
|
||||
echo "Auto-derived dev Version: ${VERSION_OVERRIDE}"
|
||||
fi
|
||||
elif [[ -n "${FEATURES}" ]]; then
|
||||
echo "Warning: --version was given with --features, so the Version carries no" >&2
|
||||
echo "feature marker and this package is indistinguishable from a default" >&2
|
||||
echo "build of the same commit. Mark it yourself if that matters." >&2
|
||||
fi
|
||||
|
||||
# Build the .deb package
|
||||
@@ -144,9 +105,6 @@ fi
|
||||
if [[ "${NO_BUILD}" -eq 1 ]]; then
|
||||
cargo_args+=(--no-build)
|
||||
fi
|
||||
if [[ -n "${FEATURES}" ]]; then
|
||||
cargo_args+=(--features "${FEATURES}")
|
||||
fi
|
||||
cargo "${cargo_args[@]}"
|
||||
|
||||
# Move output to deploy/
|
||||
|
||||
@@ -19,11 +19,6 @@ RestartSec=5
|
||||
RuntimeDirectory=fips
|
||||
RuntimeDirectoryMode=0750
|
||||
|
||||
# Log directory (/var/log/fips/), where the built-in tick-body profiler writes
|
||||
# its capture files. Declared so systemd creates it on start and removes it on
|
||||
# purge; the daemon runs as root and already has access without it.
|
||||
LogsDirectory=fips
|
||||
|
||||
# Security hardening (daemon runs as root for TUN and raw sockets)
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
|
||||
@@ -182,8 +182,6 @@ install -m 0755 "$RELEASE_DIR/fips" "$STAGE_DIR/usr/bin/fips"
|
||||
install -m 0755 "$RELEASE_DIR/fipsctl" "$STAGE_DIR/usr/bin/fipsctl"
|
||||
install -m 0755 "$RELEASE_DIR/fipstop" "$STAGE_DIR/usr/bin/fipstop"
|
||||
install -m 0755 "$RELEASE_DIR/fips-gateway" "$STAGE_DIR/usr/bin/fips-gateway"
|
||||
install -m 0755 "$FILES_DIR/usr/bin/fips-mesh-setup" "$STAGE_DIR/usr/bin/fips-mesh-setup"
|
||||
install -m 0755 "$FILES_DIR/usr/bin/fips-ap-setup" "$STAGE_DIR/usr/bin/fips-ap-setup"
|
||||
|
||||
install -d "$STAGE_DIR/etc/init.d"
|
||||
install -m 0755 "$FILES_DIR/etc/init.d/fips" "$STAGE_DIR/etc/init.d/fips"
|
||||
|
||||
@@ -96,12 +96,6 @@ define Package/fips/install
|
||||
$(INSTALL_BIN) $(RUST_RELEASE_DIR)/fipstop $(1)/usr/bin/fipstop
|
||||
$(INSTALL_BIN) $(RUST_RELEASE_DIR)/fips-gateway $(1)/usr/bin/fips-gateway
|
||||
|
||||
# 802.11s mesh backhaul setup helper
|
||||
$(INSTALL_BIN) $(CURDIR)/files/usr/bin/fips-mesh-setup $(1)/usr/bin/fips-mesh-setup
|
||||
|
||||
# Open "FIPS" access SSID setup helper
|
||||
$(INSTALL_BIN) $(CURDIR)/files/usr/bin/fips-ap-setup $(1)/usr/bin/fips-ap-setup
|
||||
|
||||
# procd init script
|
||||
$(INSTALL_DIR) $(1)/etc/init.d
|
||||
$(INSTALL_BIN) $(CURDIR)/files/etc/init.d/fips $(1)/etc/init.d/fips
|
||||
|
||||
@@ -14,7 +14,6 @@ For ad-hoc deployment without the build system, see
|
||||
| `/usr/bin/fipsctl` | CLI control tool (`fipsctl show peers`, `fipsctl show links`, …) |
|
||||
| `/usr/bin/fipstop` | Live TUI dashboard |
|
||||
| `/usr/bin/fips-gateway` | Outbound LAN gateway service (not started by default) |
|
||||
| `/usr/bin/fips-mesh-setup` | Opt-in helper — creates an open 802.11s mesh interface for router↔router backhaul |
|
||||
| `/etc/init.d/fips` | procd service for the daemon (auto-start, crash respawn) |
|
||||
| `/etc/init.d/fips-gateway` | procd service for the gateway (disabled by default) |
|
||||
| `/etc/fips/fips.yaml` | Node configuration (edit before first start) |
|
||||
|
||||
@@ -161,8 +161,6 @@ install -m 0755 "$RELEASE_DIR/fips" "$DATA_DIR/usr/bin/fips"
|
||||
install -m 0755 "$RELEASE_DIR/fipsctl" "$DATA_DIR/usr/bin/fipsctl"
|
||||
install -m 0755 "$RELEASE_DIR/fipstop" "$DATA_DIR/usr/bin/fipstop"
|
||||
install -m 0755 "$RELEASE_DIR/fips-gateway" "$DATA_DIR/usr/bin/fips-gateway"
|
||||
install -m 0755 "$FILES_DIR/usr/bin/fips-mesh-setup" "$DATA_DIR/usr/bin/fips-mesh-setup"
|
||||
install -m 0755 "$FILES_DIR/usr/bin/fips-ap-setup" "$DATA_DIR/usr/bin/fips-ap-setup"
|
||||
|
||||
install -d "$DATA_DIR/etc/init.d"
|
||||
install -m 0755 "$FILES_DIR/etc/init.d/fips" "$DATA_DIR/etc/init.d/fips"
|
||||
|
||||
@@ -10,21 +10,12 @@ node:
|
||||
#
|
||||
# Or set an explicit key (overrides persistent):
|
||||
# nsec: "nsec1..."
|
||||
# Mesh-lookup protocol (node.lookup.*): the overlay coordinate-lookup engine
|
||||
# (mesh address -> coordinates). Defaults shown; uncomment to override.
|
||||
# lookup:
|
||||
# ttl: 64
|
||||
# attempt_timeouts_secs: [1, 2, 4, 8]
|
||||
# recent_expiry_secs: 10
|
||||
# backoff_base_secs: 0
|
||||
# backoff_max_secs: 0
|
||||
# forward_min_interval_secs: 2
|
||||
rendezvous:
|
||||
# Optional Nostr-mediated overlay endpoint rendezvous.
|
||||
discovery:
|
||||
# Optional Nostr-mediated overlay endpoint discovery.
|
||||
# nostr:
|
||||
# enabled: true
|
||||
# policy: configured_only # disabled | configured_only | open
|
||||
# open_discovery_max_pending: 64 # caps queued open-rendezvous retries
|
||||
# open_discovery_max_pending: 64 # caps queued open-discovery retries
|
||||
# app: "fips-overlay-v1"
|
||||
# advertise: true
|
||||
# advert_relays:
|
||||
@@ -43,14 +34,6 @@ node:
|
||||
# - "stun:stun.cloudflare.com:3478"
|
||||
# - "stun:global.stun.twilio.com:3478"
|
||||
|
||||
# mDNS/DNS-SD peer rendezvous on the local link. Ships commented (the
|
||||
# daemon default is off); 'fips-ap-setup' uncomments it when creating
|
||||
# the access SSID — phone FIPS apps cannot see raw-Ethernet beacons,
|
||||
# so mDNS is how they find this router's daemon. Daemon-wide switch,
|
||||
# left enabled on 'fips-ap-setup remove'.
|
||||
# lan:
|
||||
# enabled: true
|
||||
|
||||
tun:
|
||||
enabled: true
|
||||
name: fips0
|
||||
@@ -72,11 +55,7 @@ dns:
|
||||
|
||||
transports:
|
||||
udp:
|
||||
# Dual-stack wildcard, not "0.0.0.0": access-SSID clients (phones) learn
|
||||
# this node's addresses from the mDNS advert and prefer the IPv6
|
||||
# link-local — a v4-only bind silently drops their Noise msg1.
|
||||
# OpenWrt is Linux (bindv6only=0), so "[::]" accepts v4 too.
|
||||
bind_addr: "[::]:2121"
|
||||
bind_addr: "0.0.0.0:2121"
|
||||
# advertise_on_nostr: true
|
||||
# public: false # false => advertise udp:nat; true => advertise bound host:port
|
||||
# accept_connections: true # default; refuse inbound msg1 when false
|
||||
@@ -95,73 +74,23 @@ transports:
|
||||
ethernet:
|
||||
wan:
|
||||
interface: "eth0"
|
||||
listen: true
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
wwan:
|
||||
interface: "phy0-sta0"
|
||||
listen: true
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
lan:
|
||||
interface: "br-lan"
|
||||
listen: true
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
|
||||
# 802.11s mesh backhaul between FIPS routers. These entries ship
|
||||
# commented out so a stock install that never creates fips-mesh*
|
||||
# logs no per-boot "interface missing" bind warning. Running
|
||||
# 'fips-mesh-setup <radio>' creates the interface AND uncomments the
|
||||
# matching block here (once per radio; radio0 -> fips-mesh0, radio1 ->
|
||||
# fips-mesh1); 'fips-mesh-setup remove' re-comments it. Restart fips
|
||||
# after — a transport whose interface is missing at startup is skipped,
|
||||
# not retried. Dual-band routers can mesh on both bands at once —
|
||||
# failover, not multipath: FIPS keeps one active link per peer, the
|
||||
# other band stands by. The mesh runs OPEN (no SAE) with 802.11s
|
||||
# forwarding off: FIPS's Noise handshake is the encryption and
|
||||
# authentication, and FIPS is the routing layer. See
|
||||
# docs/how-to/set-up-80211s-mesh-backhaul.md.
|
||||
# mesh0:
|
||||
# interface: "fips-mesh0"
|
||||
# discovery: true
|
||||
# announce: true
|
||||
# auto_connect: true
|
||||
# accept_connections: true
|
||||
# mesh1:
|
||||
# interface: "fips-mesh1"
|
||||
# discovery: true
|
||||
# announce: true
|
||||
# auto_connect: true
|
||||
# accept_connections: true
|
||||
|
||||
# Open "!FIPS" access SSID for phones and laptops running FIPS. These
|
||||
# entries ship commented out so a stock install that never creates
|
||||
# fips-ap* logs no per-boot "interface missing" bind warning. Running
|
||||
# 'fips-ap-setup <radio>' creates the interface AND uncomments the
|
||||
# matching block here (once per radio; radio0 -> fips-ap0, radio1 ->
|
||||
# fips-ap1); 'fips-ap-setup remove' re-comments it. Restart fips after
|
||||
# — a transport whose interface is missing at startup is skipped, not
|
||||
# retried. The SSID is OPEN and isolated on purpose: FIPS's Noise
|
||||
# handshake is the only security layer, and associated clients reach
|
||||
# nothing but the FIPS handshake surface. See
|
||||
# docs/how-to/set-up-open-access-ssid.md.
|
||||
# ap0:
|
||||
# interface: "fips-ap0"
|
||||
# discovery: true
|
||||
# announce: true
|
||||
# auto_connect: true
|
||||
# accept_connections: true
|
||||
# ap1:
|
||||
# interface: "fips-ap1"
|
||||
# discovery: true
|
||||
# announce: true
|
||||
# auto_connect: true
|
||||
# accept_connections: true
|
||||
|
||||
# Bluetooth Low Energy transport — requires BlueZ and the 'ble' feature.
|
||||
# ble:
|
||||
# adapter: "hci0"
|
||||
@@ -192,5 +121,5 @@ peers: []
|
||||
# - transport: udp
|
||||
# addr: "test-us01.fips.network:2121" # IP or hostname (e.g., "peer.example.com:2121")
|
||||
# - transport: udp
|
||||
# addr: "nat" # Use node.rendezvous.nostr for Nostr/STUN hole punching
|
||||
# addr: "nat" # Use node.discovery.nostr for Nostr/STUN hole punching
|
||||
# connect_policy: auto_connect
|
||||
|
||||
@@ -1,412 +0,0 @@
|
||||
#!/bin/sh
|
||||
# fips-ap-setup — configure the open "FIPS" access SSID for phones/laptops.
|
||||
#
|
||||
# Usage:
|
||||
# fips-ap-setup <radio> [ssid] e.g. fips-ap-setup radio0
|
||||
# fips-ap-setup remove [radio] no radio: remove all instances
|
||||
#
|
||||
# Creates an open AP on the given radio so client devices running FIPS can
|
||||
# reach the router. Every FIPS router broadcasts the SAME SSID ("!FIPS" by
|
||||
# default — the leading '!' sorts it to the top of alphabetically ordered
|
||||
# network pickers): same SSID + unique BSSIDs is one standard ESS, so a
|
||||
# phone saves the network once and roams between all FIPS routers natively.
|
||||
#
|
||||
# - encryption 'none' — the AP is OPEN on purpose. FIPS's Noise IK
|
||||
# handshake authenticates and encrypts everything above the radio, and
|
||||
# the security type must be uniform across ALL routers anyway: clients
|
||||
# key a saved network on SSID + security type, so one router with a PSK
|
||||
# splits the ESS into a different saved network. A stranger can
|
||||
# associate AND form a FIPS peer link — that is the point of open
|
||||
# access. The Noise handshake authenticates each link (no
|
||||
# impersonation of another identity, no MITM); it does NOT gate who
|
||||
# may peer. Admission is open up to the daemon's max-peers cap; the
|
||||
# firewall zone below is what confines every client to the FIPS
|
||||
# overlay (no path to br-lan or the WAN).
|
||||
# - DHCPv4 + RA IPv6 — dnsmasq serves DHCPv4 from a FIXED subnet,
|
||||
# 10.21.<N>.0/24 (echoes FIPS port 2121), identical on every router:
|
||||
# a roaming phone keeps its lease across routers, and dnsmasq's
|
||||
# authoritative mode (the OpenWrt default) ACKs the renew a foreign
|
||||
# router never issued. odhcpd additionally announces a ULA prefix in
|
||||
# router advertisements (stateless SLAAC); DHCPv6 stays off. FIPS
|
||||
# itself only needs link-local + mDNS, but client provisioning checks
|
||||
# (Android disconnects without an RA or a DHCP offer) and plain
|
||||
# laptops both want a real address. The network provides no internet,
|
||||
# so phones mark it unvalidated and keep cellular as the default
|
||||
# route while staying associated.
|
||||
# - ISOLATED — own network and firewall zone: no path to
|
||||
# br-lan, no forwarding to the WAN, and AP client isolation on.
|
||||
# Associated clients reach only the FIPS handshake surface.
|
||||
#
|
||||
# Interfaces are named per radio index (radio0 -> fips-ap0, radio1 ->
|
||||
# fips-ap1). Unlike the 802.11s backhaul there is NO same-channel
|
||||
# constraint — clients scan when they roam, so every router picks its
|
||||
# access channels freely.
|
||||
#
|
||||
# The shipped /etc/fips/fips.yaml carries 'ap0' and 'ap1' entries under
|
||||
# 'transports.ethernet' bound to these names, but commented out — a stock
|
||||
# install that never creates fips-ap* then logs no bind warning. This
|
||||
# helper uncomments the matching entry when it creates an interface and
|
||||
# re-comments it on remove, so the daemon binds the transport without a
|
||||
# manual config edit. It also uncomments the node.rendezvous.lan block
|
||||
# (mDNS/DNS-SD — how phone FIPS apps discover the daemon); that switch is
|
||||
# daemon-wide and stays on at remove. After an interface is up, restart
|
||||
# fips.
|
||||
# See docs/how-to/set-up-open-access-ssid.md for the full guide.
|
||||
|
||||
DEFAULT_SSID="!FIPS"
|
||||
CONFIG="/etc/fips/fips.yaml"
|
||||
|
||||
# Replace $CONFIG with the rewritten $CONFIG.tmp. Force mode 0600 first: the
|
||||
# package installs fips.yaml 0600 (it may hold an inline 'nsec' private key),
|
||||
# and a fresh tmp file would otherwise land world-readable after the move.
|
||||
ap_config_write() {
|
||||
chmod 600 "$CONFIG.tmp" && mv "$CONFIG.tmp" "$CONFIG"
|
||||
}
|
||||
|
||||
# Uncomment the 'ap<idx>' transports.ethernet block in $CONFIG (created by
|
||||
# 'fips-ap-setup'). Reversible with ap_config_disable. Returns:
|
||||
# 0 enabled (or already active) 1 no config file 2 no such block
|
||||
ap_config_enable() {
|
||||
idx="$1"
|
||||
[ -f "$CONFIG" ] || return 1
|
||||
grep -q "^ ap$idx:" "$CONFIG" && return 0
|
||||
grep -q "^ # ap$idx:" "$CONFIG" || return 2
|
||||
awk -v idx="$idx" '
|
||||
$0 ~ ("^ # ap" idx ":[ \t]*$") { blk = 1; sub(/^ # /, " "); print; next }
|
||||
blk && /^ # / { sub(/^ # /, " "); print; next }
|
||||
{ blk = 0; print }
|
||||
' "$CONFIG" > "$CONFIG.tmp" && ap_config_write
|
||||
}
|
||||
|
||||
# Uncomment the 'lan' block under node.rendezvous in $CONFIG — the daemon's
|
||||
# mDNS/DNS-SD responder+browser. Phone FIPS apps cannot open raw-Ethernet
|
||||
# sockets, so mDNS is how they find this router's daemon. The match is
|
||||
# scoped to node.rendezvous: transports.ethernet also has a 'lan' entry at
|
||||
# the same indent. Daemon-wide switch — enabled here, deliberately NOT
|
||||
# re-commented on remove (other transports use it once on). Returns:
|
||||
# 0 enabled (or already active) 1 no config file 2 no such block
|
||||
lan_rendezvous_enable() {
|
||||
[ -f "$CONFIG" ] || return 1
|
||||
state="$(awk '
|
||||
/^[A-Za-z_]/ { top = $1 }
|
||||
top == "node:" && /^ [A-Za-z_]/ { sec = $1 }
|
||||
top == "node:" && sec == "rendezvous:" && /^ lan:[ \t]*$/ { print "active"; exit }
|
||||
top == "node:" && sec == "rendezvous:" && /^ # lan:[ \t]*$/ { print "commented"; exit }
|
||||
' "$CONFIG")"
|
||||
case "$state" in
|
||||
active) return 0 ;;
|
||||
commented) ;;
|
||||
*) return 2 ;;
|
||||
esac
|
||||
awk '
|
||||
/^[A-Za-z_]/ { top = $1 }
|
||||
top == "node:" && /^ [A-Za-z_]/ { sec = $1 }
|
||||
top == "node:" && sec == "rendezvous:" && $0 ~ /^ # lan:[ \t]*$/ { blk = 1; sub(/^ # /, " "); print; next }
|
||||
blk && /^ # / { sub(/^ # /, " "); print; next }
|
||||
{ blk = 0; print }
|
||||
' "$CONFIG" > "$CONFIG.tmp" && ap_config_write
|
||||
}
|
||||
|
||||
# Re-comment the 'ap<idx>' block so the daemon stops binding it (and stops
|
||||
# warning about the now-missing interface). Inverse of ap_config_enable.
|
||||
ap_config_disable() {
|
||||
idx="$1"
|
||||
[ -f "$CONFIG" ] || return 1
|
||||
grep -q "^ ap$idx:" "$CONFIG" || return 0
|
||||
awk -v idx="$idx" '
|
||||
$0 ~ ("^ ap" idx ":[ \t]*$") { blk = 1; sub(/^ /, " # "); print; next }
|
||||
blk && /^ / { sub(/^ /, " # "); print; next }
|
||||
{ blk = 0; print }
|
||||
' "$CONFIG" > "$CONFIG.tmp" && ap_config_write
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "Usage: fips-ap-setup <radio> [ssid]" >&2
|
||||
echo " fips-ap-setup remove [radio]" >&2
|
||||
echo "Radios on this device:" >&2
|
||||
uci show wireless 2>/dev/null | sed -n "s/^wireless\.\([^.]*\)=wifi-device$/ \1/p" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# List the UCI section names of fips-managed access-point wifi-ifaces.
|
||||
ap_sections() {
|
||||
uci show wireless 2>/dev/null | sed -n "s/^wireless\.\(fips_ap[^.=]*\)=wifi-iface$/\1/p"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# remove [radio] — delete the wireless, network, dhcp, and firewall sections
|
||||
# created below
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if [ "$1" = "remove" ]; then
|
||||
if [ -n "$2" ]; then
|
||||
SECTIONS="fips_ap_$(printf '%s' "$2" | tr -c 'a-zA-Z0-9_' '_')"
|
||||
else
|
||||
SECTIONS="$(ap_sections)"
|
||||
fi
|
||||
[ -n "$SECTIONS" ] || {
|
||||
echo "No fips access-point instances configured."
|
||||
exit 0
|
||||
}
|
||||
for section in $SECTIONS; do
|
||||
ifname="$(uci -q get "wireless.$section.ifname")"
|
||||
uci -q delete "wireless.$section"
|
||||
uci -q delete "network.$section"
|
||||
uci -q delete "dhcp.$section"
|
||||
uci -q del_list "firewall.fips_ap.network=$section"
|
||||
# Re-comment the matching ap<N> transport in fips.yaml so the
|
||||
# daemon stops warning about the interface we just removed.
|
||||
idx="$(printf '%s' "$ifname" | sed -n 's/.*[^0-9]\([0-9]\{1,\}\)$/\1/p')"
|
||||
[ -n "$idx" ] && ap_config_disable "$idx"
|
||||
echo "Removed ${ifname:-$section}."
|
||||
done
|
||||
# Drop the shared zone and its rules once the last instance is gone.
|
||||
if [ -z "$(uci -q get firewall.fips_ap.network)" ]; then
|
||||
uci -q delete firewall.fips_ap
|
||||
uci -q delete firewall.fips_ap_icmpv6
|
||||
uci -q delete firewall.fips_ap_dhcpv4
|
||||
uci -q delete firewall.fips_ap_mdns
|
||||
uci -q delete firewall.fips_ap_fips_udp
|
||||
uci -q delete firewall.fips_ap_fips_tcp
|
||||
fi
|
||||
uci commit wireless
|
||||
uci commit network
|
||||
uci commit dhcp
|
||||
uci commit firewall
|
||||
wifi reload
|
||||
/etc/init.d/dnsmasq reload
|
||||
/etc/init.d/odhcpd reload
|
||||
/etc/init.d/firewall reload
|
||||
echo "Restart fips: /etc/init.d/fips restart"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RADIO="$1"
|
||||
SSID="${2:-$DEFAULT_SSID}"
|
||||
|
||||
[ -n "$RADIO" ] || usage
|
||||
|
||||
if [ "$(uci -q get "wireless.$RADIO")" != "wifi-device" ]; then
|
||||
echo "Error: '$RADIO' is not a wifi-device in /etc/config/wireless." >&2
|
||||
usage
|
||||
fi
|
||||
|
||||
# One instance per radio: section fips_ap_<radio>, netdev fips-ap<N>
|
||||
# where N is the radio's trailing index (radio0 -> fips-ap0). For radios
|
||||
# named without a trailing number, fall back to the first free index.
|
||||
SECTION="fips_ap_$(printf '%s' "$RADIO" | tr -c 'a-zA-Z0-9_' '_')"
|
||||
IDX="$(printf '%s' "$RADIO" | sed -n 's/.*[^0-9]\([0-9]\{1,\}\)$/\1/p')"
|
||||
[ -n "$IDX" ] || IDX="$(printf '%s' "$RADIO" | sed -n 's/^\([0-9]\{1,\}\)$/\1/p')"
|
||||
if [ -z "$IDX" ]; then
|
||||
IDX=0
|
||||
while uci show wireless 2>/dev/null | grep -q "\.ifname='fips-ap$IDX'"; do
|
||||
IDX=$((IDX + 1))
|
||||
done
|
||||
fi
|
||||
AP_IFNAME="fips-ap$IDX"
|
||||
|
||||
# Refuse a name collision from another radio's instance (e.g. two radios
|
||||
# whose names end in the same digit) rather than silently hijacking it.
|
||||
OWNER="$(uci show wireless 2>/dev/null \
|
||||
| sed -n "s/^wireless\.\(fips_ap[^.=]*\)\.ifname='$AP_IFNAME'$/\1/p")"
|
||||
if [ -n "$OWNER" ] && [ "$OWNER" != "$SECTION" ]; then
|
||||
echo "Error: $AP_IFNAME is already used by section '$OWNER'." >&2
|
||||
echo "Remove it first: fips-ap-setup remove" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wireless: open AP with client isolation. Clients of the same AP cannot
|
||||
# exchange L2 frames directly — two FIPS phones on one router still reach
|
||||
# each other through the router at the overlay layer. Isolation is an L2
|
||||
# control only: a stranger who peers is an overlay peer like any other, so
|
||||
# the FIPS overlay (not L2) is the trust boundary between clients.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
uci -q delete "wireless.$SECTION"
|
||||
uci set "wireless.$SECTION=wifi-iface"
|
||||
uci set "wireless.$SECTION.device=$RADIO"
|
||||
uci set "wireless.$SECTION.mode=ap"
|
||||
uci set "wireless.$SECTION.ssid=$SSID"
|
||||
uci set "wireless.$SECTION.encryption=none"
|
||||
uci set "wireless.$SECTION.isolate=1"
|
||||
uci set "wireless.$SECTION.ifname=$AP_IFNAME"
|
||||
uci set "wireless.$SECTION.network=$SECTION"
|
||||
|
||||
# Radios ship disabled on fresh OpenWrt installs; a disabled radio would
|
||||
# leave the AP down with no error anywhere visible.
|
||||
if [ "$(uci -q get "wireless.$RADIO.disabled")" = "1" ]; then
|
||||
echo "Note: enabling $RADIO (was disabled)."
|
||||
uci -q delete "wireless.$RADIO.disabled"
|
||||
fi
|
||||
|
||||
CHANNEL="$(uci -q get "wireless.$RADIO.channel")"
|
||||
BAND="$(uci -q get "wireless.$RADIO.band")"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Network: IPv4 from the fixed convention 10.21.<IDX>.1/24 — deterministic,
|
||||
# so every router serving the same radio index lands on the same subnet and
|
||||
# a roaming client's lease stays valid. IPv6 is a static ULA /64 so odhcpd
|
||||
# has a prefix to announce; that space is per-router and disposable — a
|
||||
# roaming phone SLAACs a fresh address on each router, and the FIPS overlay
|
||||
# identity (not the IP) is the mobility anchor. The ULA is derived from the
|
||||
# router's global ULA prefix; a re-run keeps the address already configured.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
AP_ADDR="$(uci -q get "network.$SECTION.ip6addr")"
|
||||
case "$AP_ADDR" in
|
||||
fd*) ;; # keep the existing address on re-run
|
||||
*)
|
||||
ULA_BASE=""
|
||||
ULA_PREFIX="$(uci -q get network.globals.ula_prefix)"
|
||||
case "$ULA_PREFIX" in
|
||||
fd*::/48) ULA_BASE="${ULA_PREFIX%::/48}" ;;
|
||||
esac
|
||||
if [ -z "$ULA_BASE" ]; then
|
||||
HEX="$(head -c 5 /dev/urandom | hexdump -e '5/1 "%02x"')"
|
||||
ULA_BASE="fd$(printf '%s' "$HEX" | cut -c1-2):$(printf '%s' "$HEX" | cut -c3-6):$(printf '%s' "$HEX" | cut -c7-10)"
|
||||
echo "Note: no usable ULA prefix in network.globals — generated $ULA_BASE::/48 for this AP."
|
||||
fi
|
||||
# 64000 = 0xfa00 — high subnet IDs keep clear of br-lan's low
|
||||
# ip6assign allocations from the same ULA prefix.
|
||||
AP_ADDR="$ULA_BASE:$(printf '%04x' $((64000 + IDX)))::1/64"
|
||||
;;
|
||||
esac
|
||||
|
||||
AP_ADDR4="10.21.$IDX.1"
|
||||
|
||||
uci -q delete "network.$SECTION"
|
||||
uci set "network.$SECTION=interface"
|
||||
uci set "network.$SECTION.proto=static"
|
||||
uci set "network.$SECTION.ipaddr=$AP_ADDR4"
|
||||
uci set "network.$SECTION.netmask=255.255.255.0"
|
||||
uci set "network.$SECTION.ip6addr=$AP_ADDR"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DHCP/RA: dnsmasq DHCPv4 leases out of 10.21.<IDX>.0/24, plus router
|
||||
# advertisements for the ULA (stateless SLAAC, no DHCPv6). ra_default '2'
|
||||
# announces a default router even without an upstream default route:
|
||||
# Android's provisioning wants address + route + DNS, and its validation
|
||||
# probe then fails by design (no internet), so the phone keeps cellular as
|
||||
# the default route. 'dhcpv4 server' is read by BOTH dnsmasq (the default
|
||||
# DHCPv4 server) and odhcpd (serves v4 only when odhcpd.maindhcp is set),
|
||||
# so either arrangement hands out leases.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
uci -q delete "dhcp.$SECTION"
|
||||
uci set "dhcp.$SECTION=dhcp"
|
||||
uci set "dhcp.$SECTION.interface=$SECTION"
|
||||
uci set "dhcp.$SECTION.ra=server"
|
||||
uci set "dhcp.$SECTION.ra_default=2"
|
||||
uci set "dhcp.$SECTION.dhcpv6=disabled"
|
||||
uci set "dhcp.$SECTION.dhcpv4=server"
|
||||
uci set "dhcp.$SECTION.start=10"
|
||||
uci set "dhcp.$SECTION.limit=200"
|
||||
|
||||
# Authoritative is the OpenWrt default, but roaming correctness depends on
|
||||
# it (a foreign router must ACK a lease it never issued), so pin it.
|
||||
[ -n "$(uci -q get dhcp.@dnsmasq[0])" ] && uci set dhcp.@dnsmasq[0].authoritative=1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Firewall: one shared 'fips_ap' zone for all instances. Everything is
|
||||
# rejected except what a FIPS client needs — DHCPv4 (addressing), ICMPv6
|
||||
# (SLAAC itself), mDNS (discovery), and the FIPS UDP/TCP transports (the
|
||||
# handshake surface).
|
||||
# The raw-Ethernet transport (EtherType 0x2121) is not IP and never
|
||||
# traverses the firewall. No forwardings exist, so there is no path to
|
||||
# br-lan or the WAN.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if [ "$(uci -q get firewall.fips_ap)" != "zone" ]; then
|
||||
uci set firewall.fips_ap=zone
|
||||
fi
|
||||
uci set firewall.fips_ap.name=fips_ap
|
||||
uci set firewall.fips_ap.input=REJECT
|
||||
uci set firewall.fips_ap.output=ACCEPT
|
||||
uci set firewall.fips_ap.forward=REJECT
|
||||
uci -q del_list "firewall.fips_ap.network=$SECTION"
|
||||
uci add_list "firewall.fips_ap.network=$SECTION"
|
||||
|
||||
# ap_rule <section-suffix> <name> <proto> [dest_port]
|
||||
ap_rule() {
|
||||
rule="firewall.fips_ap_$1"
|
||||
uci -q delete "$rule"
|
||||
uci set "$rule=rule"
|
||||
uci set "$rule.name=$2"
|
||||
uci set "$rule.src=fips_ap"
|
||||
uci set "$rule.proto=$3"
|
||||
uci set "$rule.target=ACCEPT"
|
||||
[ -z "${4:-}" ] || uci set "$rule.dest_port=$4"
|
||||
}
|
||||
|
||||
ap_rule icmpv6 "FIPS-AP-ICMPv6" icmp
|
||||
uci set firewall.fips_ap_icmpv6.family=ipv6
|
||||
ap_rule dhcpv4 "FIPS-AP-DHCPv4" udp 67
|
||||
uci set firewall.fips_ap_dhcpv4.family=ipv4
|
||||
ap_rule mdns "FIPS-AP-mDNS" udp 5353
|
||||
ap_rule fips_udp "FIPS-AP-FIPS-UDP" udp 2121
|
||||
ap_rule fips_tcp "FIPS-AP-FIPS-TCP" tcp 8443
|
||||
|
||||
uci commit wireless
|
||||
uci commit network
|
||||
uci commit dhcp
|
||||
uci commit firewall
|
||||
wifi reload
|
||||
/etc/init.d/dnsmasq reload
|
||||
/etc/init.d/odhcpd reload
|
||||
/etc/init.d/firewall reload
|
||||
|
||||
# Enable the matching ap<N> transport in the shipped fips.yaml (it ships
|
||||
# commented out). Tailor the restart hint to what we could do.
|
||||
ap_config_enable "$IDX"
|
||||
case $? in
|
||||
0) TRANSPORT_NOTE="The ap$IDX transport in $CONFIG that binds '$AP_IFNAME' is
|
||||
now uncommented and enabled." ;;
|
||||
1) TRANSPORT_NOTE="No $CONFIG found — add a transports.ethernet entry binding
|
||||
interface '$AP_IFNAME' by hand." ;;
|
||||
*) TRANSPORT_NOTE="No 'ap$IDX' entry in $CONFIG — add a transports.ethernet
|
||||
entry binding interface '$AP_IFNAME' by hand (copy the ap0 block)." ;;
|
||||
esac
|
||||
|
||||
# Phones discover the daemon via mDNS, not raw-Ethernet beacons — make sure
|
||||
# the daemon-wide mDNS rendezvous is on.
|
||||
if lan_rendezvous_enable; then
|
||||
MDNS_NOTE="node.rendezvous.lan (mDNS) is enabled — phone FIPS apps
|
||||
discover this router via DNS-SD."
|
||||
else
|
||||
MDNS_NOTE="Could not enable mDNS in $CONFIG — set
|
||||
'node.rendezvous.lan.enabled: true' by hand; phone FIPS apps rely
|
||||
on it to discover this router."
|
||||
fi
|
||||
|
||||
cat <<EOF
|
||||
Created open access SSID '$SSID' as $AP_IFNAME on $RADIO \
|
||||
(band ${BAND:-?}, channel ${CHANNEL:-auto}).
|
||||
DHCPv4 on $AP_ADDR4/24 and RA IPv6 on $AP_ADDR — no internet,
|
||||
isolated from br-lan and the WAN.
|
||||
|
||||
ALL FIPS routers must broadcast this SSID with the same security type
|
||||
(open) — phones then save it once and roam between routers as one
|
||||
network. The 10.21.$IDX.0/24 subnet is the same on every router on
|
||||
purpose: leases survive roaming. Unlike the mesh backhaul, channels
|
||||
are free per router. On a dual-band router, run fips-ap-setup for the
|
||||
other radio too so clients can pick either band.
|
||||
|
||||
On first connect a phone warns that the network has no internet —
|
||||
choose "stay connected" and "don't ask again". That choice is stored
|
||||
per SSID, so it covers every FIPS router.
|
||||
|
||||
Next steps:
|
||||
1. $TRANSPORT_NOTE
|
||||
2. $MDNS_NOTE
|
||||
Restart the daemon AFTER the interface is up — a transport whose
|
||||
interface is missing at startup is skipped, not retried:
|
||||
/etc/init.d/fips restart
|
||||
3. Associate a phone or laptop running FIPS and verify:
|
||||
iw dev $AP_IFNAME station dump
|
||||
and the FIPS link on top of it:
|
||||
fipsctl show peers
|
||||
|
||||
Run 'fips-ap-setup remove' to undo all instances, or
|
||||
'fips-ap-setup remove $RADIO' for just this one.
|
||||
EOF
|
||||
@@ -1,261 +0,0 @@
|
||||
#!/bin/sh
|
||||
# fips-mesh-setup — configure open 802.11s mesh interfaces for FIPS backhaul.
|
||||
#
|
||||
# Usage:
|
||||
# fips-mesh-setup <radio> [mesh-id] e.g. fips-mesh-setup radio1
|
||||
# fips-mesh-setup remove [radio] no radio: remove all instances
|
||||
#
|
||||
# Creates a mesh-point interface on the given radio and leaves everything
|
||||
# above L2 to FIPS. Run once per radio: dual-band routers can mesh on both
|
||||
# bands at once (2.4 GHz reaches further, 5 GHz carries more). Note this is
|
||||
# failover, not multipath — FIPS keeps one active link per peer; the other
|
||||
# band stands by and reconnects the peer if the active link dies.
|
||||
#
|
||||
# - encryption 'none' — the mesh is OPEN on purpose. FIPS's Noise IK
|
||||
# handshake authenticates and encrypts every peer link, so SAE would
|
||||
# only duplicate that (and on ath10k it forces the slower raw Tx/Rx
|
||||
# firmware mode). A stranger can form an 802.11s peering AND a FIPS
|
||||
# peer link — the Noise handshake authenticates each link (no
|
||||
# impersonation of another identity, no MITM), it does not gate who
|
||||
# may peer. Admission is open up to the daemon's max-peers cap.
|
||||
# - mesh_fwding '0' — disables 802.11s HWMP forwarding so each mesh
|
||||
# link is a plain L2 neighbor link. FIPS is the routing layer; two
|
||||
# routing layers would fight.
|
||||
#
|
||||
# Interfaces are named per radio index (radio0 -> fips-mesh0, radio1 ->
|
||||
# fips-mesh1) and are intentionally NOT bridged into br-lan: the FIPS
|
||||
# Ethernet transport binds each directly and runs discovery beacons over it.
|
||||
#
|
||||
# The shipped /etc/fips/fips.yaml carries 'mesh0' and 'mesh1' entries under
|
||||
# 'transports.ethernet' bound to these names, but commented out — a stock
|
||||
# install that never creates fips-mesh* then logs no bind warning. This
|
||||
# helper uncomments the matching entry when it creates an interface and
|
||||
# re-comments it on remove, so the daemon binds the transport without a
|
||||
# manual config edit. After an interface is up, restart fips.
|
||||
# See docs/how-to/set-up-80211s-mesh-backhaul.md for the full guide.
|
||||
|
||||
DEFAULT_MESH_ID="fips-mesh"
|
||||
CONFIG="/etc/fips/fips.yaml"
|
||||
|
||||
# Replace $CONFIG with the rewritten $CONFIG.tmp. Force mode 0600 first: the
|
||||
# package installs fips.yaml 0600 (it may hold an inline 'nsec' private key),
|
||||
# and a fresh tmp file would otherwise land world-readable after the move.
|
||||
mesh_config_write() {
|
||||
chmod 600 "$CONFIG.tmp" && mv "$CONFIG.tmp" "$CONFIG"
|
||||
}
|
||||
|
||||
# Uncomment the 'mesh<idx>' transports.ethernet block in $CONFIG (created by
|
||||
# 'fips-mesh-setup'). Reversible with mesh_config_disable. Returns:
|
||||
# 0 enabled (or already active) 1 no config file 2 no such block
|
||||
mesh_config_enable() {
|
||||
idx="$1"
|
||||
[ -f "$CONFIG" ] || return 1
|
||||
grep -q "^ mesh$idx:" "$CONFIG" && return 0
|
||||
grep -q "^ # mesh$idx:" "$CONFIG" || return 2
|
||||
awk -v idx="$idx" '
|
||||
$0 ~ ("^ # mesh" idx ":[ \t]*$") { blk = 1; sub(/^ # /, " "); print; next }
|
||||
blk && /^ # / { sub(/^ # /, " "); print; next }
|
||||
{ blk = 0; print }
|
||||
' "$CONFIG" > "$CONFIG.tmp" && mesh_config_write
|
||||
}
|
||||
|
||||
# Re-comment the 'mesh<idx>' block so the daemon stops binding it (and stops
|
||||
# warning about the now-missing interface). Inverse of mesh_config_enable.
|
||||
mesh_config_disable() {
|
||||
idx="$1"
|
||||
[ -f "$CONFIG" ] || return 1
|
||||
grep -q "^ mesh$idx:" "$CONFIG" || return 0
|
||||
awk -v idx="$idx" '
|
||||
$0 ~ ("^ mesh" idx ":[ \t]*$") { blk = 1; sub(/^ /, " # "); print; next }
|
||||
blk && /^ / { sub(/^ /, " # "); print; next }
|
||||
{ blk = 0; print }
|
||||
' "$CONFIG" > "$CONFIG.tmp" && mesh_config_write
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "Usage: fips-mesh-setup <radio> [mesh-id]" >&2
|
||||
echo " fips-mesh-setup remove [radio]" >&2
|
||||
echo "Radios on this device:" >&2
|
||||
uci show wireless 2>/dev/null | sed -n "s/^wireless\.\([^.]*\)=wifi-device$/ \1/p" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# List the UCI section names of fips-managed mesh wifi-ifaces.
|
||||
mesh_sections() {
|
||||
uci show wireless 2>/dev/null | sed -n "s/^wireless\.\(fips_mesh[^.=]*\)=wifi-iface$/\1/p"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# remove [radio] — delete the wireless and network sections created below
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if [ "$1" = "remove" ]; then
|
||||
if [ -n "$2" ]; then
|
||||
SECTIONS="fips_mesh_$(printf '%s' "$2" | tr -c 'a-zA-Z0-9_' '_')"
|
||||
else
|
||||
SECTIONS="$(mesh_sections)"
|
||||
fi
|
||||
[ -n "$SECTIONS" ] || {
|
||||
echo "No fips mesh instances configured."
|
||||
exit 0
|
||||
}
|
||||
for section in $SECTIONS; do
|
||||
ifname="$(uci -q get "wireless.$section.ifname")"
|
||||
uci -q delete "wireless.$section"
|
||||
uci -q delete "network.$section"
|
||||
# Re-comment the matching mesh<N> transport in fips.yaml so the
|
||||
# daemon stops warning about the interface we just removed.
|
||||
idx="$(printf '%s' "$ifname" | sed -n 's/.*[^0-9]\([0-9]\{1,\}\)$/\1/p')"
|
||||
[ -n "$idx" ] && mesh_config_disable "$idx"
|
||||
echo "Removed ${ifname:-$section}."
|
||||
done
|
||||
uci commit wireless
|
||||
uci commit network
|
||||
# 'wifi reload' re-applies the whole wireless config, so it briefly drops
|
||||
# every client AP on all radios (a few seconds) — expected on remove.
|
||||
wifi reload
|
||||
echo "Restart fips: /etc/init.d/fips restart"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RADIO="$1"
|
||||
MESH_ID="${2:-$DEFAULT_MESH_ID}"
|
||||
|
||||
[ -n "$RADIO" ] || usage
|
||||
|
||||
if [ "$(uci -q get "wireless.$RADIO")" != "wifi-device" ]; then
|
||||
echo "Error: '$RADIO' is not a wifi-device in /etc/config/wireless." >&2
|
||||
usage
|
||||
fi
|
||||
|
||||
# One instance per radio: section fips_mesh_<radio>, netdev fips-mesh<N>
|
||||
# where N is the radio's trailing index (radio0 -> fips-mesh0). For radios
|
||||
# named without a trailing number, fall back to the first free index.
|
||||
SECTION="fips_mesh_$(printf '%s' "$RADIO" | tr -c 'a-zA-Z0-9_' '_')"
|
||||
IDX="$(printf '%s' "$RADIO" | sed -n 's/.*[^0-9]\([0-9]\{1,\}\)$/\1/p')"
|
||||
[ -n "$IDX" ] || IDX="$(printf '%s' "$RADIO" | sed -n 's/^\([0-9]\{1,\}\)$/\1/p')"
|
||||
if [ -z "$IDX" ]; then
|
||||
IDX=0
|
||||
while uci show wireless 2>/dev/null | grep -q "\.ifname='fips-mesh$IDX'"; do
|
||||
IDX=$((IDX + 1))
|
||||
done
|
||||
fi
|
||||
MESH_IFNAME="fips-mesh$IDX"
|
||||
|
||||
# Refuse a name collision from another radio's instance (e.g. two radios
|
||||
# whose names end in the same digit) rather than silently hijacking it.
|
||||
OWNER="$(uci show wireless 2>/dev/null \
|
||||
| sed -n "s/^wireless\.\(fips_mesh[^.=]*\)\.ifname='$MESH_IFNAME'$/\1/p")"
|
||||
if [ -n "$OWNER" ] && [ "$OWNER" != "$SECTION" ]; then
|
||||
echo "Error: $MESH_IFNAME is already used by section '$OWNER'." >&2
|
||||
echo "Remove it first: fips-mesh-setup remove" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver capability check (advisory — config below is harmless either way)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if command -v iw >/dev/null 2>&1; then
|
||||
if ! iw list 2>/dev/null | grep -q "\* mesh point"; then
|
||||
echo "Warning: no radio on this device advertises 'mesh point' support" >&2
|
||||
echo "(iw list | grep 'mesh point'). The interface may fail to come up." >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wireless: open 802.11s mesh point, HWMP forwarding off
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
uci -q delete "wireless.$SECTION"
|
||||
uci set "wireless.$SECTION=wifi-iface"
|
||||
uci set "wireless.$SECTION.device=$RADIO"
|
||||
uci set "wireless.$SECTION.mode=mesh"
|
||||
uci set "wireless.$SECTION.mesh_id=$MESH_ID"
|
||||
uci set "wireless.$SECTION.encryption=none"
|
||||
uci set "wireless.$SECTION.mesh_fwding=0"
|
||||
uci set "wireless.$SECTION.ifname=$MESH_IFNAME"
|
||||
uci set "wireless.$SECTION.network=$SECTION"
|
||||
|
||||
# Radios ship disabled on fresh OpenWrt installs; a disabled radio would
|
||||
# leave the mesh interface down with no error anywhere visible.
|
||||
if [ "$(uci -q get "wireless.$RADIO.disabled")" = "1" ]; then
|
||||
echo "Note: enabling $RADIO (was disabled)."
|
||||
uci -q delete "wireless.$RADIO.disabled"
|
||||
fi
|
||||
|
||||
# The mesh inherits the radio's channel, and mesh points only peer on the
|
||||
# same channel. 'auto' lets each router pick its own — the classic silent
|
||||
# non-peering cause — so surface the setting loudly.
|
||||
CHANNEL="$(uci -q get "wireless.$RADIO.channel")"
|
||||
BAND="$(uci -q get "wireless.$RADIO.band")"
|
||||
if [ -z "$CHANNEL" ] || [ "$CHANNEL" = "auto" ]; then
|
||||
echo "Warning: $RADIO channel is '${CHANNEL:-unset}' — each router may" >&2
|
||||
echo "auto-select a different channel and mesh points only peer on the" >&2
|
||||
echo "same one. Pin the same channel on every backhaul router, e.g.:" >&2
|
||||
echo " uci set wireless.$RADIO.channel='36' && uci commit wireless && wifi reload" >&2
|
||||
fi
|
||||
|
||||
# A client (sta) interface on the same radio follows its upstream AP's
|
||||
# channel and drags every other interface with it — a mesh pinned to a
|
||||
# different channel silently never joins, and does not recover when the
|
||||
# STA disconnects.
|
||||
for s in $(uci show wireless 2>/dev/null | sed -n "s/^wireless\.\([^.]*\)\.mode='sta'$/\1/p"); do
|
||||
if [ "$(uci -q get "wireless.$s.device")" = "$RADIO" ]; then
|
||||
echo "Warning: $RADIO also carries client interface '$s' (mode 'sta')." >&2
|
||||
echo "The whole radio follows that STA's upstream channel — a mesh" >&2
|
||||
echo "pinned to a different channel stays down silently. Align the" >&2
|
||||
echo "mesh channel with the upstream AP, or put the mesh on a radio" >&2
|
||||
echo "without a STA (a roaming uplink is incompatible with a" >&2
|
||||
echo "fixed-channel mesh on the same radio)." >&2
|
||||
fi
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Network: unmanaged interface so netifd brings the netdev up. No IP config —
|
||||
# the FIPS Ethernet transport speaks raw frames on it.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
uci -q delete "network.$SECTION"
|
||||
uci set "network.$SECTION=interface"
|
||||
uci set "network.$SECTION.proto=none"
|
||||
|
||||
uci commit wireless
|
||||
uci commit network
|
||||
# 'wifi reload' re-applies the whole wireless config, so it briefly drops
|
||||
# every client AP on all radios (a few seconds) — expected when adding a mesh.
|
||||
wifi reload
|
||||
|
||||
# Enable the matching mesh<N> transport in the shipped fips.yaml (it ships
|
||||
# commented out). Tailor the restart hint to what we could do.
|
||||
mesh_config_enable "$IDX"
|
||||
case $? in
|
||||
0) TRANSPORT_NOTE="The mesh$IDX transport in $CONFIG that binds '$MESH_IFNAME' is
|
||||
now uncommented and enabled." ;;
|
||||
1) TRANSPORT_NOTE="No $CONFIG found — add a transports.ethernet entry binding
|
||||
interface '$MESH_IFNAME' by hand." ;;
|
||||
*) TRANSPORT_NOTE="No 'mesh$IDX' entry in $CONFIG — add a transports.ethernet
|
||||
entry binding interface '$MESH_IFNAME' by hand (copy the mesh0 block)." ;;
|
||||
esac
|
||||
|
||||
cat <<EOF
|
||||
Created open 802.11s mesh '$MESH_ID' as $MESH_IFNAME on $RADIO \
|
||||
(band ${BAND:-?}, channel ${CHANNEL:-auto}).
|
||||
|
||||
ALL routers in this backhaul must share this mesh ID AND channel
|
||||
(per band). On a dual-band router, run fips-mesh-setup for the other
|
||||
radio too — second band is a standby path (failover, not multipath).
|
||||
|
||||
Next steps:
|
||||
1. $TRANSPORT_NOTE
|
||||
Restart the daemon AFTER the interface is up — a transport whose
|
||||
interface is missing at startup is skipped, not retried:
|
||||
/etc/init.d/fips restart
|
||||
2. Verify L2 peering with a second FIPS router in range:
|
||||
iw dev $MESH_IFNAME station dump
|
||||
and the FIPS link on top of it:
|
||||
fipsctl show peers
|
||||
|
||||
Run 'fips-mesh-setup remove' to undo all instances, or
|
||||
'fips-mesh-setup remove $RADIO' for just this one.
|
||||
EOF
|
||||
@@ -68,7 +68,7 @@ and set the interface name:
|
||||
transports:
|
||||
ethernet:
|
||||
interface: "eth0"
|
||||
listen: true
|
||||
discovery: true
|
||||
announce: true
|
||||
auto_connect: true
|
||||
accept_connections: true
|
||||
|
||||
@@ -14,11 +14,6 @@ RestartSec=5
|
||||
RuntimeDirectory=fips
|
||||
RuntimeDirectoryMode=0750
|
||||
|
||||
# Log directory (/var/log/fips/), where the built-in tick-body profiler writes
|
||||
# its capture files. Declared so systemd creates it on start and removes it on
|
||||
# purge; the daemon runs as root and already has access without it.
|
||||
LogsDirectory=fips
|
||||
|
||||
# Security hardening (daemon runs as root for TUN and raw sockets)
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
|
||||
+17
-14
@@ -8,7 +8,7 @@ use fips::config::{IdentitySource, resolve_identity};
|
||||
use fips::version;
|
||||
use fips::{Config, Node};
|
||||
use std::path::PathBuf;
|
||||
use tracing::{debug, error, info};
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing_subscriber::{EnvFilter, fmt};
|
||||
|
||||
/// FIPS mesh network daemon
|
||||
@@ -157,23 +157,26 @@ async fn run_daemon(
|
||||
|
||||
info!("FIPS running");
|
||||
|
||||
// Serve until the shutdown signal, then drain in place before returning.
|
||||
// The rx loop observes the signal directly, so its channels are never
|
||||
// destructively cancelled — they live in the loop's locals across serve and
|
||||
// drain, and are dropped only on clean exit (after which teardown does not
|
||||
// need them). On the signal the loop broadcasts a shutdown Disconnect and
|
||||
// waits (bounded by node.drain_timeout_secs) for peers to clear.
|
||||
match node.run_rx_loop_with_shutdown(shutdown_signal).await {
|
||||
Ok(()) => info!("RX loop exited"),
|
||||
Err(e) => error!("RX loop error: {}", e),
|
||||
// Run the RX event loop until shutdown signal.
|
||||
// stop() drops the packet channel, causing run_rx_loop to exit.
|
||||
tokio::select! {
|
||||
result = node.run_rx_loop() => {
|
||||
match result {
|
||||
Ok(()) => info!("RX loop exited"),
|
||||
Err(e) => error!("RX loop error: {}", e),
|
||||
}
|
||||
}
|
||||
_ = shutdown_signal => {
|
||||
info!("Shutdown signal received");
|
||||
}
|
||||
}
|
||||
|
||||
info!("FIPS shutting down");
|
||||
|
||||
// Close the drain window (if the loop drained) and tear down. A drained
|
||||
// loop tears down without re-broadcasting; a loop that exited some other
|
||||
// way falls back to the immediate stop().
|
||||
node.finish_shutdown().await;
|
||||
// Stop the node (shuts down transports, TUN, I/O threads)
|
||||
if let Err(e) = node.stop().await {
|
||||
warn!("Error during shutdown: {}", e);
|
||||
}
|
||||
|
||||
info!("FIPS shutdown complete");
|
||||
}
|
||||
|
||||
@@ -76,37 +76,6 @@ enum Commands {
|
||||
#[command(subcommand)]
|
||||
what: StatsCommands,
|
||||
},
|
||||
/// Control the built-in profiler (requires a `--features profiling` build)
|
||||
#[cfg(feature = "profiling")]
|
||||
Profile {
|
||||
#[command(subcommand)]
|
||||
what: ProfileCommands,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(feature = "profiling")]
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum ProfileCommands {
|
||||
/// Profile the rx-loop tick body
|
||||
Tick {
|
||||
#[command(subcommand)]
|
||||
action: ProfileTickAction,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(feature = "profiling")]
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum ProfileTickAction {
|
||||
/// Start a capture
|
||||
On {
|
||||
/// Directory for the capture file (default /var/log/fips)
|
||||
#[arg(long)]
|
||||
dir: Option<PathBuf>,
|
||||
},
|
||||
/// Stop the running capture
|
||||
Off,
|
||||
/// Report capture state
|
||||
Status,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
@@ -502,20 +471,6 @@ fn main() {
|
||||
build_command("show_stats_history", params)
|
||||
}
|
||||
},
|
||||
#[cfg(feature = "profiling")]
|
||||
Commands::Profile { what } => match what {
|
||||
ProfileCommands::Tick { action } => match action {
|
||||
ProfileTickAction::On { dir } => match dir {
|
||||
Some(dir) => build_command(
|
||||
"profile_tick_on",
|
||||
serde_json::json!({"dir": dir.display().to_string()}),
|
||||
),
|
||||
None => build_query("profile_tick_on"),
|
||||
},
|
||||
ProfileTickAction::Off => build_query("profile_tick_off"),
|
||||
ProfileTickAction::Status => build_query("profile_tick_status"),
|
||||
},
|
||||
},
|
||||
Commands::Keygen { .. } => unreachable!(),
|
||||
};
|
||||
|
||||
|
||||
@@ -134,8 +134,8 @@ fn draw_routing_stats(
|
||||
let cols =
|
||||
Layout::horizontal([Constraint::Percentage(50), Constraint::Percentage(50)]).split(inner);
|
||||
|
||||
// Shorthand for a nested counter value (e.g. lookup.req_received).
|
||||
let lookup = |key: &str| helpers::nested_u64(data, "lookup", key);
|
||||
// Shorthand for a nested counter value (e.g. discovery.req_received).
|
||||
let disc = |key: &str| helpers::nested_u64(data, "discovery", key);
|
||||
let err = |key: &str| helpers::nested_u64(data, "error_signals", key);
|
||||
let cong = |key: &str| helpers::nested_u64(data, "congestion", key);
|
||||
|
||||
@@ -174,32 +174,32 @@ fn draw_routing_stats(
|
||||
));
|
||||
left.push(Line::from(""));
|
||||
left.extend(section(
|
||||
"Lookup Requests",
|
||||
"Discovery Requests",
|
||||
&[
|
||||
("Received", lookup("req_received")),
|
||||
("Forwarded", lookup("req_forwarded")),
|
||||
("Initiated", lookup("req_initiated")),
|
||||
("Deduplicated", lookup("req_deduplicated")),
|
||||
("Target Is Us", lookup("req_target_is_us")),
|
||||
("Duplicate", lookup("req_duplicate")),
|
||||
("Bloom Miss", lookup("req_bloom_miss")),
|
||||
("Backoff Suppressed", lookup("req_backoff_suppressed")),
|
||||
("Fwd Rate Limited", lookup("req_forward_rate_limited")),
|
||||
("TTL Exhausted", lookup("req_ttl_exhausted")),
|
||||
("Decode Error", lookup("req_decode_error")),
|
||||
("Received", disc("req_received")),
|
||||
("Forwarded", disc("req_forwarded")),
|
||||
("Initiated", disc("req_initiated")),
|
||||
("Deduplicated", disc("req_deduplicated")),
|
||||
("Target Is Us", disc("req_target_is_us")),
|
||||
("Duplicate", disc("req_duplicate")),
|
||||
("Bloom Miss", disc("req_bloom_miss")),
|
||||
("Backoff Suppressed", disc("req_backoff_suppressed")),
|
||||
("Fwd Rate Limited", disc("req_forward_rate_limited")),
|
||||
("TTL Exhausted", disc("req_ttl_exhausted")),
|
||||
("Decode Error", disc("req_decode_error")),
|
||||
],
|
||||
));
|
||||
left.push(Line::from(""));
|
||||
left.extend(section(
|
||||
"Lookup Responses",
|
||||
"Discovery Responses",
|
||||
&[
|
||||
("Received", lookup("resp_received")),
|
||||
("Accepted", lookup("resp_accepted")),
|
||||
("Forwarded", lookup("resp_forwarded")),
|
||||
("Timed Out", lookup("resp_timed_out")),
|
||||
("Identity Miss", lookup("resp_identity_miss")),
|
||||
("Proof Failed", lookup("resp_proof_failed")),
|
||||
("Decode Error", lookup("resp_decode_error")),
|
||||
("Received", disc("resp_received")),
|
||||
("Accepted", disc("resp_accepted")),
|
||||
("Forwarded", disc("resp_forwarded")),
|
||||
("Timed Out", disc("resp_timed_out")),
|
||||
("Identity Miss", disc("resp_identity_miss")),
|
||||
("Proof Failed", disc("resp_proof_failed")),
|
||||
("Decode Error", disc("resp_decode_error")),
|
||||
],
|
||||
));
|
||||
|
||||
|
||||
@@ -1189,15 +1189,6 @@ fn mmp_focused_pane_indicator() {
|
||||
});
|
||||
// The focused Session MMP title is cyan; the unfocused Link MMP title is not.
|
||||
assert_eq!(testkit::fg_at(&buf, "Session MMP"), Some(Color::Cyan));
|
||||
// Presence before colour. `fg_at` is `find(..)?` mapped to the cell's fg, so
|
||||
// it returns None for a title that was never drawn, and None != Some(Cyan) --
|
||||
// meaning the assertion below passed when the Link MMP pane was missing
|
||||
// entirely. Asserting it is on screen first is what makes the next line read
|
||||
// "not highlighted" rather than "not there".
|
||||
assert!(
|
||||
testkit::find(&buf, "Link MMP").is_some(),
|
||||
"the unfocused Link MMP title should still be rendered"
|
||||
);
|
||||
assert_ne!(testkit::fg_at(&buf, "Link MMP"), Some(Color::Cyan));
|
||||
}
|
||||
|
||||
|
||||
@@ -174,6 +174,10 @@ fn draw_stats(frame: &mut Frame, data: &serde_json::Value, scroll: u16, focused:
|
||||
&helpers::nested_u64(data, "stats", "sig_failed"),
|
||||
),
|
||||
helpers::kv_line("Stale", &helpers::nested_u64(data, "stats", "stale")),
|
||||
helpers::kv_line(
|
||||
"Parent Switched",
|
||||
&helpers::nested_u64(data, "stats", "parent_switched"),
|
||||
),
|
||||
helpers::kv_line(
|
||||
"Loop Detected",
|
||||
&helpers::nested_u64(data, "stats", "loop_detected"),
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
//! Generic Bloom filter data structure.
|
||||
|
||||
use core::fmt;
|
||||
use std::fmt;
|
||||
|
||||
use tracing::trace;
|
||||
|
||||
use super::{BloomError, DEFAULT_FILTER_SIZE_BITS, DEFAULT_HASH_COUNT};
|
||||
use crate::NodeAddr;
|
||||
@@ -67,18 +69,16 @@ impl BloomFilter {
|
||||
|
||||
/// Insert a NodeAddr into the filter.
|
||||
pub fn insert(&mut self, node_addr: &NodeAddr) {
|
||||
let (h1, h2) = Self::base_hashes(node_addr.as_bytes());
|
||||
for i in 0..self.hash_count {
|
||||
let bit_index = self.bit_index(h1, h2, i);
|
||||
let bit_index = self.hash(node_addr.as_bytes(), i);
|
||||
self.set_bit(bit_index);
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert raw bytes into the filter.
|
||||
pub fn insert_bytes(&mut self, data: &[u8]) {
|
||||
let (h1, h2) = Self::base_hashes(data);
|
||||
for i in 0..self.hash_count {
|
||||
let bit_index = self.bit_index(h1, h2, i);
|
||||
let bit_index = self.hash(data, i);
|
||||
self.set_bit(bit_index);
|
||||
}
|
||||
}
|
||||
@@ -93,9 +93,8 @@ impl BloomFilter {
|
||||
|
||||
/// Check if the filter might contain raw bytes.
|
||||
pub fn contains_bytes(&self, data: &[u8]) -> bool {
|
||||
let (h1, h2) = Self::base_hashes(data);
|
||||
for i in 0..self.hash_count {
|
||||
let bit_index = self.bit_index(h1, h2, i);
|
||||
let bit_index = self.hash(data, i);
|
||||
if !self.get_bit(bit_index) {
|
||||
return false;
|
||||
}
|
||||
@@ -142,11 +141,6 @@ impl BloomFilter {
|
||||
self.count_ones() as f64 / self.num_bits as f64
|
||||
}
|
||||
|
||||
/// Current false-positive rate: fill ratio raised to the hash count.
|
||||
pub fn fpr(&self) -> f64 {
|
||||
crate::proto::math::powi(self.fill_ratio(), self.hash_count as u32)
|
||||
}
|
||||
|
||||
/// Estimate the number of elements in the filter.
|
||||
///
|
||||
/// Uses the formula: n = -(m/k) * ln(1 - X/m)
|
||||
@@ -168,12 +162,13 @@ impl BloomFilter {
|
||||
}
|
||||
|
||||
let fill = x / m;
|
||||
let fpr = self.fpr();
|
||||
let fpr = fill.powi(self.hash_count as i32);
|
||||
if fpr > max_fpr {
|
||||
trace!(fill, fpr, max_fpr, "estimated_count: filter above cap");
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(-(m / k) * libm::log(1.0 - fill))
|
||||
Some(-(m / k) * (1.0 - fill).ln())
|
||||
}
|
||||
|
||||
/// Check if the filter is empty.
|
||||
@@ -201,28 +196,21 @@ impl BloomFilter {
|
||||
self.hash_count
|
||||
}
|
||||
|
||||
/// Compute the two base hashes for `data` with a single SHA-256 digest.
|
||||
/// Compute a hash index for the given data and hash function number.
|
||||
///
|
||||
/// Double hashing derives the k hash functions from two base hashes:
|
||||
/// h(x,i) = (h1(x) + i*h2(x)) mod m. Computing the digest once here and
|
||||
/// reusing `(h1, h2)` across all k functions avoids re-hashing per k.
|
||||
fn base_hashes(data: &[u8]) -> (u64, u64) {
|
||||
// Use first 16 bytes of SHA-256 for h1 and h2.
|
||||
/// Uses double hashing: h(x,i) = (h1(x) + i*h2(x)) mod m
|
||||
fn hash(&self, data: &[u8], k: u8) -> usize {
|
||||
// Use first 16 bytes of SHA-256 for h1 and h2
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data);
|
||||
let hash = hasher.finalize();
|
||||
|
||||
// h1 from first 8 bytes, h2 from next 8 bytes (little-endian).
|
||||
// h1 from first 8 bytes
|
||||
let h1 = u64::from_le_bytes(hash[0..8].try_into().unwrap());
|
||||
// h2 from next 8 bytes
|
||||
let h2 = u64::from_le_bytes(hash[8..16].try_into().unwrap());
|
||||
(h1, h2)
|
||||
}
|
||||
|
||||
/// Derive the bit index for hash function `k` from the base hashes.
|
||||
///
|
||||
/// Uses double hashing: h(x,k) = (h1(x) + k*h2(x)) mod m.
|
||||
fn bit_index(&self, h1: u64, h2: u64, k: u8) -> usize {
|
||||
let combined = h1.wrapping_add((k as u64).wrapping_mul(h2));
|
||||
(combined as usize) % self.num_bits
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
//! Bloom Filter Implementation
|
||||
//!
|
||||
//! 1KB Bloom filters for reachability in FIPS routing. Each node
|
||||
//! maintains filters that summarize which destinations are reachable
|
||||
//! through each peer, enabling efficient routing decisions without
|
||||
//! global network knowledge.
|
||||
//!
|
||||
//! ## v1 Parameters
|
||||
//!
|
||||
//! - Size: 1 KB (8,192 bits) - sized for actual ~400-800 entry occupancy
|
||||
//! - Hash functions: k=5 - optimal at ~1,200 entries, good for 800-1,600
|
||||
//! - Bandwidth: 1 KB/announce (75% reduction from original 4KB design)
|
||||
//!
|
||||
//! These parameters are right-sized for typical network occupancy of
|
||||
//! ~250-800 entries per node.
|
||||
|
||||
mod filter;
|
||||
mod state;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
pub use filter::BloomFilter;
|
||||
pub use state::BloomState;
|
||||
|
||||
/// Default filter size in bits (1KB = 8,192 bits).
|
||||
///
|
||||
/// Sized for ~800-1,600 entries. FPR ~0.05% at 400 entries, ~0.9% at 800.
|
||||
/// This is v1 protocol default (size_class=1).
|
||||
pub const DEFAULT_FILTER_SIZE_BITS: usize = 8192;
|
||||
|
||||
/// Default filter size in bytes (1KB).
|
||||
pub const DEFAULT_FILTER_SIZE_BYTES: usize = DEFAULT_FILTER_SIZE_BITS / 8;
|
||||
|
||||
/// Default number of hash functions.
|
||||
///
|
||||
/// k=5 is optimal at ~1,200 entries and a good compromise for 800-1,600.
|
||||
/// At 400 entries: FPR ~0.05%. At 800 entries: FPR ~0.9%.
|
||||
pub const DEFAULT_HASH_COUNT: u8 = 5;
|
||||
|
||||
/// Size class for v1 protocol (1 KB filters).
|
||||
pub const V1_SIZE_CLASS: u8 = 1;
|
||||
|
||||
/// Filter sizes by size_class: bytes = 512 << size_class
|
||||
pub const SIZE_CLASS_BYTES: [usize; 4] = [512, 1024, 2048, 4096];
|
||||
|
||||
/// Errors related to Bloom filter operations.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum BloomError {
|
||||
#[error("invalid filter size: expected {expected} bits, got {got}")]
|
||||
InvalidSize { expected: usize, got: usize },
|
||||
|
||||
#[error("filter size must be a multiple of 8, got {0}")]
|
||||
SizeNotByteAligned(usize),
|
||||
|
||||
#[error("hash count must be positive")]
|
||||
ZeroHashCount,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -1,6 +1,6 @@
|
||||
//! FIPS-specific Bloom filter announcement state management.
|
||||
|
||||
use alloc::collections::{BTreeMap, BTreeSet};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use super::BloomFilter;
|
||||
use crate::NodeAddr;
|
||||
@@ -13,19 +13,19 @@ pub struct BloomState {
|
||||
/// This node's NodeAddr (always included in outgoing filters).
|
||||
own_node_addr: NodeAddr,
|
||||
/// Leaf-only nodes we speak for (included in our filter).
|
||||
leaf_dependents: BTreeSet<NodeAddr>,
|
||||
leaf_dependents: HashSet<NodeAddr>,
|
||||
/// Whether this node operates in leaf-only mode.
|
||||
is_leaf_only: bool,
|
||||
/// Rate limiting: minimum interval between outgoing updates (milliseconds).
|
||||
update_debounce_ms: u64,
|
||||
/// Timestamp of last update sent (per peer, in milliseconds).
|
||||
last_update_sent: BTreeMap<NodeAddr, u64>,
|
||||
last_update_sent: HashMap<NodeAddr, u64>,
|
||||
/// Peers that need a filter update.
|
||||
pending_updates: BTreeSet<NodeAddr>,
|
||||
pending_updates: HashSet<NodeAddr>,
|
||||
/// Current sequence number for outgoing filters.
|
||||
sequence: u64,
|
||||
/// Last outgoing filter sent to each peer (for change detection).
|
||||
last_sent_filters: BTreeMap<NodeAddr, BloomFilter>,
|
||||
last_sent_filters: HashMap<NodeAddr, BloomFilter>,
|
||||
}
|
||||
|
||||
impl BloomState {
|
||||
@@ -33,13 +33,13 @@ impl BloomState {
|
||||
pub fn new(own_node_addr: NodeAddr) -> Self {
|
||||
Self {
|
||||
own_node_addr,
|
||||
leaf_dependents: BTreeSet::new(),
|
||||
leaf_dependents: HashSet::new(),
|
||||
is_leaf_only: false,
|
||||
update_debounce_ms: 500,
|
||||
last_update_sent: BTreeMap::new(),
|
||||
pending_updates: BTreeSet::new(),
|
||||
last_update_sent: HashMap::new(),
|
||||
pending_updates: HashSet::new(),
|
||||
sequence: 0,
|
||||
last_sent_filters: BTreeMap::new(),
|
||||
last_sent_filters: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ impl BloomState {
|
||||
}
|
||||
|
||||
/// Get the set of leaf dependents.
|
||||
pub fn leaf_dependents(&self) -> &BTreeSet<NodeAddr> {
|
||||
pub fn leaf_dependents(&self) -> &HashSet<NodeAddr> {
|
||||
&self.leaf_dependents
|
||||
}
|
||||
|
||||
@@ -170,81 +170,23 @@ impl BloomState {
|
||||
&mut self,
|
||||
exclude_from: &NodeAddr,
|
||||
peer_addrs: &[NodeAddr],
|
||||
peer_filters: &BTreeMap<NodeAddr, BloomFilter>,
|
||||
peer_filters: &HashMap<NodeAddr, BloomFilter>,
|
||||
) {
|
||||
let targets: Vec<NodeAddr> = peer_addrs
|
||||
.iter()
|
||||
.filter(|addr| *addr != exclude_from)
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
for (peer_addr, new_filter) in self.compute_outgoing_filters(&targets, peer_filters) {
|
||||
let changed = match self.last_sent_filters.get(&peer_addr) {
|
||||
for peer_addr in peer_addrs {
|
||||
if peer_addr == exclude_from {
|
||||
continue;
|
||||
}
|
||||
let new_filter = self.compute_outgoing_filter(peer_addr, peer_filters);
|
||||
let changed = match self.last_sent_filters.get(peer_addr) {
|
||||
Some(last) => *last != new_filter,
|
||||
None => true, // never sent → must send
|
||||
};
|
||||
if changed {
|
||||
self.pending_updates.insert(peer_addr);
|
||||
self.pending_updates.insert(*peer_addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the outgoing filter for many peers in one pass.
|
||||
///
|
||||
/// Equivalent to calling [`compute_outgoing_filter`](Self::compute_outgoing_filter)
|
||||
/// once per target, but linear in the number of contributing peer
|
||||
/// filters instead of quadratic. The per-peer call rebuilds the whole
|
||||
/// union from scratch, so computing it for every peer costs
|
||||
/// O(targets × filters) 1 KB merges; announce fan-out on a
|
||||
/// large node does exactly that, once per tick and again on every
|
||||
/// inbound announce.
|
||||
///
|
||||
/// The split-horizon exclusion is the only thing that differs between
|
||||
/// targets, so the union of "everything except peer i" is assembled
|
||||
/// from a running prefix union and a precomputed suffix union. Merging
|
||||
/// is a bytewise OR, which is commutative and associative, so the
|
||||
/// result is bit-identical to the per-peer computation.
|
||||
pub fn compute_outgoing_filters(
|
||||
&self,
|
||||
targets: &[NodeAddr],
|
||||
peer_filters: &BTreeMap<NodeAddr, BloomFilter>,
|
||||
) -> BTreeMap<NodeAddr, BloomFilter> {
|
||||
let base = self.base_filter();
|
||||
let keys: Vec<NodeAddr> = peer_filters.keys().copied().collect();
|
||||
let n = keys.len();
|
||||
|
||||
// suffix[i] = union of peer_filters[keys[i..]]; suffix[n] is empty.
|
||||
let mut suffix = vec![BloomFilter::new(); n + 1];
|
||||
for i in (0..n).rev() {
|
||||
let mut acc = suffix[i + 1].clone();
|
||||
// Size mismatches are skipped, exactly as in the per-peer path.
|
||||
let _ = acc.merge(&peer_filters[&keys[i]]);
|
||||
suffix[i] = acc;
|
||||
}
|
||||
|
||||
// Filter for a target that contributes nothing: everything merged.
|
||||
let mut all = base.clone();
|
||||
let _ = all.merge(&suffix[0]);
|
||||
|
||||
let mut per_key: BTreeMap<NodeAddr, BloomFilter> = BTreeMap::new();
|
||||
let mut prefix = BloomFilter::new();
|
||||
for i in 0..n {
|
||||
let mut outgoing = base.clone();
|
||||
let _ = outgoing.merge(&prefix);
|
||||
let _ = outgoing.merge(&suffix[i + 1]);
|
||||
per_key.insert(keys[i], outgoing);
|
||||
let _ = prefix.merge(&peer_filters[&keys[i]]);
|
||||
}
|
||||
|
||||
targets
|
||||
.iter()
|
||||
.map(|target| {
|
||||
let filter = per_key.get(target).cloned().unwrap_or_else(|| all.clone());
|
||||
(*target, filter)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Compute the outgoing filter for a specific peer.
|
||||
///
|
||||
/// The filter includes:
|
||||
@@ -257,7 +199,7 @@ impl BloomState {
|
||||
pub fn compute_outgoing_filter(
|
||||
&self,
|
||||
exclude_peer: &NodeAddr,
|
||||
peer_filters: &BTreeMap<NodeAddr, BloomFilter>,
|
||||
peer_filters: &HashMap<NodeAddr, BloomFilter>,
|
||||
) -> BloomFilter {
|
||||
let mut filter = BloomFilter::new();
|
||||
|
||||
@@ -1,9 +1,302 @@
|
||||
//! Tests for `BloomState` (announcement state management).
|
||||
use super::*;
|
||||
use crate::NodeAddr;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use alloc::collections::BTreeMap;
|
||||
fn make_node_addr(val: u8) -> NodeAddr {
|
||||
let mut bytes = [0u8; 16];
|
||||
bytes[0] = val;
|
||||
NodeAddr::from_bytes(bytes)
|
||||
}
|
||||
|
||||
use crate::proto::bloom::{BloomFilter, BloomState};
|
||||
use crate::testutil::make_node_addr;
|
||||
// ===== BloomFilter Tests =====
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_new() {
|
||||
let filter = BloomFilter::new();
|
||||
assert_eq!(filter.num_bits(), DEFAULT_FILTER_SIZE_BITS);
|
||||
assert_eq!(filter.hash_count(), DEFAULT_HASH_COUNT);
|
||||
assert_eq!(filter.count_ones(), 0);
|
||||
assert!(filter.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_insert_contains() {
|
||||
let mut filter = BloomFilter::new();
|
||||
let node1 = make_node_addr(1);
|
||||
let node2 = make_node_addr(2);
|
||||
|
||||
assert!(!filter.contains(&node1));
|
||||
assert!(!filter.contains(&node2));
|
||||
|
||||
filter.insert(&node1);
|
||||
|
||||
assert!(filter.contains(&node1));
|
||||
// node2 might have false positive, but very unlikely with single insert
|
||||
assert!(!filter.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_multiple_inserts() {
|
||||
let mut filter = BloomFilter::new();
|
||||
|
||||
for i in 0..100 {
|
||||
let node = make_node_addr(i);
|
||||
filter.insert(&node);
|
||||
}
|
||||
|
||||
// All inserted items should be found
|
||||
for i in 0..100 {
|
||||
let node = make_node_addr(i);
|
||||
assert!(filter.contains(&node), "Node {} not found", i);
|
||||
}
|
||||
|
||||
// Fill ratio should be reasonable
|
||||
let fill = filter.fill_ratio();
|
||||
assert!(fill > 0.0 && fill < 0.5, "Unexpected fill ratio: {}", fill);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_merge() {
|
||||
let mut filter1 = BloomFilter::new();
|
||||
let mut filter2 = BloomFilter::new();
|
||||
|
||||
let node1 = make_node_addr(1);
|
||||
let node2 = make_node_addr(2);
|
||||
|
||||
filter1.insert(&node1);
|
||||
filter2.insert(&node2);
|
||||
|
||||
filter1.merge(&filter2).unwrap();
|
||||
|
||||
assert!(filter1.contains(&node1));
|
||||
assert!(filter1.contains(&node2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_union() {
|
||||
let mut filter1 = BloomFilter::new();
|
||||
let mut filter2 = BloomFilter::new();
|
||||
|
||||
let node1 = make_node_addr(1);
|
||||
let node2 = make_node_addr(2);
|
||||
|
||||
filter1.insert(&node1);
|
||||
filter2.insert(&node2);
|
||||
|
||||
let union = filter1.union(&filter2).unwrap();
|
||||
|
||||
assert!(union.contains(&node1));
|
||||
assert!(union.contains(&node2));
|
||||
// Original filters unchanged
|
||||
assert!(!filter1.contains(&node2));
|
||||
assert!(!filter2.contains(&node1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_clear() {
|
||||
let mut filter = BloomFilter::new();
|
||||
let node = make_node_addr(1);
|
||||
|
||||
filter.insert(&node);
|
||||
assert!(!filter.is_empty());
|
||||
|
||||
filter.clear();
|
||||
assert!(filter.is_empty());
|
||||
assert_eq!(filter.count_ones(), 0);
|
||||
assert!(!filter.contains(&node));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_merge_size_mismatch() {
|
||||
let mut filter1 = BloomFilter::with_params(1024, 7).unwrap();
|
||||
let filter2 = BloomFilter::with_params(2048, 7).unwrap();
|
||||
|
||||
let result = filter1.merge(&filter2);
|
||||
assert!(matches!(result, Err(BloomError::InvalidSize { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_custom_params() {
|
||||
let filter = BloomFilter::with_params(1024, 5).unwrap();
|
||||
assert_eq!(filter.num_bits(), 1024);
|
||||
assert_eq!(filter.num_bytes(), 128);
|
||||
assert_eq!(filter.hash_count(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_invalid_params() {
|
||||
// Not byte-aligned (1001 is not divisible by 8)
|
||||
assert!(matches!(
|
||||
BloomFilter::with_params(1001, 7),
|
||||
Err(BloomError::SizeNotByteAligned(1001))
|
||||
));
|
||||
|
||||
// Zero size
|
||||
assert!(matches!(
|
||||
BloomFilter::with_params(0, 7),
|
||||
Err(BloomError::SizeNotByteAligned(0))
|
||||
));
|
||||
|
||||
// Zero hash count
|
||||
assert!(matches!(
|
||||
BloomFilter::with_params(1024, 0),
|
||||
Err(BloomError::ZeroHashCount)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_from_bytes() {
|
||||
let original = BloomFilter::new();
|
||||
let bytes = original.as_bytes().to_vec();
|
||||
|
||||
let restored = BloomFilter::from_bytes(bytes, original.hash_count()).unwrap();
|
||||
|
||||
assert_eq!(original, restored);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_estimated_count() {
|
||||
let mut filter = BloomFilter::new();
|
||||
|
||||
// Empty filter
|
||||
assert_eq!(filter.estimated_count(f64::INFINITY), Some(0.0));
|
||||
|
||||
// Insert some items
|
||||
for i in 0..50 {
|
||||
filter.insert(&make_node_addr(i));
|
||||
}
|
||||
|
||||
// Estimate should be reasonably close to 50
|
||||
let estimate = filter.estimated_count(f64::INFINITY).unwrap();
|
||||
assert!(
|
||||
estimate > 30.0 && estimate < 100.0,
|
||||
"Unexpected estimate: {}",
|
||||
estimate
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_equality() {
|
||||
let mut filter1 = BloomFilter::new();
|
||||
let mut filter2 = BloomFilter::new();
|
||||
|
||||
assert_eq!(filter1, filter2);
|
||||
|
||||
filter1.insert(&make_node_addr(1));
|
||||
assert_ne!(filter1, filter2);
|
||||
|
||||
filter2.insert(&make_node_addr(1));
|
||||
assert_eq!(filter1, filter2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_from_bytes_empty() {
|
||||
let result = BloomFilter::from_bytes(vec![], 5);
|
||||
assert!(matches!(result, Err(BloomError::SizeNotByteAligned(0))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_from_bytes_zero_hash_count() {
|
||||
let result = BloomFilter::from_bytes(vec![0u8; 128], 0);
|
||||
assert!(matches!(result, Err(BloomError::ZeroHashCount)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_from_slice() {
|
||||
let mut original = BloomFilter::new();
|
||||
original.insert(&make_node_addr(42));
|
||||
let bytes = original.as_bytes();
|
||||
|
||||
let restored = BloomFilter::from_slice(bytes, original.hash_count()).unwrap();
|
||||
assert_eq!(original, restored);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_insert_bytes_contains_bytes() {
|
||||
let mut filter = BloomFilter::new();
|
||||
let data1 = b"hello world";
|
||||
let data2 = b"goodbye";
|
||||
|
||||
assert!(!filter.contains_bytes(data1));
|
||||
|
||||
filter.insert_bytes(data1);
|
||||
assert!(filter.contains_bytes(data1));
|
||||
assert!(!filter.contains_bytes(data2));
|
||||
|
||||
filter.insert_bytes(data2);
|
||||
assert!(filter.contains_bytes(data1));
|
||||
assert!(filter.contains_bytes(data2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_estimated_count_saturated() {
|
||||
// Create a small filter with all bits set
|
||||
let bytes = vec![0xFF; 8]; // all bits set
|
||||
let filter = BloomFilter::from_bytes(bytes, 3).unwrap();
|
||||
|
||||
// Saturated filter returns None regardless of cap (defense in depth).
|
||||
// Previously returned f64::INFINITY.
|
||||
assert_eq!(filter.estimated_count(f64::INFINITY), None);
|
||||
assert_eq!(filter.estimated_count(0.05), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_estimated_count_fpr_cap_boundary() {
|
||||
// Cap boundary: FPR = fill^k = 0.05 at k=5 ⇒ fill ≈ 0.5493
|
||||
// 1KB filter (8192 bits). 560 bytes of 0xFF = 4480 bits set =
|
||||
// fill 0.5469, FPR ≈ 0.04877 — just below cap.
|
||||
// 564 bytes of 0xFF = 4512 bits set = fill 0.5508, FPR ≈ 0.05060 —
|
||||
// just above cap.
|
||||
|
||||
let mut below = vec![0x00u8; 1024];
|
||||
below[..560].fill(0xFF);
|
||||
let below_filter = BloomFilter::from_bytes(below, DEFAULT_HASH_COUNT).unwrap();
|
||||
assert!(
|
||||
below_filter.estimated_count(0.05).is_some(),
|
||||
"fill 0.5469 (FPR ≈ 0.049) must be accepted by cap 0.05"
|
||||
);
|
||||
|
||||
let mut above = vec![0x00u8; 1024];
|
||||
above[..564].fill(0xFF);
|
||||
let above_filter = BloomFilter::from_bytes(above, DEFAULT_HASH_COUNT).unwrap();
|
||||
assert_eq!(
|
||||
above_filter.estimated_count(0.05),
|
||||
None,
|
||||
"fill 0.5508 (FPR ≈ 0.051) must be rejected by cap 0.05"
|
||||
);
|
||||
|
||||
// Same above-cap filter with a looser cap is accepted.
|
||||
assert!(
|
||||
above_filter.estimated_count(0.10).is_some(),
|
||||
"fill 0.5508 (FPR ≈ 0.051) must be accepted by cap 0.10"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_default() {
|
||||
let default: BloomFilter = Default::default();
|
||||
let explicit = BloomFilter::new();
|
||||
assert_eq!(default, explicit);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_debug_format() {
|
||||
let mut filter = BloomFilter::new();
|
||||
let debug = format!("{:?}", filter);
|
||||
assert!(debug.contains("BloomFilter"));
|
||||
assert!(debug.contains("8192"));
|
||||
assert!(debug.contains("hash_count"));
|
||||
|
||||
// With some entries
|
||||
for i in 0..10 {
|
||||
filter.insert(&make_node_addr(i));
|
||||
}
|
||||
let debug = format!("{:?}", filter);
|
||||
assert!(debug.contains("fill_ratio"));
|
||||
assert!(debug.contains("est_count"));
|
||||
}
|
||||
|
||||
// ===== BloomState Tests =====
|
||||
|
||||
#[test]
|
||||
fn test_bloom_state_new() {
|
||||
@@ -133,7 +426,7 @@ fn test_bloom_state_compute_outgoing_filter() {
|
||||
let mut filter2 = BloomFilter::new();
|
||||
filter2.insert(&make_node_addr(200));
|
||||
|
||||
let mut peer_filters = BTreeMap::new();
|
||||
let mut peer_filters = HashMap::new();
|
||||
peer_filters.insert(peer1, filter1);
|
||||
peer_filters.insert(peer2, filter2);
|
||||
|
||||
@@ -184,7 +477,7 @@ fn test_bloom_state_record_sent_filter() {
|
||||
state.record_sent_filter(peer, filter);
|
||||
|
||||
// Compute what would be sent to peer (just our own node, no peer filters)
|
||||
let peer_filters = BTreeMap::new();
|
||||
let peer_filters = HashMap::new();
|
||||
let peer_addrs = vec![peer];
|
||||
state.mark_changed_peers(&make_node_addr(99), &peer_addrs, &peer_filters);
|
||||
|
||||
@@ -221,7 +514,7 @@ fn test_bloom_state_remove_peer_state() {
|
||||
|
||||
// Sent filter cleared — mark_changed_peers should treat as "never sent"
|
||||
state.clear_pending_updates();
|
||||
let peer_filters = BTreeMap::new();
|
||||
let peer_filters = HashMap::new();
|
||||
let peer_addrs = vec![peer];
|
||||
state.mark_changed_peers(&make_node_addr(99), &peer_addrs, &peer_filters);
|
||||
assert!(state.needs_update(&peer)); // never sent → must send
|
||||
@@ -235,7 +528,7 @@ fn test_bloom_state_mark_changed_peers_never_sent() {
|
||||
let peer1 = make_node_addr(1);
|
||||
let peer2 = make_node_addr(2);
|
||||
|
||||
let peer_filters = BTreeMap::new();
|
||||
let peer_filters = HashMap::new();
|
||||
let peer_addrs = vec![peer1, peer2];
|
||||
|
||||
// No filters ever sent — all peers should be marked
|
||||
@@ -252,7 +545,7 @@ fn test_bloom_state_mark_changed_peers_unchanged() {
|
||||
|
||||
let peer1 = make_node_addr(1);
|
||||
let peer2 = make_node_addr(2);
|
||||
let peer_filters = BTreeMap::new();
|
||||
let peer_filters = HashMap::new();
|
||||
let peer_addrs = vec![peer1, peer2];
|
||||
|
||||
// Compute and record what would be sent to each peer
|
||||
@@ -275,7 +568,7 @@ fn test_bloom_state_mark_changed_peers_one_changed() {
|
||||
|
||||
let peer1 = make_node_addr(1);
|
||||
let peer2 = make_node_addr(2);
|
||||
let peer_filters = BTreeMap::new();
|
||||
let peer_filters = HashMap::new();
|
||||
let peer_addrs = vec![peer1, peer2];
|
||||
|
||||
// Record current outgoing filters for both peers
|
||||
@@ -287,7 +580,7 @@ fn test_bloom_state_mark_changed_peers_one_changed() {
|
||||
// Now peer1 sends us a filter with new entries
|
||||
let mut inbound_from_peer1 = BloomFilter::new();
|
||||
inbound_from_peer1.insert(&make_node_addr(100));
|
||||
let mut updated_peer_filters = BTreeMap::new();
|
||||
let mut updated_peer_filters = HashMap::new();
|
||||
updated_peer_filters.insert(peer1, inbound_from_peer1);
|
||||
|
||||
// mark_changed_peers triggered by receiving from peer1
|
||||
@@ -305,7 +598,7 @@ fn test_bloom_state_mark_changed_peers_excludes_source() {
|
||||
let mut state = BloomState::new(node);
|
||||
|
||||
let peer1 = make_node_addr(1);
|
||||
let peer_filters = BTreeMap::new();
|
||||
let peer_filters = HashMap::new();
|
||||
let peer_addrs = vec![peer1];
|
||||
|
||||
// peer1 is both the source and the only peer — should be skipped
|
||||
Vendored
+1
-1
@@ -9,7 +9,7 @@ use std::collections::HashMap;
|
||||
use super::CacheStats;
|
||||
use super::entry::CacheEntry;
|
||||
use crate::NodeAddr;
|
||||
use crate::proto::stp::TreeCoordinate;
|
||||
use crate::tree::TreeCoordinate;
|
||||
|
||||
/// Default maximum entries in coordinate cache.
|
||||
pub const DEFAULT_COORD_CACHE_SIZE: usize = 50_000;
|
||||
|
||||
Vendored
+1
-1
@@ -1,6 +1,6 @@
|
||||
//! Cache entry with TTL and LRU tracking.
|
||||
|
||||
use crate::proto::stp::TreeCoordinate;
|
||||
use crate::tree::TreeCoordinate;
|
||||
|
||||
/// A cached coordinate entry.
|
||||
#[derive(Clone, Debug)]
|
||||
|
||||
+31
-317
@@ -24,7 +24,6 @@ mod node;
|
||||
mod peer;
|
||||
mod transport;
|
||||
|
||||
use crate::node::REKEY_JITTER_SECS;
|
||||
use crate::upper::config::{DnsConfig, TunConfig};
|
||||
use crate::{Identity, IdentityError};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -34,9 +33,9 @@ use thiserror::Error;
|
||||
#[cfg(target_os = "linux")]
|
||||
pub use gateway::{ConntrackConfig, GatewayConfig, GatewayDnsConfig, PortForward, Proto};
|
||||
pub use node::{
|
||||
BloomConfig, BuffersConfig, CacheConfig, ControlConfig, LimitsConfig, LookupConfig, MmpConfig,
|
||||
NodeConfig, NostrRendezvousConfig, NostrRendezvousPolicy, RateLimitConfig, RekeyConfig,
|
||||
RendezvousConfig, RetryConfig, SessionConfig, SessionMmpConfig, TreeConfig,
|
||||
BloomConfig, BuffersConfig, CacheConfig, ControlConfig, DiscoveryConfig, LimitsConfig,
|
||||
NodeConfig, NostrDiscoveryConfig, NostrDiscoveryPolicy, RateLimitConfig, RekeyConfig,
|
||||
RetryConfig, SessionConfig, SessionMmpConfig, TreeConfig,
|
||||
};
|
||||
pub use peer::{ConnectPolicy, PeerAddress, PeerConfig};
|
||||
pub use transport::{
|
||||
@@ -490,59 +489,10 @@ impl Config {
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let mut config: Config =
|
||||
serde_yaml::from_str(&contents).map_err(|e| ConfigError::ParseYaml {
|
||||
path: path.to_path_buf(),
|
||||
source: e,
|
||||
})?;
|
||||
config.normalize_deprecated_keys();
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// COMPAT (drop at the v2 cutover): fold a deprecated `node.discovery:`
|
||||
/// block into the `node.lookup.*` (mesh-lookup scalars) and
|
||||
/// `node.rendezvous.*` (nostr/LAN peer rendezvous) tables that replaced it.
|
||||
///
|
||||
/// Runs at every deserialize boundary (see `load_file`). A present legacy
|
||||
/// field fills the corresponding new-table field, so a config that predates
|
||||
/// the split keeps behaving identically. When a legacy block is seen, a
|
||||
/// one-time deprecation warning names the old→new key moves. Exposed to the
|
||||
/// crate so config tests that deserialize directly can invoke it.
|
||||
pub(crate) fn normalize_deprecated_keys(&mut self) {
|
||||
let Some(compat) = self.node.discovery.take() else {
|
||||
return;
|
||||
};
|
||||
tracing::warn!(
|
||||
target: "fips::config",
|
||||
"`node.discovery.*` is deprecated and will be removed: mesh-lookup \
|
||||
scalars moved to `node.lookup.*`, and peer-rendezvous keys moved to \
|
||||
`node.rendezvous.nostr.*` / `node.rendezvous.lan.*`. Please migrate; \
|
||||
a legacy `node.discovery` block still applies for now."
|
||||
);
|
||||
if let Some(v) = compat.ttl {
|
||||
self.node.lookup.ttl = v;
|
||||
}
|
||||
if let Some(v) = compat.attempt_timeouts_secs {
|
||||
self.node.lookup.attempt_timeouts_secs = v;
|
||||
}
|
||||
if let Some(v) = compat.recent_expiry_secs {
|
||||
self.node.lookup.recent_expiry_secs = v;
|
||||
}
|
||||
if let Some(v) = compat.backoff_base_secs {
|
||||
self.node.lookup.backoff_base_secs = v;
|
||||
}
|
||||
if let Some(v) = compat.backoff_max_secs {
|
||||
self.node.lookup.backoff_max_secs = v;
|
||||
}
|
||||
if let Some(v) = compat.forward_min_interval_secs {
|
||||
self.node.lookup.forward_min_interval_secs = v;
|
||||
}
|
||||
if let Some(v) = compat.nostr {
|
||||
self.node.rendezvous.nostr = v;
|
||||
}
|
||||
if let Some(v) = compat.lan {
|
||||
self.node.rendezvous.lan = v;
|
||||
}
|
||||
serde_yaml::from_str(&contents).map_err(|e| ConfigError::ParseYaml {
|
||||
path: path.to_path_buf(),
|
||||
source: e,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the standard search paths in priority order (lowest to highest).
|
||||
@@ -650,7 +600,7 @@ impl Config {
|
||||
|
||||
/// Validate cross-field configuration invariants.
|
||||
pub fn validate(&self) -> Result<(), ConfigError> {
|
||||
let nostr = &self.node.rendezvous.nostr;
|
||||
let nostr = &self.node.discovery.nostr;
|
||||
|
||||
let any_transport_advertises_on_nostr = self
|
||||
.transports
|
||||
@@ -670,13 +620,13 @@ impl Config {
|
||||
|
||||
if any_transport_advertises_on_nostr && !nostr.enabled {
|
||||
return Err(ConfigError::Validation(
|
||||
"at least one transport has `advertise_on_nostr = true`, but `node.rendezvous.nostr.enabled` is false".to_string(),
|
||||
"at least one transport has `advertise_on_nostr = true`, but `node.discovery.nostr.enabled` is false".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if self.peers.iter().any(|peer| peer.via_nostr) && !nostr.enabled {
|
||||
return Err(ConfigError::Validation(
|
||||
"at least one peer has `via_nostr = true`, but `node.rendezvous.nostr.enabled` is false".to_string(),
|
||||
"at least one peer has `via_nostr = true`, but `node.discovery.nostr.enabled` is false".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -698,12 +648,12 @@ impl Config {
|
||||
if nostr.enabled && has_nat_udp_advert {
|
||||
if nostr.dm_relays.is_empty() {
|
||||
return Err(ConfigError::Validation(
|
||||
"NAT UDP advert publishing requires `node.rendezvous.nostr.dm_relays` to be non-empty".to_string(),
|
||||
"NAT UDP advert publishing requires `node.discovery.nostr.dm_relays` to be non-empty".to_string(),
|
||||
));
|
||||
}
|
||||
if nostr.stun_servers.is_empty() {
|
||||
return Err(ConfigError::Validation(
|
||||
"NAT UDP advert publishing requires `node.rendezvous.nostr.stun_servers` to be non-empty".to_string(),
|
||||
"NAT UDP advert publishing requires `node.discovery.nostr.stun_servers` to be non-empty".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -736,32 +686,6 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
// Reject rekey triggers that fire immediately and forever. Both
|
||||
// arms are checked regardless of `node.rekey.enabled` so that
|
||||
// turning rekey on later cannot surface a config error at a
|
||||
// surprising moment. There is deliberately no upper bound:
|
||||
// u64::MAX is the idiom for disabling one arm of the trigger.
|
||||
let rekey = &self.node.rekey;
|
||||
|
||||
if rekey.after_messages == 0 {
|
||||
return Err(ConfigError::Validation(
|
||||
"`node.rekey.after_messages` must be at least 1; 0 fires the message-count trigger on every poll instead of disabling it. \
|
||||
Use a very large value to effectively disable the message-count trigger."
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let jitter_secs = REKEY_JITTER_SECS.unsigned_abs();
|
||||
if rekey.after_secs <= jitter_secs {
|
||||
return Err(ConfigError::Validation(format!(
|
||||
"`node.rekey.after_secs` is {}, but must be greater than the per-session rekey jitter of {jitter_secs}s; \
|
||||
each session offsets the interval by a random value in [-{jitter_secs}, +{jitter_secs}] seconds, so a smaller interval saturates to zero \
|
||||
and rekeys on sight for roughly half of sessions. \
|
||||
Use a very large value to effectively disable the timer trigger.",
|
||||
rekey.after_secs
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -797,84 +721,6 @@ node:
|
||||
assert!(config.has_identity());
|
||||
}
|
||||
|
||||
/// The fips.yaml shipped in the OpenWrt package must keep parsing as the
|
||||
/// config schema evolves. Both the 802.11s mesh backhaul entries
|
||||
/// (docs/how-to/set-up-80211s-mesh-backhaul.md) and the open-access SSID
|
||||
/// entries (docs/how-to/set-up-open-access-ssid.md) ship commented out —
|
||||
/// one per radio, so dual-band routers can run either on both bands — so
|
||||
/// a stock install that never creates fips-mesh*/fips-ap* logs no
|
||||
/// per-boot bind warning; `fips-mesh-setup`/`fips-ap-setup` uncomment the
|
||||
/// matching block when they create the interface. Verify both states
|
||||
/// parse: as shipped (both inactive), and after the uncomment the helpers
|
||||
/// perform.
|
||||
#[test]
|
||||
fn shipped_openwrt_config_parses() {
|
||||
let yaml = include_str!("../../packaging/openwrt-ipk/files/etc/fips/fips.yaml");
|
||||
|
||||
// As shipped: parses, and the mesh/ap entries are commented out (a
|
||||
// running daemon binds no fips-mesh*/fips-ap* transport, no warning).
|
||||
let config: Config = serde_yaml::from_str(yaml).expect("shipped OpenWrt fips.yaml");
|
||||
for name in ["mesh0", "mesh1", "ap0", "ap1"] {
|
||||
assert!(
|
||||
!config
|
||||
.transports
|
||||
.ethernet
|
||||
.iter()
|
||||
.any(|(n, _)| n == Some(name)),
|
||||
"{name} must ship commented out, not active, in fips.yaml"
|
||||
);
|
||||
}
|
||||
|
||||
// What `fips-mesh-setup`/`fips-ap-setup` produce: uncomment each
|
||||
// block, which must still parse into a transport bound to the right
|
||||
// netdev.
|
||||
let uncommented =
|
||||
uncomment_transport_blocks(&uncomment_transport_blocks(yaml, "mesh"), "ap");
|
||||
let config: Config = serde_yaml::from_str(&uncommented)
|
||||
.expect("fips.yaml with mesh and ap transports uncommented");
|
||||
for (name, interface) in [
|
||||
("mesh0", "fips-mesh0"),
|
||||
("mesh1", "fips-mesh1"),
|
||||
("ap0", "fips-ap0"),
|
||||
("ap1", "fips-ap1"),
|
||||
] {
|
||||
assert!(
|
||||
config
|
||||
.transports
|
||||
.ethernet
|
||||
.iter()
|
||||
.any(|(n, eth)| n == Some(name) && eth.interface == interface),
|
||||
"{name} entry missing after uncommenting shipped fips.yaml"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirror the setup helpers' block uncomment: strip the ` # ` prefix
|
||||
/// from each `# <prefix><N>:` header and its ` # ` continuation
|
||||
/// lines, leaving every other comment untouched.
|
||||
fn uncomment_transport_blocks(yaml: &str, prefix: &str) -> String {
|
||||
let header = format!(" # {prefix}");
|
||||
let mut out = String::new();
|
||||
let mut in_block = false;
|
||||
for line in yaml.lines() {
|
||||
let is_header = line
|
||||
.strip_prefix(&header)
|
||||
.and_then(|r| r.strip_suffix(':'))
|
||||
.is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit()));
|
||||
if is_header {
|
||||
in_block = true;
|
||||
out.push_str(&line.replacen(" # ", " ", 1));
|
||||
} else if in_block && line.starts_with(" # ") {
|
||||
out.push_str(&line.replacen(" # ", " ", 1));
|
||||
} else {
|
||||
in_block = false;
|
||||
out.push_str(line);
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_yaml_with_hex() {
|
||||
let yaml = r#"
|
||||
@@ -1415,9 +1261,7 @@ peers:
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_legacy_discovery_nostr_config_compat() {
|
||||
// COMPAT (drop at the v2 cutover): a deprecated `node.discovery.nostr`
|
||||
// block must fold into `node.rendezvous.nostr` via normalize.
|
||||
fn test_parse_nostr_discovery_config() {
|
||||
let yaml = r#"
|
||||
node:
|
||||
discovery:
|
||||
@@ -1441,27 +1285,26 @@ peers:
|
||||
- transport: udp
|
||||
addr: "nat"
|
||||
"#;
|
||||
let mut config: Config = serde_yaml::from_str(yaml).unwrap();
|
||||
config.normalize_deprecated_keys();
|
||||
assert!(config.node.rendezvous.nostr.enabled);
|
||||
assert!(!config.node.rendezvous.nostr.advertise);
|
||||
assert_eq!(config.node.rendezvous.nostr.app, "fips.nat.test.v1");
|
||||
assert_eq!(config.node.rendezvous.nostr.signal_ttl_secs, 45);
|
||||
let config: Config = serde_yaml::from_str(yaml).unwrap();
|
||||
assert!(config.node.discovery.nostr.enabled);
|
||||
assert!(!config.node.discovery.nostr.advertise);
|
||||
assert_eq!(config.node.discovery.nostr.app, "fips.nat.test.v1");
|
||||
assert_eq!(config.node.discovery.nostr.signal_ttl_secs, 45);
|
||||
assert_eq!(
|
||||
config.node.rendezvous.nostr.policy,
|
||||
NostrRendezvousPolicy::ConfiguredOnly
|
||||
config.node.discovery.nostr.policy,
|
||||
NostrDiscoveryPolicy::ConfiguredOnly
|
||||
);
|
||||
assert_eq!(config.node.rendezvous.nostr.open_discovery_max_pending, 12);
|
||||
assert_eq!(config.node.discovery.nostr.open_discovery_max_pending, 12);
|
||||
assert_eq!(
|
||||
config.node.rendezvous.nostr.advert_relays,
|
||||
config.node.discovery.nostr.advert_relays,
|
||||
vec!["wss://relay-a.example".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
config.node.rendezvous.nostr.dm_relays,
|
||||
config.node.discovery.nostr.dm_relays,
|
||||
vec!["wss://relay-b.example".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
config.node.rendezvous.nostr.stun_servers,
|
||||
config.node.discovery.nostr.stun_servers,
|
||||
vec!["stun:stun.example.org:3478".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -1471,55 +1314,6 @@ peers:
|
||||
assert!(config.peers[0].via_nostr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_lookup_and_rendezvous_new_keys() {
|
||||
// The post-split keys parse directly, with no deprecated block and no
|
||||
// normalize warning.
|
||||
let yaml = r#"
|
||||
node:
|
||||
lookup:
|
||||
ttl: 7
|
||||
attempt_timeouts_secs: [3, 6]
|
||||
forward_min_interval_secs: 9
|
||||
rendezvous:
|
||||
nostr:
|
||||
enabled: true
|
||||
app: "fips.new.keys.v1"
|
||||
"#;
|
||||
let mut config: Config = serde_yaml::from_str(yaml).unwrap();
|
||||
config.normalize_deprecated_keys();
|
||||
assert_eq!(config.node.lookup.ttl, 7);
|
||||
assert_eq!(config.node.lookup.attempt_timeouts_secs, vec![3, 6]);
|
||||
assert_eq!(config.node.lookup.forward_min_interval_secs, 9);
|
||||
// Unset scalar keeps its default.
|
||||
assert_eq!(config.node.lookup.recent_expiry_secs, 10);
|
||||
assert!(config.node.rendezvous.nostr.enabled);
|
||||
assert_eq!(config.node.rendezvous.nostr.app, "fips.new.keys.v1");
|
||||
assert!(config.node.discovery.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_legacy_discovery_lookup_scalars_compat() {
|
||||
// COMPAT (drop at the v2 cutover): legacy `node.discovery` mesh-lookup
|
||||
// scalars must fold into `node.lookup`; unset keys keep their defaults.
|
||||
let yaml = r#"
|
||||
node:
|
||||
discovery:
|
||||
ttl: 5
|
||||
backoff_base_secs: 4
|
||||
backoff_max_secs: 30
|
||||
"#;
|
||||
let mut config: Config = serde_yaml::from_str(yaml).unwrap();
|
||||
config.normalize_deprecated_keys();
|
||||
assert_eq!(config.node.lookup.ttl, 5);
|
||||
assert_eq!(config.node.lookup.backoff_base_secs, 4);
|
||||
assert_eq!(config.node.lookup.backoff_max_secs, 30);
|
||||
// Unset legacy scalar leaves the new-table default intact.
|
||||
assert_eq!(config.node.lookup.attempt_timeouts_secs, vec![1, 2, 4, 8]);
|
||||
// The compat block is consumed by normalize.
|
||||
assert!(config.node.discovery.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_transport_advert_requires_nostr_enabled() {
|
||||
let mut config = Config::default();
|
||||
@@ -1527,7 +1321,7 @@ node:
|
||||
advertise_on_nostr: Some(true),
|
||||
..Default::default()
|
||||
});
|
||||
config.node.rendezvous.nostr.enabled = false;
|
||||
config.node.discovery.nostr.enabled = false;
|
||||
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
assert!(err.to_string().contains("advertise_on_nostr"));
|
||||
@@ -1543,7 +1337,7 @@ node:
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
config.node.rendezvous.nostr.enabled = false;
|
||||
config.node.discovery.nostr.enabled = false;
|
||||
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
assert!(err.to_string().contains("via_nostr"));
|
||||
@@ -1564,7 +1358,7 @@ node:
|
||||
|
||||
// Empty addresses + via_nostr=true + nostr.enabled=true → ok.
|
||||
config.peers[0].via_nostr = true;
|
||||
config.node.rendezvous.nostr.enabled = true;
|
||||
config.node.discovery.nostr.enabled = true;
|
||||
config
|
||||
.validate()
|
||||
.expect("via_nostr should allow empty addresses");
|
||||
@@ -1573,8 +1367,8 @@ node:
|
||||
#[test]
|
||||
fn test_validate_nat_udp_advert_requires_relays_and_stun() {
|
||||
let mut config = Config::default();
|
||||
config.node.rendezvous.nostr.enabled = true;
|
||||
config.node.rendezvous.nostr.dm_relays.clear();
|
||||
config.node.discovery.nostr.enabled = true;
|
||||
config.node.discovery.nostr.dm_relays.clear();
|
||||
config.transports.udp = TransportInstances::Single(UdpConfig {
|
||||
advertise_on_nostr: Some(true),
|
||||
public: Some(false),
|
||||
@@ -1584,8 +1378,8 @@ node:
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
assert!(err.to_string().contains("dm_relays"));
|
||||
|
||||
config.node.rendezvous.nostr.dm_relays = vec!["wss://relay.example".to_string()];
|
||||
config.node.rendezvous.nostr.stun_servers.clear();
|
||||
config.node.discovery.nostr.dm_relays = vec!["wss://relay.example".to_string()];
|
||||
config.node.discovery.nostr.stun_servers.clear();
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
assert!(err.to_string().contains("stun_servers"));
|
||||
}
|
||||
@@ -1665,86 +1459,6 @@ node:
|
||||
.expect("outbound_only should be exempt from the loopback check");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_default_rekey_settings_ok() {
|
||||
Config::default()
|
||||
.validate()
|
||||
.expect("shipped default rekey settings must validate");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_after_messages_zero_rejected() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_messages = 0;
|
||||
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("after_messages"), "got: {msg}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_after_messages_one_accepted() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_messages = 1;
|
||||
|
||||
config
|
||||
.validate()
|
||||
.expect("after_messages = 1 rekeys every message, which is wasteful but well defined");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_after_secs_at_or_below_jitter_rejected() {
|
||||
let jitter = REKEY_JITTER_SECS.unsigned_abs();
|
||||
|
||||
for after_secs in [0, 1, jitter - 1, jitter] {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_secs = after_secs;
|
||||
|
||||
match config.validate() {
|
||||
Err(e) => assert!(e.to_string().contains("after_secs"), "got: {e}"),
|
||||
Ok(()) => panic!("after_secs = {after_secs} should be rejected"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_after_secs_just_above_jitter_accepted() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_secs = REKEY_JITTER_SECS.unsigned_abs() + 1;
|
||||
|
||||
config
|
||||
.validate()
|
||||
.expect("one second above the jitter bound leaves a non-zero effective interval");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_unbounded_values_accepted() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_secs = u64::MAX;
|
||||
config.node.rekey.after_messages = u64::MAX;
|
||||
|
||||
config
|
||||
.validate()
|
||||
.expect("u64::MAX disables an arm of the trigger and must stay legal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_checked_even_when_disabled() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.enabled = false;
|
||||
config.node.rekey.after_messages = 0;
|
||||
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
assert!(err.to_string().contains("after_messages"));
|
||||
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.enabled = false;
|
||||
config.node.rekey.after_secs = REKEY_JITTER_SECS.unsigned_abs();
|
||||
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
assert!(err.to_string().contains("after_secs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_outbound_only_forces_ephemeral_bind() {
|
||||
let cfg = UdpConfig {
|
||||
|
||||
+80
-226
@@ -7,7 +7,7 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::IdentityConfig;
|
||||
use crate::proto::mmp::{DEFAULT_LOG_INTERVAL_SECS, DEFAULT_OWD_WINDOW_SIZE, MmpMode};
|
||||
use crate::mmp::{DEFAULT_LOG_INTERVAL_SECS, DEFAULT_OWD_WINDOW_SIZE, MmpConfig, MmpMode};
|
||||
|
||||
// ============================================================================
|
||||
// Node Configuration Subsections
|
||||
@@ -186,42 +186,48 @@ impl CacheConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Mesh-lookup protocol (`node.lookup.*`): the overlay coordinate-lookup
|
||||
/// engine (address → coordinates). The peer-rendezvous keys that used to
|
||||
/// share this table (`nostr`/`lan`) now live under [`RendezvousConfig`]
|
||||
/// (`node.rendezvous.*`).
|
||||
/// Discovery protocol (`node.discovery.*`).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LookupConfig {
|
||||
/// Hop limit for LookupRequest flood (`node.lookup.ttl`).
|
||||
#[serde(default = "LookupConfig::default_ttl")]
|
||||
pub struct DiscoveryConfig {
|
||||
/// Hop limit for LookupRequest flood (`node.discovery.ttl`).
|
||||
#[serde(default = "DiscoveryConfig::default_ttl")]
|
||||
pub ttl: u8,
|
||||
/// Per-attempt timeouts in seconds (`node.lookup.attempt_timeouts_secs`).
|
||||
/// Per-attempt timeouts in seconds (`node.discovery.attempt_timeouts_secs`).
|
||||
/// Each entry is the time to wait for a response before sending the next
|
||||
/// LookupRequest (with a fresh request_id). Sequence length determines the
|
||||
/// total number of attempts before declaring the destination unreachable.
|
||||
/// Default `[1, 2, 4, 8]` gives 4 attempts and a 15s total budget.
|
||||
#[serde(default = "LookupConfig::default_attempt_timeouts_secs")]
|
||||
#[serde(default = "DiscoveryConfig::default_attempt_timeouts_secs")]
|
||||
pub attempt_timeouts_secs: Vec<u64>,
|
||||
/// Dedup cache expiry in seconds (`node.lookup.recent_expiry_secs`).
|
||||
#[serde(default = "LookupConfig::default_recent_expiry_secs")]
|
||||
/// Dedup cache expiry in seconds (`node.discovery.recent_expiry_secs`).
|
||||
#[serde(default = "DiscoveryConfig::default_recent_expiry_secs")]
|
||||
pub recent_expiry_secs: u64,
|
||||
/// Base backoff after lookup failure in seconds (`node.lookup.backoff_base_secs`).
|
||||
/// Base backoff after lookup failure in seconds (`node.discovery.backoff_base_secs`).
|
||||
/// Doubles per consecutive failure up to `backoff_max_secs`. Defaults to 0
|
||||
/// (no post-failure suppression); the per-attempt sequence in
|
||||
/// `attempt_timeouts_secs` provides the only retry pacing.
|
||||
#[serde(default = "LookupConfig::default_backoff_base_secs")]
|
||||
#[serde(default = "DiscoveryConfig::default_backoff_base_secs")]
|
||||
pub backoff_base_secs: u64,
|
||||
/// Maximum backoff cap in seconds (`node.lookup.backoff_max_secs`).
|
||||
#[serde(default = "LookupConfig::default_backoff_max_secs")]
|
||||
/// Maximum backoff cap in seconds (`node.discovery.backoff_max_secs`).
|
||||
#[serde(default = "DiscoveryConfig::default_backoff_max_secs")]
|
||||
pub backoff_max_secs: u64,
|
||||
/// Minimum interval between forwarded lookups for the same target in seconds
|
||||
/// (`node.lookup.forward_min_interval_secs`).
|
||||
/// (`node.discovery.forward_min_interval_secs`).
|
||||
/// Defense-in-depth against misbehaving nodes.
|
||||
#[serde(default = "LookupConfig::default_forward_min_interval_secs")]
|
||||
#[serde(default = "DiscoveryConfig::default_forward_min_interval_secs")]
|
||||
pub forward_min_interval_secs: u64,
|
||||
/// Nostr-mediated overlay endpoint discovery.
|
||||
#[serde(default = "DiscoveryConfig::default_nostr")]
|
||||
pub nostr: NostrDiscoveryConfig,
|
||||
/// mDNS / DNS-SD peer discovery on the local link. Identity surface
|
||||
/// is a strict subset of what `nostr.advertise` already publishes
|
||||
/// publicly, so there's no marginal privacy cost; the latency win
|
||||
/// for same-LAN peers is large (sub-second pairing, no relay).
|
||||
#[serde(default = "DiscoveryConfig::default_lan")]
|
||||
pub lan: crate::discovery::lan::LanDiscoveryConfig,
|
||||
}
|
||||
|
||||
impl Default for LookupConfig {
|
||||
impl Default for DiscoveryConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
ttl: 64,
|
||||
@@ -230,11 +236,13 @@ impl Default for LookupConfig {
|
||||
backoff_base_secs: 0,
|
||||
backoff_max_secs: 0,
|
||||
forward_min_interval_secs: 2,
|
||||
nostr: NostrDiscoveryConfig::default(),
|
||||
lan: crate::discovery::lan::LanDiscoveryConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LookupConfig {
|
||||
impl DiscoveryConfig {
|
||||
fn default_ttl() -> u8 {
|
||||
64
|
||||
}
|
||||
@@ -253,45 +261,12 @@ impl LookupConfig {
|
||||
fn default_forward_min_interval_secs() -> u64 {
|
||||
2
|
||||
}
|
||||
}
|
||||
|
||||
/// Peer rendezvous (`node.rendezvous.*`): how the node finds peers to connect
|
||||
/// to at all — Nostr-mediated overlay endpoints and mDNS/DNS-SD on the local
|
||||
/// link. Distinct from mesh lookup ([`LookupConfig`]), which finds coordinates
|
||||
/// for an already-known mesh address.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct RendezvousConfig {
|
||||
/// Nostr-mediated overlay endpoint rendezvous (`node.rendezvous.nostr.*`).
|
||||
#[serde(default)]
|
||||
pub nostr: NostrRendezvousConfig,
|
||||
/// mDNS / DNS-SD peer rendezvous on the local link (`node.rendezvous.lan.*`).
|
||||
/// Identity surface is a strict subset of what `nostr.advertise` already
|
||||
/// publishes publicly, so there's no marginal privacy cost; the latency
|
||||
/// win for same-LAN peers is large (sub-second pairing, no relay).
|
||||
#[serde(default)]
|
||||
pub lan: crate::mdns::LanRendezvousConfig,
|
||||
}
|
||||
|
||||
/// COMPAT (drop at the v2 cutover): a deprecated legacy `node.discovery:` block.
|
||||
///
|
||||
/// The `node.discovery.*` table was split into `node.lookup.*` (mesh-lookup
|
||||
/// scalars) and `node.rendezvous.*` (nostr/LAN peer rendezvous). Because
|
||||
/// `NodeConfig` does not deny unknown fields, a still-deployed `node.discovery:`
|
||||
/// block would otherwise deserialize into nothing and silently revert every
|
||||
/// lookup/rendezvous setting to its default. This all-`Option` mirror captures
|
||||
/// it so [`Config::normalize_deprecated_keys`] can fold it into the new tables
|
||||
/// with a one-time deprecation warning; unset legacy keys stay `None` and leave
|
||||
/// the new-table defaults intact.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub(crate) struct DiscoveryConfigCompat {
|
||||
pub ttl: Option<u8>,
|
||||
pub attempt_timeouts_secs: Option<Vec<u64>>,
|
||||
pub recent_expiry_secs: Option<u64>,
|
||||
pub backoff_base_secs: Option<u64>,
|
||||
pub backoff_max_secs: Option<u64>,
|
||||
pub forward_min_interval_secs: Option<u64>,
|
||||
pub nostr: Option<NostrRendezvousConfig>,
|
||||
pub lan: Option<crate::mdns::LanRendezvousConfig>,
|
||||
fn default_nostr() -> NostrDiscoveryConfig {
|
||||
NostrDiscoveryConfig::default()
|
||||
}
|
||||
fn default_lan() -> crate::discovery::lan::LanDiscoveryConfig {
|
||||
crate::discovery::lan::LanDiscoveryConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Nostr advert discovery policy.
|
||||
@@ -303,33 +278,33 @@ pub(crate) struct DiscoveryConfigCompat {
|
||||
/// - `open`: also consider adverts for non-configured peers
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NostrRendezvousPolicy {
|
||||
pub enum NostrDiscoveryPolicy {
|
||||
Disabled,
|
||||
#[default]
|
||||
ConfiguredOnly,
|
||||
Open,
|
||||
}
|
||||
|
||||
/// Nostr-mediated overlay endpoint discovery (`node.rendezvous.nostr.*`).
|
||||
/// Nostr-mediated overlay endpoint discovery (`node.discovery.nostr.*`).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct NostrRendezvousConfig {
|
||||
pub struct NostrDiscoveryConfig {
|
||||
/// Enable Nostr-signaled traversal bootstrap.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Publish service advertisements so remote peers can bootstrap inbound.
|
||||
#[serde(default = "NostrRendezvousConfig::default_advertise")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_advertise")]
|
||||
pub advertise: bool,
|
||||
/// Relay URLs used for service advertisements.
|
||||
#[serde(default = "NostrRendezvousConfig::default_advert_relays")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_advert_relays")]
|
||||
pub advert_relays: Vec<String>,
|
||||
/// Relay URLs used for encrypted signaling events.
|
||||
#[serde(default = "NostrRendezvousConfig::default_dm_relays")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_dm_relays")]
|
||||
pub dm_relays: Vec<String>,
|
||||
/// STUN servers used for local reflexive address discovery.
|
||||
/// Outbound observation uses only this local list; peer-advertised STUN
|
||||
/// values are informational and are not treated as egress targets.
|
||||
#[serde(default = "NostrRendezvousConfig::default_stun_servers")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_stun_servers")]
|
||||
pub stun_servers: Vec<String>,
|
||||
/// Whether to advertise local (RFC 1918 / ULA) interface addresses as
|
||||
/// host candidates in the traversal offer.
|
||||
@@ -343,85 +318,85 @@ pub struct NostrRendezvousConfig {
|
||||
#[serde(default)]
|
||||
pub share_local_candidates: bool,
|
||||
/// Traversal application namespace and advert identifier suffix.
|
||||
#[serde(default = "NostrRendezvousConfig::default_app")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_app")]
|
||||
pub app: String,
|
||||
/// Signaling TTL in seconds.
|
||||
#[serde(default = "NostrRendezvousConfig::default_signal_ttl_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_signal_ttl_secs")]
|
||||
pub signal_ttl_secs: u64,
|
||||
/// Policy for advert-derived endpoint discovery.
|
||||
#[serde(default)]
|
||||
pub policy: NostrRendezvousPolicy,
|
||||
pub policy: NostrDiscoveryPolicy,
|
||||
/// Max number of open-discovery peers queued for outbound retry/connection
|
||||
/// at once. Prevents unbounded queue growth from ambient advert traffic.
|
||||
#[serde(default = "NostrRendezvousConfig::default_open_discovery_max_pending")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_open_discovery_max_pending")]
|
||||
pub open_discovery_max_pending: usize,
|
||||
/// Max concurrent inbound traversal offers processed at once.
|
||||
/// Acts as a rate limit against offer spam from relays.
|
||||
#[serde(default = "NostrRendezvousConfig::default_max_concurrent_incoming_offers")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_max_concurrent_incoming_offers")]
|
||||
pub max_concurrent_incoming_offers: usize,
|
||||
/// Max cached overlay adverts retained from relay traffic.
|
||||
/// Bounds memory under ambient advert volume.
|
||||
#[serde(default = "NostrRendezvousConfig::default_advert_cache_max_entries")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_advert_cache_max_entries")]
|
||||
pub advert_cache_max_entries: usize,
|
||||
/// Max seen-session IDs retained for replay detection.
|
||||
/// Oldest entries are evicted when the cap is exceeded.
|
||||
#[serde(default = "NostrRendezvousConfig::default_seen_sessions_max_entries")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_seen_sessions_max_entries")]
|
||||
pub seen_sessions_max_entries: usize,
|
||||
/// Overall punch attempt timeout in seconds.
|
||||
#[serde(default = "NostrRendezvousConfig::default_attempt_timeout_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_attempt_timeout_secs")]
|
||||
pub attempt_timeout_secs: u64,
|
||||
/// Replay tracking retention window in seconds.
|
||||
#[serde(default = "NostrRendezvousConfig::default_replay_window_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_replay_window_secs")]
|
||||
pub replay_window_secs: u64,
|
||||
/// Delay before punch traffic starts.
|
||||
#[serde(default = "NostrRendezvousConfig::default_punch_start_delay_ms")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_punch_start_delay_ms")]
|
||||
pub punch_start_delay_ms: u64,
|
||||
/// Interval between punch packets.
|
||||
#[serde(default = "NostrRendezvousConfig::default_punch_interval_ms")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_punch_interval_ms")]
|
||||
pub punch_interval_ms: u64,
|
||||
/// How long to keep punching before failure.
|
||||
#[serde(default = "NostrRendezvousConfig::default_punch_duration_ms")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_punch_duration_ms")]
|
||||
pub punch_duration_ms: u64,
|
||||
/// Advert TTL in seconds.
|
||||
#[serde(default = "NostrRendezvousConfig::default_advert_ttl_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_advert_ttl_secs")]
|
||||
pub advert_ttl_secs: u64,
|
||||
/// How often adverts are refreshed in seconds.
|
||||
#[serde(default = "NostrRendezvousConfig::default_advert_refresh_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_advert_refresh_secs")]
|
||||
pub advert_refresh_secs: u64,
|
||||
/// Settle delay in seconds after Nostr discovery starts before the
|
||||
/// one-shot startup sweep of cached adverts runs. Allows the relay
|
||||
/// subscription backlog to populate the in-memory advert cache.
|
||||
/// Only used under `policy: open`. Default: 5.
|
||||
#[serde(default = "NostrRendezvousConfig::default_startup_sweep_delay_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_startup_sweep_delay_secs")]
|
||||
pub startup_sweep_delay_secs: u64,
|
||||
/// Maximum age in seconds for cached adverts considered by the
|
||||
/// one-shot startup sweep. Adverts whose `created_at` is older than
|
||||
/// `now - startup_sweep_max_age_secs` are skipped. Only used under
|
||||
/// `policy: open`. Default: 3600 (1 hour).
|
||||
#[serde(default = "NostrRendezvousConfig::default_startup_sweep_max_age_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_startup_sweep_max_age_secs")]
|
||||
pub startup_sweep_max_age_secs: u64,
|
||||
/// Number of consecutive NAT-traversal failures against a peer before
|
||||
/// an extended cooldown is applied to throttle further offer publishes.
|
||||
/// At this threshold the daemon also actively re-fetches the peer's
|
||||
/// advert from `advert_relays` to evict cache entries for peers that
|
||||
/// have gone away. Default: 5.
|
||||
#[serde(default = "NostrRendezvousConfig::default_failure_streak_threshold")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_failure_streak_threshold")]
|
||||
pub failure_streak_threshold: u32,
|
||||
/// Cooldown applied to a peer once `failure_streak_threshold` is hit.
|
||||
/// Suppresses both open-discovery sweep enqueues and per-attempt
|
||||
/// retry firings until elapsed. Default: 1800 (30 minutes).
|
||||
#[serde(default = "NostrRendezvousConfig::default_extended_cooldown_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_extended_cooldown_secs")]
|
||||
pub extended_cooldown_secs: u64,
|
||||
/// Minimum interval between `NAT traversal failed` WARN log lines for
|
||||
/// the same peer. Subsequent failures inside the window log at DEBUG.
|
||||
/// Reduces log spam on public-test nodes with many cache-learned
|
||||
/// peers. Default: 300 (5 minutes).
|
||||
#[serde(default = "NostrRendezvousConfig::default_warn_log_interval_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_warn_log_interval_secs")]
|
||||
pub warn_log_interval_secs: u64,
|
||||
/// Maximum entries retained in the per-npub failure-state map.
|
||||
/// Bounds memory under high cache turnover. Oldest entries (by last
|
||||
/// failure time) evicted when the cap is exceeded. Default: 4096.
|
||||
#[serde(default = "NostrRendezvousConfig::default_failure_state_max_entries")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_failure_state_max_entries")]
|
||||
pub failure_state_max_entries: usize,
|
||||
/// Cooldown applied after observing a fatal protocol mismatch on a
|
||||
/// Nostr-adopted bootstrap transport (e.g. `Unknown FMP version`
|
||||
@@ -429,11 +404,11 @@ pub struct NostrRendezvousConfig {
|
||||
/// of `extended_cooldown_secs` and much longer because the mismatch
|
||||
/// is structural — re-traversing the peer is wasted effort until one
|
||||
/// side upgrades. Default: 86400 (24 hours).
|
||||
#[serde(default = "NostrRendezvousConfig::default_protocol_mismatch_cooldown_secs")]
|
||||
#[serde(default = "NostrDiscoveryConfig::default_protocol_mismatch_cooldown_secs")]
|
||||
pub protocol_mismatch_cooldown_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for NostrRendezvousConfig {
|
||||
impl Default for NostrDiscoveryConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
@@ -444,7 +419,7 @@ impl Default for NostrRendezvousConfig {
|
||||
share_local_candidates: false,
|
||||
app: Self::default_app(),
|
||||
signal_ttl_secs: Self::default_signal_ttl_secs(),
|
||||
policy: NostrRendezvousPolicy::default(),
|
||||
policy: NostrDiscoveryPolicy::default(),
|
||||
open_discovery_max_pending: Self::default_open_discovery_max_pending(),
|
||||
max_concurrent_incoming_offers: Self::default_max_concurrent_incoming_offers(),
|
||||
advert_cache_max_entries: Self::default_advert_cache_max_entries(),
|
||||
@@ -467,7 +442,7 @@ impl Default for NostrRendezvousConfig {
|
||||
}
|
||||
}
|
||||
|
||||
impl NostrRendezvousConfig {
|
||||
impl NostrDiscoveryConfig {
|
||||
fn default_advertise() -> bool {
|
||||
true
|
||||
}
|
||||
@@ -660,8 +635,8 @@ pub struct BloomConfig {
|
||||
pub update_debounce_ms: u64,
|
||||
/// Antipoison cap: reject inbound FilterAnnounce whose FPR exceeds
|
||||
/// this value (`node.bloom.max_inbound_fpr`). Valid range `(0.0, 1.0)`.
|
||||
/// Default `0.20` ≈ fill 0.7248 at k=5 ≈ ~2,114 entries on the 1 KB
|
||||
/// filter (Swamidass–Baldi). Raised from 0.10 so aggregates that are
|
||||
/// Default `0.10` ≈ fill 0.631 at k=5 ≈ ~1,630 entries on the 1 KB
|
||||
/// filter (Swamidass–Baldi). Raised from 0.05 so aggregates that are
|
||||
/// legitimately near their operating ceiling are not rejected before
|
||||
/// the network reaches the fixed-filter capacity limit; conceptually
|
||||
/// distinct from future autoscaling hysteresis setpoints — same unit,
|
||||
@@ -673,8 +648,8 @@ pub struct BloomConfig {
|
||||
impl Default for BloomConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
update_debounce_ms: Self::default_update_debounce_ms(),
|
||||
max_inbound_fpr: Self::default_max_inbound_fpr(),
|
||||
update_debounce_ms: 500,
|
||||
max_inbound_fpr: 0.10,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -684,7 +659,7 @@ impl BloomConfig {
|
||||
500
|
||||
}
|
||||
fn default_max_inbound_fpr() -> f64 {
|
||||
0.20
|
||||
0.10
|
||||
}
|
||||
}
|
||||
|
||||
@@ -751,41 +726,6 @@ impl SessionConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// MMP configuration (`node.mmp.*`).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MmpConfig {
|
||||
/// Operating mode (`node.mmp.mode`).
|
||||
#[serde(default)]
|
||||
pub mode: MmpMode,
|
||||
|
||||
/// Periodic operator log interval in seconds (`node.mmp.log_interval_secs`).
|
||||
#[serde(default = "MmpConfig::default_log_interval_secs")]
|
||||
pub log_interval_secs: u64,
|
||||
|
||||
/// OWD trend ring buffer size (`node.mmp.owd_window_size`).
|
||||
#[serde(default = "MmpConfig::default_owd_window_size")]
|
||||
pub owd_window_size: usize,
|
||||
}
|
||||
|
||||
impl Default for MmpConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
mode: MmpMode::default(),
|
||||
log_interval_secs: DEFAULT_LOG_INTERVAL_SECS,
|
||||
owd_window_size: DEFAULT_OWD_WINDOW_SIZE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MmpConfig {
|
||||
fn default_log_interval_secs() -> u64 {
|
||||
DEFAULT_LOG_INTERVAL_SECS
|
||||
}
|
||||
fn default_owd_window_size() -> usize {
|
||||
DEFAULT_OWD_WINDOW_SIZE
|
||||
}
|
||||
}
|
||||
|
||||
/// Session-layer Metrics Measurement Protocol (`node.session_mmp.*`).
|
||||
///
|
||||
/// Separate from link-layer `node.mmp.*` to allow independent mode/interval
|
||||
@@ -1030,19 +970,6 @@ pub struct NodeConfig {
|
||||
#[serde(default = "NodeConfig::default_link_dead_timeout_secs")]
|
||||
pub link_dead_timeout_secs: u64,
|
||||
|
||||
/// Graceful-shutdown drain deadline in seconds (`node.drain_timeout_secs`).
|
||||
/// The bounded `Draining` phase broadcasts a shutdown `Disconnect` and then
|
||||
/// waits up to this long for peers to clear before tearing down, early-
|
||||
/// exiting as soon as all peers are gone. `None` selects the 2-second
|
||||
/// default (see [`NodeConfig::drain_timeout`]).
|
||||
///
|
||||
/// Kept `Option` deliberately: `NodeConfig` has no `deny_unknown_fields`, so
|
||||
/// a naive non-`Option` add with a `default` fn would silently rewrite the
|
||||
/// value into deployed configs on the next serialize. The `Option` +
|
||||
/// `skip_serializing_if` keeps absent configs absent.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub drain_timeout_secs: Option<u64>,
|
||||
|
||||
/// Resource limits (`node.limits.*`).
|
||||
#[serde(default)]
|
||||
pub limits: LimitsConfig,
|
||||
@@ -1059,19 +986,9 @@ pub struct NodeConfig {
|
||||
#[serde(default)]
|
||||
pub cache: CacheConfig,
|
||||
|
||||
/// Mesh-lookup protocol (`node.lookup.*`).
|
||||
/// Discovery protocol (`node.discovery.*`).
|
||||
#[serde(default)]
|
||||
pub lookup: LookupConfig,
|
||||
|
||||
/// Peer rendezvous (`node.rendezvous.*`).
|
||||
#[serde(default)]
|
||||
pub rendezvous: RendezvousConfig,
|
||||
|
||||
/// COMPAT (drop at the v2 cutover): a deprecated legacy `node.discovery:`
|
||||
/// block, folded into `lookup`/`rendezvous` by
|
||||
/// [`Config::normalize_deprecated_keys`]. Never re-serialized.
|
||||
#[serde(default, skip_serializing)]
|
||||
pub(crate) discovery: Option<DiscoveryConfigCompat>,
|
||||
pub discovery: DiscoveryConfig,
|
||||
|
||||
/// Spanning tree (`node.tree.*`).
|
||||
#[serde(default)]
|
||||
@@ -1124,14 +1041,11 @@ impl Default for NodeConfig {
|
||||
base_rtt_ms: 100,
|
||||
heartbeat_interval_secs: 10,
|
||||
link_dead_timeout_secs: 30,
|
||||
drain_timeout_secs: None,
|
||||
limits: LimitsConfig::default(),
|
||||
rate_limit: RateLimitConfig::default(),
|
||||
retry: RetryConfig::default(),
|
||||
cache: CacheConfig::default(),
|
||||
lookup: LookupConfig::default(),
|
||||
rendezvous: RendezvousConfig::default(),
|
||||
discovery: None,
|
||||
discovery: DiscoveryConfig::default(),
|
||||
tree: TreeConfig::default(),
|
||||
bloom: BloomConfig::default(),
|
||||
session: SessionConfig::default(),
|
||||
@@ -1175,72 +1089,12 @@ impl NodeConfig {
|
||||
fn default_link_dead_timeout_secs() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
/// Graceful-shutdown drain deadline as a `Duration`.
|
||||
///
|
||||
/// Returns the configured `drain_timeout_secs`, or the 2-second default
|
||||
/// when unset. Used by the daemon's bounded `Draining` phase.
|
||||
pub fn drain_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(self.drain_timeout_secs.unwrap_or(2))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_config_default() {
|
||||
let config = MmpConfig::default();
|
||||
assert_eq!(config.mode, MmpMode::Full);
|
||||
assert_eq!(config.log_interval_secs, 30);
|
||||
assert_eq!(config.owd_window_size, 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_yaml_parse() {
|
||||
let yaml = r#"
|
||||
mode: lightweight
|
||||
log_interval_secs: 60
|
||||
owd_window_size: 48
|
||||
"#;
|
||||
let config: MmpConfig = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(config.mode, MmpMode::Lightweight);
|
||||
assert_eq!(config.log_interval_secs, 60);
|
||||
assert_eq!(config.owd_window_size, 48);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_yaml_partial() {
|
||||
let yaml = "mode: minimal";
|
||||
let config: MmpConfig = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(config.mode, MmpMode::Minimal);
|
||||
assert_eq!(config.log_interval_secs, DEFAULT_LOG_INTERVAL_SECS);
|
||||
assert_eq!(config.owd_window_size, DEFAULT_OWD_WINDOW_SIZE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_drain_timeout_default_and_override() {
|
||||
// Unset → the 2-second default.
|
||||
let c = NodeConfig::default();
|
||||
assert_eq!(c.drain_timeout_secs, None);
|
||||
assert_eq!(c.drain_timeout(), std::time::Duration::from_secs(2));
|
||||
|
||||
// Explicit override is honored.
|
||||
let c2 = NodeConfig {
|
||||
drain_timeout_secs: Some(10),
|
||||
..NodeConfig::default()
|
||||
};
|
||||
assert_eq!(c2.drain_timeout(), std::time::Duration::from_secs(10));
|
||||
|
||||
// A zero override is a valid (immediate) drain, not the default.
|
||||
let c3 = NodeConfig {
|
||||
drain_timeout_secs: Some(0),
|
||||
..NodeConfig::default()
|
||||
};
|
||||
assert_eq!(c3.drain_timeout(), std::time::Duration::from_secs(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ecn_config_defaults() {
|
||||
let c = EcnConfig::default();
|
||||
@@ -1269,27 +1123,27 @@ owd_window_size: 48
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nostr_rendezvous_startup_sweep_defaults() {
|
||||
let c = NostrRendezvousConfig::default();
|
||||
fn test_nostr_discovery_startup_sweep_defaults() {
|
||||
let c = NostrDiscoveryConfig::default();
|
||||
assert_eq!(c.startup_sweep_delay_secs, 5);
|
||||
assert_eq!(c.startup_sweep_max_age_secs, 3_600);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nostr_rendezvous_startup_sweep_yaml_override() {
|
||||
fn test_nostr_discovery_startup_sweep_yaml_override() {
|
||||
let yaml = "enabled: true\npolicy: open\nstartup_sweep_delay_secs: 10\nstartup_sweep_max_age_secs: 1800\n";
|
||||
let c: NostrRendezvousConfig = serde_yaml::from_str(yaml).unwrap();
|
||||
let c: NostrDiscoveryConfig = serde_yaml::from_str(yaml).unwrap();
|
||||
assert!(c.enabled);
|
||||
assert_eq!(c.policy, NostrRendezvousPolicy::Open);
|
||||
assert_eq!(c.policy, NostrDiscoveryPolicy::Open);
|
||||
assert_eq!(c.startup_sweep_delay_secs, 10);
|
||||
assert_eq!(c.startup_sweep_max_age_secs, 1_800);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nostr_rendezvous_startup_sweep_partial_yaml_uses_defaults() {
|
||||
fn test_nostr_discovery_startup_sweep_partial_yaml_uses_defaults() {
|
||||
// Only override delay; max_age should fall back to default.
|
||||
let yaml = "enabled: true\nstartup_sweep_delay_secs: 30\n";
|
||||
let c: NostrRendezvousConfig = serde_yaml::from_str(yaml).unwrap();
|
||||
let c: NostrDiscoveryConfig = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(c.startup_sweep_delay_secs, 30);
|
||||
assert_eq!(c.startup_sweep_max_age_secs, 3_600);
|
||||
}
|
||||
|
||||
+6
-25
@@ -282,10 +282,9 @@ pub struct EthernetConfig {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub send_buf_size: Option<usize>,
|
||||
|
||||
/// Listen for neighbor beacons from other nodes. Default: true.
|
||||
/// (Renamed from `discovery`; the old key is still accepted.)
|
||||
#[serde(default, alias = "discovery", skip_serializing_if = "Option::is_none")]
|
||||
pub listen: Option<bool>,
|
||||
/// Listen for discovery beacons from other nodes. Default: true.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub discovery: Option<bool>,
|
||||
|
||||
/// Broadcast announcement beacons on the LAN. Default: false.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -320,9 +319,9 @@ impl EthernetConfig {
|
||||
self.send_buf_size.unwrap_or(DEFAULT_ETHERNET_SEND_BUF)
|
||||
}
|
||||
|
||||
/// Whether to listen for neighbor beacons. Default: true.
|
||||
pub fn listen(&self) -> bool {
|
||||
self.listen.unwrap_or(true)
|
||||
/// Whether to listen for discovery beacons. Default: true.
|
||||
pub fn discovery(&self) -> bool {
|
||||
self.discovery.unwrap_or(true)
|
||||
}
|
||||
|
||||
/// Whether to broadcast announcement beacons. Default: false.
|
||||
@@ -1047,22 +1046,4 @@ mod tests {
|
||||
assert_eq!(parse_bind_port("[::]:443"), Some(443));
|
||||
assert_eq!(parse_bind_port("not-a-socket-addr"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ethernet_listen_accepts_legacy_discovery_alias_and_rejects_unknown() {
|
||||
// (a) The legacy `discovery:` key is still accepted via serde alias.
|
||||
let legacy: EthernetConfig =
|
||||
serde_yaml::from_str("interface: eth0\ndiscovery: true\n").unwrap();
|
||||
assert_eq!(legacy.listen, Some(true));
|
||||
|
||||
// (b) The new canonical `listen:` key parses into the renamed field.
|
||||
let renamed: EthernetConfig =
|
||||
serde_yaml::from_str("interface: eth0\nlisten: true\n").unwrap();
|
||||
assert_eq!(renamed.listen, Some(true));
|
||||
|
||||
// (c) `deny_unknown_fields` still rejects an unknown ethernet key.
|
||||
let bogus: Result<EthernetConfig, _> =
|
||||
serde_yaml::from_str("interface: eth0\nbogus: true\n");
|
||||
assert!(bogus.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
+14
-23
@@ -236,7 +236,7 @@ pub fn show_peers(node: &Node) -> Value {
|
||||
// Per-npub Nostr-traversal failure-state snapshot, indexed by npub
|
||||
// for O(1) per-peer lookup. Empty if Nostr discovery is disabled.
|
||||
let nostr_state: std::collections::HashMap<String, _> = node
|
||||
.nostr_rendezvous_handle()
|
||||
.nostr_discovery_handle()
|
||||
.map(|d| {
|
||||
d.failure_state_snapshot()
|
||||
.into_iter()
|
||||
@@ -1303,18 +1303,17 @@ pub fn show_connections(node: &Node) -> Value {
|
||||
let now = now_ms();
|
||||
let connections: Vec<Value> = node
|
||||
.connections()
|
||||
.map(|(_, machine)| {
|
||||
let link_id = machine.link_id();
|
||||
.map(|conn| {
|
||||
let mut conn_json = json!({
|
||||
"link_id": link_id.as_u64(),
|
||||
"direction": format!("{}", machine.conn_direction()),
|
||||
"handshake_state": node.connection_handshake_state(link_id),
|
||||
"started_at_ms": node.connection_started_at(link_id),
|
||||
"idle_ms": now.saturating_sub(node.connection_last_activity(link_id)),
|
||||
"resend_count": node.connection_resend_count(link_id),
|
||||
"link_id": conn.link_id().as_u64(),
|
||||
"direction": format!("{}", conn.direction()),
|
||||
"handshake_state": format!("{}", conn.handshake_state()),
|
||||
"started_at_ms": conn.started_at(),
|
||||
"idle_ms": now.saturating_sub(conn.last_activity()),
|
||||
"resend_count": conn.resend_count(),
|
||||
});
|
||||
|
||||
if let Some(identity) = node.connection_expected_identity(link_id) {
|
||||
if let Some(identity) = conn.expected_identity() {
|
||||
conn_json["expected_peer"] = json!(identity.npub());
|
||||
}
|
||||
|
||||
@@ -1488,9 +1487,7 @@ pub fn show_routing(node: &Node) -> Value {
|
||||
"recent_requests": node.recent_request_count(),
|
||||
"retries": retries,
|
||||
"forwarding": serde_json::to_value(metrics.forwarding.snapshot()).unwrap_or_default(),
|
||||
// COMPAT: `discovery` is the deprecated alias for `lookup`; drop at the v2 cutover.
|
||||
"discovery": serde_json::to_value(metrics.lookup.snapshot()).unwrap_or_default(),
|
||||
"lookup": serde_json::to_value(metrics.lookup.snapshot()).unwrap_or_default(),
|
||||
"discovery": serde_json::to_value(metrics.discovery.snapshot()).unwrap_or_default(),
|
||||
"error_signals": serde_json::to_value(metrics.errors.snapshot()).unwrap_or_default(),
|
||||
"congestion": serde_json::to_value(metrics.congestion.snapshot()).unwrap_or_default(),
|
||||
})
|
||||
@@ -1546,9 +1543,7 @@ pub(crate) fn show_routing_from_handle(handle: &super::read_handle::ControlReadH
|
||||
"recent_requests": view.recent_requests,
|
||||
"retries": retries,
|
||||
"forwarding": serde_json::to_value(metrics.forwarding.snapshot()).unwrap_or_default(),
|
||||
// COMPAT: `discovery` is the deprecated alias for `lookup`; drop at the v2 cutover.
|
||||
"discovery": serde_json::to_value(metrics.lookup.snapshot()).unwrap_or_default(),
|
||||
"lookup": serde_json::to_value(metrics.lookup.snapshot()).unwrap_or_default(),
|
||||
"discovery": serde_json::to_value(metrics.discovery.snapshot()).unwrap_or_default(),
|
||||
"error_signals": serde_json::to_value(metrics.errors.snapshot()).unwrap_or_default(),
|
||||
"congestion": serde_json::to_value(metrics.congestion.snapshot()).unwrap_or_default(),
|
||||
})
|
||||
@@ -2333,9 +2328,7 @@ pub(crate) fn show_metrics_from_handle(handle: &super::read_handle::ControlReadH
|
||||
let m = handle.metrics();
|
||||
json!({
|
||||
"forwarding": m.forwarding.snapshot(),
|
||||
// COMPAT: `discovery` is the deprecated alias for `lookup`; drop at the v2 cutover.
|
||||
"discovery": m.lookup.snapshot(),
|
||||
"lookup": m.lookup.snapshot(),
|
||||
"discovery": m.discovery.snapshot(),
|
||||
"tree": m.tree.snapshot(),
|
||||
"bloom": m.bloom.snapshot(),
|
||||
"congestion": m.congestion.snapshot(),
|
||||
@@ -2768,12 +2761,12 @@ mod tests {
|
||||
/// Structural confirmation that the rx_loop no longer dispatches `show_*`:
|
||||
/// the rx_loop source carries no `queries::dispatch` call and no
|
||||
/// `starts_with("show_")` routing branch. Reads the committed source of
|
||||
/// `src/node/dataplane/rx_loop.rs` and asserts both markers are absent. This
|
||||
/// `src/node/handlers/rx_loop.rs` and asserts both markers are absent. This
|
||||
/// is the milestone's "remove `show_*` from the data-plane dispatch path"
|
||||
/// invariant, guarded against regression.
|
||||
#[test]
|
||||
fn rx_loop_has_no_show_dispatch() {
|
||||
let src = include_str!("../node/dataplane/rx_loop.rs");
|
||||
let src = include_str!("../node/handlers/rx_loop.rs");
|
||||
assert!(
|
||||
!src.contains("queries::dispatch"),
|
||||
"rx_loop must not call queries::dispatch (show_* served off-loop)"
|
||||
@@ -2801,9 +2794,7 @@ mod tests {
|
||||
|
||||
let expected_families = [
|
||||
("forwarding", "received_packets"),
|
||||
// `discovery` is the deprecated dual-emit alias for `lookup`; drop at the v2 cutover.
|
||||
("discovery", "req_received"),
|
||||
("lookup", "req_received"),
|
||||
("tree", "accepted"),
|
||||
("bloom", "accepted"),
|
||||
("congestion", "ce_forwarded"),
|
||||
|
||||
+35
-32
@@ -41,7 +41,7 @@ use super::snapshot::{EntitySnapshot, RoutingSnapshot, StatsSnapshot};
|
||||
/// starting R1 as `show_*` queries cut over to off-loop rendering; until then
|
||||
/// they are wired but unread.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ControlReadHandle {
|
||||
pub struct ControlReadHandle {
|
||||
/// Effectively-immutable node context (config, identity, limits).
|
||||
context: Arc<NodeContext>,
|
||||
/// Metrics registry (counters / gauges) for `show_stats_*`.
|
||||
@@ -108,6 +108,40 @@ impl ControlReadHandle {
|
||||
}
|
||||
}
|
||||
|
||||
/// A minimal, public view of one peer for embedders (e.g. an app UI), read
|
||||
/// lock-free from the tick-published snapshot. See [`ControlReadHandle::peer_views`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PeerView {
|
||||
/// The peer's `node_addr`, hex-encoded.
|
||||
pub node_addr_hex: String,
|
||||
/// Resolved npub (or the `node_addr` hex when not yet resolved to a peer).
|
||||
pub npub: String,
|
||||
/// Whether the peer is currently in the live authenticated-peer table.
|
||||
pub connected: bool,
|
||||
}
|
||||
|
||||
impl ControlReadHandle {
|
||||
/// A lock-free snapshot of known peers (node_addr / npub / connected),
|
||||
/// read from the tick-published stats snapshot.
|
||||
///
|
||||
/// Intended for embedders that run [`crate::Node::run_rx_loop`] on a
|
||||
/// background task (so the node is exclusively borrowed there) and poll peer
|
||||
/// state from a clone of this handle — the read touches only an `ArcSwap`
|
||||
/// load, never the `Node`. See the Myco app for the reference embedding.
|
||||
pub fn peer_views(&self) -> Vec<PeerView> {
|
||||
self.stats
|
||||
.load()
|
||||
.peer_meta
|
||||
.iter()
|
||||
.map(|(addr, meta)| PeerView {
|
||||
node_addr_hex: addr.to_string(),
|
||||
npub: meta.npub.clone(),
|
||||
connected: meta.is_active,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Attempt to serve a request entirely from the read handle, off the rx_loop.
|
||||
///
|
||||
/// Returns `Some(response)` when the command is a pure-snapshot query that has
|
||||
@@ -118,41 +152,10 @@ impl ControlReadHandle {
|
||||
/// Cutover queries (R1) read only `NodeContext` / `MetricsRegistry` (the state
|
||||
/// the read handle already bundles) plus host-OS facts (`/proc`, nftables), so
|
||||
/// they render entirely in the control task without touching `Node`.
|
||||
///
|
||||
/// **It now also carries mutating commands**, namely the `profile_tick_*`
|
||||
/// family under the `profiling` feature. They are served here rather than on
|
||||
/// the rx_loop deliberately: all of their state is process statics, they need
|
||||
/// no `&mut Node`, and routing them through the loop would make the toggle
|
||||
/// queue behind the very behavior it exists to measure.
|
||||
pub(crate) fn snapshot_dispatch(request: &Request, handle: &ControlReadHandle) -> Option<Response> {
|
||||
use crate::control::queries;
|
||||
|
||||
match request.command.as_str() {
|
||||
// Tick-body profiler toggle. Present only in a `--features profiling`
|
||||
// build; otherwise these fall through to the rx_loop dispatch, which
|
||||
// reports them as unknown commands.
|
||||
#[cfg(feature = "profiling")]
|
||||
"profile_tick_on" => {
|
||||
let dir = request
|
||||
.params
|
||||
.as_ref()
|
||||
.and_then(|p| p.get("dir"))
|
||||
.and_then(|v| v.as_str());
|
||||
let context = handle.context();
|
||||
let npub = context.identity.npub();
|
||||
let period = context.config.node.tick_interval_secs;
|
||||
Some(match crate::instr::capture::start(dir, &npub, period) {
|
||||
Ok(value) => Response::ok(value),
|
||||
Err(e) => Response::error(e),
|
||||
})
|
||||
}
|
||||
#[cfg(feature = "profiling")]
|
||||
"profile_tick_off" => Some(match crate::instr::capture::stop() {
|
||||
Ok(value) => Response::ok(value),
|
||||
Err(e) => Response::error(e),
|
||||
}),
|
||||
#[cfg(feature = "profiling")]
|
||||
"profile_tick_status" => Some(Response::ok(crate::instr::capture::status())),
|
||||
"show_listening_sockets" => Some(Response::ok(
|
||||
queries::show_listening_sockets_from_handle(handle),
|
||||
)),
|
||||
|
||||
@@ -63,30 +63,6 @@
|
||||
"ttl_exhausted_packets": 0
|
||||
},
|
||||
"identity_cache_entries": 0,
|
||||
"lookup": {
|
||||
"req_backoff_suppressed": 0,
|
||||
"req_bloom_miss": 0,
|
||||
"req_decode_error": 0,
|
||||
"req_dedup_cache_full": 0,
|
||||
"req_deduplicated": 0,
|
||||
"req_duplicate": 0,
|
||||
"req_fallback_forwarded": 0,
|
||||
"req_forward_rate_limited": 0,
|
||||
"req_forwarded": 0,
|
||||
"req_initiated": 0,
|
||||
"req_no_tree_peer": 0,
|
||||
"req_received": 0,
|
||||
"req_target_is_us": 0,
|
||||
"req_ttl_exhausted": 0,
|
||||
"resp_accepted": 0,
|
||||
"resp_decode_error": 0,
|
||||
"resp_forwarded": 0,
|
||||
"resp_identity_miss": 0,
|
||||
"resp_no_route": 0,
|
||||
"resp_proof_failed": 0,
|
||||
"resp_received": 0,
|
||||
"resp_timed_out": 0
|
||||
},
|
||||
"pending_lookups": [],
|
||||
"pending_tun_destinations": 0,
|
||||
"pending_tun_packets": 0,
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"loop_detected": 0,
|
||||
"outbound_sign_failed": 0,
|
||||
"parent_losses": 0,
|
||||
"parent_switched": 0,
|
||||
"parent_switches": 0,
|
||||
"rate_limited": 0,
|
||||
"received": 0,
|
||||
|
||||
@@ -6,6 +6,9 @@
|
||||
//! it hands the live socket and selected remote endpoint to FIPS so the
|
||||
//! existing Noise/FMP transport path can take over.
|
||||
|
||||
pub mod lan;
|
||||
pub mod nostr;
|
||||
|
||||
use crate::config::UdpConfig;
|
||||
use crate::{NodeAddr, TransportId};
|
||||
use std::net::{SocketAddr, UdpSocket};
|
||||
@@ -13,9 +16,9 @@ use std::net::{SocketAddr, UdpSocket};
|
||||
/// Punch-probe magic ("NPTC", network byte order). First byte `0x4E`
|
||||
/// collides with FMP's prefix-version high-nibble check, so the UDP
|
||||
/// transport silently filters packets carrying this magic to keep
|
||||
/// post-adoption handshake logs clean. Defined in the nostr rendezvous
|
||||
/// home so the UDP filter and the nostr submodule's punch sender share
|
||||
/// the same constant.
|
||||
/// post-adoption handshake logs clean. Defined at the top-level
|
||||
/// `discovery` module so the UDP filter and the nostr submodule's
|
||||
/// punch sender share the same constant.
|
||||
pub const PUNCH_MAGIC: u32 = 0x4E505443;
|
||||
|
||||
/// Punch-probe-ack magic ("NPTA", network byte order). Same filter as
|
||||
@@ -54,12 +54,8 @@ pub const TXT_KEY_SCOPE: &str = "scope";
|
||||
/// `PROTOCOL_VERSION`).
|
||||
pub const TXT_KEY_VERSION: &str = "v";
|
||||
|
||||
/// FIPS protocol version advertised in the mDNS TXT `v` key. Kept in sync
|
||||
/// with the Nostr rendezvous `PROTOCOL_VERSION` (same value, `"1"`).
|
||||
const TXT_PROTOCOL_VERSION: &str = "1";
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LanRendezvousError {
|
||||
pub enum LanDiscoveryError {
|
||||
#[error("mDNS daemon init failed: {0}")]
|
||||
Daemon(String),
|
||||
#[error("mDNS register failed: {0}")]
|
||||
@@ -83,7 +79,7 @@ pub struct LanDiscoveredPeer {
|
||||
pub observed_at: Instant,
|
||||
}
|
||||
|
||||
/// Browser-side events surfaced by `LanRendezvous::drain_events`.
|
||||
/// Browser-side events surfaced by `LanDiscovery::drain_events`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum LanEvent {
|
||||
Discovered(LanDiscoveredPeer),
|
||||
@@ -91,17 +87,17 @@ pub enum LanEvent {
|
||||
|
||||
/// Runtime configuration for the mDNS responder + browser.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct LanRendezvousConfig {
|
||||
pub struct LanDiscoveryConfig {
|
||||
/// Master switch. Default: `false` — LAN discovery is opt-in. Operators
|
||||
/// who want sub-second same-LAN pairing enable it via
|
||||
/// `node.rendezvous.lan.enabled: true`. Default-off avoids reintroducing
|
||||
/// `node.discovery.lan.enabled: true`. Default-off avoids reintroducing
|
||||
/// a per-LAN identity broadcast on nodes that have deliberately disabled
|
||||
/// other discovery channels, and avoids any multicast surprise on upgrade.
|
||||
#[serde(default = "LanRendezvousConfig::default_enabled")]
|
||||
#[serde(default = "LanDiscoveryConfig::default_enabled")]
|
||||
pub enabled: bool,
|
||||
/// Overridable service type, primarily so integration tests can run
|
||||
/// multiple isolated services on the same loopback interface.
|
||||
#[serde(default = "LanRendezvousConfig::default_service_type")]
|
||||
#[serde(default = "LanDiscoveryConfig::default_service_type")]
|
||||
pub service_type: String,
|
||||
/// Optional application/network scope carried in the LAN-only TXT
|
||||
/// record. Browsers that set a scope ignore adverts for other scopes.
|
||||
@@ -113,7 +109,7 @@ pub struct LanRendezvousConfig {
|
||||
pub scope: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for LanRendezvousConfig {
|
||||
impl Default for LanDiscoveryConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: Self::default_enabled(),
|
||||
@@ -123,7 +119,7 @@ impl Default for LanRendezvousConfig {
|
||||
}
|
||||
}
|
||||
|
||||
impl LanRendezvousConfig {
|
||||
impl LanDiscoveryConfig {
|
||||
fn default_enabled() -> bool {
|
||||
false
|
||||
}
|
||||
@@ -133,7 +129,7 @@ impl LanRendezvousConfig {
|
||||
}
|
||||
|
||||
/// Running mDNS responder + browser bound to the node's UDP advert port.
|
||||
pub struct LanRendezvous {
|
||||
pub struct LanDiscovery {
|
||||
daemon: ServiceDaemon,
|
||||
own_npub: String,
|
||||
instance_fullname: String,
|
||||
@@ -141,12 +137,7 @@ pub struct LanRendezvous {
|
||||
event_pump: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl LanRendezvous {
|
||||
/// Whether the mDNS event-pump task has exited (runtime liveness).
|
||||
pub fn is_finished(&self) -> bool {
|
||||
self.event_pump.is_finished()
|
||||
}
|
||||
|
||||
impl LanDiscovery {
|
||||
/// Start the mDNS responder and browser.
|
||||
///
|
||||
/// `advertised_port` is the UDP port the operational UDP transport
|
||||
@@ -157,16 +148,16 @@ impl LanRendezvous {
|
||||
identity: &Identity,
|
||||
scope: Option<String>,
|
||||
advertised_port: u16,
|
||||
config: LanRendezvousConfig,
|
||||
) -> Result<Arc<Self>, LanRendezvousError> {
|
||||
config: LanDiscoveryConfig,
|
||||
) -> Result<Arc<Self>, LanDiscoveryError> {
|
||||
if !config.enabled {
|
||||
return Err(LanRendezvousError::Disabled);
|
||||
return Err(LanDiscoveryError::Disabled);
|
||||
}
|
||||
if advertised_port == 0 {
|
||||
return Err(LanRendezvousError::NoAdvertisedPort);
|
||||
return Err(LanDiscoveryError::NoAdvertisedPort);
|
||||
}
|
||||
|
||||
let daemon = ServiceDaemon::new().map_err(|e| LanRendezvousError::Daemon(e.to_string()))?;
|
||||
let daemon = ServiceDaemon::new().map_err(|e| LanDiscoveryError::Daemon(e.to_string()))?;
|
||||
|
||||
let npub = identity.npub();
|
||||
// mDNS DNS labels are capped at 63 bytes. 16 bech32 chars of npub
|
||||
@@ -185,7 +176,7 @@ impl LanRendezvous {
|
||||
}
|
||||
props.insert(
|
||||
TXT_KEY_VERSION.to_string(),
|
||||
TXT_PROTOCOL_VERSION.to_string(),
|
||||
super::nostr::PROTOCOL_VERSION.to_string(),
|
||||
);
|
||||
|
||||
// host_ipv4 is set to "127.0.0.1" *and* enable_addr_auto() is
|
||||
@@ -202,18 +193,18 @@ impl LanRendezvous {
|
||||
advertised_port,
|
||||
Some(props),
|
||||
)
|
||||
.map_err(|e| LanRendezvousError::Register(e.to_string()))?
|
||||
.map_err(|e| LanDiscoveryError::Register(e.to_string()))?
|
||||
.enable_addr_auto();
|
||||
|
||||
let instance_fullname = service_info.get_fullname().to_string();
|
||||
|
||||
daemon
|
||||
.register(service_info)
|
||||
.map_err(|e| LanRendezvousError::Register(e.to_string()))?;
|
||||
.map_err(|e| LanDiscoveryError::Register(e.to_string()))?;
|
||||
|
||||
let browse_rx = daemon
|
||||
.browse(&config.service_type)
|
||||
.map_err(|e| LanRendezvousError::Browse(e.to_string()))?;
|
||||
.map_err(|e| LanDiscoveryError::Browse(e.to_string()))?;
|
||||
|
||||
let (events_tx, events_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let own_npub = npub.clone();
|
||||
@@ -4,7 +4,7 @@ use std::time::Duration;
|
||||
use crate::Identity;
|
||||
use mdns_sd::ScopedIp;
|
||||
|
||||
use super::{LanEvent, LanRendezvous, LanRendezvousConfig};
|
||||
use super::{LanDiscovery, LanDiscoveryConfig, LanEvent};
|
||||
|
||||
/// Distinct service type per test run so concurrent cargo-test workers
|
||||
/// on the same machine don't cross-feed each other's adverts via the
|
||||
@@ -15,8 +15,8 @@ fn isolated_service_type(tag: &str) -> String {
|
||||
format!("_fipstest-{tag}-{rand:08x}._udp.local.")
|
||||
}
|
||||
|
||||
fn config_for(service_type: String) -> LanRendezvousConfig {
|
||||
LanRendezvousConfig {
|
||||
fn config_for(service_type: String) -> LanDiscoveryConfig {
|
||||
LanDiscoveryConfig {
|
||||
enabled: true,
|
||||
service_type,
|
||||
scope: None,
|
||||
@@ -47,7 +47,7 @@ fn non_link_local_ipv6_advert_is_preserved() {
|
||||
}
|
||||
|
||||
async fn wait_for_peer(
|
||||
discovery: &LanRendezvous,
|
||||
discovery: &LanDiscovery,
|
||||
expected_npub: &str,
|
||||
timeout: Duration,
|
||||
) -> Option<super::LanDiscoveredPeer> {
|
||||
@@ -64,7 +64,7 @@ async fn wait_for_peer(
|
||||
None
|
||||
}
|
||||
|
||||
/// Two LanRendezvous instances on isolated service types — `a` browses
|
||||
/// Two LanDiscovery instances on isolated service types — `a` browses
|
||||
/// only its own type and never sees `b`, and vice versa. Sanity check
|
||||
/// that the scope-isolation defense works (we'd lose isolation if mdns-
|
||||
/// sd ever leaked across service types).
|
||||
@@ -76,7 +76,7 @@ async fn isolated_service_types_do_not_cross_feed() {
|
||||
let service_a = isolated_service_type("isolated-a");
|
||||
let service_b = isolated_service_type("isolated-b");
|
||||
|
||||
let lan_a = LanRendezvous::start(
|
||||
let lan_a = LanDiscovery::start(
|
||||
&identity_a,
|
||||
Some("scope-x".to_string()),
|
||||
61001,
|
||||
@@ -84,7 +84,7 @@ async fn isolated_service_types_do_not_cross_feed() {
|
||||
)
|
||||
.await
|
||||
.expect("start a");
|
||||
let lan_b = LanRendezvous::start(
|
||||
let lan_b = LanDiscovery::start(
|
||||
&identity_b,
|
||||
Some("scope-x".to_string()),
|
||||
61002,
|
||||
@@ -118,7 +118,7 @@ async fn isolated_service_types_do_not_cross_feed() {
|
||||
assert!(!saw_a_from_b, "isolated service types must not cross-feed");
|
||||
}
|
||||
|
||||
/// Two LanRendezvous instances on the same service type and the same
|
||||
/// Two LanDiscovery instances on the same service type and the same
|
||||
/// scope: each should observe the other's advert within a few seconds.
|
||||
/// Exercises the responder + browser + TXT plumbing end-to-end.
|
||||
///
|
||||
@@ -136,7 +136,7 @@ async fn matched_scope_peers_observe_each_other() {
|
||||
|
||||
let service = isolated_service_type("matched");
|
||||
|
||||
let lan_a = LanRendezvous::start(
|
||||
let lan_a = LanDiscovery::start(
|
||||
&identity_a,
|
||||
Some("scope-shared".to_string()),
|
||||
61101,
|
||||
@@ -144,7 +144,7 @@ async fn matched_scope_peers_observe_each_other() {
|
||||
)
|
||||
.await
|
||||
.expect("start a");
|
||||
let lan_b = LanRendezvous::start(
|
||||
let lan_b = LanDiscovery::start(
|
||||
&identity_b,
|
||||
Some("scope-shared".to_string()),
|
||||
61102,
|
||||
@@ -181,7 +181,7 @@ async fn cross_scope_advert_is_filtered() {
|
||||
|
||||
let service = isolated_service_type("cross-scope");
|
||||
|
||||
let lan_a = LanRendezvous::start(
|
||||
let lan_a = LanDiscovery::start(
|
||||
&identity_a,
|
||||
Some("scope-a".to_string()),
|
||||
61201,
|
||||
@@ -189,7 +189,7 @@ async fn cross_scope_advert_is_filtered() {
|
||||
)
|
||||
.await
|
||||
.expect("start a");
|
||||
let lan_b = LanRendezvous::start(
|
||||
let lan_b = LanDiscovery::start(
|
||||
&identity_b,
|
||||
Some("scope-b".to_string()),
|
||||
61202,
|
||||
@@ -1,20 +1,14 @@
|
||||
mod advert;
|
||||
mod driver;
|
||||
mod failure_state;
|
||||
mod handoff;
|
||||
mod runtime;
|
||||
mod signal;
|
||||
mod stun;
|
||||
mod traversal;
|
||||
mod traversal_machine;
|
||||
mod types;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use driver::{AdvertTransportSnapshot, RendezvousDriver};
|
||||
pub use handoff::{BootstrapHandoffResult, EstablishedTraversal, is_punch_packet};
|
||||
pub use runtime::NostrRendezvous;
|
||||
pub use runtime::NostrDiscovery;
|
||||
pub use types::{
|
||||
ADVERT_IDENTIFIER, ADVERT_KIND, ADVERT_VERSION, BootstrapError, BootstrapEvent,
|
||||
CachedOverlayAdvert, NostrFailureDecision, NostrPeerFailureView, NostrRefetchOutcome,
|
||||
@@ -16,9 +16,7 @@ use tokio::sync::{Mutex, Notify, RwLock, Semaphore, broadcast, mpsc, oneshot};
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
use super::advert::{AdvertMachine, PublishPlan};
|
||||
use super::failure_state::FailureState;
|
||||
use super::handoff::EstablishedTraversal;
|
||||
use super::signal::{
|
||||
FreshnessOutcome, SignalEnvelope, build_signal_event, create_traversal_answer,
|
||||
create_traversal_offer, estimate_clock_skew, unwrap_signal_event, validate_offer_freshness,
|
||||
@@ -26,15 +24,15 @@ use super::signal::{
|
||||
};
|
||||
use super::stun::observe_traversal_addresses;
|
||||
use super::traversal::{nonce, now_ms, planned_remote_endpoints, run_punch_attempt};
|
||||
use super::traversal_machine::{OfferDisposition, SeenDecision, TraversalMachine};
|
||||
use super::types::{
|
||||
ADVERT_IDENTIFIER, ADVERT_KIND, ADVERT_VERSION, BootstrapError, BootstrapEvent,
|
||||
CachedOverlayAdvert, NostrFailureDecision, NostrPeerFailureView, NostrRefetchOutcome,
|
||||
OverlayAdvert, OverlayEndpointAdvert, PROTOCOL_VERSION, PunchHint, SIGNAL_KIND,
|
||||
TraversalAnswer, TraversalOffer,
|
||||
};
|
||||
use crate::PeerIdentity;
|
||||
use crate::config::{NostrRendezvousConfig, PeerConfig};
|
||||
use crate::config::{NostrDiscoveryConfig, PeerConfig};
|
||||
use crate::discovery::EstablishedTraversal;
|
||||
use crate::{NodeAddr, PeerIdentity};
|
||||
|
||||
const ADVERT_CACHE_STALE_GRACE_MULTIPLIER: u64 = 2;
|
||||
|
||||
@@ -53,6 +51,42 @@ fn short_id(id: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide whether an incoming-offer responder session should be suppressed
|
||||
/// in favour of our own already-running outbound initiator session.
|
||||
///
|
||||
/// Two peers that each have the other as `auto_connect` simultaneously run an
|
||||
/// initiator traversal *and* a responder traversal for the same peer, binding a
|
||||
/// separate UDP socket per session. Each node then emits two
|
||||
/// `BootstrapEvent::Established` events and `adopt_established_traversal` keeps
|
||||
/// only the first on a non-deterministic race; when the two nodes' independent
|
||||
/// races resolve to mismatched sessions, each side's Noise msg1 lands on a peer
|
||||
/// port the peer already stopped draining and both handshakes stall (root cause
|
||||
/// of ISSUE-2026-0031).
|
||||
///
|
||||
/// To collapse the four-socket dance to a single, guaranteed-matching socket
|
||||
/// pair, both nodes deterministically keep the session **initiated by the
|
||||
/// smaller `NodeAddr`** — reusing the project's existing NodeAddr tie-breaker
|
||||
/// convention (`cross_connection_winner`, the rekey dual-init resolution, and
|
||||
/// the dual-cross-init adopt path in `lifecycle.rs`).
|
||||
///
|
||||
/// This is evaluated on the responder path, where the session being handled is
|
||||
/// *peer-initiated*. It returns `true` (suppress this responder session) only
|
||||
/// when genuine duplication exists — i.e. we also have an in-flight outbound
|
||||
/// initiator for this same peer (`have_active_initiator`) — and our own
|
||||
/// initiator session is the preferred one (`our_addr < peer_addr`). When there
|
||||
/// is no co-active initiator (the asymmetric / one-sided `auto_connect` case,
|
||||
/// where only one session exists at all) it never suppresses, so connectivity
|
||||
/// is preserved. The `our_addr == peer_addr` case (self / loopback) and any
|
||||
/// caller that cannot derive a peer `NodeAddr` likewise fall through to "do not
|
||||
/// suppress".
|
||||
pub(super) fn suppress_responder_for_own_initiator(
|
||||
our_addr: &NodeAddr,
|
||||
peer_addr: &NodeAddr,
|
||||
have_active_initiator: bool,
|
||||
) -> bool {
|
||||
have_active_initiator && our_addr < peer_addr
|
||||
}
|
||||
|
||||
fn endpoint_summary(endpoints: &[OverlayEndpointAdvert]) -> String {
|
||||
endpoints
|
||||
.iter()
|
||||
@@ -83,7 +117,7 @@ fn is_unroutable_direct_advert_ip(ip: std::net::IpAddr) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn endpoint_advert_is_publicly_usable(endpoint: &OverlayEndpointAdvert) -> bool {
|
||||
fn endpoint_advert_is_publicly_usable(endpoint: &OverlayEndpointAdvert) -> bool {
|
||||
let addr = endpoint.addr.trim();
|
||||
if addr.is_empty() {
|
||||
return false;
|
||||
@@ -123,7 +157,7 @@ pub(super) fn endpoint_advert_is_publicly_usable(endpoint: &OverlayEndpointAdver
|
||||
}
|
||||
|
||||
/// Cached STUN-derived public address for an advert-eligible UDP transport
|
||||
/// bound to a wildcard. Lives on `NostrRendezvous` so the freshness window
|
||||
/// bound to a wildcard. Lives on `NostrDiscovery` so the freshness window
|
||||
/// survives advert refresh cycles.
|
||||
struct CachedPublicUdpAddr {
|
||||
/// Most recent STUN observation. `None` means the last attempt failed
|
||||
@@ -143,15 +177,18 @@ const PUBLIC_UDP_ADDR_FAILURE_TTL: Duration = Duration::from_secs(60);
|
||||
const RELAY_STARTUP_OP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const ADVERT_PUBLISH_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
pub struct NostrRendezvous {
|
||||
pub struct NostrDiscovery {
|
||||
client: Client,
|
||||
keys: nostr::Keys,
|
||||
pubkey: PublicKey,
|
||||
npub: String,
|
||||
config: NostrRendezvousConfig,
|
||||
advert: AdvertMachine,
|
||||
traversal: TraversalMachine,
|
||||
config: NostrDiscoveryConfig,
|
||||
advert_cache: RwLock<HashMap<String, CachedOverlayAdvert>>,
|
||||
local_advert: RwLock<Option<OverlayAdvert>>,
|
||||
current_advert_event_id: RwLock<Option<EventId>>,
|
||||
pending_answers: Mutex<HashMap<String, oneshot::Sender<SignalEnvelope<TraversalAnswer>>>>,
|
||||
active_initiators: Mutex<HashSet<String>>,
|
||||
seen_sessions: Mutex<HashMap<String, u64>>,
|
||||
offer_slots: Arc<Semaphore>,
|
||||
event_tx: mpsc::UnboundedSender<BootstrapEvent>,
|
||||
event_rx: Mutex<mpsc::UnboundedReceiver<BootstrapEvent>>,
|
||||
@@ -175,59 +212,10 @@ pub struct NostrRendezvous {
|
||||
outbound_admission: AtomicBool,
|
||||
}
|
||||
|
||||
impl NostrRendezvous {
|
||||
/// Whether the Nostr subsystem has stopped being able to do its job
|
||||
/// (runtime liveness).
|
||||
///
|
||||
/// "Nostr exited" is defined as *any* of the three service loops that never
|
||||
/// return by design having finished:
|
||||
///
|
||||
/// - `notify_task` — the inbound receive loop. Without it no advert and no
|
||||
/// traversal signal is ever observed again.
|
||||
/// - `publish_task` — the advert publisher. Without it this node stops being
|
||||
/// discoverable.
|
||||
/// - `advertise_task` — the refresh ticker that drives the publisher.
|
||||
///
|
||||
/// Each of these is an unconditional `loop` (the notify loop's only `break`
|
||||
/// is the relay-pool broadcast channel closing, i.e. the pool itself is
|
||||
/// gone), so a finished handle means the task panicked or was aborted:
|
||||
/// unrecoverable, which matches the one-way `ChildExited` → `Degraded`
|
||||
/// latch in the supervisor FSM.
|
||||
///
|
||||
/// Deliberately *not* watched: `connect_task` and `relay_startup_task`.
|
||||
/// `Client::connect()` only spawns a per-relay background connection task
|
||||
/// and returns, so `connect_task` finishes moments after start on a
|
||||
/// perfectly healthy node; `relay_startup_task` breaks out of its retry loop
|
||||
/// on the first successful subscribe. Watching either reports Degraded on
|
||||
/// every node forever.
|
||||
///
|
||||
/// Each handle is `Some` for the engine's whole running life (installed in
|
||||
/// `start`); `shutdown` takes them all, leaving `None` — a taken handle
|
||||
/// means the engine has been shut down, which counts as finished, so a
|
||||
/// `None` inner maps to `true` (this lets the liveness poll monitor
|
||||
/// terminate after a stop rather than spinning forever). The slots are
|
||||
/// `tokio::sync::Mutex`es, so this sync accessor uses the non-blocking
|
||||
/// `try_lock`: a momentarily-contended lock (only start/stop hold it,
|
||||
/// briefly) reports "not finished", the safe direction — the 2s liveness
|
||||
/// poll re-checks next tick and never spuriously degrades a healthy node.
|
||||
pub fn is_finished(&self) -> bool {
|
||||
Self::task_finished(&self.notify_task)
|
||||
|| Self::task_finished(&self.publish_task)
|
||||
|| Self::task_finished(&self.advertise_task)
|
||||
}
|
||||
|
||||
/// Liveness of one task slot: finished if the handle is gone (shut down) or
|
||||
/// the task has completed; "not finished" when the slot is momentarily
|
||||
/// locked by start/stop.
|
||||
fn task_finished(slot: &Mutex<Option<JoinHandle<()>>>) -> bool {
|
||||
slot.try_lock()
|
||||
.map(|g| g.as_ref().is_none_or(|h| h.is_finished()))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
impl NostrDiscovery {
|
||||
pub async fn start(
|
||||
identity: &crate::Identity,
|
||||
config: NostrRendezvousConfig,
|
||||
config: NostrDiscoveryConfig,
|
||||
) -> Result<Arc<Self>, BootstrapError> {
|
||||
if !config.enabled {
|
||||
return Err(BootstrapError::Disabled);
|
||||
@@ -261,25 +249,18 @@ impl NostrRendezvous {
|
||||
config.failure_state_max_entries,
|
||||
);
|
||||
|
||||
let advert = AdvertMachine::new(
|
||||
npub.clone(),
|
||||
config.advertise,
|
||||
config.advert_ttl_secs * 1000 * ADVERT_CACHE_STALE_GRACE_MULTIPLIER,
|
||||
config.advert_cache_max_entries,
|
||||
);
|
||||
let traversal = TraversalMachine::new(
|
||||
config.replay_window_secs * 1000,
|
||||
config.seen_sessions_max_entries,
|
||||
);
|
||||
let runtime = Arc::new(Self {
|
||||
client,
|
||||
keys,
|
||||
pubkey,
|
||||
npub,
|
||||
config,
|
||||
advert,
|
||||
traversal,
|
||||
advert_cache: RwLock::new(HashMap::new()),
|
||||
local_advert: RwLock::new(None),
|
||||
current_advert_event_id: RwLock::new(None),
|
||||
pending_answers: Mutex::new(HashMap::new()),
|
||||
active_initiators: Mutex::new(HashSet::new()),
|
||||
seen_sessions: Mutex::new(HashMap::new()),
|
||||
offer_slots,
|
||||
event_tx,
|
||||
event_rx: Mutex::new(event_rx),
|
||||
@@ -328,8 +309,11 @@ impl NostrRendezvous {
|
||||
|
||||
pub async fn request_connect(self: &Arc<Self>, peer_config: PeerConfig) {
|
||||
let peer_npub = peer_config.npub.clone();
|
||||
if !self.traversal.begin_initiator(&peer_npub) {
|
||||
return;
|
||||
{
|
||||
let mut active = self.active_initiators.lock().await;
|
||||
if !active.insert(peer_npub.clone()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let runtime = Arc::clone(self);
|
||||
@@ -342,7 +326,7 @@ impl NostrRendezvous {
|
||||
},
|
||||
};
|
||||
let _ = runtime.event_tx.send(event);
|
||||
runtime.traversal.end_initiator(&peer_npub);
|
||||
runtime.active_initiators.lock().await.remove(&peer_npub);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -523,7 +507,12 @@ impl NostrRendezvous {
|
||||
if self.config.advert_relays.is_empty() {
|
||||
return NostrRefetchOutcome::Skipped;
|
||||
}
|
||||
let cached_created_at = self.advert.cached_created_at(peer_npub);
|
||||
let cached_created_at = self
|
||||
.advert_cache
|
||||
.read()
|
||||
.await
|
||||
.get(peer_npub)
|
||||
.map(|c| c.created_at);
|
||||
|
||||
let events = match self
|
||||
.client
|
||||
@@ -552,7 +541,7 @@ impl NostrRendezvous {
|
||||
|
||||
let Some((relay_created_at, ev)) = newest else {
|
||||
// Absent on relays. Evict any stale cache entry.
|
||||
self.advert.remove(peer_npub);
|
||||
self.advert_cache.write().await.remove(peer_npub);
|
||||
self.failure_state.reset_streak_after_refresh(peer_npub);
|
||||
return NostrRefetchOutcome::Evicted;
|
||||
};
|
||||
@@ -572,7 +561,10 @@ impl NostrRendezvous {
|
||||
created_at: relay_created_at,
|
||||
valid_until_ms,
|
||||
};
|
||||
self.advert.insert_fetched(peer_npub, updated);
|
||||
self.advert_cache
|
||||
.write()
|
||||
.await
|
||||
.insert(peer_npub.to_string(), updated);
|
||||
self.failure_state.reset_streak_after_refresh(peer_npub);
|
||||
NostrRefetchOutcome::Refreshed
|
||||
}
|
||||
@@ -592,9 +584,19 @@ impl NostrRendezvous {
|
||||
self: &Arc<Self>,
|
||||
advert: Option<OverlayAdvert>,
|
||||
) -> Result<(), BootstrapError> {
|
||||
if self.advert.set_local_advert(advert) {
|
||||
self.request_publish_advert();
|
||||
let changed = {
|
||||
let mut slot = self.local_advert.write().await;
|
||||
if *slot == advert {
|
||||
false
|
||||
} else {
|
||||
*slot = advert;
|
||||
true
|
||||
}
|
||||
};
|
||||
if !changed {
|
||||
return Ok(());
|
||||
}
|
||||
self.request_publish_advert();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -615,8 +617,22 @@ impl NostrRendezvous {
|
||||
&self,
|
||||
max: usize,
|
||||
) -> Vec<(String, Vec<OverlayEndpointAdvert>, u64)> {
|
||||
self.prune_advert_cache();
|
||||
self.advert.open_discovery_candidates(max, now_ms())
|
||||
self.prune_advert_cache().await;
|
||||
let now = now_ms();
|
||||
let cache = self.advert_cache.read().await;
|
||||
cache
|
||||
.values()
|
||||
.filter(|entry| entry.author_npub != self.npub)
|
||||
.filter(|entry| entry.valid_until_ms > now)
|
||||
.map(|entry| {
|
||||
(
|
||||
entry.author_npub.clone(),
|
||||
entry.advert.endpoints.clone(),
|
||||
entry.created_at,
|
||||
)
|
||||
})
|
||||
.take(max)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn shutdown(&self) -> Result<(), BootstrapError> {
|
||||
@@ -639,7 +655,7 @@ impl NostrRendezvous {
|
||||
// permanent shutdown. An explicit retraction races with the next
|
||||
// daemon's republish on strict relays (e.g. Damus rate-limits the
|
||||
// burst, leaving the advert deleted and never restored).
|
||||
let _ = self.advert.take_event_id();
|
||||
let _ = self.current_advert_event_id.write().await.take();
|
||||
|
||||
if let Some(handle) = self.notify_task.lock().await.take() {
|
||||
handle.abort();
|
||||
@@ -685,23 +701,32 @@ impl NostrRendezvous {
|
||||
&& let Ok(advert) =
|
||||
Self::parse_overlay_advert_event(&event, &self.config.app)
|
||||
{
|
||||
let endpoints = endpoint_summary(&advert.endpoints);
|
||||
let created_at = event.created_at.as_secs();
|
||||
if self.advert.observe_advert(
|
||||
&author_npub,
|
||||
advert,
|
||||
created_at,
|
||||
valid_until_ms,
|
||||
) {
|
||||
let mut cache = self.advert_cache.write().await;
|
||||
let should_replace = cache
|
||||
.get(&author_npub)
|
||||
.map(|existing| existing.created_at <= event.created_at.as_secs())
|
||||
.unwrap_or(true);
|
||||
if should_replace && author_npub != self.npub {
|
||||
debug!(
|
||||
peer = %short_npub(&author_npub),
|
||||
endpoints = %endpoints,
|
||||
endpoints = %endpoint_summary(&advert.endpoints),
|
||||
event = %short_id(&event.id.to_string()),
|
||||
"advert: peer cached"
|
||||
);
|
||||
}
|
||||
if should_replace {
|
||||
cache.insert(
|
||||
author_npub.clone(),
|
||||
CachedOverlayAdvert {
|
||||
author_npub,
|
||||
advert,
|
||||
created_at: event.created_at.as_secs(),
|
||||
valid_until_ms,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
self.prune_advert_cache();
|
||||
self.prune_advert_cache().await;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -924,17 +949,59 @@ impl NostrRendezvous {
|
||||
}
|
||||
|
||||
async fn publish_advert(&self) -> Result<(), BootstrapError> {
|
||||
let advert = match self.advert.plan_publish()? {
|
||||
PublishPlan::Nothing => return Ok(()),
|
||||
PublishPlan::Delete(event_id) => {
|
||||
let previous_event_id = self.current_advert_event_id.read().await.to_owned();
|
||||
if !self.config.advertise {
|
||||
if let Some(event_id) = previous_event_id {
|
||||
self.publish_delete(&self.config.advert_relays, [event_id])
|
||||
.await?;
|
||||
self.advert.clear_event_id();
|
||||
return Ok(());
|
||||
*self.current_advert_event_id.write().await = None;
|
||||
}
|
||||
PublishPlan::Publish(advert) => advert,
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut advert = match self.local_advert.read().await.clone() {
|
||||
Some(advert) => advert,
|
||||
// Transient absence (e.g., a single tick during startup where
|
||||
// build_overlay_advert briefly returns None). Don't proactively
|
||||
// emit a NIP-09 delete: the next publish supersedes the old
|
||||
// event via parameterized-replaceable semantics, and the NIP-40
|
||||
// expiration tag bounds the worst case if we never re-publish.
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
advert.identifier = ADVERT_IDENTIFIER.to_string();
|
||||
advert.version = ADVERT_VERSION;
|
||||
advert.endpoints.retain(endpoint_advert_is_publicly_usable);
|
||||
// Defensive: build_overlay_advert returns None on empty endpoints,
|
||||
// so this is only reachable from non-lifecycle callers.
|
||||
if advert.endpoints.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if advert.has_udp_nat_endpoint() {
|
||||
if advert
|
||||
.signal_relays
|
||||
.as_ref()
|
||||
.is_none_or(|relays| relays.is_empty())
|
||||
{
|
||||
return Err(BootstrapError::InvalidAdvert(
|
||||
"udp:nat endpoint requires non-empty signalRelays".to_string(),
|
||||
));
|
||||
}
|
||||
if advert
|
||||
.stun_servers
|
||||
.as_ref()
|
||||
.is_none_or(|servers| servers.is_empty())
|
||||
{
|
||||
return Err(BootstrapError::InvalidAdvert(
|
||||
"udp:nat endpoint requires non-empty stunServers".to_string(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
advert.signal_relays = None;
|
||||
advert.stun_servers = None;
|
||||
}
|
||||
|
||||
let expires_at = now_ms() + self.config.advert_ttl_secs * 1000;
|
||||
let tags = vec![
|
||||
Tag::identifier(ADVERT_IDENTIFIER.to_string()),
|
||||
@@ -964,7 +1031,7 @@ impl NostrRendezvous {
|
||||
// NIP-09 delete here is redundant and races with the replacement
|
||||
// publish, which strict relays (e.g. Damus) honor by removing the
|
||||
// new advert too.
|
||||
self.advert.set_event_id(event.id);
|
||||
*self.current_advert_event_id.write().await = Some(event.id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1203,17 +1270,17 @@ impl NostrRendezvous {
|
||||
// single matching socket pair survives on both sides. Asymmetric /
|
||||
// one-sided `auto_connect` (no co-active initiator) is never suppressed,
|
||||
// preserving connectivity. See `suppress_responder_for_own_initiator`.
|
||||
match (
|
||||
PeerIdentity::from_npub(&self.npub),
|
||||
PeerIdentity::from_npub(&sender_npub),
|
||||
) {
|
||||
(Ok(ours), Ok(theirs)) => {
|
||||
match self.traversal.classify_incoming_offer(
|
||||
&sender_npub,
|
||||
ours.node_addr(),
|
||||
theirs.node_addr(),
|
||||
) {
|
||||
OfferDisposition::Suppress => {
|
||||
if self.active_initiators.lock().await.contains(&sender_npub) {
|
||||
match (
|
||||
PeerIdentity::from_npub(&self.npub),
|
||||
PeerIdentity::from_npub(&sender_npub),
|
||||
) {
|
||||
(Ok(ours), Ok(theirs)) => {
|
||||
if suppress_responder_for_own_initiator(
|
||||
ours.node_addr(),
|
||||
theirs.node_addr(),
|
||||
true,
|
||||
) {
|
||||
debug!(
|
||||
peer = %peer_short,
|
||||
session = %short_id(&offer.session_id),
|
||||
@@ -1221,47 +1288,19 @@ impl NostrRendezvous {
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
OfferDisposition::Proceed => {}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Could not derive a NodeAddr for one side; fall through and
|
||||
// answer rather than risk suppressing the only session.
|
||||
trace!(
|
||||
peer = %peer_short,
|
||||
"traversal: could not derive NodeAddr for dedup, answering offer"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
match self
|
||||
.traversal
|
||||
.note_session_seen(&offer.session_id, now_ms())
|
||||
{
|
||||
SeenDecision::Replay => {
|
||||
return Err(BootstrapError::Replay(offer.session_id.clone()));
|
||||
}
|
||||
SeenDecision::Fresh { evicted } => {
|
||||
if let Some((evicted, retained)) = evicted {
|
||||
debug!(
|
||||
evicted = evicted,
|
||||
retained = retained,
|
||||
cap = self.config.seen_sessions_max_entries,
|
||||
"seen-sessions cache overflow; evicted oldest entries"
|
||||
_ => {
|
||||
// Could not derive a NodeAddr for one side; fall through and
|
||||
// answer rather than risk suppressing the only session.
|
||||
trace!(
|
||||
peer = %peer_short,
|
||||
"traversal: could not derive NodeAddr for dedup, answering offer"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the answer's relays before binding a socket and running STUN.
|
||||
// Nothing in the relay choice depends on what STUN observes, and an offer
|
||||
// from a peer we share no relay with cannot be answered at all — doing it
|
||||
// in this order spends a STUN round trip, and holds an offer slot for its
|
||||
// duration, only to discard the result.
|
||||
let relays = self.preferred_signal_relays(sender, None).await?;
|
||||
if relays.is_empty() {
|
||||
return Err(BootstrapError::MissingRelays(offer.sender_npub.clone()));
|
||||
}
|
||||
self.mark_session_seen(&offer.session_id).await?;
|
||||
|
||||
let base_socket = std::net::UdpSocket::bind(("0.0.0.0", 0))?;
|
||||
base_socket.set_nonblocking(true)?;
|
||||
@@ -1297,6 +1336,7 @@ impl NostrRendezvous {
|
||||
(!accepted).then_some("no-usable-addresses".to_string()),
|
||||
Some(offer_received_at),
|
||||
);
|
||||
let relays = self.preferred_signal_relays(sender, None).await?;
|
||||
let answer_event = self.send_signal(&relays, sender, &answer).await?;
|
||||
debug!(
|
||||
peer = %peer_short,
|
||||
@@ -1356,15 +1396,15 @@ impl NostrRendezvous {
|
||||
peer_npub: &str,
|
||||
target_pubkey: PublicKey,
|
||||
) -> Result<OverlayAdvert, BootstrapError> {
|
||||
self.prune_advert_cache();
|
||||
if let Some(advert) = self.advert.cached_advert(peer_npub) {
|
||||
self.prune_advert_cache().await;
|
||||
if let Some(cached) = self.advert_cache.read().await.get(peer_npub).cloned() {
|
||||
debug!(
|
||||
peer = %short_npub(peer_npub),
|
||||
source = "cache",
|
||||
endpoints = %endpoint_summary(&advert.endpoints),
|
||||
endpoints = %endpoint_summary(&cached.advert.endpoints),
|
||||
"advert: resolved"
|
||||
);
|
||||
return Ok(advert);
|
||||
return Ok(cached.advert);
|
||||
}
|
||||
|
||||
let events = self
|
||||
@@ -1413,8 +1453,11 @@ impl NostrRendezvous {
|
||||
endpoints = %endpoint_summary(&cached.advert.endpoints),
|
||||
"advert: resolved"
|
||||
);
|
||||
self.advert.insert_fetched(peer_npub, cached.clone());
|
||||
self.prune_advert_cache();
|
||||
self.advert_cache
|
||||
.write()
|
||||
.await
|
||||
.insert(peer_npub.to_string(), cached.clone());
|
||||
self.prune_advert_cache().await;
|
||||
Ok(cached.advert)
|
||||
}
|
||||
|
||||
@@ -1423,21 +1466,22 @@ impl NostrRendezvous {
|
||||
target_pubkey: PublicKey,
|
||||
advert: Option<&OverlayAdvert>,
|
||||
) -> Result<Vec<String>, BootstrapError> {
|
||||
let inbox = self.find_recipient_inbox_relays(target_pubkey).await?;
|
||||
let pool: HashSet<RelayUrl> = self.client.pool().all_relays().await.into_keys().collect();
|
||||
let usable = signal_relays(
|
||||
&inbox,
|
||||
advert.and_then(|advert| advert.signal_relays.as_deref()),
|
||||
&self.config.dm_relays,
|
||||
&pool,
|
||||
);
|
||||
debug!(
|
||||
peer = %target_pubkey.to_bech32().map(|npub| short_npub(&npub)).unwrap_or_default(),
|
||||
inbox = inbox.len(),
|
||||
usable = usable.len(),
|
||||
"traversal: signal relays resolved against the client pool"
|
||||
);
|
||||
Ok(usable)
|
||||
let mut merged = self.find_recipient_inbox_relays(target_pubkey).await?;
|
||||
if let Some(advert) = advert
|
||||
&& let Some(relays) = advert.signal_relays.as_ref()
|
||||
{
|
||||
for relay in relays {
|
||||
if !merged.contains(relay) {
|
||||
merged.push(relay.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
for relay in &self.config.dm_relays {
|
||||
if !merged.contains(relay) {
|
||||
merged.push(relay.clone());
|
||||
}
|
||||
}
|
||||
Ok(merged)
|
||||
}
|
||||
|
||||
async fn find_recipient_inbox_relays(
|
||||
@@ -1559,19 +1603,39 @@ impl NostrRendezvous {
|
||||
Ok(advert)
|
||||
}
|
||||
|
||||
fn prune_advert_cache(&self) {
|
||||
if let Some((evicted, retained)) = self.advert.prune(now_ms()) {
|
||||
debug!(
|
||||
evicted,
|
||||
retained,
|
||||
cap = self.config.advert_cache_max_entries,
|
||||
"advert cache overflow; evicted oldest entries"
|
||||
);
|
||||
async fn prune_advert_cache(&self) {
|
||||
let now = now_ms();
|
||||
let mut cache = self.advert_cache.write().await;
|
||||
cache.retain(|_, entry| entry.valid_until_ms > now);
|
||||
if cache.len() <= self.config.advert_cache_max_entries {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut oldest = cache
|
||||
.iter()
|
||||
.map(|(npub, entry)| (npub.clone(), entry.valid_until_ms))
|
||||
.collect::<Vec<_>>();
|
||||
oldest.sort_by_key(|(_, ts)| *ts);
|
||||
let overflow = cache
|
||||
.len()
|
||||
.saturating_sub(self.config.advert_cache_max_entries);
|
||||
for (npub, _) in oldest.into_iter().take(overflow) {
|
||||
cache.remove(&npub);
|
||||
}
|
||||
debug!(
|
||||
evicted = overflow,
|
||||
retained = cache.len(),
|
||||
cap = self.config.advert_cache_max_entries,
|
||||
"advert cache overflow; evicted oldest entries"
|
||||
);
|
||||
}
|
||||
|
||||
fn advert_max_age_ms(&self) -> u64 {
|
||||
self.config.advert_ttl_secs * 1000 * ADVERT_CACHE_STALE_GRACE_MULTIPLIER
|
||||
}
|
||||
|
||||
fn event_valid_until_ms(&self, event: &Event) -> Option<u64> {
|
||||
self.advert.event_valid_until_ms(event, now_ms())
|
||||
Self::compute_advert_valid_until_ms(event, self.advert_max_age_ms(), now_ms())
|
||||
}
|
||||
|
||||
pub(super) fn compute_advert_valid_until_ms(
|
||||
@@ -1635,61 +1699,42 @@ impl NostrRendezvous {
|
||||
.map_err(|e| BootstrapError::Nostr(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Retain only the candidates the client pool actually holds.
|
||||
///
|
||||
/// `send_event_to` rejects the whole send with `RelayNotFound` if any single URL
|
||||
/// is outside the pool, so a signal addressed to a peer's advertised relays fails
|
||||
/// entirely on one relay we are not configured with. Filtering first turns that
|
||||
/// into a send to the relays we share.
|
||||
///
|
||||
/// Comparison is on the normalized `RelayUrl` rather than the raw string, because
|
||||
/// the pool is keyed that way: a candidate spelled `wss://relay.example/` matches
|
||||
/// a configured `wss://relay.example`. Order is preserved, candidates that fail
|
||||
/// to parse are dropped, and duplicates that normalize alike are collapsed.
|
||||
fn retain_pooled_relays(candidates: &[String], pool: &HashSet<RelayUrl>) -> Vec<String> {
|
||||
let mut seen: HashSet<RelayUrl> = HashSet::new();
|
||||
let mut usable = Vec::with_capacity(candidates.len());
|
||||
for candidate in candidates {
|
||||
let Ok(url) = RelayUrl::parse(candidate) else {
|
||||
continue;
|
||||
};
|
||||
if pool.contains(&url) && seen.insert(url.clone()) {
|
||||
usable.push(url.to_string());
|
||||
async fn mark_session_seen(&self, session_id: &str) -> Result<(), BootstrapError> {
|
||||
let now = now_ms();
|
||||
let expiry = now + self.config.replay_window_secs * 1000;
|
||||
let mut seen = self.seen_sessions.lock().await;
|
||||
seen.retain(|_, expires_at| *expires_at > now);
|
||||
if seen.contains_key(session_id) {
|
||||
return Err(BootstrapError::Replay(session_id.to_string()));
|
||||
}
|
||||
}
|
||||
usable
|
||||
}
|
||||
|
||||
/// Choose the relays a traversal signal for one peer should be sent to.
|
||||
///
|
||||
/// The candidates are the peer's NIP-17 inbox relays, then the relays its advert
|
||||
/// nominates for signaling, then our own DM relays — remote-supplied first, ours
|
||||
/// last, so a peer's preference is honored where we can act on it. The result is
|
||||
/// whatever survives [`retain_pooled_relays`].
|
||||
///
|
||||
/// This is the whole decision, kept synchronous so it can be exercised without a
|
||||
/// relay client: the caller's only job is to supply the fetched inbox list and
|
||||
/// the pool.
|
||||
pub(super) fn signal_relays(
|
||||
inbox: &[String],
|
||||
advert_signal: Option<&[String]>,
|
||||
dm_relays: &[String],
|
||||
pool: &HashSet<RelayUrl>,
|
||||
) -> Vec<String> {
|
||||
let mut merged: Vec<String> = inbox.to_vec();
|
||||
for relay in advert_signal.unwrap_or_default().iter().chain(dm_relays) {
|
||||
if !merged.contains(relay) {
|
||||
merged.push(relay.clone());
|
||||
seen.insert(session_id.to_string(), expiry);
|
||||
if seen.len() > self.config.seen_sessions_max_entries {
|
||||
let mut oldest = seen
|
||||
.iter()
|
||||
.map(|(session, expires_at)| (session.clone(), *expires_at))
|
||||
.collect::<Vec<_>>();
|
||||
oldest.sort_by_key(|(_, expires_at)| *expires_at);
|
||||
let overflow = seen
|
||||
.len()
|
||||
.saturating_sub(self.config.seen_sessions_max_entries);
|
||||
for (session, _) in oldest.into_iter().take(overflow) {
|
||||
seen.remove(&session);
|
||||
}
|
||||
debug!(
|
||||
evicted = overflow,
|
||||
retained = seen.len(),
|
||||
cap = self.config.seen_sessions_max_entries,
|
||||
"seen-sessions cache overflow; evicted oldest entries"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
retain_pooled_relays(&merged, pool)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl NostrRendezvous {
|
||||
/// Build a minimal `NostrRendezvous` for unit tests. No relay client is
|
||||
impl NostrDiscovery {
|
||||
/// Build a minimal `NostrDiscovery` for unit tests. No relay client is
|
||||
/// connected and no background tasks are spawned; only the in-memory
|
||||
/// `advert_cache` and `npub` are usable. Intended for cache-injection
|
||||
/// tests of consumers (e.g. `Node::run_open_discovery_sweep`).
|
||||
@@ -1701,7 +1746,7 @@ impl NostrRendezvous {
|
||||
.signer(keys.clone())
|
||||
.opts(ClientOptions::new().autoconnect(false))
|
||||
.build();
|
||||
let config = NostrRendezvousConfig::default();
|
||||
let config = NostrDiscoveryConfig::default();
|
||||
let offer_slots = Arc::new(Semaphore::new(config.max_concurrent_incoming_offers));
|
||||
let (event_tx, event_rx) = mpsc::unbounded_channel();
|
||||
let failure_state = FailureState::new(
|
||||
@@ -1710,25 +1755,18 @@ impl NostrRendezvous {
|
||||
config.warn_log_interval_secs,
|
||||
config.failure_state_max_entries,
|
||||
);
|
||||
let advert = AdvertMachine::new(
|
||||
npub.clone(),
|
||||
config.advertise,
|
||||
config.advert_ttl_secs * 1000 * ADVERT_CACHE_STALE_GRACE_MULTIPLIER,
|
||||
config.advert_cache_max_entries,
|
||||
);
|
||||
let traversal = TraversalMachine::new(
|
||||
config.replay_window_secs * 1000,
|
||||
config.seen_sessions_max_entries,
|
||||
);
|
||||
Self {
|
||||
client,
|
||||
keys,
|
||||
pubkey,
|
||||
npub,
|
||||
config,
|
||||
advert,
|
||||
traversal,
|
||||
advert_cache: RwLock::new(HashMap::new()),
|
||||
local_advert: RwLock::new(None),
|
||||
current_advert_event_id: RwLock::new(None),
|
||||
pending_answers: Mutex::new(HashMap::new()),
|
||||
active_initiators: Mutex::new(HashSet::new()),
|
||||
seen_sessions: Mutex::new(HashMap::new()),
|
||||
offer_slots,
|
||||
event_tx,
|
||||
event_rx: Mutex::new(event_rx),
|
||||
@@ -1744,24 +1782,6 @@ impl NostrRendezvous {
|
||||
}
|
||||
}
|
||||
|
||||
/// Install the five background-task handles that `start` would install, so
|
||||
/// liveness tests can drive `is_finished()` without live relays. Each
|
||||
/// argument is the handle to place in the matching slot.
|
||||
pub(crate) async fn install_tasks_for_test(
|
||||
&self,
|
||||
connect: JoinHandle<()>,
|
||||
relay_startup: JoinHandle<()>,
|
||||
notify: JoinHandle<()>,
|
||||
publish: JoinHandle<()>,
|
||||
advertise: JoinHandle<()>,
|
||||
) {
|
||||
*self.connect_task.lock().await = Some(connect);
|
||||
*self.relay_startup_task.lock().await = Some(relay_startup);
|
||||
*self.notify_task.lock().await = Some(notify);
|
||||
*self.publish_task.lock().await = Some(publish);
|
||||
*self.advertise_task.lock().await = Some(advertise);
|
||||
}
|
||||
|
||||
/// Build a `CachedOverlayAdvert` for tests with a single endpoint and
|
||||
/// a generous validity window (one hour from `now_ms()`).
|
||||
pub(crate) fn cached_advert_for_test(
|
||||
@@ -1783,22 +1803,11 @@ impl NostrRendezvous {
|
||||
}
|
||||
}
|
||||
|
||||
/// Point the test instance's advert relays at explicit URLs. Unit tests
|
||||
/// that exercise `refetch_advert_for_stale_check` use this to replace the
|
||||
/// default public relay list with a local blackhole, so the refetch runs
|
||||
/// its full 2s timeout without touching the network.
|
||||
pub(crate) async fn set_advert_relays_for_test(&mut self, relays: Vec<String>) {
|
||||
for url in &relays {
|
||||
let _ = self.client.add_relay(url.as_str()).await;
|
||||
}
|
||||
self.client.connect().await;
|
||||
self.config.advert_relays = relays;
|
||||
}
|
||||
|
||||
/// Insert a cached advert directly into the in-memory cache. Used by
|
||||
/// unit tests to set up consumer-side state without needing live relays.
|
||||
pub(crate) async fn insert_advert_for_test(&self, npub: String, advert: CachedOverlayAdvert) {
|
||||
self.advert.insert_fetched(&npub, advert);
|
||||
let mut cache = self.advert_cache.write().await;
|
||||
cache.insert(npub, advert);
|
||||
}
|
||||
|
||||
/// Queue a bootstrap event directly for lifecycle tests without live relays
|
||||
@@ -1,18 +1,15 @@
|
||||
use std::collections::HashSet;
|
||||
use nostr::prelude::{EventBuilder, Kind, Tag, Timestamp};
|
||||
|
||||
use nostr::prelude::{EventBuilder, Kind, RelayUrl, Tag, Timestamp};
|
||||
|
||||
use super::runtime::{NostrRendezvous, signal_relays};
|
||||
use super::runtime::{NostrDiscovery, suppress_responder_for_own_initiator};
|
||||
use super::signal::{
|
||||
FreshnessOutcome, build_signal_event, create_traversal_answer, create_traversal_offer,
|
||||
estimate_clock_skew, validate_offer_freshness, validate_traversal_answer_for_offer,
|
||||
};
|
||||
use super::stun::{parse_stun_binding_success, parse_stun_url};
|
||||
use super::traversal::{
|
||||
PunchStrategy, build_punch_packet, now_ms, parse_punch_packet, plan_punch_targets,
|
||||
PunchStrategy, build_punch_packet, parse_punch_packet, plan_punch_targets,
|
||||
planned_remote_endpoints, session_hash,
|
||||
};
|
||||
use super::traversal_machine::suppress_responder_for_own_initiator;
|
||||
use super::{
|
||||
ADVERT_IDENTIFIER, ADVERT_KIND, ADVERT_VERSION, OverlayAdvert, OverlayEndpointAdvert,
|
||||
OverlayTransportKind, PunchHint, PunchPacketKind, TraversalAddress,
|
||||
@@ -107,7 +104,7 @@ fn rejects_invalid_overlay_adverts() {
|
||||
signal_relays: None,
|
||||
stun_servers: None,
|
||||
};
|
||||
assert!(NostrRendezvous::validate_overlay_advert(missing_nat_metadata).is_err());
|
||||
assert!(NostrDiscovery::validate_overlay_advert(missing_nat_metadata).is_err());
|
||||
|
||||
let wrong_identifier = OverlayAdvert {
|
||||
identifier: "not-fips-overlay".to_string(),
|
||||
@@ -119,7 +116,7 @@ fn rejects_invalid_overlay_adverts() {
|
||||
signal_relays: None,
|
||||
stun_servers: None,
|
||||
};
|
||||
assert!(NostrRendezvous::validate_overlay_advert(wrong_identifier).is_err());
|
||||
assert!(NostrDiscovery::validate_overlay_advert(wrong_identifier).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -145,7 +142,7 @@ fn validate_overlay_advert_filters_unroutable_direct_endpoints() {
|
||||
stun_servers: None,
|
||||
};
|
||||
|
||||
let validated = NostrRendezvous::validate_overlay_advert(advert).unwrap();
|
||||
let validated = NostrDiscovery::validate_overlay_advert(advert).unwrap();
|
||||
assert_eq!(validated.endpoints.len(), 1);
|
||||
assert_eq!(validated.endpoints[0].addr, "8.8.8.8:443");
|
||||
}
|
||||
@@ -169,7 +166,7 @@ fn validate_overlay_advert_rejects_only_unroutable_direct_endpoints() {
|
||||
stun_servers: None,
|
||||
};
|
||||
|
||||
let err = NostrRendezvous::validate_overlay_advert(advert).unwrap_err();
|
||||
let err = NostrDiscovery::validate_overlay_advert(advert).unwrap_err();
|
||||
assert!(err.to_string().contains("missing publicly routable"));
|
||||
}
|
||||
|
||||
@@ -178,7 +175,7 @@ fn advert_freshness_rejects_expired_events() {
|
||||
let now_secs = Timestamp::now().as_secs();
|
||||
let event = signed_overlay_advert_event(now_secs, Some(now_secs.saturating_sub(1)));
|
||||
let valid_until =
|
||||
NostrRendezvous::compute_advert_valid_until_ms(&event, 600_000, now_secs * 1000);
|
||||
NostrDiscovery::compute_advert_valid_until_ms(&event, 600_000, now_secs * 1000);
|
||||
assert!(valid_until.is_none());
|
||||
}
|
||||
|
||||
@@ -188,7 +185,7 @@ fn advert_freshness_rejects_stale_created_at_without_expiration() {
|
||||
let stale_created = now_secs.saturating_sub(10_000);
|
||||
let event = signed_overlay_advert_event(stale_created, None);
|
||||
let valid_until =
|
||||
NostrRendezvous::compute_advert_valid_until_ms(&event, 600_000, now_secs * 1000);
|
||||
NostrDiscovery::compute_advert_valid_until_ms(&event, 600_000, now_secs * 1000);
|
||||
assert!(valid_until.is_none());
|
||||
}
|
||||
|
||||
@@ -197,7 +194,7 @@ fn advert_freshness_uses_earliest_expiration_bound() {
|
||||
let now_secs = Timestamp::now().as_secs();
|
||||
let event = signed_overlay_advert_event(now_secs.saturating_sub(10), Some(now_secs + 30));
|
||||
let valid_until =
|
||||
NostrRendezvous::compute_advert_valid_until_ms(&event, 3_600_000, now_secs * 1000)
|
||||
NostrDiscovery::compute_advert_valid_until_ms(&event, 3_600_000, now_secs * 1000)
|
||||
.expect("event should be fresh");
|
||||
assert_eq!(valid_until, (now_secs + 30) * 1000);
|
||||
}
|
||||
@@ -673,290 +670,3 @@ fn responder_suppression_election() {
|
||||
&smaller, &smaller, true
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn now_ms_tracks_the_wall_clock() {
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn wall_ms() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock is after the Unix epoch")
|
||||
.as_millis() as u64
|
||||
}
|
||||
|
||||
// Bracket a sample between two independent wall-clock reads taken either
|
||||
// side of it. This is the property the traversal clock has to hold for the
|
||||
// NIP-40 expiration tags it computes to be in the future when published.
|
||||
//
|
||||
// Read this for what it is: it pins the contract (Unix epoch, milliseconds,
|
||||
// tracking real time) and it fires on a host that has actually suspended,
|
||||
// where the sample falls below `before` by the suspend duration. It is NOT a
|
||||
// regression guard for the anchored-clock defect. Nothing reachable from a
|
||||
// unit test can simulate a suspend, so on a machine that has not slept, an
|
||||
// anchored implementation passes this -- deterministically when this is the
|
||||
// first caller of `now_ms()` in the binary, and otherwise with a probability
|
||||
// set by the fractional millisecond the anchor happened to capture.
|
||||
let before = wall_ms();
|
||||
let sampled = now_ms();
|
||||
let after = wall_ms();
|
||||
|
||||
assert!(
|
||||
sampled >= before,
|
||||
"now_ms() is behind the wall clock: {sampled} < {before}"
|
||||
);
|
||||
assert!(
|
||||
sampled <= after,
|
||||
"now_ms() is ahead of the wall clock: {sampled} > {after}"
|
||||
);
|
||||
}
|
||||
|
||||
fn pool(urls: &[&str]) -> HashSet<RelayUrl> {
|
||||
urls.iter()
|
||||
.map(|url| RelayUrl::parse(url).expect("test pool url parses"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn candidates(urls: &[&str]) -> Vec<String> {
|
||||
urls.iter().map(|url| url.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_pool_relay_does_not_suppress_the_shared_ones() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&[
|
||||
"wss://relay.damus.io",
|
||||
"wss://temp.iris.to",
|
||||
"wss://nos.lol",
|
||||
]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&[
|
||||
"wss://relay.damus.io",
|
||||
"wss://nos.lol",
|
||||
"wss://offchain.pub",
|
||||
]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable,
|
||||
vec![
|
||||
"wss://relay.damus.io".to_string(),
|
||||
"wss://nos.lol".to_string()
|
||||
],
|
||||
"the unknown relay must be dropped without taking the shared ones with it"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_slash_and_host_case_variants_are_retained() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://Relay.Damus.io/", "wss://nos.lol"]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&["wss://relay.damus.io", "wss://nos.lol"]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable.len(),
|
||||
2,
|
||||
"normalized spellings of a configured relay are the same relay: {usable:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicates_that_normalize_alike_are_collapsed() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://nos.lol", "wss://nos.lol/", "wss://NOS.LOL"]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&["wss://nos.lol"]),
|
||||
);
|
||||
assert_eq!(usable, vec!["wss://nos.lol".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unparseable_candidates_are_dropped_rather_than_failing_the_set() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["not a url", "wss://nos.lol"]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&["wss://nos.lol"]),
|
||||
);
|
||||
assert_eq!(usable, vec!["wss://nos.lol".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_shared_relay_yields_an_empty_set_for_the_caller_to_reject() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://temp.iris.to"]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&["wss://nos.lol"]),
|
||||
);
|
||||
assert!(
|
||||
usable.is_empty(),
|
||||
"with no overlap the caller must see nothing to send to, not a doomed send"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_relays_merges_all_three_sources_then_filters() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://temp.iris.to", "wss://nos.lol"]),
|
||||
Some(&candidates(&[
|
||||
"wss://relay.damus.io",
|
||||
"wss://unknown.example",
|
||||
])),
|
||||
&candidates(&["wss://offchain.pub"]),
|
||||
&pool(&[
|
||||
"wss://nos.lol",
|
||||
"wss://relay.damus.io",
|
||||
"wss://offchain.pub",
|
||||
]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable,
|
||||
vec![
|
||||
"wss://nos.lol".to_string(),
|
||||
"wss://relay.damus.io".to_string(),
|
||||
"wss://offchain.pub".to_string(),
|
||||
],
|
||||
"every source must contribute, and only the out-of-pool entries drop out"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_relays_keeps_our_dm_relays_when_the_peer_shares_nothing() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://temp.iris.to"]),
|
||||
Some(&candidates(&["wss://also.unknown"])),
|
||||
&candidates(&["wss://nos.lol"]),
|
||||
&pool(&["wss://nos.lol"]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable,
|
||||
vec!["wss://nos.lol".to_string()],
|
||||
"our own DM relays are always in the pool, so the result is never empty \
|
||||
while any are configured"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_relays_without_an_advert_still_resolves() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://nos.lol", "wss://temp.iris.to"]),
|
||||
None,
|
||||
&candidates(&["wss://offchain.pub"]),
|
||||
&pool(&["wss://nos.lol", "wss://offchain.pub"]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable,
|
||||
vec![
|
||||
"wss://nos.lol".to_string(),
|
||||
"wss://offchain.pub".to_string()
|
||||
],
|
||||
"the responder path passes no advert and must still produce a target set"
|
||||
);
|
||||
}
|
||||
|
||||
/// A `JoinHandle` for a task that has definitely completed. Yields until the
|
||||
/// runtime has polled the no-op task to completion, so `is_finished()` is
|
||||
/// deterministically `true` on return.
|
||||
async fn finished_handle() -> tokio::task::JoinHandle<()> {
|
||||
let handle = tokio::spawn(async {});
|
||||
while !handle.is_finished() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
handle
|
||||
}
|
||||
|
||||
/// A `JoinHandle` for a task that never completes.
|
||||
fn live_handle() -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(std::future::pending::<()>())
|
||||
}
|
||||
|
||||
/// The regression case: `connect_task` and `relay_startup_task` both return by
|
||||
/// design (`Client::connect()` only kicks off per-relay connection tasks;
|
||||
/// the startup loop breaks on the first successful subscribe), so a healthy
|
||||
/// node has two finished handles and must still report live.
|
||||
#[tokio::test]
|
||||
async fn nostr_liveness_ignores_the_tasks_that_return_by_design() {
|
||||
let runtime = NostrRendezvous::new_for_test();
|
||||
runtime
|
||||
.install_tasks_for_test(
|
||||
finished_handle().await,
|
||||
finished_handle().await,
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
!runtime.is_finished(),
|
||||
"a node whose connect/relay-startup tasks have returned normally is healthy"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nostr_liveness_fires_when_the_notify_loop_dies() {
|
||||
let runtime = NostrRendezvous::new_for_test();
|
||||
runtime
|
||||
.install_tasks_for_test(
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
finished_handle().await,
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
runtime.is_finished(),
|
||||
"a dead inbound notify loop means no advert or signal is ever received again"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nostr_liveness_fires_when_the_publish_loop_dies() {
|
||||
let runtime = NostrRendezvous::new_for_test();
|
||||
runtime
|
||||
.install_tasks_for_test(
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
finished_handle().await,
|
||||
live_handle(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
runtime.is_finished(),
|
||||
"a dead publish loop means this node stops being discoverable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nostr_liveness_fires_when_the_advertise_loop_dies() {
|
||||
let runtime = NostrRendezvous::new_for_test();
|
||||
runtime
|
||||
.install_tasks_for_test(
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
live_handle(),
|
||||
finished_handle().await,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
runtime.is_finished(),
|
||||
"a dead advertise ticker means the advert is never refreshed"
|
||||
);
|
||||
}
|
||||
|
||||
/// `shutdown` takes every handle, leaving `None`. That must read as finished so
|
||||
/// the 2s liveness poll monitor terminates instead of spinning after a stop.
|
||||
#[tokio::test]
|
||||
async fn nostr_liveness_reports_finished_once_the_handles_are_taken() {
|
||||
let runtime = NostrRendezvous::new_for_test();
|
||||
assert!(
|
||||
runtime.is_finished(),
|
||||
"no installed handles (post-shutdown) reads as finished"
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::net::UdpSocket;
|
||||
@@ -177,15 +177,17 @@ pub(super) async fn run_punch_attempt(
|
||||
let Ok(Ok((len, remote))) = recv else {
|
||||
break Err(BootstrapError::PunchTimeout(session_id.to_string()));
|
||||
};
|
||||
match classify_punch_packet(&buf[..len], expected_hash) {
|
||||
PunchAction::Ignore => continue,
|
||||
PunchAction::Ack { sequence } => {
|
||||
let ack = build_punch_packet(PunchPacketKind::Ack, sequence, session_id);
|
||||
let _ = udp.send_to(&ack, remote).await;
|
||||
break Ok(remote);
|
||||
}
|
||||
PunchAction::Matched => break Ok(remote),
|
||||
let Ok(packet) = parse_punch_packet(&buf[..len]) else {
|
||||
continue;
|
||||
};
|
||||
if packet.session_hash != expected_hash {
|
||||
continue;
|
||||
}
|
||||
if packet.kind == PunchPacketKind::Probe {
|
||||
let ack = build_punch_packet(PunchPacketKind::Ack, packet.sequence, session_id);
|
||||
let _ = udp.send_to(&ack, remote).await;
|
||||
}
|
||||
break Ok(remote);
|
||||
};
|
||||
send_handle.abort();
|
||||
result
|
||||
@@ -195,35 +197,25 @@ pub(super) fn nonce() -> String {
|
||||
format!("{}-{:016x}", now_ms(), rand::random::<u64>())
|
||||
}
|
||||
|
||||
/// Current Unix time in milliseconds, read from the wall clock on every call.
|
||||
///
|
||||
/// This deliberately does not cache a start-of-process anchor and advance it
|
||||
/// with a monotonic `Instant`. A monotonic clock does not advance while the host
|
||||
/// is suspended, so an anchored value trails real time by the suspend duration
|
||||
/// for the remaining life of the process. Every expiry computed from it is then
|
||||
/// published already in the past, the relay drops the event as expired, and
|
||||
/// traversal signalling fails until the daemon is restarted.
|
||||
///
|
||||
/// About half the consumers publish or serialize the value as an absolute
|
||||
/// timestamp: the NIP-40 expiration tags on adverts and traversal signals, and
|
||||
/// the `issuedAt`/`expiresAt` fields of offers and answers. The rest compare it
|
||||
/// against timestamps on the same basis, including the peer-authored, signed
|
||||
/// `created_at` of a received advert, so they need it to track real time too.
|
||||
///
|
||||
/// The interval-shaped consumers survive a step in the wall clock. A forward
|
||||
/// step, which is what a resume produces, saturates the punch start delay to
|
||||
/// zero so punching begins immediately; the attempt's own bounds are monotonic
|
||||
/// `Instant` deadlines, so its length is unaffected. A backward step lengthens
|
||||
/// that delay instead and can cost a single punch attempt, which retries. Early
|
||||
/// eviction from the replay window cannot admit a replay under the shipped
|
||||
/// defaults, because the freshness window a replayed offer would also have to
|
||||
/// satisfy (`signal_ttl_secs` plus `FRESHNESS_SKEW_TOLERANCE_MS`, 180s) is
|
||||
/// strictly narrower than the replay window itself (`replay_window_secs`, 300s).
|
||||
pub(super) fn now_ms() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
struct ClockAnchor {
|
||||
started_at: Instant,
|
||||
started_unix_ms: u64,
|
||||
}
|
||||
|
||||
static ANCHOR: OnceLock<ClockAnchor> = OnceLock::new();
|
||||
|
||||
let anchor = ANCHOR.get_or_init(|| ClockAnchor {
|
||||
started_at: Instant::now(),
|
||||
started_unix_ms: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_millis() as u64)
|
||||
.unwrap_or(0),
|
||||
});
|
||||
|
||||
anchor
|
||||
.started_unix_ms
|
||||
.saturating_add(anchor.started_at.elapsed().as_millis() as u64)
|
||||
}
|
||||
|
||||
pub(super) fn session_hash(session_id: &str) -> [u8; 16] {
|
||||
@@ -282,82 +274,3 @@ pub(super) fn parse_punch_packet(bytes: &[u8]) -> Result<PunchPacket, BootstrapE
|
||||
session_hash: hash,
|
||||
})
|
||||
}
|
||||
|
||||
/// Classification of a received UDP datagram on the punch socket. Returned
|
||||
/// by [`classify_punch_packet`]; the timing loop performs the actual ack
|
||||
/// send / break described by the variant.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum PunchAction {
|
||||
/// Not a valid punch packet for this session — keep listening.
|
||||
Ignore,
|
||||
/// A matching probe: the driver builds and sends an ack for `sequence`,
|
||||
/// then treats the peer as reached.
|
||||
Ack { sequence: u32 },
|
||||
/// A matching non-probe (ack) packet: the peer is reached, no ack to send.
|
||||
Matched,
|
||||
}
|
||||
|
||||
/// Pure classification of a received datagram against the expected session
|
||||
/// hash. No I/O: the caller sends any ack and decides control flow.
|
||||
pub(super) fn classify_punch_packet(bytes: &[u8], expected_hash: [u8; 16]) -> PunchAction {
|
||||
let Ok(packet) = parse_punch_packet(bytes) else {
|
||||
return PunchAction::Ignore;
|
||||
};
|
||||
if packet.session_hash != expected_hash {
|
||||
return PunchAction::Ignore;
|
||||
}
|
||||
if packet.kind == PunchPacketKind::Probe {
|
||||
PunchAction::Ack {
|
||||
sequence: packet.sequence,
|
||||
}
|
||||
} else {
|
||||
PunchAction::Matched
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SESSION: &str = "session-classify-vectors";
|
||||
|
||||
#[test]
|
||||
fn classify_ignores_unparseable_bytes() {
|
||||
// P1: too short to parse.
|
||||
assert_eq!(
|
||||
classify_punch_packet(&[0u8; 4], session_hash(SESSION)),
|
||||
PunchAction::Ignore
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_ignores_mismatched_session_hash() {
|
||||
// P2: parseable, but hash is for a different session.
|
||||
let packet = build_punch_packet(PunchPacketKind::Probe, 7, SESSION);
|
||||
let other_hash = session_hash("some-other-session");
|
||||
assert_eq!(
|
||||
classify_punch_packet(&packet, other_hash),
|
||||
PunchAction::Ignore
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_probe_matching_hash_acks_with_sequence() {
|
||||
// P3: matching probe -> Ack carrying the packet's sequence.
|
||||
let packet = build_punch_packet(PunchPacketKind::Probe, 42, SESSION);
|
||||
assert_eq!(
|
||||
classify_punch_packet(&packet, session_hash(SESSION)),
|
||||
PunchAction::Ack { sequence: 42 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_ack_matching_hash_is_matched() {
|
||||
// P4: matching non-probe (ack) -> Matched.
|
||||
let packet = build_punch_packet(PunchPacketKind::Ack, 3, SESSION);
|
||||
assert_eq!(
|
||||
classify_punch_packet(&packet, session_hash(SESSION)),
|
||||
PunchAction::Matched
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,14 @@
|
||||
use super::handoff::EstablishedTraversal;
|
||||
use crate::config::PeerConfig;
|
||||
use crate::discovery::EstablishedTraversal;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const ADVERT_KIND: u16 = 37195;
|
||||
pub const ADVERT_IDENTIFIER: &str = "fips-overlay-v1";
|
||||
pub const ADVERT_VERSION: u32 = 1;
|
||||
pub const SIGNAL_KIND: u16 = 21059;
|
||||
// Defined in the nostr `handoff` submodule; re-exported here so the
|
||||
// Defined at the top-level `discovery` module; re-exported here so the
|
||||
// existing punch sender / receiver imports remain unchanged.
|
||||
pub use super::handoff::{PUNCH_ACK_MAGIC, PUNCH_MAGIC};
|
||||
pub use crate::discovery::{PUNCH_ACK_MAGIC, PUNCH_MAGIC};
|
||||
pub const PROTOCOL_VERSION: &str = "1";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -189,7 +189,7 @@ pub struct PunchPacket {
|
||||
pub session_hash: [u8; 16],
|
||||
}
|
||||
|
||||
/// Outcome of `NostrRendezvous::record_traversal_failure`.
|
||||
/// Outcome of `NostrDiscovery::record_traversal_failure`.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct NostrFailureDecision {
|
||||
pub consecutive_failures: u32,
|
||||
@@ -212,7 +212,7 @@ pub struct NostrPeerFailureView {
|
||||
pub last_observed_skew_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Outcome of `NostrRendezvous::refetch_advert_for_stale_check` (B6).
|
||||
/// Outcome of `NostrDiscovery::refetch_advert_for_stale_check` (B6).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum NostrRefetchOutcome {
|
||||
Evicted,
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Authentication challenge-response protocol.
|
||||
|
||||
use rand::Rng;
|
||||
use secp256k1::XOnlyPublicKey;
|
||||
use secp256k1::{Secp256k1, XOnlyPublicKey};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{IdentityError, NodeAddr};
|
||||
@@ -34,9 +34,9 @@ impl AuthChallenge {
|
||||
/// Verify a response to this challenge.
|
||||
pub fn verify(&self, response: &AuthResponse) -> Result<NodeAddr, IdentityError> {
|
||||
let digest = auth_challenge_digest(&self.0, response.timestamp);
|
||||
let secp = Secp256k1::new();
|
||||
|
||||
super::SECP
|
||||
.verify_schnorr(&response.signature, &digest, &response.pubkey)
|
||||
secp.verify_schnorr(&response.signature, &digest, &response.pubkey)
|
||||
.map_err(|_| IdentityError::SignatureVerificationFailed)?;
|
||||
|
||||
Ok(NodeAddr::from_pubkey(&response.pubkey))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Local node identity with signing capability.
|
||||
|
||||
use secp256k1::{Keypair, PublicKey, SecretKey, XOnlyPublicKey};
|
||||
use secp256k1::{Keypair, PublicKey, Secp256k1, SecretKey, XOnlyPublicKey};
|
||||
use std::fmt;
|
||||
|
||||
use super::auth::{AuthResponse, auth_challenge_digest};
|
||||
@@ -42,7 +42,8 @@ impl Identity {
|
||||
|
||||
/// Create an identity from a secret key.
|
||||
pub fn from_secret_key(secret_key: SecretKey) -> Self {
|
||||
let keypair = Keypair::from_secret_key(&super::SECP, &secret_key);
|
||||
let secp = Secp256k1::new();
|
||||
let keypair = Keypair::from_secret_key(&secp, &secret_key);
|
||||
Self::from_keypair(keypair)
|
||||
}
|
||||
|
||||
@@ -92,8 +93,9 @@ impl Identity {
|
||||
|
||||
/// Sign arbitrary data with this identity's secret key.
|
||||
pub fn sign(&self, data: &[u8]) -> secp256k1::schnorr::Signature {
|
||||
let secp = Secp256k1::new();
|
||||
let digest = sha256(data);
|
||||
super::SECP.sign_schnorr(&digest, &self.keypair)
|
||||
secp.sign_schnorr(&digest, &self.keypair)
|
||||
}
|
||||
|
||||
/// Create an authentication response for a challenge.
|
||||
@@ -101,7 +103,8 @@ impl Identity {
|
||||
/// The response signs: SHA256("fips-auth-v1" || challenge || timestamp)
|
||||
pub fn sign_challenge(&self, challenge: &[u8; 32], timestamp: u64) -> AuthResponse {
|
||||
let digest = auth_challenge_digest(challenge, timestamp);
|
||||
let signature = super::SECP.sign_schnorr(&digest, &self.keypair);
|
||||
let secp = Secp256k1::new();
|
||||
let signature = secp.sign_schnorr(&digest, &self.keypair);
|
||||
AuthResponse {
|
||||
pubkey: self.pubkey(),
|
||||
timestamp,
|
||||
|
||||
@@ -11,9 +11,6 @@ mod local;
|
||||
mod node_addr;
|
||||
mod peer;
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use secp256k1::{All, Secp256k1};
|
||||
use sha2::{Digest, Sha256};
|
||||
use thiserror::Error;
|
||||
|
||||
@@ -24,15 +21,6 @@ pub use local::Identity;
|
||||
pub use node_addr::NodeAddr;
|
||||
pub use peer::PeerIdentity;
|
||||
|
||||
/// Shared secp256k1 context reused across all identity operations.
|
||||
///
|
||||
/// `Secp256k1::new()` allocates a `Secp256k1<All>` and runs randomization /
|
||||
/// blinding table setup; it is designed to be created once and reused rather
|
||||
/// than rebuilt per sign / verify / key-derive call. This single `All` context
|
||||
/// serves both signing and verification across the identity module and still
|
||||
/// performs the standard construction-time blinding.
|
||||
pub(crate) static SECP: LazyLock<Secp256k1<All>> = LazyLock::new(Secp256k1::new);
|
||||
|
||||
/// FIPS address prefix (IPv6 ULA range).
|
||||
pub const FIPS_ADDRESS_PREFIX: u8 = 0xfd;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Remote peer identity (public key only, no signing capability).
|
||||
|
||||
use secp256k1::{Parity, PublicKey, XOnlyPublicKey};
|
||||
use secp256k1::{Parity, PublicKey, Secp256k1, XOnlyPublicKey};
|
||||
use std::fmt;
|
||||
|
||||
use super::encoding::{decode_npub, encode_npub};
|
||||
@@ -107,9 +107,9 @@ impl PeerIdentity {
|
||||
|
||||
/// Verify a signature from this peer.
|
||||
pub fn verify(&self, data: &[u8], signature: &secp256k1::schnorr::Signature) -> bool {
|
||||
let secp = Secp256k1::new();
|
||||
let digest = sha256(data);
|
||||
super::SECP
|
||||
.verify_schnorr(signature, &digest, &self.pubkey)
|
||||
secp.verify_schnorr(signature, &digest, &self.pubkey)
|
||||
.is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::collections::HashSet;
|
||||
use std::net::Ipv6Addr;
|
||||
|
||||
use secp256k1::{Keypair, SecretKey};
|
||||
use secp256k1::{Keypair, Secp256k1, SecretKey};
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -161,10 +161,10 @@ fn test_identity_sign() {
|
||||
let sig = identity.sign(data);
|
||||
|
||||
// Verify the signature manually
|
||||
let secp = secp256k1::Secp256k1::new();
|
||||
let digest = super::sha256(data);
|
||||
assert!(
|
||||
super::SECP
|
||||
.verify_schnorr(&sig, &digest, &identity.pubkey())
|
||||
secp.verify_schnorr(&sig, &digest, &identity.pubkey())
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
@@ -580,12 +580,13 @@ fn test_peer_identity_pubkey_full_even_parity_fallback() {
|
||||
#[test]
|
||||
fn test_peer_identity_pubkey_full_preserved_parity() {
|
||||
// Create two identities and find one with odd parity to make this test meaningful
|
||||
let secp = Secp256k1::new();
|
||||
let secret_bytes: [u8; 32] = [
|
||||
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
|
||||
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e,
|
||||
0x1f, 0x20,
|
||||
];
|
||||
let keypair = Keypair::from_seckey_slice(&super::SECP, &secret_bytes).unwrap();
|
||||
let keypair = Keypair::from_seckey_slice(&secp, &secret_bytes).unwrap();
|
||||
let full_pubkey = keypair.public_key();
|
||||
|
||||
let peer = PeerIdentity::from_pubkey_full(full_pubkey);
|
||||
|
||||
@@ -1,418 +0,0 @@
|
||||
//! Capture lifecycle: the arm/disarm state machine, the sink file, and the
|
||||
//! `fipsctl`-facing operations.
|
||||
//!
|
||||
//! The toggle — not the writer — creates and opens the sink and publishes its
|
||||
//! path, so an unwritable directory fails the `on` command loudly instead of
|
||||
//! being discovered later by a background thread with nobody to report to.
|
||||
//!
|
||||
//! Capture state is a single atomic state machine (`Idle`, `Running`,
|
||||
//! `StoppedByCap`) transitioned by `compare_exchange`. Every accepted control
|
||||
//! connection is served by its own spawned task, so two simultaneous `on`
|
||||
//! requests are genuinely concurrent and must not both create a writer.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use super::recorder;
|
||||
use super::writer;
|
||||
|
||||
/// Default sink directory. Overridable per capture with `--dir`.
|
||||
pub(crate) const DEFAULT_DIR: &str = "/var/log/fips";
|
||||
|
||||
/// Writer flush interval.
|
||||
pub(crate) const INTERVAL: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Size at which a capture stops itself. Reaching it stops the capture rather
|
||||
/// than rotating: the point of a capture is a bounded, self-describing window.
|
||||
pub(crate) const BYTE_CAP: u64 = 32 * 1024 * 1024;
|
||||
|
||||
pub(crate) const IDLE: u8 = 0;
|
||||
pub(crate) const RUNNING: u8 = 1;
|
||||
pub(crate) const STOPPED_BY_CAP: u8 = 2;
|
||||
/// The writer could not write and stopped itself. Distinct from a cap stop:
|
||||
/// a capture that died on a full disk produced a truncated window, and calling
|
||||
/// that "stopped_by_cap" tells the operator it ran to its limit when it did
|
||||
/// not. The trailer line explaining it goes to the same failing file, so the
|
||||
/// state is the only signal that survives.
|
||||
pub(crate) const STOPPED_BY_ERROR: u8 = 3;
|
||||
|
||||
static STATE: AtomicU8 = AtomicU8::new(IDLE);
|
||||
static GATE: AtomicBool = AtomicBool::new(false);
|
||||
static BYTES: AtomicU64 = AtomicU64::new(0);
|
||||
static ACTIVE_PATH: Mutex<Option<PathBuf>> = Mutex::new(None);
|
||||
static WRITER: Mutex<Option<writer::Handle>> = Mutex::new(None);
|
||||
|
||||
/// The per-tick gate. One relaxed load per tick when the feature is compiled in
|
||||
/// and no capture is running.
|
||||
#[inline]
|
||||
pub(crate) fn gate() -> bool {
|
||||
GATE.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(crate) fn bytes_written() -> u64 {
|
||||
BYTES.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(crate) fn add_bytes(n: u64) -> u64 {
|
||||
BYTES.fetch_add(n, Ordering::Relaxed) + n
|
||||
}
|
||||
|
||||
fn active_path() -> Option<PathBuf> {
|
||||
ACTIVE_PATH
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn path_display() -> String {
|
||||
active_path()
|
||||
.map(|p| p.display().to_string())
|
||||
.unwrap_or_else(|| "<none>".to_string())
|
||||
}
|
||||
|
||||
fn state_name(state: u8) -> &'static str {
|
||||
match state {
|
||||
RUNNING => "running",
|
||||
STOPPED_BY_CAP => "stopped_by_cap",
|
||||
STOPPED_BY_ERROR => "stopped_by_error",
|
||||
_ => "idle",
|
||||
}
|
||||
}
|
||||
|
||||
/// Called by the writer when it stops itself. `terminal` is `STOPPED_BY_CAP`
|
||||
/// or `STOPPED_BY_ERROR`. Returns true if this call is the one that stopped it.
|
||||
pub(crate) fn mark_stopped(terminal: u8) -> bool {
|
||||
debug_assert!(terminal == STOPPED_BY_CAP || terminal == STOPPED_BY_ERROR);
|
||||
GATE.store(false, Ordering::Relaxed);
|
||||
STATE
|
||||
.compare_exchange(RUNNING, terminal, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// Join the writer thread, if one exists. Never called while holding another
|
||||
/// lock the writer might want.
|
||||
fn reap() {
|
||||
let handle = WRITER.lock().unwrap_or_else(|e| e.into_inner()).take();
|
||||
if let Some(handle) = handle {
|
||||
handle.stop_and_join();
|
||||
}
|
||||
}
|
||||
|
||||
/// Arm a capture.
|
||||
///
|
||||
/// Opens the sink first and only then starts the writer, so a bad `--dir` is
|
||||
/// reported to the caller rather than logged into the void.
|
||||
pub(crate) fn start(
|
||||
dir: Option<&str>,
|
||||
node_npub: &str,
|
||||
tick_period_secs: u64,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
claim()?;
|
||||
|
||||
match open_sink(dir, node_npub, tick_period_secs) {
|
||||
Ok((file, path, header_len)) => {
|
||||
recorder::reset();
|
||||
BYTES.store(header_len, Ordering::Relaxed);
|
||||
match writer::spawn(file) {
|
||||
Ok(handle) => {
|
||||
*WRITER.lock().unwrap_or_else(|e| e.into_inner()) = Some(handle);
|
||||
*ACTIVE_PATH.lock().unwrap_or_else(|e| e.into_inner()) = Some(path.clone());
|
||||
GATE.store(true, Ordering::Release);
|
||||
Ok(serde_json::json!({
|
||||
"state": "running",
|
||||
"path": path.display().to_string(),
|
||||
"interval_secs": INTERVAL.as_secs(),
|
||||
"byte_cap": BYTE_CAP,
|
||||
}))
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = std::fs::remove_file(&path);
|
||||
BYTES.store(0, Ordering::Relaxed);
|
||||
STATE.store(IDLE, Ordering::Release);
|
||||
Err(format!("cannot start profile writer thread: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
BYTES.store(0, Ordering::Relaxed);
|
||||
STATE.store(IDLE, Ordering::Release);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Take the capture slot, reaping a cap-stopped predecessor if that is what is
|
||||
/// in the way.
|
||||
fn claim() -> Result<(), String> {
|
||||
match STATE.compare_exchange(IDLE, RUNNING, Ordering::AcqRel, Ordering::Acquire) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(RUNNING) => Err(format!("capture already running: {}", path_display())),
|
||||
Err(stopped @ (STOPPED_BY_CAP | STOPPED_BY_ERROR)) => {
|
||||
reap();
|
||||
*ACTIVE_PATH.lock().unwrap_or_else(|e| e.into_inner()) = None;
|
||||
STATE
|
||||
.compare_exchange(stopped, RUNNING, Ordering::AcqRel, Ordering::Acquire)
|
||||
.map(|_| ())
|
||||
.map_err(|_| "capture state changed concurrently; retry".to_string())
|
||||
}
|
||||
Err(_) => Err("capture in an unexpected state".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Disarm the capture. Succeeds when nothing is running, reporting so.
|
||||
pub(crate) fn stop() -> Result<serde_json::Value, String> {
|
||||
let previous = STATE.load(Ordering::Acquire);
|
||||
if previous == IDLE {
|
||||
return Ok(serde_json::json!({"state": "idle", "stopped": false}));
|
||||
}
|
||||
GATE.store(false, Ordering::Release);
|
||||
// The writer wakes on the stop message rather than after the interval, so
|
||||
// this join returns promptly instead of parking the caller for up to one
|
||||
// flush interval.
|
||||
reap();
|
||||
let path = path_display();
|
||||
*ACTIVE_PATH.lock().unwrap_or_else(|e| e.into_inner()) = None;
|
||||
let bytes = bytes_written();
|
||||
// Clear the counter with the slot: a later `status` while idle must not
|
||||
// report the previous capture's byte total as though a capture were live.
|
||||
BYTES.store(0, Ordering::Relaxed);
|
||||
STATE.store(IDLE, Ordering::Release);
|
||||
Ok(serde_json::json!({
|
||||
"state": "idle",
|
||||
"stopped": true,
|
||||
"stopped_by_cap": previous == STOPPED_BY_CAP,
|
||||
"stopped_by_error": previous == STOPPED_BY_ERROR,
|
||||
"path": path,
|
||||
"bytes": bytes,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Report capture state. Distinguishes all four states.
|
||||
pub(crate) fn status() -> serde_json::Value {
|
||||
let state = STATE.load(Ordering::Acquire);
|
||||
serde_json::json!({
|
||||
"state": state_name(state),
|
||||
"path": active_path().map(|p| p.display().to_string()),
|
||||
"bytes": bytes_written(),
|
||||
"byte_cap": BYTE_CAP,
|
||||
"interval_secs": INTERVAL.as_secs(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Stop and reap at daemon teardown. Idempotent.
|
||||
pub(crate) fn shutdown() {
|
||||
if STATE.load(Ordering::Acquire) != IDLE {
|
||||
let _ = stop();
|
||||
}
|
||||
}
|
||||
|
||||
/// Create the sink file and write its header block. Returns the open file, its
|
||||
/// path, and the number of header bytes written.
|
||||
fn open_sink(
|
||||
dir: Option<&str>,
|
||||
node_npub: &str,
|
||||
tick_period_secs: u64,
|
||||
) -> Result<(File, PathBuf, u64), String> {
|
||||
let dir = PathBuf::from(dir.unwrap_or(DEFAULT_DIR));
|
||||
std::fs::create_dir_all(&dir)
|
||||
.map_err(|e| format!("cannot use profile directory {}: {e}", dir.display()))?;
|
||||
|
||||
let start_unix = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
let path = dir.join(format!("profile-{}.tsv", compact_utc(start_unix)));
|
||||
|
||||
let mut file = File::create(&path)
|
||||
.map_err(|e| format!("cannot create profile file {}: {e}", path.display()))?;
|
||||
|
||||
let header = format!(
|
||||
"# fips tick profile\n\
|
||||
# node\t{node}\n\
|
||||
# build\t{build}\n\
|
||||
# platform\t{platform}\n\
|
||||
# tick_period_secs\t{period}\n\
|
||||
# interval_secs\t{interval}\n\
|
||||
# byte_cap\t{cap}\n\
|
||||
# start_utc\t{start_utc}\n\
|
||||
# start_unix\t{start_unix}\n\
|
||||
# NOTE\tstep durations are WALL CLOCK across await points, not CPU time:\n\
|
||||
# NOTE\ta step that awaits I/O accrues the wait, and other tasks may run\n\
|
||||
# NOTE\tinside that span. That is the intended measure for head-of-line\n\
|
||||
# NOTE\tdelay; do not read a large step as CPU cost.\n\
|
||||
# NOTE\tarm_starvation is measured directly as (entry time - the deadline\n\
|
||||
# NOTE\tthe interval scheduled the tick for). It is NOT derived from\n\
|
||||
# NOTE\ttick_entry_gap, which carries no starvation signal by itself:\n\
|
||||
# NOTE\tunder a steady delay every gap is exactly one tick period.\n\
|
||||
ts_unix\tkind\tdomain\tname\tcount\tmax\ttotal\tunit\n",
|
||||
node = node_npub,
|
||||
build = crate::version::short_version(),
|
||||
platform = std::env::consts::OS,
|
||||
period = tick_period_secs,
|
||||
interval = INTERVAL.as_secs(),
|
||||
cap = BYTE_CAP,
|
||||
start_utc = iso_utc(start_unix),
|
||||
start_unix = start_unix,
|
||||
);
|
||||
file.write_all(header.as_bytes())
|
||||
.map_err(|e| format!("cannot write profile header to {}: {e}", path.display()))?;
|
||||
|
||||
Ok((file, path, header.len() as u64))
|
||||
}
|
||||
|
||||
/// Break a Unix timestamp into UTC `(year, month, day, hour, minute, second)`.
|
||||
///
|
||||
/// Hinnant's `civil_from_days`, era-based. No date crate is in the dependency
|
||||
/// set and one filename stamp does not justify adding one.
|
||||
fn utc_parts(unix: u64) -> (i64, u32, u32, u32, u32, u32) {
|
||||
let days = (unix / 86_400) as i64;
|
||||
let secs = unix % 86_400;
|
||||
let z = days + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z.rem_euclid(146_097);
|
||||
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let y = yoe + era * 400;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
|
||||
let m = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32;
|
||||
let y = if m <= 2 { y + 1 } else { y };
|
||||
(
|
||||
y,
|
||||
m,
|
||||
d,
|
||||
(secs / 3_600) as u32,
|
||||
((secs % 3_600) / 60) as u32,
|
||||
(secs % 60) as u32,
|
||||
)
|
||||
}
|
||||
|
||||
/// `20260727T191500Z` — filename-safe.
|
||||
fn compact_utc(unix: u64) -> String {
|
||||
let (y, mo, d, h, mi, s) = utc_parts(unix);
|
||||
format!("{y:04}{mo:02}{d:02}T{h:02}{mi:02}{s:02}Z")
|
||||
}
|
||||
|
||||
/// `2026-07-27T19:15:00Z` — for the header block.
|
||||
fn iso_utc(unix: u64) -> String {
|
||||
let (y, mo, d, h, mi, s) = utc_parts(unix);
|
||||
format!("{y:04}-{mo:02}-{d:02}T{h:02}:{mi:02}:{s:02}Z")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn utc_parts_matches_known_instants() {
|
||||
assert_eq!(utc_parts(0), (1970, 1, 1, 0, 0, 0));
|
||||
assert_eq!(utc_parts(946_684_800), (2000, 1, 1, 0, 0, 0));
|
||||
// 2026-07-27T19:15:00Z
|
||||
assert_eq!(utc_parts(1_785_179_700), (2026, 7, 27, 19, 15, 0));
|
||||
// Leap day.
|
||||
assert_eq!(utc_parts(1_709_164_800), (2024, 2, 29, 0, 0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stamps_render_expected_shapes() {
|
||||
assert_eq!(compact_utc(1_785_179_700), "20260727T191500Z");
|
||||
assert_eq!(iso_utc(1_785_179_700), "2026-07-27T19:15:00Z");
|
||||
}
|
||||
|
||||
// The lock these tests take is shared with the recorder tests, which
|
||||
// mutate the same statics. See `crate::instr::test_serial`.
|
||||
|
||||
#[test]
|
||||
fn capture_round_trip_writes_header_and_rows() {
|
||||
let _guard = crate::instr::test_serial();
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let dir_str = dir.path().to_str().unwrap().to_string();
|
||||
|
||||
let started = start(Some(&dir_str), "npub1test", 1).expect("start");
|
||||
assert_eq!(started["state"], "running");
|
||||
assert!(gate(), "gate must be armed while running");
|
||||
let path = PathBuf::from(started["path"].as_str().unwrap());
|
||||
|
||||
// A second `on` is refused while one is running, and names the file.
|
||||
let refused = start(Some(&dir_str), "npub1test", 1).unwrap_err();
|
||||
assert!(refused.contains(&path.display().to_string()), "{refused}");
|
||||
|
||||
// Feed one observation so the drained rows are not all zero.
|
||||
recorder::record(
|
||||
recorder::Domain::Tick,
|
||||
recorder::Step::WholeTick,
|
||||
Duration::from_millis(7),
|
||||
);
|
||||
|
||||
// Stopping wakes the writer immediately; it drains once more and joins.
|
||||
let stopped = stop().expect("stop");
|
||||
assert_eq!(stopped["stopped"], true);
|
||||
assert_eq!(stopped["stopped_by_cap"], false);
|
||||
assert!(!gate(), "gate must be clear after stop");
|
||||
|
||||
let text = std::fs::read_to_string(&path).expect("read capture");
|
||||
assert!(text.starts_with("# fips tick profile\n"), "{text}");
|
||||
assert!(text.contains("# node\tnpub1test\n"), "{text}");
|
||||
assert!(
|
||||
text.contains("ts_unix\tkind\tdomain\tname\tcount\tmax\ttotal\tunit\n"),
|
||||
"{text}"
|
||||
);
|
||||
// The final drain emitted one row per emitted step, plus the gauges.
|
||||
let rows: Vec<&str> = text
|
||||
.lines()
|
||||
.filter(|l| l.starts_with(|c: char| c.is_ascii_digit()))
|
||||
.collect();
|
||||
let expected_steps = recorder::STEPS.iter().filter(|s| s.emitted()).count();
|
||||
assert_eq!(rows.len(), expected_steps + recorder::N_GAUGES);
|
||||
// The 7 ms observation above is in the whole-tick row, converted to
|
||||
// microseconds. Bounds rather than equality: the gate is process-wide,
|
||||
// so a node under test elsewhere in this binary may have ticked into
|
||||
// the same capture window.
|
||||
let whole_tick = rows
|
||||
.iter()
|
||||
.find(|r| r.contains("\tstep\ttick\twhole_tick\t"))
|
||||
.expect("whole_tick row");
|
||||
let fields: Vec<&str> = whole_tick.split('\t').collect();
|
||||
assert_eq!(fields.last(), Some(&"us"), "{whole_tick}");
|
||||
assert!(
|
||||
fields[4].parse::<u64>().unwrap() >= 1,
|
||||
"count: {whole_tick}"
|
||||
);
|
||||
assert!(
|
||||
fields[5].parse::<u64>().unwrap() >= 7_000,
|
||||
"max: {whole_tick}"
|
||||
);
|
||||
assert!(
|
||||
rows.iter()
|
||||
.any(|r| r.contains("\tgauge\ttick\tarm_starvation\t")),
|
||||
"{text}"
|
||||
);
|
||||
|
||||
// A stop with nothing running is not an error.
|
||||
let again = stop().expect("second stop");
|
||||
assert_eq!(again["stopped"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn start_fails_loudly_on_an_unwritable_directory() {
|
||||
let _guard = crate::instr::test_serial();
|
||||
let err = start(Some("/proc/fips-profile-should-not-exist"), "npub1test", 1)
|
||||
.expect_err("must fail");
|
||||
assert!(err.contains("profile directory"), "{err}");
|
||||
// The failed attempt must leave the slot free for the next try.
|
||||
assert_eq!(STATE.load(Ordering::Acquire), IDLE);
|
||||
assert!(!gate());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_reports_the_bounds_it_is_enforcing() {
|
||||
let _guard = crate::instr::test_serial();
|
||||
let value = status();
|
||||
assert_eq!(value["byte_cap"], BYTE_CAP);
|
||||
assert_eq!(value["interval_secs"], INTERVAL.as_secs());
|
||||
}
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
//! Tick-body instrumentation.
|
||||
//!
|
||||
//! A purpose-built, feature-gated profiler for the rx-loop tick arm. It exists
|
||||
//! to answer one question with field data: which subsystem step dominates the
|
||||
//! tick body, and how long does the tick arm wait behind the other `select!`
|
||||
//! arms before it runs at all.
|
||||
//!
|
||||
//! # Shape
|
||||
//!
|
||||
//! - Everything that costs anything at runtime is behind the `profiling` Cargo
|
||||
//! feature, which is **off by default**. The default build's neutrality is a
|
||||
//! property of the generated code, not of a runtime check.
|
||||
//! - The instrumentation macro is defined twice, once per feature state. The
|
||||
//! feature-off definition is a pure pass-through: it expands to the measured
|
||||
//! expression and nothing else, so no timing code exists in a default build.
|
||||
//! - The always-present surface — [`gate`], [`tick_entry`], [`tick_gauges`],
|
||||
//! [`shutdown`] — exists in both feature states because the call sites in
|
||||
//! `rx_loop.rs` and the lifecycle teardown must compile either way. Their
|
||||
//! feature-off forms are empty (and [`gate`] is a `const fn` returning
|
||||
//! `false`), so they cost nothing.
|
||||
//! - The module is named `instr` rather than `profiling` so that it sorts
|
||||
//! before `node` in `lib.rs`'s alphabetical module list: a `#[macro_use]`
|
||||
//! module must be declared before the modules that use its macros.
|
||||
//!
|
||||
//! # Data model
|
||||
//!
|
||||
//! Domain above step: [`Domain`] carries exactly one variant today
|
||||
//! (`Domain::Tick`). The primitive, the recorder, the writer and the `fipsctl`
|
||||
//! surface all take a domain, so adding a data-path domain later is additive.
|
||||
//! No second domain is declared until something records into it.
|
||||
//!
|
||||
//! Per (domain, step) the recorder keeps an exact count, max and total in fixed
|
||||
//! static `AtomicU64` arrays — no histogram, no accumulation, a fixed footprint
|
||||
//! regardless of run length. Gauges (ticks per interval, peer count, and the
|
||||
//! arm-starvation figures) live in a parallel array and are emitted with an
|
||||
//! explicit row kind so a gauge value never lands under a duration column.
|
||||
|
||||
#[cfg(feature = "profiling")]
|
||||
pub(crate) mod capture;
|
||||
#[cfg(feature = "profiling")]
|
||||
mod recorder;
|
||||
#[cfg(feature = "profiling")]
|
||||
mod writer;
|
||||
|
||||
#[cfg(feature = "profiling")]
|
||||
pub(crate) use recorder::{Domain, Step, now, record};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The macro pair.
|
||||
//
|
||||
// Every path in the body is `$crate::`-qualified. `macro_rules!` bodies are not
|
||||
// path-hygienic: an unqualified `Instant::now()` or `record(..)` would resolve
|
||||
// at the *call site* (`rx_loop.rs`), where neither name is in scope. Importing
|
||||
// them there is worse still, because the imports would be unused in the
|
||||
// feature-off build and red it under `-D warnings`.
|
||||
//
|
||||
// `$e` is evaluated exactly once in both forms, which is what makes nesting the
|
||||
// whole-tick span around the per-step spans safe.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Time `$e` as one step of `$domain`, when `$on` is true.
|
||||
///
|
||||
/// `$on` is the per-tick gate hoist: the enable flag is read once at the top of
|
||||
/// the tick arm into a local, and that local is passed explicitly to every
|
||||
/// invocation, because macro hygiene makes a call-site local invisible inside
|
||||
/// the macro body.
|
||||
#[cfg(feature = "profiling")]
|
||||
macro_rules! instr_step {
|
||||
($on:expr, $domain:expr, $step:expr, $e:expr) => {{
|
||||
let t0 = if $on {
|
||||
Some($crate::instr::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let r = $e;
|
||||
if let Some(t) = t0 {
|
||||
$crate::instr::record($domain, $step, t.elapsed());
|
||||
}
|
||||
r
|
||||
}};
|
||||
}
|
||||
|
||||
/// Feature-off form: a pure pass-through. The expansion contains no clock read,
|
||||
/// no counter update and no reference to the recorder — only the measured
|
||||
/// expression, plus a discard of the gate local so it is not unused.
|
||||
#[cfg(not(feature = "profiling"))]
|
||||
macro_rules! instr_step {
|
||||
($on:expr, $domain:expr, $step:expr, $e:expr) => {{
|
||||
let _ = &$on;
|
||||
$e
|
||||
}};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Always-present surface.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Whether a capture is armed. Read **once per tick** into a local that is then
|
||||
/// passed to each `instr_step!` invocation, so the feature-on-but-idle cost of
|
||||
/// the whole tick arm is a single relaxed load.
|
||||
#[cfg(feature = "profiling")]
|
||||
#[inline]
|
||||
pub(crate) fn gate() -> bool {
|
||||
capture::gate()
|
||||
}
|
||||
|
||||
/// Feature-off gate: a `const fn` returning `false`, so the whole tick arm
|
||||
/// folds to the uninstrumented sequence at compile time.
|
||||
#[cfg(not(feature = "profiling"))]
|
||||
#[inline]
|
||||
pub(crate) const fn gate() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Record how late this tick-arm entry is against its scheduled deadline.
|
||||
///
|
||||
/// The arm is polled **last** under `biased;`, so its lateness is the time it
|
||||
/// spent waiting behind the packet, TUN and control arms. `tokio::time::
|
||||
/// interval::tick` returns the deadline it was scheduled for, so this is a
|
||||
/// direct subtraction rather than a model. Two earlier designs derived it from
|
||||
/// the inter-entry gap instead and both under-reported: one by the previous
|
||||
/// body, the other by reporting only the first difference of the delay, so a
|
||||
/// sustained stall read as zero. The inter-entry gap is still recorded as its
|
||||
/// own gauge, but it carries no starvation signal on its own.
|
||||
#[cfg(feature = "profiling")]
|
||||
#[inline]
|
||||
pub(crate) fn tick_entry(on: bool, deadline: std::time::Instant, now: std::time::Instant) {
|
||||
recorder::tick_entry(on, deadline, now);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "profiling"))]
|
||||
#[inline]
|
||||
pub(crate) fn tick_entry(_on: bool, _deadline: std::time::Instant, _now: std::time::Instant) {}
|
||||
|
||||
/// Sample the per-tick gauges taken from node state.
|
||||
#[cfg(feature = "profiling")]
|
||||
#[inline]
|
||||
pub(crate) fn tick_gauges(on: bool, peers: u64) {
|
||||
recorder::tick_gauges(on, peers);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "profiling"))]
|
||||
#[inline]
|
||||
pub(crate) fn tick_gauges(_on: bool, _peers: u64) {}
|
||||
|
||||
/// Stop and reap any running capture at daemon teardown. Idempotent.
|
||||
#[cfg(feature = "profiling")]
|
||||
pub(crate) fn shutdown() {
|
||||
capture::shutdown();
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "profiling"))]
|
||||
pub(crate) fn shutdown() {}
|
||||
|
||||
/// One serialization lock for every test in this module tree.
|
||||
///
|
||||
/// The recorder counters and the capture state machine are the *same* process
|
||||
/// statics: `capture::start` calls `recorder::reset`, and `capture::stop`
|
||||
/// drains every slot. Two suites with their own locks therefore do not
|
||||
/// serialize against each other, and the feature-on stage runs tests as
|
||||
/// threads in one process, so a capture round-trip can zero the counters a
|
||||
/// recorder test is mid-way through asserting on. One lock for both.
|
||||
#[cfg(all(test, feature = "profiling"))]
|
||||
pub(crate) static TEST_SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
/// Take the shared test lock, recovering from a poisoned mutex so one failing
|
||||
/// test does not cascade into every other one.
|
||||
#[cfg(all(test, feature = "profiling"))]
|
||||
pub(crate) fn test_serial() -> std::sync::MutexGuard<'static, ()> {
|
||||
TEST_SERIAL.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
@@ -1,478 +0,0 @@
|
||||
//! Fixed-footprint recorder: exact count / max / total per (domain, step).
|
||||
//!
|
||||
//! All state is process statics, not `Node` state, because the `fipsctl`
|
||||
//! handler that arms and disarms a capture runs in the control accept task and
|
||||
//! has no `&Node` — that is the whole point of serving it off-loop, so it
|
||||
//! cannot queue behind the behavior it is measuring.
|
||||
//!
|
||||
//! The writer thread is the only reader. It takes each interval's figures with
|
||||
//! `swap(0)`, so there are no "previous value" arrays to carry and the counters
|
||||
//! are per-interval by construction.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Measurement domain. Structural only: one variant today.
|
||||
///
|
||||
/// A data-path domain is deliberately **not** declared until something records
|
||||
/// into it. What generalizes here is the enum, the counter table and the
|
||||
/// writer; the per-tick gate hoist does not, so a data-path domain will need
|
||||
/// its own gate strategy.
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
#[repr(usize)]
|
||||
pub(crate) enum Domain {
|
||||
Tick = 0,
|
||||
}
|
||||
|
||||
pub(crate) const N_DOMAINS: usize = 1;
|
||||
pub(crate) const DOMAINS: [Domain; N_DOMAINS] = [Domain::Tick];
|
||||
|
||||
impl Domain {
|
||||
pub(crate) const fn name(self) -> &'static str {
|
||||
match self {
|
||||
Domain::Tick => "tick",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One measured step of the rx-loop tick arm, in call order, plus the
|
||||
/// whole-body span.
|
||||
///
|
||||
/// `as usize` indexes the counter arrays, so the discriminants are dense and
|
||||
/// `WholeTick` is last (it defines `N_STEPS`). Variants are declared
|
||||
/// unconditionally — see [`Step::emitted`] for how the two platform- and
|
||||
/// profile-conditional steps are kept out of the emitted table.
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
#[repr(usize)]
|
||||
pub(crate) enum Step {
|
||||
CheckTimeouts = 0,
|
||||
ReloadPeerAcl,
|
||||
ReloadHostMap,
|
||||
PollPendingConnects,
|
||||
PollNostrRendezvous,
|
||||
PollLanRendezvous,
|
||||
DrivePeerTimers,
|
||||
ResendPendingRekeys,
|
||||
ResendPendingSessionHandshakes,
|
||||
ResendPendingSessionMsg3,
|
||||
PurgeIdleSessions,
|
||||
ProcessPendingRetries,
|
||||
CheckTreeState,
|
||||
CheckBloomState,
|
||||
ComputeMeshSize,
|
||||
RecordStatsHistory,
|
||||
CheckMmpReports,
|
||||
CheckSessionMmpReports,
|
||||
CheckLinkHeartbeats,
|
||||
CheckRekey,
|
||||
CheckSessionRekey,
|
||||
CheckPendingLookups,
|
||||
PollTransportDiscovery,
|
||||
SampleTransportCongestion,
|
||||
ActivateConnectedUdpSessions,
|
||||
DebugAssertPeerMapsCoherent,
|
||||
/// The whole tick-arm body, from before `check_timeouts` to after the last
|
||||
/// step. Composes safely with the per-step spans because the macro
|
||||
/// evaluates its measured expression exactly once.
|
||||
WholeTick,
|
||||
}
|
||||
|
||||
pub(crate) const N_STEPS: usize = Step::WholeTick as usize + 1;
|
||||
|
||||
/// Every step, in emission order. Index `i` of this table is `STEPS[i] as
|
||||
/// usize`; `steps_table_is_dense` asserts it.
|
||||
pub(crate) const STEPS: [Step; N_STEPS] = [
|
||||
Step::CheckTimeouts,
|
||||
Step::ReloadPeerAcl,
|
||||
Step::ReloadHostMap,
|
||||
Step::PollPendingConnects,
|
||||
Step::PollNostrRendezvous,
|
||||
Step::PollLanRendezvous,
|
||||
Step::DrivePeerTimers,
|
||||
Step::ResendPendingRekeys,
|
||||
Step::ResendPendingSessionHandshakes,
|
||||
Step::ResendPendingSessionMsg3,
|
||||
Step::PurgeIdleSessions,
|
||||
Step::ProcessPendingRetries,
|
||||
Step::CheckTreeState,
|
||||
Step::CheckBloomState,
|
||||
Step::ComputeMeshSize,
|
||||
Step::RecordStatsHistory,
|
||||
Step::CheckMmpReports,
|
||||
Step::CheckSessionMmpReports,
|
||||
Step::CheckLinkHeartbeats,
|
||||
Step::CheckRekey,
|
||||
Step::CheckSessionRekey,
|
||||
Step::CheckPendingLookups,
|
||||
Step::PollTransportDiscovery,
|
||||
Step::SampleTransportCongestion,
|
||||
Step::ActivateConnectedUdpSessions,
|
||||
Step::DebugAssertPeerMapsCoherent,
|
||||
Step::WholeTick,
|
||||
];
|
||||
|
||||
impl Step {
|
||||
pub(crate) const fn name(self) -> &'static str {
|
||||
match self {
|
||||
Step::CheckTimeouts => "check_timeouts",
|
||||
Step::ReloadPeerAcl => "reload_peer_acl",
|
||||
Step::ReloadHostMap => "reload_host_map",
|
||||
Step::PollPendingConnects => "poll_pending_connects",
|
||||
Step::PollNostrRendezvous => "poll_nostr_rendezvous",
|
||||
Step::PollLanRendezvous => "poll_lan_rendezvous",
|
||||
Step::DrivePeerTimers => "drive_peer_timers",
|
||||
Step::ResendPendingRekeys => "resend_pending_rekeys",
|
||||
Step::ResendPendingSessionHandshakes => "resend_pending_session_handshakes",
|
||||
Step::ResendPendingSessionMsg3 => "resend_pending_session_msg3",
|
||||
Step::PurgeIdleSessions => "purge_idle_sessions",
|
||||
Step::ProcessPendingRetries => "process_pending_retries",
|
||||
Step::CheckTreeState => "check_tree_state",
|
||||
Step::CheckBloomState => "check_bloom_state",
|
||||
Step::ComputeMeshSize => "compute_mesh_size",
|
||||
Step::RecordStatsHistory => "record_stats_history",
|
||||
Step::CheckMmpReports => "check_mmp_reports",
|
||||
Step::CheckSessionMmpReports => "check_session_mmp_reports",
|
||||
Step::CheckLinkHeartbeats => "check_link_heartbeats",
|
||||
Step::CheckRekey => "check_rekey",
|
||||
Step::CheckSessionRekey => "check_session_rekey",
|
||||
Step::CheckPendingLookups => "check_pending_lookups",
|
||||
Step::PollTransportDiscovery => "poll_transport_discovery",
|
||||
Step::SampleTransportCongestion => "sample_transport_congestion",
|
||||
Step::ActivateConnectedUdpSessions => "activate_connected_udp_sessions",
|
||||
Step::DebugAssertPeerMapsCoherent => "debug_assert_peer_maps_coherent",
|
||||
Step::WholeTick => "whole_tick",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this step gets a row in this build.
|
||||
///
|
||||
/// Two steps are conditionally compiled at their call sites. Emitting a row
|
||||
/// for them in a build where the call site does not exist would publish a
|
||||
/// count that is structurally zero forever, which reads as "this step never
|
||||
/// runs" rather than "this step is not in this build". The predicates below
|
||||
/// are the same `cfg` expressions that gate the call sites in
|
||||
/// `node::dataplane::rx_loop`; keep them in step.
|
||||
pub(crate) const fn emitted(self) -> bool {
|
||||
match self {
|
||||
Step::ActivateConnectedUdpSessions => {
|
||||
cfg!(any(target_os = "linux", target_os = "macos"))
|
||||
}
|
||||
Step::DebugAssertPeerMapsCoherent => cfg!(debug_assertions),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A scalar sampled once per tick, as opposed to a duration.
|
||||
///
|
||||
/// Gauges carry their own row kind and their own unit in the output so a gauge
|
||||
/// value can never be read as a duration.
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
#[repr(usize)]
|
||||
pub(crate) enum Gauge {
|
||||
Ticks = 0,
|
||||
Peers,
|
||||
TickGap,
|
||||
ArmStarvation,
|
||||
}
|
||||
|
||||
pub(crate) const N_GAUGES: usize = Gauge::ArmStarvation as usize + 1;
|
||||
|
||||
pub(crate) const GAUGES: [Gauge; N_GAUGES] = [
|
||||
Gauge::Ticks,
|
||||
Gauge::Peers,
|
||||
Gauge::TickGap,
|
||||
Gauge::ArmStarvation,
|
||||
];
|
||||
|
||||
impl Gauge {
|
||||
pub(crate) const fn name(self) -> &'static str {
|
||||
match self {
|
||||
Gauge::Ticks => "ticks",
|
||||
Gauge::Peers => "peers",
|
||||
Gauge::TickGap => "tick_entry_gap",
|
||||
Gauge::ArmStarvation => "arm_starvation",
|
||||
}
|
||||
}
|
||||
|
||||
/// Unit of the `max` and `total` columns for this gauge.
|
||||
pub(crate) const fn unit(self) -> &'static str {
|
||||
match self {
|
||||
Gauge::Ticks => "ticks",
|
||||
Gauge::Peers => "peers",
|
||||
Gauge::TickGap | Gauge::ArmStarvation => "us",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the gauge's stored values are nanosecond durations that the
|
||||
/// writer converts to microseconds.
|
||||
pub(crate) const fn is_duration(self) -> bool {
|
||||
matches!(self, Gauge::TickGap | Gauge::ArmStarvation)
|
||||
}
|
||||
}
|
||||
|
||||
const N_SLOTS: usize = N_DOMAINS * N_STEPS;
|
||||
|
||||
static COUNT: [AtomicU64; N_SLOTS] = [const { AtomicU64::new(0) }; N_SLOTS];
|
||||
static MAX_NS: [AtomicU64; N_SLOTS] = [const { AtomicU64::new(0) }; N_SLOTS];
|
||||
static TOTAL_NS: [AtomicU64; N_SLOTS] = [const { AtomicU64::new(0) }; N_SLOTS];
|
||||
|
||||
static G_COUNT: [AtomicU64; N_GAUGES] = [const { AtomicU64::new(0) }; N_GAUGES];
|
||||
static G_MAX: [AtomicU64; N_GAUGES] = [const { AtomicU64::new(0) }; N_GAUGES];
|
||||
static G_TOTAL: [AtomicU64; N_GAUGES] = [const { AtomicU64::new(0) }; N_GAUGES];
|
||||
|
||||
/// Monotonic baseline so tick-arm entry times fit in an atomic. Offset by one
|
||||
/// on store so that zero can mean "no previous entry".
|
||||
static BASE: LazyLock<Instant> = LazyLock::new(Instant::now);
|
||||
static PREV_ENTRY_NS: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[inline]
|
||||
const fn slot(domain: Domain, step: Step) -> usize {
|
||||
(domain as usize * N_STEPS) + step as usize
|
||||
}
|
||||
|
||||
/// Read the clock for a step span.
|
||||
#[inline]
|
||||
pub(crate) fn now() -> Instant {
|
||||
Instant::now()
|
||||
}
|
||||
|
||||
/// Record one observation of `step`.
|
||||
#[inline]
|
||||
pub(crate) fn record(domain: Domain, step: Step, elapsed: Duration) {
|
||||
let ns = elapsed.as_nanos() as u64;
|
||||
let idx = slot(domain, step);
|
||||
COUNT[idx].fetch_add(1, Relaxed);
|
||||
TOTAL_NS[idx].fetch_add(ns, Relaxed);
|
||||
MAX_NS[idx].fetch_max(ns, Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn record_gauge(gauge: Gauge, value: u64) {
|
||||
let idx = gauge as usize;
|
||||
G_COUNT[idx].fetch_add(1, Relaxed);
|
||||
G_TOTAL[idx].fetch_add(value, Relaxed);
|
||||
G_MAX[idx].fetch_max(value, Relaxed);
|
||||
}
|
||||
|
||||
/// Sample the inter-entry gap and the measured arm-starvation delay.
|
||||
pub(crate) fn tick_entry(on: bool, deadline: Instant, now: Instant) {
|
||||
if !on {
|
||||
return;
|
||||
}
|
||||
// Offset by one so that a stored zero unambiguously means "no previous
|
||||
// entry", even for an entry that lands on the baseline instant.
|
||||
let stamp = BASE.elapsed().as_nanos() as u64 + 1;
|
||||
let late = now.saturating_duration_since(deadline).as_nanos() as u64;
|
||||
tick_entry_at(stamp, late);
|
||||
}
|
||||
|
||||
/// The clock-free half of [`tick_entry`]: both times are inputs, so the
|
||||
/// arithmetic can be driven with synthetic stamps in a test.
|
||||
///
|
||||
/// `late_ns` is how far past its scheduled deadline this entry was, measured
|
||||
/// directly rather than derived. Two earlier designs derived it from the
|
||||
/// inter-entry gap and were both wrong: subtracting the previous body
|
||||
/// understated it by exactly the body, and subtracting `max(period, body)`
|
||||
/// reported the *first difference* of the delay, so a sustained stall — the
|
||||
/// overload regime this measurement exists to characterize — read as zero
|
||||
/// forever. `tokio::time::interval::tick` hands back the deadline it was
|
||||
/// scheduled for, so the delay is a subtraction with no model behind it.
|
||||
fn tick_entry_at(stamp: u64, late_ns: u64) {
|
||||
let prev = PREV_ENTRY_NS.swap(stamp, Relaxed);
|
||||
record_gauge(Gauge::Ticks, 1);
|
||||
record_gauge(Gauge::ArmStarvation, late_ns);
|
||||
if prev == 0 {
|
||||
// First entry of this capture: there is no previous entry to measure a
|
||||
// gap against, and the idle interval before arming is not a gap. The
|
||||
// lateness above does not depend on a previous entry, so it still counts.
|
||||
return;
|
||||
}
|
||||
record_gauge(Gauge::TickGap, stamp.saturating_sub(prev));
|
||||
}
|
||||
|
||||
/// Sample the gauges that come from node state.
|
||||
pub(crate) fn tick_gauges(on: bool, peers: u64) {
|
||||
if !on {
|
||||
return;
|
||||
}
|
||||
record_gauge(Gauge::Peers, peers);
|
||||
}
|
||||
|
||||
/// Take (and clear) this interval's figures for one step: count, max ns, total
|
||||
/// ns. Called only by the writer thread.
|
||||
pub(crate) fn take_step(domain: Domain, step: Step) -> (u64, u64, u64) {
|
||||
let idx = slot(domain, step);
|
||||
(
|
||||
COUNT[idx].swap(0, Relaxed),
|
||||
MAX_NS[idx].swap(0, Relaxed),
|
||||
TOTAL_NS[idx].swap(0, Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
/// Take (and clear) this interval's figures for one gauge.
|
||||
pub(crate) fn take_gauge(gauge: Gauge) -> (u64, u64, u64) {
|
||||
let idx = gauge as usize;
|
||||
(
|
||||
G_COUNT[idx].swap(0, Relaxed),
|
||||
G_MAX[idx].swap(0, Relaxed),
|
||||
G_TOTAL[idx].swap(0, Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
/// Zero every counter so a capture starts from a clean slate.
|
||||
pub(crate) fn reset() {
|
||||
for i in 0..N_SLOTS {
|
||||
COUNT[i].store(0, Relaxed);
|
||||
MAX_NS[i].store(0, Relaxed);
|
||||
TOTAL_NS[i].store(0, Relaxed);
|
||||
}
|
||||
for i in 0..N_GAUGES {
|
||||
G_COUNT[i].store(0, Relaxed);
|
||||
G_MAX[i].store(0, Relaxed);
|
||||
G_TOTAL[i].store(0, Relaxed);
|
||||
}
|
||||
PREV_ENTRY_NS.store(0, Relaxed);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::MutexGuard;
|
||||
|
||||
/// Shared with the capture tests: they mutate the same statics. See
|
||||
/// `crate::instr::test_serial`.
|
||||
fn serial() -> MutexGuard<'static, ()> {
|
||||
crate::instr::test_serial()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn steps_table_is_dense() {
|
||||
for (i, step) in STEPS.iter().enumerate() {
|
||||
assert_eq!(*step as usize, i, "step {} is out of order", step.name());
|
||||
}
|
||||
assert_eq!(STEPS.len(), N_STEPS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gauges_table_is_dense() {
|
||||
for (i, gauge) in GAUGES.iter().enumerate() {
|
||||
assert_eq!(*gauge as usize, i, "gauge {} is out of order", gauge.name());
|
||||
}
|
||||
assert_eq!(GAUGES.len(), N_GAUGES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_names_are_unique() {
|
||||
let mut names: Vec<&str> = STEPS.iter().map(|s| s.name()).collect();
|
||||
names.sort_unstable();
|
||||
let before = names.len();
|
||||
names.dedup();
|
||||
assert_eq!(before, names.len(), "duplicate step name");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emitted_row_count_matches_build() {
|
||||
let emitted = STEPS.iter().filter(|s| s.emitted()).count();
|
||||
// 24 unconditional subsystem steps + the whole-tick span, plus the two
|
||||
// conditionally-compiled steps where this build has them.
|
||||
let mut expected = 25;
|
||||
if cfg!(any(target_os = "linux", target_os = "macos")) {
|
||||
expected += 1;
|
||||
}
|
||||
if cfg!(debug_assertions) {
|
||||
expected += 1;
|
||||
}
|
||||
assert_eq!(emitted, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_accumulates_count_max_and_total() {
|
||||
let _guard = serial();
|
||||
reset();
|
||||
record(Domain::Tick, Step::CheckRekey, Duration::from_nanos(10));
|
||||
record(Domain::Tick, Step::CheckRekey, Duration::from_nanos(30));
|
||||
let (count, max, total) = take_step(Domain::Tick, Step::CheckRekey);
|
||||
assert_eq!((count, max, total), (2, 30, 40));
|
||||
// Taking clears the slot.
|
||||
assert_eq!(take_step(Domain::Tick, Step::CheckRekey), (0, 0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn starvation_is_the_measured_lateness_of_the_entry() {
|
||||
let _guard = serial();
|
||||
reset();
|
||||
// The interval hands back the deadline it was scheduled for, so the
|
||||
// delay is `now - deadline` and nothing is derived from the period, the
|
||||
// previous entry, or the previous body.
|
||||
PREV_ENTRY_NS.store(1_000_000_000, Relaxed);
|
||||
tick_entry_at(1_100_000_000, 50_000_000);
|
||||
assert_eq!(take_gauge(Gauge::TickGap), (1, 100_000_000, 100_000_000));
|
||||
assert_eq!(
|
||||
take_gauge(Gauge::ArmStarvation),
|
||||
(1, 50_000_000, 50_000_000)
|
||||
);
|
||||
assert_eq!(take_gauge(Gauge::Ticks).0, 1);
|
||||
reset();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sustained_lateness_is_reported_on_every_tick() {
|
||||
let _guard = serial();
|
||||
reset();
|
||||
// The regime the two earlier designs both hid. Three consecutive entries
|
||||
// each 50 ms past their deadline, one period apart, i.e. the arm waiting
|
||||
// a constant amount behind the other select arms every round. The gaps
|
||||
// are all exactly one period, so any formula derived from the
|
||||
// inter-entry gap reports zero here; measured lateness reports 50 ms
|
||||
// three times, which is the truth.
|
||||
PREV_ENTRY_NS.store(1_000_000_000, Relaxed);
|
||||
tick_entry_at(1_050_000_000, 50_000_000);
|
||||
tick_entry_at(1_100_000_000, 50_000_000);
|
||||
tick_entry_at(1_150_000_000, 50_000_000);
|
||||
let (count, max, total) = take_gauge(Gauge::ArmStarvation);
|
||||
assert_eq!(count, 3);
|
||||
assert_eq!(max, 50_000_000);
|
||||
assert_eq!(total, 150_000_000);
|
||||
// ...and the gap alone carries no signal about it: every gap is one
|
||||
// period, exactly as it would be on a perfectly healthy node.
|
||||
assert_eq!(take_gauge(Gauge::TickGap), (3, 50_000_000, 150_000_000));
|
||||
reset();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_entry_of_a_capture_records_no_gap_but_still_records_lateness() {
|
||||
let _guard = serial();
|
||||
reset();
|
||||
tick_entry_at(500, 7_000_000);
|
||||
assert_eq!(take_gauge(Gauge::Ticks).0, 1);
|
||||
assert_eq!(take_gauge(Gauge::TickGap), (0, 0, 0));
|
||||
// Lateness does not depend on a previous entry, so the first tick of a
|
||||
// capture still contributes one.
|
||||
assert_eq!(take_gauge(Gauge::ArmStarvation), (1, 7_000_000, 7_000_000));
|
||||
reset();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_on_schedule_entry_reports_no_starvation() {
|
||||
let _guard = serial();
|
||||
reset();
|
||||
PREV_ENTRY_NS.store(1_000_000_000, Relaxed);
|
||||
tick_entry_at(1_050_000_000, 0);
|
||||
assert_eq!(take_gauge(Gauge::ArmStarvation), (1, 0, 0));
|
||||
assert_eq!(take_gauge(Gauge::TickGap), (1, 50_000_000, 50_000_000));
|
||||
reset();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_off_records_nothing() {
|
||||
let _guard = serial();
|
||||
reset();
|
||||
let t = Instant::now();
|
||||
tick_entry(false, t, t);
|
||||
tick_gauges(false, 42);
|
||||
assert_eq!(take_gauge(Gauge::Ticks), (0, 0, 0));
|
||||
assert_eq!(take_gauge(Gauge::Peers), (0, 0, 0));
|
||||
}
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
//! The capture writer: a dedicated, named OS thread with an explicit
|
||||
//! lifecycle.
|
||||
//!
|
||||
//! There is no worker-thread lifecycle in this codebase to copy — the crypto
|
||||
//! worker pools are never torn down and drop their join handles at spawn — so
|
||||
//! this one is designed here.
|
||||
//!
|
||||
//! Two properties matter:
|
||||
//!
|
||||
//! - **It is an OS thread, not a spawned task.** The runtime is
|
||||
//! `current_thread`, so file I/O on a task would run on the rx loop's own
|
||||
//! thread and stall every `select!` arm, including the one being measured.
|
||||
//! - **It waits on a channel with a timeout, not on a sleep.** `recv_timeout`
|
||||
//! returns immediately when the toggle sends stop, so `off` performs a final
|
||||
//! drain and joins promptly instead of parking the caller for up to a full
|
||||
//! flush interval.
|
||||
//!
|
||||
//! The thread is created lazily when a capture starts, so a node that never
|
||||
//! arms one never has the thread.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use super::capture::{self, BYTE_CAP, INTERVAL};
|
||||
use super::recorder::{self, DOMAINS, GAUGES, STEPS};
|
||||
|
||||
/// Owner-side handle to the writer thread.
|
||||
pub(crate) struct Handle {
|
||||
stop_tx: Sender<()>,
|
||||
join: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl Handle {
|
||||
/// Wake the writer, let it drain once more, and join it.
|
||||
pub(crate) fn stop_and_join(self) {
|
||||
// A send error means the thread already exited (cap stop); joining is
|
||||
// still correct and returns at once.
|
||||
let _ = self.stop_tx.send(());
|
||||
let _ = self.join.join();
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the writer thread on an already-open sink.
|
||||
pub(crate) fn spawn(file: File) -> std::io::Result<Handle> {
|
||||
let (stop_tx, stop_rx) = mpsc::channel();
|
||||
let join = thread::Builder::new()
|
||||
.name("fips-profile".to_string())
|
||||
.spawn(move || run(file, stop_rx))?;
|
||||
Ok(Handle { stop_tx, join })
|
||||
}
|
||||
|
||||
/// What one flush cycle decided. Separated from [`run`] so the terminal paths
|
||||
/// can be driven in a test with a failing sink: the loop below owns the waiting
|
||||
/// and the state transition, this owns the decision.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum Cycle {
|
||||
Continue,
|
||||
CapReached,
|
||||
WriteFailed,
|
||||
}
|
||||
|
||||
/// Drain one interval into the sink and decide whether the capture goes on.
|
||||
fn flush_cycle<W: Write>(file: &mut W) -> Cycle {
|
||||
if flush(file).is_err() {
|
||||
// The sink is gone or full; stop rather than spinning on a broken file
|
||||
// for the rest of the run.
|
||||
let _ = note(file, "capture stopped: write error");
|
||||
return Cycle::WriteFailed;
|
||||
}
|
||||
if capture::bytes_written() >= BYTE_CAP {
|
||||
let _ = note(
|
||||
file,
|
||||
&format!("capture stopped: byte cap {BYTE_CAP} reached"),
|
||||
);
|
||||
let _ = file.flush();
|
||||
return Cycle::CapReached;
|
||||
}
|
||||
Cycle::Continue
|
||||
}
|
||||
|
||||
fn run<W: Write>(mut file: W, stop_rx: Receiver<()>) {
|
||||
loop {
|
||||
match stop_rx.recv_timeout(INTERVAL) {
|
||||
// Stop requested, or the owner went away: final drain, then exit.
|
||||
Ok(()) | Err(RecvTimeoutError::Disconnected) => {
|
||||
let _ = flush(&mut file);
|
||||
let _ = file.flush();
|
||||
return;
|
||||
}
|
||||
Err(RecvTimeoutError::Timeout) => match flush_cycle(&mut file) {
|
||||
Cycle::Continue => {}
|
||||
Cycle::WriteFailed => {
|
||||
// The trailer went to the same failing file, so it is not a
|
||||
// signal that survives. `stop` and a subsequent `on` both
|
||||
// clear the state without surfacing it, so an operator would
|
||||
// otherwise never learn the window was truncated.
|
||||
tracing::warn!(
|
||||
target: "fips::instr",
|
||||
"profile capture stopped: write error on the sink"
|
||||
);
|
||||
capture::mark_stopped(capture::STOPPED_BY_ERROR);
|
||||
return;
|
||||
}
|
||||
Cycle::CapReached => {
|
||||
capture::mark_stopped(capture::STOPPED_BY_CAP);
|
||||
return;
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a `#`-prefixed trailer line.
|
||||
fn note<W: Write>(file: &mut W, text: &str) -> std::io::Result<()> {
|
||||
let line = format!("# {text}\n");
|
||||
file.write_all(line.as_bytes())?;
|
||||
capture::add_bytes(line.len() as u64);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Emit one interval: every step of every domain, then the gauges.
|
||||
///
|
||||
/// Every emitted step gets a row every interval, including zero-count rows, so
|
||||
/// "this step did not run" is visible rather than absent. The two steps whose
|
||||
/// call sites are conditionally compiled are excluded in builds that do not
|
||||
/// have them, so no row is structurally zero forever.
|
||||
fn flush<W: Write>(file: &mut W) -> std::io::Result<()> {
|
||||
let ts = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
|
||||
let mut out = String::with_capacity(4096);
|
||||
for domain in DOMAINS {
|
||||
for step in STEPS {
|
||||
if !step.emitted() {
|
||||
continue;
|
||||
}
|
||||
let (count, max_ns, total_ns) = recorder::take_step(domain, step);
|
||||
out.push_str(&format!(
|
||||
"{ts}\tstep\t{domain}\t{name}\t{count}\t{max}\t{total}\tus\n",
|
||||
domain = domain.name(),
|
||||
name = step.name(),
|
||||
max = max_ns / 1_000,
|
||||
total = total_ns / 1_000,
|
||||
));
|
||||
}
|
||||
}
|
||||
// Gauges carry the tick domain today; the row kind and the unit column keep
|
||||
// them distinguishable from the duration rows above.
|
||||
for gauge in GAUGES {
|
||||
let (count, mut max, mut total) = recorder::take_gauge(gauge);
|
||||
if gauge.is_duration() {
|
||||
max /= 1_000;
|
||||
total /= 1_000;
|
||||
}
|
||||
out.push_str(&format!(
|
||||
"{ts}\tgauge\t{domain}\t{name}\t{count}\t{max}\t{total}\t{unit}\n",
|
||||
domain = recorder::Domain::Tick.name(),
|
||||
name = gauge.name(),
|
||||
unit = gauge.unit(),
|
||||
));
|
||||
}
|
||||
|
||||
file.write_all(out.as_bytes())?;
|
||||
capture::add_bytes(out.len() as u64);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A sink that fails every write, so the writer's error path is driven by a
|
||||
/// real `Err` rather than asserted about.
|
||||
struct AlwaysFails;
|
||||
|
||||
impl Write for AlwaysFails {
|
||||
fn write(&mut self, _buf: &[u8]) -> std::io::Result<usize> {
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::StorageFull,
|
||||
"no space left on device",
|
||||
))
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// A failing sink ends the capture as a write error, which `run` turns into
|
||||
/// `STOPPED_BY_ERROR` — not into a byte-cap stop. The distinction is what
|
||||
/// tells an operator that a window is truncated rather than complete.
|
||||
///
|
||||
/// **Coverage note.** This drives the decision, not the filesystem
|
||||
/// condition. The mesh rehearsal cannot produce one: removing the file
|
||||
/// leaves the writer's descriptor valid and writes keep succeeding into the
|
||||
/// unlinked inode, and mounting a tiny filesystem inside the test container
|
||||
/// is refused. So "a real ENOSPC reaches this branch" stays unexercised;
|
||||
/// what is covered is that an `Err` from the sink produces the error
|
||||
/// outcome and not the cap outcome.
|
||||
#[test]
|
||||
fn a_failing_sink_ends_the_cycle_as_an_error_not_a_cap() {
|
||||
let _guard = crate::instr::test_serial();
|
||||
assert_eq!(flush_cycle(&mut AlwaysFails), Cycle::WriteFailed);
|
||||
}
|
||||
|
||||
/// The healthy path must not be reported as either terminal state, or the
|
||||
/// test above would pass for a writer that always stops.
|
||||
#[test]
|
||||
fn a_working_sink_continues() {
|
||||
let _guard = crate::instr::test_serial();
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
assert_eq!(flush_cycle(&mut sink), Cycle::Continue);
|
||||
}
|
||||
}
|
||||
+20
-53
@@ -3,34 +3,22 @@
|
||||
//! A distributed, decentralized network routing protocol for mesh nodes
|
||||
//! connecting over arbitrary transports.
|
||||
|
||||
// Name the `alloc` crate directly so the sans-IO protocol cores can spell their
|
||||
// heap-type imports in `no_std`-forward form (`alloc::sync::Arc`,
|
||||
// `alloc::collections::BTreeMap`). The crate remains `std`; this only reduces the
|
||||
// distance to extracting the pure cores into a `no_std` crate later.
|
||||
extern crate alloc;
|
||||
|
||||
pub mod bloom;
|
||||
pub mod cache;
|
||||
pub mod config;
|
||||
pub mod control;
|
||||
pub mod discovery;
|
||||
#[cfg(target_os = "linux")]
|
||||
pub mod gateway;
|
||||
pub mod identity;
|
||||
// Declared before `node` (and named to sort there) because it carries
|
||||
// `#[macro_use]`: the tick instrumentation macro must be in scope for the
|
||||
// modules that follow.
|
||||
#[macro_use]
|
||||
pub(crate) mod instr;
|
||||
pub mod mdns;
|
||||
pub mod mmp;
|
||||
pub mod node;
|
||||
pub mod noise;
|
||||
pub mod nostr;
|
||||
pub mod peer;
|
||||
pub mod perf_profile;
|
||||
pub(crate) mod proto;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod testutil;
|
||||
mod time;
|
||||
pub mod protocol;
|
||||
pub mod transport;
|
||||
pub mod tree;
|
||||
pub mod upper;
|
||||
pub mod utils;
|
||||
pub mod version;
|
||||
@@ -45,16 +33,14 @@ pub use identity::{
|
||||
pub use config::{Config, ConfigError, IdentityConfig, NymConfig, TorConfig, UdpConfig};
|
||||
pub use upper::config::{DnsConfig, TunConfig};
|
||||
|
||||
// Re-export nostr rendezvous handoff types
|
||||
pub use nostr::{BootstrapHandoffResult, EstablishedTraversal, is_punch_packet};
|
||||
// Re-export discovery types
|
||||
pub use discovery::{BootstrapHandoffResult, EstablishedTraversal};
|
||||
|
||||
// Re-export tree types (relocated from tree:: to proto::stp)
|
||||
pub use proto::stp::{
|
||||
CoordEntry, CoordError, ParentDeclaration, TreeCoordinate, TreeError, TreeState,
|
||||
};
|
||||
// Re-export tree types
|
||||
pub use tree::{CoordEntry, ParentDeclaration, TreeCoordinate, TreeError, TreeState};
|
||||
|
||||
// Re-export bloom filter types (relocated from bloom:: to proto::bloom)
|
||||
pub use proto::bloom::{BloomError, BloomFilter, BloomState};
|
||||
// Re-export bloom filter types
|
||||
pub use bloom::{BloomError, BloomFilter, BloomState};
|
||||
|
||||
// Re-export transport types
|
||||
pub use transport::udp::UdpTransport;
|
||||
@@ -64,40 +50,21 @@ pub use transport::{
|
||||
TransportState, TransportType, packet_channel,
|
||||
};
|
||||
|
||||
// Re-export link-layer types (relocated from protocol:: to proto::link)
|
||||
pub use proto::link::{LinkMessageType, SessionDatagram};
|
||||
|
||||
// Re-export the shared protocol error (relocated from protocol:: to proto::Error)
|
||||
pub use proto::Error;
|
||||
|
||||
// Re-export FSP session wire types (relocated from protocol:: to proto::fsp)
|
||||
pub use proto::fsp::{SessionAck, SessionFlags, SessionMessageType, SessionSetup};
|
||||
|
||||
// Re-export STP wire types (relocated from protocol:: to proto::stp)
|
||||
pub use proto::stp::TreeAnnounce;
|
||||
|
||||
// Re-export bloom wire types (relocated from protocol:: to proto::bloom)
|
||||
pub use proto::bloom::FilterAnnounce;
|
||||
|
||||
// Re-export discovery wire types (relocated from protocol:: to proto::lookup)
|
||||
pub use proto::lookup::{LookupRequest, LookupResponse};
|
||||
|
||||
// Re-export routing wire types (relocated from protocol:: to proto::routing)
|
||||
pub use proto::routing::{
|
||||
COORDS_REQUIRED_SIZE, CoordsRequired, MTU_EXCEEDED_SIZE, MtuExceeded, PathBroken,
|
||||
// Re-export protocol types
|
||||
pub use protocol::{
|
||||
CoordsRequired, FilterAnnounce, HandshakeMessageType, LinkMessageType, LookupRequest,
|
||||
LookupResponse, PathBroken, ProtocolError, SessionAck, SessionDatagram, SessionFlags,
|
||||
SessionMessageType, SessionSetup, TreeAnnounce,
|
||||
};
|
||||
|
||||
// Re-export FMP link-framing wire type (relocated from protocol:: to proto::fmp)
|
||||
pub use proto::fmp::HandshakeMessageType;
|
||||
|
||||
// Re-export cache types
|
||||
pub use cache::{CacheEntry, CacheError, CacheStats, CoordCache};
|
||||
|
||||
// Re-export FMP tie-break helper and promotion result (relocated from peer:: to proto::fmp)
|
||||
pub use proto::fmp::{PromotionResult, cross_connection_winner};
|
||||
|
||||
// Re-export peer types
|
||||
pub use peer::{ActivePeer, ConnectivityState, PeerError};
|
||||
pub use peer::{
|
||||
ActivePeer, ConnectivityState, HandshakeState, PeerConnection, PeerError, PeerSlot,
|
||||
PromotionResult, cross_connection_winner,
|
||||
};
|
||||
|
||||
// Re-export node types
|
||||
pub use node::{Node, NodeError, NodeState, UpdatePeersOutcome};
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
//! MMP algorithmic building blocks.
|
||||
//!
|
||||
//! Pure computational types with no dependency on peer or node state.
|
||||
//! Each is independently testable. `no_std`+`alloc`-clean: the ring buffer
|
||||
//! comes from `alloc`, all arithmetic is `core`, and the spin-bit RTT clock is
|
||||
//! an injected `u64` millisecond value (never a `std::time` read).
|
||||
//! Each is independently testable.
|
||||
|
||||
use alloc::collections::VecDeque;
|
||||
use std::collections::VecDeque;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::{EWMA_LONG_ALPHA, EWMA_SHORT_ALPHA};
|
||||
use crate::mmp::{EWMA_LONG_ALPHA, EWMA_SHORT_ALPHA};
|
||||
|
||||
// ============================================================================
|
||||
// Jitter Estimator (RFC 3550 §6.4.1)
|
||||
@@ -236,10 +235,7 @@ impl OwdTrendDetector {
|
||||
den += dx * dx;
|
||||
}
|
||||
|
||||
// `den` is a sum of squares, so it is always non-negative; comparing it
|
||||
// directly against EPSILON is equivalent to the original `den.abs()`
|
||||
// guard while staying `core`-only (no `libm`).
|
||||
if den < f64::EPSILON {
|
||||
if den.abs() < f64::EPSILON {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -250,6 +246,14 @@ impl OwdTrendDetector {
|
||||
let slope_per_packet = num / den;
|
||||
(slope_per_packet * 1000.0) as i32
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.samples.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.samples.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -286,9 +290,8 @@ pub struct SpinBitState {
|
||||
current_value: bool,
|
||||
/// Highest counter observed with a spin edge (responder guard).
|
||||
highest_counter_for_spin: u64,
|
||||
/// Time of last spin edge in injected `u64` milliseconds (initiator only,
|
||||
/// for RTT measurement).
|
||||
last_edge_ms: Option<u64>,
|
||||
/// Time of last spin edge (initiator only, for RTT measurement).
|
||||
last_edge_time: Option<Instant>,
|
||||
}
|
||||
|
||||
impl SpinBitState {
|
||||
@@ -297,7 +300,7 @@ impl SpinBitState {
|
||||
is_initiator,
|
||||
current_value: false,
|
||||
highest_counter_for_spin: 0,
|
||||
last_edge_ms: None,
|
||||
last_edge_time: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -313,16 +316,20 @@ impl SpinBitState {
|
||||
|
||||
/// Process a received frame's spin bit.
|
||||
///
|
||||
/// `now_ms` is the injected monotonic time in milliseconds. Returns an RTT
|
||||
/// sample in milliseconds if an edge was detected (initiator only).
|
||||
pub fn rx_observe(&mut self, received_bit: bool, counter: u64, now_ms: u64) -> Option<u64> {
|
||||
/// Returns an RTT sample duration if an edge was detected (initiator only).
|
||||
pub fn rx_observe(
|
||||
&mut self,
|
||||
received_bit: bool,
|
||||
counter: u64,
|
||||
now: Instant,
|
||||
) -> Option<std::time::Duration> {
|
||||
if self.is_initiator {
|
||||
// Initiator: when the reflected bit matches what we sent,
|
||||
// that completes a round trip. Record the edge time, then
|
||||
// flip for the next cycle.
|
||||
if received_bit == self.current_value {
|
||||
let rtt = self.last_edge_ms.map(|t| now_ms.saturating_sub(t));
|
||||
self.last_edge_ms = Some(now_ms);
|
||||
let rtt = self.last_edge_time.map(|t| now.duration_since(t));
|
||||
self.last_edge_time = Some(now);
|
||||
self.current_value = !self.current_value;
|
||||
rtt
|
||||
} else {
|
||||
@@ -339,3 +346,181 @@ impl SpinBitState {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_jitter_zero_input() {
|
||||
let mut j = JitterEstimator::new();
|
||||
j.update(0);
|
||||
assert_eq!(j.jitter_us(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jitter_convergence() {
|
||||
let mut j = JitterEstimator::new();
|
||||
// Feed constant transit delta of 1000µs
|
||||
for _ in 0..200 {
|
||||
j.update(1000);
|
||||
}
|
||||
// Should converge near 1000µs
|
||||
let jitter = j.jitter_us();
|
||||
assert!(
|
||||
jitter > 900 && jitter < 1100,
|
||||
"jitter={jitter}, expected ~1000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_srtt_first_sample() {
|
||||
let mut s = SrttEstimator::new();
|
||||
s.update(10_000); // 10ms
|
||||
assert_eq!(s.srtt_us(), 10_000);
|
||||
assert_eq!(s.rttvar_us(), 5_000);
|
||||
assert!(s.initialized());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_srtt_convergence() {
|
||||
let mut s = SrttEstimator::new();
|
||||
// Feed constant 50ms RTT
|
||||
for _ in 0..100 {
|
||||
s.update(50_000);
|
||||
}
|
||||
let srtt = s.srtt_us();
|
||||
assert!((srtt - 50_000).abs() < 1000, "srtt={srtt}, expected ~50000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dual_ewma_initialization() {
|
||||
let mut e = DualEwma::new();
|
||||
assert!(!e.initialized());
|
||||
e.update(100.0);
|
||||
assert!(e.initialized());
|
||||
assert_eq!(e.short(), 100.0);
|
||||
assert_eq!(e.long(), 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dual_ewma_short_tracks_faster() {
|
||||
let mut e = DualEwma::new();
|
||||
// Initialize at 0
|
||||
e.update(0.0);
|
||||
// Jump to 100
|
||||
for _ in 0..20 {
|
||||
e.update(100.0);
|
||||
}
|
||||
// Short should be closer to 100 than long
|
||||
assert!(
|
||||
e.short() > e.long(),
|
||||
"short={} long={}",
|
||||
e.short(),
|
||||
e.long()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_owd_trend_flat() {
|
||||
let mut d = OwdTrendDetector::new(32);
|
||||
for i in 0..20 {
|
||||
d.push(i, 5000); // constant OWD
|
||||
}
|
||||
let trend = d.trend_us_per_sec();
|
||||
assert_eq!(trend, 0, "flat OWD should have zero trend");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_owd_trend_increasing() {
|
||||
let mut d = OwdTrendDetector::new(32);
|
||||
for i in 0..20 {
|
||||
d.push(i, 5000 + (i as i64) * 100); // increasing by 100µs per packet
|
||||
}
|
||||
let trend = d.trend_us_per_sec();
|
||||
assert!(
|
||||
trend > 0,
|
||||
"increasing OWD should have positive trend, got {trend}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_owd_trend_insufficient_samples() {
|
||||
let mut d = OwdTrendDetector::new(32);
|
||||
d.push(0, 5000);
|
||||
assert_eq!(d.trend_us_per_sec(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_etx_perfect_link() {
|
||||
assert!((compute_etx(1.0, 1.0) - 1.0).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_etx_lossy_link() {
|
||||
// 10% forward loss, 5% reverse loss
|
||||
let etx = compute_etx(0.9, 0.95);
|
||||
assert!(etx > 1.0 && etx < 2.0, "etx={etx}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_etx_zero_delivery() {
|
||||
assert_eq!(compute_etx(0.0, 1.0), 100.0);
|
||||
assert_eq!(compute_etx(1.0, 0.0), 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_spin_bit_initiator_rtt() {
|
||||
let mut initiator = SpinBitState::new(true);
|
||||
let mut responder = SpinBitState::new(false);
|
||||
|
||||
let t0 = Instant::now();
|
||||
let t1 = t0 + std::time::Duration::from_millis(10);
|
||||
let t2 = t0 + std::time::Duration::from_millis(20);
|
||||
|
||||
// Initiator sends with spin=false (initial)
|
||||
let bit_to_send = initiator.tx_bit();
|
||||
assert!(!bit_to_send);
|
||||
|
||||
// Responder receives, copies bit
|
||||
responder.rx_observe(bit_to_send, 1, t0);
|
||||
assert!(!responder.tx_bit());
|
||||
|
||||
// Responder sends back, initiator receives
|
||||
let resp_bit = responder.tx_bit();
|
||||
let rtt1 = initiator.rx_observe(resp_bit, 2, t1);
|
||||
// First edge: no previous edge to compare
|
||||
assert!(rtt1.is_none());
|
||||
|
||||
// Now initiator's spin flipped to true
|
||||
let bit2 = initiator.tx_bit();
|
||||
assert!(bit2);
|
||||
|
||||
// Responder receives new bit
|
||||
responder.rx_observe(bit2, 3, t1);
|
||||
assert!(responder.tx_bit());
|
||||
|
||||
// Responder sends back, initiator receives
|
||||
let resp_bit2 = responder.tx_bit();
|
||||
let rtt2 = initiator.rx_observe(resp_bit2, 4, t2);
|
||||
// Second edge: should produce an RTT sample
|
||||
assert!(rtt2.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_spin_bit_responder_counter_guard() {
|
||||
let mut responder = SpinBitState::new(false);
|
||||
|
||||
// Receive counter=5 with spin=true
|
||||
responder.rx_observe(true, 5, Instant::now());
|
||||
assert!(responder.tx_bit());
|
||||
|
||||
// Reordered packet with counter=3 and spin=false should be ignored
|
||||
responder.rx_observe(false, 3, Instant::now());
|
||||
assert!(responder.tx_bit()); // unchanged
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
//! MMP derived metrics.
|
||||
//!
|
||||
//! `MmpMetrics` processes incoming ReceiverReports (from our peer) and
|
||||
//! maintains derived metrics: SRTT, loss rate, goodput, ETX, and dual
|
||||
//! EWMA trend indicators. Updated by the sender side when it receives
|
||||
//! a ReceiverReport about its own traffic.
|
||||
|
||||
use crate::mmp::algorithms::{DualEwma, SrttEstimator, compute_etx};
|
||||
use crate::mmp::report::ReceiverReport;
|
||||
use std::time::Instant;
|
||||
use tracing::trace;
|
||||
|
||||
/// Derived MMP metrics, updated from incoming ReceiverReports.
|
||||
///
|
||||
/// This lives on the sender side: when we receive a ReceiverReport from
|
||||
/// our peer describing what they observed about our traffic, we process
|
||||
/// it here to compute RTT, loss, goodput, and trend indicators.
|
||||
pub struct MmpMetrics {
|
||||
/// Smoothed RTT from timestamp echo.
|
||||
pub srtt: SrttEstimator,
|
||||
|
||||
/// Dual EWMA trend detectors.
|
||||
pub rtt_trend: DualEwma,
|
||||
pub loss_trend: DualEwma,
|
||||
pub goodput_trend: DualEwma,
|
||||
pub jitter_trend: DualEwma,
|
||||
pub etx_trend: DualEwma,
|
||||
|
||||
/// Forward delivery ratio (what fraction of our frames the peer received).
|
||||
pub delivery_ratio_forward: f64,
|
||||
/// Reverse delivery ratio (set when we compute from our own receiver state).
|
||||
pub delivery_ratio_reverse: f64,
|
||||
/// ETX computed from bidirectional delivery ratios.
|
||||
pub etx: f64,
|
||||
|
||||
/// Smoothed goodput in bytes/sec (forward direction: what the peer received from us).
|
||||
pub goodput_bps: f64,
|
||||
|
||||
// --- State for delta computation ---
|
||||
/// Previous ReceiverReport's cumulative counters (for computing interval deltas).
|
||||
prev_rr_cum_packets: u64,
|
||||
prev_rr_cum_bytes: u64,
|
||||
prev_rr_highest_counter: u64,
|
||||
prev_rr_ecn_ce: u32,
|
||||
prev_rr_reorder: u32,
|
||||
/// Time of previous ReceiverReport (for goodput rate computation).
|
||||
prev_rr_time: Option<Instant>,
|
||||
/// Whether we have a previous ReceiverReport for delta computation.
|
||||
has_prev_rr: bool,
|
||||
|
||||
// --- State for reverse delivery ratio delta computation ---
|
||||
/// Previous reverse-side cumulative packets received (our receiver state).
|
||||
prev_reverse_packets: u64,
|
||||
/// Previous reverse-side highest counter (our receiver state).
|
||||
prev_reverse_highest: u64,
|
||||
/// Whether we have a previous reverse-side snapshot for delta computation.
|
||||
has_prev_reverse: bool,
|
||||
}
|
||||
|
||||
impl MmpMetrics {
|
||||
/// Reset state derived from ReceiverReport counters for rekey cutover.
|
||||
///
|
||||
/// The new session starts with counter 0, so the prev_rr deltas must
|
||||
/// be reset to avoid computing bogus loss/goodput from the counter
|
||||
/// discontinuity. RTT (SRTT) is preserved since it remains valid.
|
||||
pub fn reset_for_rekey(&mut self) {
|
||||
self.prev_rr_cum_packets = 0;
|
||||
self.prev_rr_cum_bytes = 0;
|
||||
self.prev_rr_highest_counter = 0;
|
||||
self.prev_rr_ecn_ce = 0;
|
||||
self.prev_rr_reorder = 0;
|
||||
self.prev_rr_time = None;
|
||||
self.has_prev_rr = false;
|
||||
self.delivery_ratio_forward = 1.0;
|
||||
self.prev_reverse_packets = 0;
|
||||
self.prev_reverse_highest = 0;
|
||||
self.has_prev_reverse = false;
|
||||
// Keep srtt, etx, trends, goodput_bps — they'll refresh from data
|
||||
}
|
||||
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
srtt: SrttEstimator::new(),
|
||||
rtt_trend: DualEwma::new(),
|
||||
loss_trend: DualEwma::new(),
|
||||
goodput_trend: DualEwma::new(),
|
||||
jitter_trend: DualEwma::new(),
|
||||
etx_trend: DualEwma::new(),
|
||||
delivery_ratio_forward: 1.0,
|
||||
delivery_ratio_reverse: 1.0,
|
||||
etx: 1.0,
|
||||
goodput_bps: 0.0,
|
||||
prev_rr_cum_packets: 0,
|
||||
prev_rr_cum_bytes: 0,
|
||||
prev_rr_highest_counter: 0,
|
||||
prev_rr_ecn_ce: 0,
|
||||
prev_rr_reorder: 0,
|
||||
prev_rr_time: None,
|
||||
has_prev_rr: false,
|
||||
prev_reverse_packets: 0,
|
||||
prev_reverse_highest: 0,
|
||||
has_prev_reverse: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Process an incoming ReceiverReport (from the peer about our traffic).
|
||||
///
|
||||
/// `our_timestamp_ms` is the current session-relative time in ms (for RTT).
|
||||
/// `now` is the current monotonic time (for goodput rate computation).
|
||||
///
|
||||
/// Returns `true` if this report produced the first SRTT measurement
|
||||
/// (transition from uninitialized to initialized).
|
||||
pub fn process_receiver_report(
|
||||
&mut self,
|
||||
rr: &ReceiverReport,
|
||||
our_timestamp_ms: u32,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
let had_srtt = self.srtt.initialized();
|
||||
|
||||
if self.has_prev_rr {
|
||||
let counters_regressed = rr.highest_counter < self.prev_rr_highest_counter
|
||||
|| rr.cumulative_packets_recv < self.prev_rr_cum_packets
|
||||
|| rr.cumulative_bytes_recv < self.prev_rr_cum_bytes
|
||||
|| rr.ecn_ce_count < self.prev_rr_ecn_ce
|
||||
|| rr.cumulative_reorder_count < self.prev_rr_reorder;
|
||||
let duplicate_counters = rr.highest_counter == self.prev_rr_highest_counter
|
||||
&& rr.cumulative_packets_recv == self.prev_rr_cum_packets
|
||||
&& rr.cumulative_bytes_recv == self.prev_rr_cum_bytes
|
||||
&& rr.ecn_ce_count == self.prev_rr_ecn_ce
|
||||
&& rr.cumulative_reorder_count == self.prev_rr_reorder;
|
||||
// Safe to drop: reports are only built after interval data, so
|
||||
// a fresh report always advances at least one cumulative counter.
|
||||
if counters_regressed || duplicate_counters {
|
||||
trace!(
|
||||
highest_counter = rr.highest_counter,
|
||||
prev_highest_counter = self.prev_rr_highest_counter,
|
||||
cumulative_packets_recv = rr.cumulative_packets_recv,
|
||||
prev_cumulative_packets_recv = self.prev_rr_cum_packets,
|
||||
cumulative_bytes_recv = rr.cumulative_bytes_recv,
|
||||
prev_cumulative_bytes_recv = self.prev_rr_cum_bytes,
|
||||
"Ignoring stale MMP ReceiverReport"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// --- RTT from timestamp echo ---
|
||||
// RTT = now - echoed_timestamp - dwell_time
|
||||
if rr.timestamp_echo > 0 {
|
||||
let echo_ms = rr.timestamp_echo;
|
||||
let dwell_ms = u32::from(rr.dwell_time);
|
||||
let rtt_sample_ms = echo_ms
|
||||
.checked_add(dwell_ms)
|
||||
.and_then(|send_done_ms| our_timestamp_ms.checked_sub(send_done_ms));
|
||||
|
||||
match rtt_sample_ms {
|
||||
Some(rtt_ms) if rtt_ms > 0 => {
|
||||
let rtt_us = (rtt_ms as i64) * 1000;
|
||||
trace!(
|
||||
our_ts = our_timestamp_ms,
|
||||
echo = echo_ms,
|
||||
dwell = dwell_ms,
|
||||
rtt_ms = rtt_ms,
|
||||
srtt_ms = self.srtt.srtt_us() as f64 / 1000.0,
|
||||
"RTT sample from timestamp echo"
|
||||
);
|
||||
self.srtt.update(rtt_us);
|
||||
self.rtt_trend.update(rtt_us as f64);
|
||||
}
|
||||
_ => {
|
||||
trace!(
|
||||
our_ts = our_timestamp_ms,
|
||||
echo = echo_ms,
|
||||
dwell = dwell_ms,
|
||||
"Ignoring invalid MMP RTT sample"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Loss rate from cumulative counters ---
|
||||
// Delta: frames the peer should have received vs. actually received
|
||||
if self.has_prev_rr {
|
||||
let counter_span = rr
|
||||
.highest_counter
|
||||
.saturating_sub(self.prev_rr_highest_counter);
|
||||
let packets_delta = rr
|
||||
.cumulative_packets_recv
|
||||
.saturating_sub(self.prev_rr_cum_packets);
|
||||
|
||||
if counter_span > 0 {
|
||||
let delivery = (packets_delta as f64) / (counter_span as f64);
|
||||
self.delivery_ratio_forward = delivery.clamp(0.0, 1.0);
|
||||
let loss_rate = 1.0 - self.delivery_ratio_forward;
|
||||
self.loss_trend.update(loss_rate);
|
||||
self.etx = compute_etx(self.delivery_ratio_forward, self.delivery_ratio_reverse);
|
||||
self.etx_trend.update(self.etx);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Goodput from cumulative bytes + time delta ---
|
||||
if self.has_prev_rr {
|
||||
let bytes_delta = rr
|
||||
.cumulative_bytes_recv
|
||||
.saturating_sub(self.prev_rr_cum_bytes);
|
||||
self.goodput_trend.update(bytes_delta as f64);
|
||||
|
||||
// Compute bytes/sec if we have a time reference
|
||||
if let Some(prev_time) = self.prev_rr_time {
|
||||
let elapsed = now.duration_since(prev_time);
|
||||
let secs = elapsed.as_secs_f64();
|
||||
if secs > 0.0 {
|
||||
let bps = bytes_delta as f64 / secs;
|
||||
// EWMA smoothing: α = 1/4
|
||||
if self.goodput_bps == 0.0 {
|
||||
self.goodput_bps = bps;
|
||||
} else {
|
||||
self.goodput_bps += (bps - self.goodput_bps) * 0.25;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Jitter trend ---
|
||||
self.jitter_trend.update(rr.jitter as f64);
|
||||
|
||||
// --- Save for next delta ---
|
||||
self.prev_rr_cum_packets = rr.cumulative_packets_recv;
|
||||
self.prev_rr_cum_bytes = rr.cumulative_bytes_recv;
|
||||
self.prev_rr_highest_counter = rr.highest_counter;
|
||||
self.prev_rr_ecn_ce = rr.ecn_ce_count;
|
||||
self.prev_rr_reorder = rr.cumulative_reorder_count;
|
||||
self.prev_rr_time = Some(now);
|
||||
self.has_prev_rr = true;
|
||||
|
||||
!had_srtt && self.srtt.initialized()
|
||||
}
|
||||
|
||||
/// Update the reverse delivery ratio from our own receiver state.
|
||||
///
|
||||
/// Computes a per-interval delta (same as forward ratio) rather than
|
||||
/// a lifetime cumulative ratio, so ETX responds to recent conditions.
|
||||
pub fn update_reverse_delivery(&mut self, our_recv_packets: u64, peer_highest: u64) {
|
||||
if self.has_prev_reverse {
|
||||
let counter_span = peer_highest.saturating_sub(self.prev_reverse_highest);
|
||||
let packets_delta = our_recv_packets.saturating_sub(self.prev_reverse_packets);
|
||||
|
||||
if counter_span > 0 {
|
||||
let delivery = (packets_delta as f64) / (counter_span as f64);
|
||||
self.delivery_ratio_reverse = delivery.clamp(0.0, 1.0);
|
||||
self.etx = compute_etx(self.delivery_ratio_forward, self.delivery_ratio_reverse);
|
||||
self.etx_trend.update(self.etx);
|
||||
}
|
||||
}
|
||||
|
||||
self.prev_reverse_packets = our_recv_packets;
|
||||
self.prev_reverse_highest = peer_highest;
|
||||
self.has_prev_reverse = true;
|
||||
}
|
||||
|
||||
/// Current smoothed RTT in milliseconds, or `None` if not yet measured.
|
||||
pub fn srtt_ms(&self) -> Option<f64> {
|
||||
if self.srtt.initialized() {
|
||||
Some(self.srtt.srtt_us() as f64 / 1000.0)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Current loss rate (0.0 = no loss, 1.0 = total loss).
|
||||
pub fn loss_rate(&self) -> f64 {
|
||||
1.0 - self.delivery_ratio_forward
|
||||
}
|
||||
|
||||
/// Smoothed loss rate (long-term EWMA), or `None` if not yet initialized.
|
||||
pub fn smoothed_loss(&self) -> Option<f64> {
|
||||
if self.loss_trend.initialized() {
|
||||
Some(self.loss_trend.long())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Smoothed ETX (long-term EWMA), or `None` if not yet initialized.
|
||||
pub fn smoothed_etx(&self) -> Option<f64> {
|
||||
if self.etx_trend.initialized() {
|
||||
Some(self.etx_trend.long())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Current smoothed goodput in bytes/sec, or 0 if not yet measured.
|
||||
pub fn goodput_bps(&self) -> f64 {
|
||||
self.goodput_bps
|
||||
}
|
||||
|
||||
/// Cumulative ECN CE count from the most recent ReceiverReport.
|
||||
pub fn last_ecn_ce_count(&self) -> u32 {
|
||||
self.prev_rr_ecn_ce
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MmpMetrics {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
|
||||
fn make_rr(
|
||||
highest_counter: u64,
|
||||
cum_packets: u64,
|
||||
cum_bytes: u64,
|
||||
timestamp_echo: u32,
|
||||
dwell: u16,
|
||||
jitter: u32,
|
||||
) -> ReceiverReport {
|
||||
ReceiverReport {
|
||||
highest_counter,
|
||||
cumulative_packets_recv: cum_packets,
|
||||
cumulative_bytes_recv: cum_bytes,
|
||||
timestamp_echo,
|
||||
dwell_time: dwell,
|
||||
max_burst_loss: 0,
|
||||
mean_burst_loss: 0,
|
||||
jitter,
|
||||
ecn_ce_count: 0,
|
||||
owd_trend: 0,
|
||||
burst_loss_count: 0,
|
||||
cumulative_reorder_count: 0,
|
||||
interval_packets_recv: 0,
|
||||
interval_bytes_recv: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rtt_from_echo() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let now = Instant::now();
|
||||
// Peer echoes timestamp 1000ms, dwell=5ms, our current time=1050ms
|
||||
let rr = make_rr(10, 10, 5000, 1000, 5, 0);
|
||||
m.process_receiver_report(&rr, 1050, now);
|
||||
|
||||
assert!(m.srtt.initialized());
|
||||
// RTT = 1050 - 1000 - 5 = 45ms
|
||||
let srtt_ms = m.srtt_ms().unwrap();
|
||||
assert!((srtt_ms - 45.0).abs() < 1.0, "srtt={srtt_ms}, expected ~45");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ignores_duplicate_receiver_report_after_valid_sample() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let t0 = Instant::now();
|
||||
|
||||
let rr1 = make_rr(10, 10, 5_000, 1_000, 5, 0);
|
||||
m.process_receiver_report(&rr1, 1_050, t0);
|
||||
|
||||
let rr2 = make_rr(20, 18, 14_000, 1_100, 5, 0);
|
||||
m.process_receiver_report(&rr2, 1_150, t0 + Duration::from_secs(1));
|
||||
let baseline_srtt_ms = m.srtt_ms().unwrap();
|
||||
let baseline_loss = m.loss_rate();
|
||||
let baseline_goodput = m.goodput_bps();
|
||||
|
||||
assert!(baseline_loss > 0.0);
|
||||
assert!(baseline_goodput > 0.0);
|
||||
|
||||
// A duplicate of the same counters arriving later would be a 4.895s
|
||||
// RTT sample if accepted. It is stale and must not move metrics.
|
||||
m.process_receiver_report(&rr2, 6_000, t0 + Duration::from_secs(5));
|
||||
|
||||
assert_eq!(m.srtt_ms().unwrap(), baseline_srtt_ms);
|
||||
assert_eq!(m.loss_rate(), baseline_loss);
|
||||
assert_eq!(m.goodput_bps(), baseline_goodput);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ignores_out_of_order_receiver_report_after_valid_sample() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let now = Instant::now();
|
||||
|
||||
let valid_rr = make_rr(20, 20, 10000, 1000, 5, 0);
|
||||
m.process_receiver_report(&valid_rr, 1050, now);
|
||||
let baseline_srtt_ms = m.srtt_ms().unwrap();
|
||||
|
||||
let old_rr = make_rr(10, 10, 5000, 1000, 0, 0);
|
||||
m.process_receiver_report(&old_rr, 6000, now + Duration::from_secs(5));
|
||||
|
||||
let srtt_ms = m.srtt_ms().unwrap();
|
||||
assert_eq!(srtt_ms, baseline_srtt_ms);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ignores_wrapped_rtt_sample() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let now = Instant::now();
|
||||
|
||||
let wrapped_rr = make_rr(10, 10, 5000, u32::MAX - 10, 20, 0);
|
||||
m.process_receiver_report(&wrapped_rr, 15, now);
|
||||
|
||||
assert!(m.srtt_ms().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ignores_future_rtt_sample() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let now = Instant::now();
|
||||
|
||||
let future_rr = make_rr(10, 10, 5_000, 2_000, 5, 0);
|
||||
m.process_receiver_report(&future_rr, 1_000, now);
|
||||
|
||||
assert!(m.srtt_ms().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_loss_rate_computation() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let t0 = Instant::now();
|
||||
|
||||
// First report: baseline
|
||||
let rr1 = make_rr(100, 100, 50000, 0, 0, 0);
|
||||
m.process_receiver_report(&rr1, 0, t0);
|
||||
|
||||
// Second report: 200 counters sent, 190 received (5% loss)
|
||||
let rr2 = make_rr(300, 290, 145000, 0, 0, 0);
|
||||
m.process_receiver_report(&rr2, 0, t0 + Duration::from_secs(1));
|
||||
|
||||
let loss = m.loss_rate();
|
||||
assert!((loss - 0.05).abs() < 0.01, "loss={loss}, expected ~0.05");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_etx_updates() {
|
||||
let mut m = MmpMetrics::new();
|
||||
assert_eq!(m.etx, 1.0); // initial: perfect
|
||||
|
||||
// Simulate some loss via forward ratio
|
||||
m.delivery_ratio_forward = 0.9;
|
||||
|
||||
// First call establishes the baseline (no ETX update yet)
|
||||
m.update_reverse_delivery(100, 100);
|
||||
assert_eq!(m.etx, 1.0); // still perfect — baseline only
|
||||
|
||||
// Second call: 190 of 200 frames received (5% loss)
|
||||
m.update_reverse_delivery(290, 300);
|
||||
assert!(m.etx > 1.0);
|
||||
assert!(m.etx < 2.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_rtt_without_echo() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let now = Instant::now();
|
||||
let rr = make_rr(10, 10, 5000, 0, 0, 0);
|
||||
m.process_receiver_report(&rr, 1000, now);
|
||||
assert!(m.srtt_ms().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jitter_trend() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let t0 = Instant::now();
|
||||
let rr1 = make_rr(10, 10, 5000, 0, 0, 100);
|
||||
m.process_receiver_report(&rr1, 0, t0);
|
||||
|
||||
let rr2 = make_rr(20, 20, 10000, 0, 0, 500);
|
||||
m.process_receiver_report(&rr2, 0, t0 + Duration::from_secs(1));
|
||||
|
||||
assert!(m.jitter_trend.initialized());
|
||||
// Short-term should be closer to 500 than long-term
|
||||
assert!(m.jitter_trend.short() > m.jitter_trend.long());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_goodput_bps() {
|
||||
let mut m = MmpMetrics::new();
|
||||
let t0 = Instant::now();
|
||||
|
||||
// First report: baseline (50KB received)
|
||||
let rr1 = make_rr(100, 100, 50_000, 0, 0, 0);
|
||||
m.process_receiver_report(&rr1, 0, t0);
|
||||
assert_eq!(m.goodput_bps(), 0.0); // no rate yet (first report)
|
||||
|
||||
// Second report 1s later: 150KB total (100KB delta in 1s = 100KB/s)
|
||||
let rr2 = make_rr(300, 290, 150_000, 0, 0, 0);
|
||||
m.process_receiver_report(&rr2, 0, t0 + Duration::from_secs(1));
|
||||
assert!(
|
||||
m.goodput_bps() > 90_000.0,
|
||||
"goodput={}, expected ~100000",
|
||||
m.goodput_bps()
|
||||
);
|
||||
assert!(
|
||||
m.goodput_bps() < 110_000.0,
|
||||
"goodput={}, expected ~100000",
|
||||
m.goodput_bps()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reverse_delivery_delta() {
|
||||
let mut m = MmpMetrics::new();
|
||||
|
||||
// First call: baseline only, no ratio update
|
||||
m.update_reverse_delivery(100, 100);
|
||||
assert_eq!(m.delivery_ratio_reverse, 1.0); // unchanged from default
|
||||
|
||||
// Second call: perfect delivery (200 new frames, all received)
|
||||
m.update_reverse_delivery(300, 300);
|
||||
assert!((m.delivery_ratio_reverse - 1.0).abs() < 0.001);
|
||||
|
||||
// Third call: 50% loss (100 frames sent, 50 received)
|
||||
m.update_reverse_delivery(350, 400);
|
||||
assert!(
|
||||
(m.delivery_ratio_reverse - 0.5).abs() < 0.001,
|
||||
"reverse={}, expected 0.5",
|
||||
m.delivery_ratio_reverse
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reverse_delivery_rekey_reset() {
|
||||
let mut m = MmpMetrics::new();
|
||||
|
||||
// Establish baseline and one measurement
|
||||
m.update_reverse_delivery(100, 100);
|
||||
m.update_reverse_delivery(300, 300);
|
||||
assert!((m.delivery_ratio_reverse - 1.0).abs() < 0.001);
|
||||
|
||||
// Rekey resets reverse state
|
||||
m.reset_for_rekey();
|
||||
|
||||
// First call after rekey: baseline only
|
||||
m.update_reverse_delivery(50, 50);
|
||||
// delivery_ratio_reverse was reset to 1.0 by reset_for_rekey's
|
||||
// clearing of delivery_ratio_forward; reverse is not explicitly
|
||||
// reset — but the delta state is, so next call computes fresh.
|
||||
assert_eq!(m.delivery_ratio_reverse, 1.0);
|
||||
|
||||
// Second call after rekey: 80% delivery
|
||||
m.update_reverse_delivery(90, 100);
|
||||
assert!(
|
||||
(m.delivery_ratio_reverse - 0.8).abs() < 0.001,
|
||||
"reverse={}, expected 0.8",
|
||||
m.delivery_ratio_reverse
|
||||
);
|
||||
}
|
||||
}
|
||||
+555
@@ -0,0 +1,555 @@
|
||||
//! Metrics Measurement Protocol (MMP) — link-layer instantiation.
|
||||
//!
|
||||
//! Measures link quality between adjacent peers: RTT, loss, jitter,
|
||||
//! throughput, one-way delay trend, and ETX. Operates on the per-frame
|
||||
//! hooks (counter, timestamp, flags) introduced by the FMP wire format
|
||||
//! revision.
|
||||
//!
|
||||
//! Three operating modes trade measurement fidelity for overhead:
|
||||
//! - **Full**: sender + receiver reports at RTT-adaptive intervals
|
||||
//! - **Lightweight**: receiver reports only (infer loss from counters)
|
||||
//! - **Minimal**: spin bit + CE echo only, no reports
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt::{self, Debug};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
// Sub-modules
|
||||
pub mod algorithms;
|
||||
pub mod metrics;
|
||||
pub mod receiver;
|
||||
pub mod report;
|
||||
pub mod sender;
|
||||
|
||||
// Re-exports
|
||||
pub use algorithms::{
|
||||
DualEwma, JitterEstimator, OwdTrendDetector, SpinBitState, SrttEstimator, compute_etx,
|
||||
};
|
||||
pub use metrics::MmpMetrics;
|
||||
pub use receiver::ReceiverState;
|
||||
pub use report::{ReceiverReport, SenderReport};
|
||||
pub use sender::SenderState;
|
||||
|
||||
// Session-layer re-exports
|
||||
// MmpSessionState and PathMtuState are defined in this file
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
/// SenderReport body size (after msg_type byte): 3 reserved + 44 payload = 47.
|
||||
pub const SENDER_REPORT_BODY_SIZE: usize = 47;
|
||||
|
||||
/// ReceiverReport body size (after msg_type byte): 3 reserved + 64 payload = 67.
|
||||
pub const RECEIVER_REPORT_BODY_SIZE: usize = 67;
|
||||
|
||||
/// SenderReport total wire size including inner header: 5 + 47 = 52.
|
||||
pub const SENDER_REPORT_WIRE_SIZE: usize = 52;
|
||||
|
||||
/// ReceiverReport total wire size including inner header: 5 + 67 = 72.
|
||||
pub const RECEIVER_REPORT_WIRE_SIZE: usize = 72;
|
||||
|
||||
// --- EWMA parameters (as shift amounts for integer arithmetic) ---
|
||||
|
||||
/// Jitter EWMA: α = 1/16 (RFC 3550 §6.4.1).
|
||||
pub const JITTER_ALPHA_SHIFT: u32 = 4;
|
||||
|
||||
/// SRTT: α = 1/8 (Jacobson, RFC 6298).
|
||||
pub const SRTT_ALPHA_SHIFT: u32 = 3;
|
||||
|
||||
/// RTTVAR: β = 1/4 (Jacobson, RFC 6298).
|
||||
pub const RTTVAR_BETA_SHIFT: u32 = 2;
|
||||
|
||||
/// Dual EWMA short-term: α = 1/4.
|
||||
pub const EWMA_SHORT_ALPHA: f64 = 0.25;
|
||||
|
||||
/// Dual EWMA long-term: α = 1/32.
|
||||
pub const EWMA_LONG_ALPHA: f64 = 1.0 / 32.0;
|
||||
|
||||
// --- Timing defaults (milliseconds) ---
|
||||
|
||||
/// Default report interval before SRTT is available (cold start).
|
||||
pub const DEFAULT_COLD_START_INTERVAL_MS: u64 = 200;
|
||||
|
||||
/// Minimum report interval (SRTT clamp floor).
|
||||
///
|
||||
/// Raised from 100ms to 1000ms: parent re-evaluation runs every 60s,
|
||||
/// so 60 samples/cycle is more than sufficient for EWMA convergence (~10).
|
||||
/// The cold-start phase uses `DEFAULT_COLD_START_INTERVAL_MS` (200ms) for
|
||||
/// fast initial SRTT convergence before transitioning to this floor.
|
||||
pub const MIN_REPORT_INTERVAL_MS: u64 = 1_000;
|
||||
|
||||
/// Maximum report interval (SRTT clamp ceiling).
|
||||
pub const MAX_REPORT_INTERVAL_MS: u64 = 5_000;
|
||||
|
||||
/// Number of SRTT samples before transitioning from cold-start to normal floor.
|
||||
///
|
||||
/// During cold-start, report intervals use `DEFAULT_COLD_START_INTERVAL_MS` as
|
||||
/// the floor to gather SRTT samples quickly. After this many updates, the floor
|
||||
/// switches to `MIN_REPORT_INTERVAL_MS`.
|
||||
pub const COLD_START_SAMPLES: u32 = 5;
|
||||
|
||||
/// Default OWD ring buffer capacity.
|
||||
pub const DEFAULT_OWD_WINDOW_SIZE: usize = 32;
|
||||
|
||||
/// Default operator log interval in seconds.
|
||||
pub const DEFAULT_LOG_INTERVAL_SECS: u64 = 30;
|
||||
|
||||
// --- Session-layer timing defaults ---
|
||||
// Session reports are routed end-to-end (bandwidth cost on every transit link),
|
||||
// so intervals are higher than link-layer.
|
||||
|
||||
/// Session-layer minimum report interval.
|
||||
pub const MIN_SESSION_REPORT_INTERVAL_MS: u64 = 500;
|
||||
|
||||
/// Session-layer maximum report interval.
|
||||
pub const MAX_SESSION_REPORT_INTERVAL_MS: u64 = 10_000;
|
||||
|
||||
/// Session-layer cold-start report interval (before SRTT is available).
|
||||
pub const SESSION_COLD_START_INTERVAL_MS: u64 = 1_000;
|
||||
|
||||
// ============================================================================
|
||||
// Operating Mode
|
||||
// ============================================================================
|
||||
|
||||
/// MMP operating mode.
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum MmpMode {
|
||||
/// Sender + receiver reports at RTT-adaptive intervals. Maximum fidelity.
|
||||
#[default]
|
||||
Full,
|
||||
/// Receiver reports only. Loss inferred from counter gaps.
|
||||
Lightweight,
|
||||
/// Spin bit + CE echo only. No reports exchanged.
|
||||
Minimal,
|
||||
}
|
||||
|
||||
impl fmt::Display for MmpMode {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
MmpMode::Full => write!(f, "full"),
|
||||
MmpMode::Lightweight => write!(f, "lightweight"),
|
||||
MmpMode::Minimal => write!(f, "minimal"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Configuration
|
||||
// ============================================================================
|
||||
|
||||
/// MMP configuration (`node.mmp.*`).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MmpConfig {
|
||||
/// Operating mode (`node.mmp.mode`).
|
||||
#[serde(default)]
|
||||
pub mode: MmpMode,
|
||||
|
||||
/// Periodic operator log interval in seconds (`node.mmp.log_interval_secs`).
|
||||
#[serde(default = "MmpConfig::default_log_interval_secs")]
|
||||
pub log_interval_secs: u64,
|
||||
|
||||
/// OWD trend ring buffer size (`node.mmp.owd_window_size`).
|
||||
#[serde(default = "MmpConfig::default_owd_window_size")]
|
||||
pub owd_window_size: usize,
|
||||
}
|
||||
|
||||
impl Default for MmpConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
mode: MmpMode::default(),
|
||||
log_interval_secs: DEFAULT_LOG_INTERVAL_SECS,
|
||||
owd_window_size: DEFAULT_OWD_WINDOW_SIZE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MmpConfig {
|
||||
fn default_log_interval_secs() -> u64 {
|
||||
DEFAULT_LOG_INTERVAL_SECS
|
||||
}
|
||||
fn default_owd_window_size() -> usize {
|
||||
DEFAULT_OWD_WINDOW_SIZE
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Per-Peer MMP State
|
||||
// ============================================================================
|
||||
|
||||
/// Combined MMP state for a single peer link.
|
||||
///
|
||||
/// Wraps sender, receiver, metrics, and spin bit state. One instance
|
||||
/// per `ActivePeer`.
|
||||
pub struct MmpPeerState {
|
||||
pub sender: SenderState,
|
||||
pub receiver: ReceiverState,
|
||||
pub metrics: MmpMetrics,
|
||||
pub spin_bit: SpinBitState,
|
||||
mode: MmpMode,
|
||||
log_interval: Duration,
|
||||
last_log_time: Option<Instant>,
|
||||
}
|
||||
|
||||
impl MmpPeerState {
|
||||
/// Create MMP state for a new peer link.
|
||||
///
|
||||
/// `is_initiator`: true if this node initiated the Noise handshake
|
||||
/// (determines spin bit role).
|
||||
pub fn new(config: &MmpConfig, is_initiator: bool) -> Self {
|
||||
Self {
|
||||
sender: SenderState::new(),
|
||||
receiver: ReceiverState::new(config.owd_window_size),
|
||||
metrics: MmpMetrics::new(),
|
||||
spin_bit: SpinBitState::new(is_initiator),
|
||||
mode: config.mode,
|
||||
log_interval: Duration::from_secs(config.log_interval_secs),
|
||||
last_log_time: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset counter-dependent state for rekey cutover.
|
||||
pub fn reset_for_rekey(&mut self, now: Instant) {
|
||||
self.receiver.reset_for_rekey(now);
|
||||
self.metrics.reset_for_rekey();
|
||||
}
|
||||
|
||||
/// Current operating mode.
|
||||
pub fn mode(&self) -> MmpMode {
|
||||
self.mode
|
||||
}
|
||||
|
||||
/// Check if it's time to emit a periodic metrics log.
|
||||
pub fn should_log(&self, now: Instant) -> bool {
|
||||
match self.last_log_time {
|
||||
None => true,
|
||||
Some(last) => now.duration_since(last) >= self.log_interval,
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark that a periodic log was emitted.
|
||||
pub fn mark_logged(&mut self, now: Instant) {
|
||||
self.last_log_time = Some(now);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Per-Session MMP State (session-layer instantiation)
|
||||
// ============================================================================
|
||||
|
||||
/// Combined MMP state for a single end-to-end session.
|
||||
///
|
||||
/// Wraps sender, receiver, metrics, spin bit, and path MTU state.
|
||||
/// One instance per established `SessionEntry`.
|
||||
pub struct MmpSessionState {
|
||||
pub sender: SenderState,
|
||||
pub receiver: ReceiverState,
|
||||
pub metrics: MmpMetrics,
|
||||
pub spin_bit: SpinBitState,
|
||||
mode: MmpMode,
|
||||
log_interval: Duration,
|
||||
last_log_time: Option<Instant>,
|
||||
pub path_mtu: PathMtuState,
|
||||
}
|
||||
|
||||
impl MmpSessionState {
|
||||
/// Create MMP state for a new session.
|
||||
///
|
||||
/// `is_initiator`: true if this node initiated the Noise handshake
|
||||
/// (determines spin bit role).
|
||||
pub fn new(config: &crate::config::SessionMmpConfig, is_initiator: bool) -> Self {
|
||||
Self {
|
||||
sender: SenderState::new_with_cold_start(SESSION_COLD_START_INTERVAL_MS),
|
||||
receiver: ReceiverState::new_with_cold_start(
|
||||
config.owd_window_size,
|
||||
SESSION_COLD_START_INTERVAL_MS,
|
||||
),
|
||||
metrics: MmpMetrics::new(),
|
||||
spin_bit: SpinBitState::new(is_initiator),
|
||||
mode: config.mode,
|
||||
log_interval: Duration::from_secs(config.log_interval_secs),
|
||||
last_log_time: None,
|
||||
path_mtu: PathMtuState::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset counter-dependent state for rekey cutover.
|
||||
pub fn reset_for_rekey(&mut self, now: Instant) {
|
||||
self.receiver.reset_for_rekey(now);
|
||||
self.metrics.reset_for_rekey();
|
||||
}
|
||||
|
||||
/// Current operating mode.
|
||||
pub fn mode(&self) -> MmpMode {
|
||||
self.mode
|
||||
}
|
||||
|
||||
/// Check if it's time to emit a periodic metrics log.
|
||||
pub fn should_log(&self, now: Instant) -> bool {
|
||||
match self.last_log_time {
|
||||
None => true,
|
||||
Some(last) => now.duration_since(last) >= self.log_interval,
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark that a periodic log was emitted.
|
||||
pub fn mark_logged(&mut self, now: Instant) {
|
||||
self.last_log_time = Some(now);
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for MmpSessionState {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("MmpSessionState")
|
||||
.field("mode", &self.mode)
|
||||
.field("path_mtu", &self.path_mtu.current_mtu())
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Path MTU State (session-layer only)
|
||||
// ============================================================================
|
||||
|
||||
/// Path MTU tracking for a single session.
|
||||
///
|
||||
/// Destination side: observes `path_mtu` from incoming SessionDatagram envelopes
|
||||
/// and generates PathMtuNotification messages back to the source.
|
||||
///
|
||||
/// Source side: applies received PathMtuNotification to limit outbound datagram
|
||||
/// size. Decrease is immediate; increase requires 3 consecutive notifications.
|
||||
pub struct PathMtuState {
|
||||
/// Current effective path MTU (what we use for sending).
|
||||
current_mtu: u16,
|
||||
/// Last observed path MTU from incoming datagrams (destination-side).
|
||||
last_observed_mtu: u16,
|
||||
/// Whether the observed MTU has changed since the last notification.
|
||||
observed_changed: bool,
|
||||
/// Last time a PathMtuNotification was sent.
|
||||
last_notification_time: Option<Instant>,
|
||||
/// Notification interval: max(10s, 5 * SRTT). Default 10s.
|
||||
notification_interval: Duration,
|
||||
/// For source-side increase tracking: consecutive higher-value notifications.
|
||||
consecutive_increase_count: u8,
|
||||
/// Time of the first notification in the current increase sequence.
|
||||
first_increase_time: Option<Instant>,
|
||||
/// The MTU value being proposed for increase.
|
||||
pending_increase_mtu: u16,
|
||||
}
|
||||
|
||||
impl PathMtuState {
|
||||
/// Create path MTU state with no initial measurement.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
current_mtu: u16::MAX,
|
||||
last_observed_mtu: u16::MAX,
|
||||
observed_changed: false,
|
||||
last_notification_time: None,
|
||||
notification_interval: Duration::from_secs(10),
|
||||
consecutive_increase_count: 0,
|
||||
first_increase_time: None,
|
||||
pending_increase_mtu: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Current effective path MTU (source-side, for sending).
|
||||
pub fn current_mtu(&self) -> u16 {
|
||||
self.current_mtu
|
||||
}
|
||||
|
||||
/// Last observed incoming path MTU (destination-side).
|
||||
pub fn last_observed_mtu(&self) -> u16 {
|
||||
self.last_observed_mtu
|
||||
}
|
||||
|
||||
/// Update notification interval from SRTT: max(10s, 5 * SRTT).
|
||||
pub fn update_interval_from_srtt(&mut self, srtt_ms: f64) {
|
||||
let five_srtt = Duration::from_millis((srtt_ms * 5.0) as u64);
|
||||
self.notification_interval = five_srtt.max(Duration::from_secs(10));
|
||||
}
|
||||
|
||||
/// Seed source-side current_mtu from outbound transport MTU.
|
||||
///
|
||||
/// Called on each send. Only decreases (never increases) the current_mtu
|
||||
/// so the destination's PathMtuNotification can still raise it later.
|
||||
/// Ensures current_mtu doesn't stay at u16::MAX before any notification
|
||||
/// arrives from the destination.
|
||||
pub fn seed_source_mtu(&mut self, outbound_mtu: u16) {
|
||||
if outbound_mtu < self.current_mtu {
|
||||
self.current_mtu = outbound_mtu;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Destination side ---
|
||||
|
||||
/// Observe the path_mtu from an incoming SessionDatagram envelope.
|
||||
///
|
||||
/// Called on the destination (receiver) side for every session message.
|
||||
pub fn observe_incoming_mtu(&mut self, path_mtu: u16) {
|
||||
if path_mtu != self.last_observed_mtu {
|
||||
self.observed_changed = true;
|
||||
self.last_observed_mtu = path_mtu;
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a PathMtuNotification should be sent.
|
||||
///
|
||||
/// Send on first measurement, on decrease (immediate), or periodic
|
||||
/// confirmation at the notification interval.
|
||||
pub fn should_send_notification(&self, now: Instant) -> bool {
|
||||
if self.last_observed_mtu == u16::MAX {
|
||||
return false; // No measurement yet
|
||||
}
|
||||
match self.last_notification_time {
|
||||
None => true, // First measurement
|
||||
Some(last) => {
|
||||
// Immediate on decrease
|
||||
if self.observed_changed && self.last_observed_mtu < self.current_mtu {
|
||||
return true;
|
||||
}
|
||||
// Periodic confirmation
|
||||
now.duration_since(last) >= self.notification_interval
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a PathMtuNotification from current state.
|
||||
///
|
||||
/// Returns the path_mtu value to send. Caller handles encoding.
|
||||
pub fn build_notification(&mut self, now: Instant) -> Option<u16> {
|
||||
if self.last_observed_mtu == u16::MAX {
|
||||
return None;
|
||||
}
|
||||
self.last_notification_time = Some(now);
|
||||
self.observed_changed = false;
|
||||
Some(self.last_observed_mtu)
|
||||
}
|
||||
|
||||
// --- Source side ---
|
||||
|
||||
/// Apply a received PathMtuNotification.
|
||||
///
|
||||
/// - Decrease: immediate (take the lower value).
|
||||
/// - Increase: require 3 consecutive notifications with the same higher
|
||||
/// value, spanning at least 2 * notification_interval.
|
||||
///
|
||||
/// Returns `true` if the effective MTU changed.
|
||||
pub fn apply_notification(&mut self, reported_mtu: u16, now: Instant) -> bool {
|
||||
if reported_mtu < self.current_mtu {
|
||||
// Decrease: immediate
|
||||
self.current_mtu = reported_mtu;
|
||||
self.consecutive_increase_count = 0;
|
||||
self.first_increase_time = None;
|
||||
return true;
|
||||
}
|
||||
|
||||
if reported_mtu > self.current_mtu {
|
||||
// Increase: track consecutive notifications
|
||||
if reported_mtu == self.pending_increase_mtu {
|
||||
self.consecutive_increase_count += 1;
|
||||
} else {
|
||||
// Different value: reset sequence
|
||||
self.pending_increase_mtu = reported_mtu;
|
||||
self.consecutive_increase_count = 1;
|
||||
self.first_increase_time = Some(now);
|
||||
}
|
||||
|
||||
// Accept increase after 3 consecutive spanning 2 * interval
|
||||
if self.consecutive_increase_count >= 3
|
||||
&& let Some(first_time) = self.first_increase_time
|
||||
{
|
||||
let required = self.notification_interval * 2;
|
||||
if now.duration_since(first_time) >= required {
|
||||
self.current_mtu = reported_mtu;
|
||||
self.consecutive_increase_count = 0;
|
||||
self.first_increase_time = None;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No change (equal or increase not yet confirmed)
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PathMtuState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for MmpPeerState {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("MmpPeerState")
|
||||
.field("mode", &self.mode)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_mode_default() {
|
||||
assert_eq!(MmpMode::default(), MmpMode::Full);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mode_display() {
|
||||
assert_eq!(MmpMode::Full.to_string(), "full");
|
||||
assert_eq!(MmpMode::Lightweight.to_string(), "lightweight");
|
||||
assert_eq!(MmpMode::Minimal.to_string(), "minimal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mode_serde_roundtrip() {
|
||||
let yaml = "full";
|
||||
let mode: MmpMode = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(mode, MmpMode::Full);
|
||||
|
||||
let yaml = "lightweight";
|
||||
let mode: MmpMode = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(mode, MmpMode::Lightweight);
|
||||
|
||||
let yaml = "minimal";
|
||||
let mode: MmpMode = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(mode, MmpMode::Minimal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_default() {
|
||||
let config = MmpConfig::default();
|
||||
assert_eq!(config.mode, MmpMode::Full);
|
||||
assert_eq!(config.log_interval_secs, 30);
|
||||
assert_eq!(config.owd_window_size, 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_yaml_parse() {
|
||||
let yaml = r#"
|
||||
mode: lightweight
|
||||
log_interval_secs: 60
|
||||
owd_window_size: 48
|
||||
"#;
|
||||
let config: MmpConfig = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(config.mode, MmpMode::Lightweight);
|
||||
assert_eq!(config.log_interval_secs, 60);
|
||||
assert_eq!(config.owd_window_size, 48);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_yaml_partial() {
|
||||
let yaml = "mode: minimal";
|
||||
let config: MmpConfig = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(config.mode, MmpMode::Minimal);
|
||||
assert_eq!(config.log_interval_secs, DEFAULT_LOG_INTERVAL_SECS);
|
||||
assert_eq!(config.owd_window_size, DEFAULT_OWD_WINDOW_SIZE);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,13 @@
|
||||
//! Per-peer receiver-side MMP state (sans-IO).
|
||||
//! MMP receiver state machine.
|
||||
//!
|
||||
//! Accumulates per-frame observations (loss bursts, jitter, OWD trend, ECN)
|
||||
//! and produces `ReceiverReport` snapshots. All time inputs are injected `u64`
|
||||
//! milliseconds.
|
||||
//! Tracks what this node has received from a specific peer and produces
|
||||
//! ReceiverReport messages on demand. One `ReceiverState` per active peer.
|
||||
|
||||
use super::algorithms::{JitterEstimator, OwdTrendDetector};
|
||||
use super::wire::ReceiverReport;
|
||||
use super::{
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::mmp::algorithms::{JitterEstimator, OwdTrendDetector};
|
||||
use crate::mmp::report::ReceiverReport;
|
||||
use crate::mmp::{
|
||||
COLD_START_SAMPLES, DEFAULT_COLD_START_INTERVAL_MS, DEFAULT_OWD_WINDOW_SIZE,
|
||||
MAX_REPORT_INTERVAL_MS, MIN_REPORT_INTERVAL_MS,
|
||||
};
|
||||
@@ -167,18 +168,17 @@ pub struct ReceiverState {
|
||||
// --- Timestamp echo ---
|
||||
/// Sender timestamp from the most recent frame (for echo).
|
||||
last_sender_timestamp: u32,
|
||||
/// Local time (injected `u64` ms) when the most recent frame was received
|
||||
/// (for dwell / jitter computation).
|
||||
last_recv_ms: Option<u64>,
|
||||
/// Local time when the most recent frame was received (for dwell computation).
|
||||
last_recv_time: Option<Instant>,
|
||||
|
||||
// --- Rekey grace ---
|
||||
/// When set, jitter updates are suppressed until this injected-ms instant
|
||||
/// passes. Prevents drain-window frames from spiking the jitter estimator.
|
||||
rekey_jitter_grace_until_ms: Option<u64>,
|
||||
/// When set, jitter updates are suppressed until this instant passes.
|
||||
/// Prevents drain-window frames from spiking the jitter estimator.
|
||||
rekey_jitter_grace_until: Option<Instant>,
|
||||
|
||||
// --- Report timing (injected `u64` ms) ---
|
||||
last_report_ms: Option<u64>,
|
||||
report_interval_ms: u64,
|
||||
// --- Report timing ---
|
||||
last_report_time: Option<Instant>,
|
||||
report_interval: Duration,
|
||||
/// Whether any frames have been received since the last report.
|
||||
interval_has_data: bool,
|
||||
|
||||
@@ -210,10 +210,10 @@ impl ReceiverState {
|
||||
gap_tracker: GapTracker::new(),
|
||||
ecn_ce_count: 0,
|
||||
last_sender_timestamp: 0,
|
||||
last_recv_ms: None,
|
||||
rekey_jitter_grace_until_ms: None,
|
||||
last_report_ms: None,
|
||||
report_interval_ms: cold_start_ms,
|
||||
last_recv_time: None,
|
||||
rekey_jitter_grace_until: None,
|
||||
last_report_time: None,
|
||||
report_interval: Duration::from_millis(cold_start_ms),
|
||||
interval_has_data: false,
|
||||
srtt_sample_count: 0,
|
||||
}
|
||||
@@ -224,8 +224,7 @@ impl ReceiverState {
|
||||
/// After cutover, the new session starts with counter 0 and reset
|
||||
/// timestamps. Without resetting, the old `highest_counter` and
|
||||
/// `GapTracker.expected_next` cause false reorder/loss detection.
|
||||
/// `now_ms` is the injected monotonic time in milliseconds.
|
||||
pub fn reset_for_rekey(&mut self, now_ms: u64) {
|
||||
pub fn reset_for_rekey(&mut self, now: Instant) {
|
||||
self.highest_counter = 0;
|
||||
self.cumulative_reorder_count = 0;
|
||||
self.gap_tracker = GapTracker::new();
|
||||
@@ -235,12 +234,12 @@ impl ReceiverState {
|
||||
self.owd_trend.clear();
|
||||
self.owd_seq = 0;
|
||||
self.last_sender_timestamp = 0;
|
||||
self.last_recv_ms = None;
|
||||
self.rekey_jitter_grace_until_ms = Some(now_ms + REKEY_JITTER_GRACE_SECS * 1000);
|
||||
self.last_recv_time = None;
|
||||
self.rekey_jitter_grace_until = Some(now + Duration::from_secs(REKEY_JITTER_GRACE_SECS));
|
||||
self.ecn_ce_count = 0;
|
||||
self.interval_has_data = false;
|
||||
// Keep cumulative_packets_recv, cumulative_bytes_recv (lifetime stats)
|
||||
// Keep last_report_ms, report_interval_ms (report scheduling)
|
||||
// Keep last_report_time, report_interval (report scheduling)
|
||||
}
|
||||
|
||||
/// Record a received frame from this peer.
|
||||
@@ -251,14 +250,14 @@ impl ReceiverState {
|
||||
/// - `sender_timestamp_ms`: session-relative timestamp from inner header (ms)
|
||||
/// - `bytes`: wire payload size
|
||||
/// - `ce_flag`: CE bit from flags byte
|
||||
/// - `now_ms`: injected monotonic local time in milliseconds
|
||||
/// - `now`: current local time
|
||||
pub fn record_recv(
|
||||
&mut self,
|
||||
counter: u64,
|
||||
sender_timestamp_ms: u32,
|
||||
bytes: usize,
|
||||
ce_flag: bool,
|
||||
now_ms: u64,
|
||||
now: Instant,
|
||||
) {
|
||||
self.interval_has_data = true;
|
||||
self.cumulative_packets_recv += 1;
|
||||
@@ -283,18 +282,18 @@ impl ReceiverState {
|
||||
|
||||
// Jitter: compute transit time delta
|
||||
// Transit = recv_local - sender_timestamp (in µs for precision)
|
||||
// We use the injected monotonic ms clock for the local reference.
|
||||
// We use a monotonic local reference derived from Instant offsets.
|
||||
let sender_us = (sender_timestamp_ms as i64) * 1000;
|
||||
// We compute the delta between consecutive transits using relative
|
||||
// millisecond differences (scaled to µs to match the estimator input).
|
||||
// We can't get absolute µs from Instant, but we can compute the delta
|
||||
// between consecutive transits using relative Instant differences.
|
||||
// Skip during post-rekey grace period to avoid drain-window spikes.
|
||||
let in_grace = self
|
||||
.rekey_jitter_grace_until_ms
|
||||
.is_some_and(|deadline| now_ms < deadline);
|
||||
.rekey_jitter_grace_until
|
||||
.is_some_and(|deadline| now < deadline);
|
||||
if !in_grace {
|
||||
self.rekey_jitter_grace_until_ms = None; // clear expired grace
|
||||
if let Some(prev_recv) = self.last_recv_ms {
|
||||
let recv_delta_us = (now_ms.saturating_sub(prev_recv) as i64) * 1000;
|
||||
self.rekey_jitter_grace_until = None; // clear expired grace
|
||||
if let Some(prev_recv) = self.last_recv_time {
|
||||
let recv_delta_us = now.duration_since(prev_recv).as_micros() as i64;
|
||||
let send_delta_us = sender_us - (self.last_sender_timestamp as i64 * 1000);
|
||||
let transit_delta = (recv_delta_us - send_delta_us) as i32;
|
||||
self.jitter.update(transit_delta);
|
||||
@@ -302,10 +301,10 @@ impl ReceiverState {
|
||||
}
|
||||
|
||||
// OWD trend: use sender timestamp as a proxy for send time
|
||||
// and the injected ms delta from a fixed reference as receive time.
|
||||
// and Instant delta from a fixed reference as receive time.
|
||||
// Since we only need the *trend* (slope), absolute offsets cancel out.
|
||||
if let Some(first_recv) = self.last_recv_ms.or(Some(now_ms)) {
|
||||
let recv_offset_us = (now_ms.saturating_sub(first_recv) as i64) * 1000;
|
||||
if let Some(first_recv) = self.last_recv_time.or(Some(now)) {
|
||||
let recv_offset_us = now.duration_since(first_recv).as_micros() as i64;
|
||||
let owd_us = recv_offset_us - sender_us;
|
||||
self.owd_seq = self.owd_seq.wrapping_add(1);
|
||||
self.owd_trend.push(self.owd_seq, owd_us);
|
||||
@@ -313,14 +312,13 @@ impl ReceiverState {
|
||||
|
||||
// Timestamp echo state
|
||||
self.last_sender_timestamp = sender_timestamp_ms;
|
||||
self.last_recv_ms = Some(now_ms);
|
||||
self.last_recv_time = Some(now);
|
||||
}
|
||||
|
||||
/// Build a ReceiverReport from current state and reset the interval.
|
||||
///
|
||||
/// Returns `None` if no frames have been received since the last report.
|
||||
/// `now_ms` is the injected monotonic time in milliseconds.
|
||||
pub fn build_report(&mut self, now_ms: u64) -> Option<ReceiverReport> {
|
||||
pub fn build_report(&mut self, now: Instant) -> Option<ReceiverReport> {
|
||||
if !self.interval_has_data {
|
||||
return None;
|
||||
}
|
||||
@@ -329,10 +327,10 @@ impl ReceiverState {
|
||||
// If it no longer fits on the wire, the timestamp echo cannot produce
|
||||
// a valid RTT sample. Preserve the counters but suppress the echo.
|
||||
let (timestamp_echo, dwell_time) = self
|
||||
.last_recv_ms
|
||||
.last_recv_time
|
||||
.map(|t| {
|
||||
let dwell_ms = now_ms.saturating_sub(t);
|
||||
if dwell_ms > u64::from(u16::MAX) {
|
||||
let dwell_ms = now.duration_since(t).as_millis();
|
||||
if dwell_ms > u128::from(u16::MAX) {
|
||||
(0, u16::MAX)
|
||||
} else {
|
||||
(self.last_sender_timestamp, dwell_ms as u16)
|
||||
@@ -363,19 +361,19 @@ impl ReceiverState {
|
||||
self.interval_packets_recv = 0;
|
||||
self.interval_bytes_recv = 0;
|
||||
self.interval_has_data = false;
|
||||
self.last_report_ms = Some(now_ms);
|
||||
self.last_report_time = Some(now);
|
||||
|
||||
Some(report)
|
||||
}
|
||||
|
||||
/// Check if it's time to send a report.
|
||||
pub fn should_send_report(&self, now_ms: u64) -> bool {
|
||||
pub fn should_send_report(&self, now: Instant) -> bool {
|
||||
if !self.interval_has_data {
|
||||
return false;
|
||||
}
|
||||
match self.last_report_ms {
|
||||
match self.last_report_time {
|
||||
None => true,
|
||||
Some(last) => now_ms.saturating_sub(last) >= self.report_interval_ms,
|
||||
Some(last) => now.duration_since(last) >= self.report_interval,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -404,7 +402,7 @@ impl ReceiverState {
|
||||
return;
|
||||
}
|
||||
let interval_ms = ((srtt_us as u64) / 1000).clamp(min_ms, max_ms);
|
||||
self.report_interval_ms = interval_ms;
|
||||
self.report_interval = Duration::from_millis(interval_ms);
|
||||
}
|
||||
|
||||
// --- Accessors ---
|
||||
@@ -425,14 +423,12 @@ impl ReceiverState {
|
||||
self.jitter.jitter_us()
|
||||
}
|
||||
|
||||
pub fn report_interval_ms(&self) -> u64 {
|
||||
self.report_interval_ms
|
||||
pub fn report_interval(&self) -> Duration {
|
||||
self.report_interval
|
||||
}
|
||||
|
||||
/// Local monotonic time (injected `u64` ms) of the most recent received
|
||||
/// frame, or `None` if none has been received.
|
||||
pub fn last_recv_ms(&self) -> Option<u64> {
|
||||
self.last_recv_ms
|
||||
pub fn last_recv_time(&self) -> Option<Instant> {
|
||||
self.last_recv_time
|
||||
}
|
||||
|
||||
pub fn ecn_ce_count(&self) -> u32 {
|
||||
@@ -445,3 +441,233 @@ impl Default for ReceiverState {
|
||||
Self::new(DEFAULT_OWD_WINDOW_SIZE)
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_new_receiver_state() {
|
||||
let r = ReceiverState::new(32);
|
||||
assert_eq!(r.cumulative_packets_recv(), 0);
|
||||
assert_eq!(r.cumulative_bytes_recv(), 0);
|
||||
assert_eq!(r.highest_counter(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_recv_basic() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
let now = Instant::now();
|
||||
r.record_recv(1, 100, 500, false, now);
|
||||
r.record_recv(2, 200, 600, false, now + Duration::from_millis(100));
|
||||
|
||||
assert_eq!(r.cumulative_packets_recv(), 2);
|
||||
assert_eq!(r.cumulative_bytes_recv(), 1100);
|
||||
assert_eq!(r.highest_counter(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reorder_detection() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
let now = Instant::now();
|
||||
r.record_recv(5, 500, 100, false, now);
|
||||
r.record_recv(3, 300, 100, false, now + Duration::from_millis(10));
|
||||
|
||||
assert_eq!(r.cumulative_reorder_count, 1);
|
||||
assert_eq!(r.highest_counter(), 5); // not changed by out-of-order
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ecn_counting() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
let now = Instant::now();
|
||||
r.record_recv(1, 100, 100, true, now);
|
||||
r.record_recv(2, 200, 100, false, now);
|
||||
r.record_recv(3, 300, 100, true, now);
|
||||
|
||||
assert_eq!(r.ecn_ce_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_report_empty() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
assert!(r.build_report(Instant::now()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_report() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
let t0 = Instant::now();
|
||||
r.record_recv(1, 100, 500, false, t0);
|
||||
r.record_recv(2, 200, 600, false, t0 + Duration::from_millis(100));
|
||||
|
||||
let report = r.build_report(t0 + Duration::from_millis(150)).unwrap();
|
||||
assert_eq!(report.highest_counter, 2);
|
||||
assert_eq!(report.cumulative_packets_recv, 2);
|
||||
assert_eq!(report.cumulative_bytes_recv, 1100);
|
||||
assert_eq!(report.timestamp_echo, 200); // last sender timestamp
|
||||
assert_eq!(report.interval_packets_recv, 2);
|
||||
assert_eq!(report.interval_bytes_recv, 1100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_report_suppresses_rtt_echo_when_dwell_overflows() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
let t0 = Instant::now();
|
||||
r.record_recv(1, 100, 500, false, t0);
|
||||
|
||||
let report = r
|
||||
.build_report(t0 + Duration::from_millis(u64::from(u16::MAX) + 1))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(report.timestamp_echo, 0);
|
||||
assert_eq!(report.dwell_time, u16::MAX);
|
||||
assert_eq!(report.cumulative_packets_recv, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_report_resets_interval() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
let t0 = Instant::now();
|
||||
r.record_recv(1, 100, 500, false, t0);
|
||||
let _ = r.build_report(t0);
|
||||
|
||||
// No new data
|
||||
assert!(r.build_report(t0).is_none());
|
||||
|
||||
// New data
|
||||
r.record_recv(2, 200, 300, false, t0 + Duration::from_millis(100));
|
||||
let report = r.build_report(t0 + Duration::from_millis(150)).unwrap();
|
||||
assert_eq!(report.interval_packets_recv, 1);
|
||||
assert_eq!(report.interval_bytes_recv, 300);
|
||||
// Cumulative continues
|
||||
assert_eq!(report.cumulative_packets_recv, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gap_tracker_no_loss() {
|
||||
let mut g = GapTracker::new();
|
||||
g.observe(1);
|
||||
g.observe(2);
|
||||
g.observe(3);
|
||||
let (count, max, mean) = g.take_interval_stats();
|
||||
assert_eq!(count, 0);
|
||||
assert_eq!(max, 0);
|
||||
assert_eq!(mean, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gap_tracker_single_burst() {
|
||||
let mut g = GapTracker::new();
|
||||
g.observe(1);
|
||||
// frames 2, 3 lost
|
||||
g.observe(4);
|
||||
g.observe(5);
|
||||
let (count, max, _mean) = g.take_interval_stats();
|
||||
assert_eq!(count, 1);
|
||||
assert_eq!(max, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gap_tracker_multiple_bursts() {
|
||||
let mut g = GapTracker::new();
|
||||
g.observe(1);
|
||||
g.observe(4); // burst of 2 (frames 2,3 lost)
|
||||
g.observe(5);
|
||||
g.observe(8); // burst of 2 (frames 6,7 lost)
|
||||
g.observe(9);
|
||||
let (count, max, mean) = g.take_interval_stats();
|
||||
assert_eq!(count, 2);
|
||||
assert_eq!(max, 2);
|
||||
// mean = 2.0 in u8.8 = 512
|
||||
assert_eq!(mean, 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_send_report_timing() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
let t0 = Instant::now();
|
||||
|
||||
assert!(!r.should_send_report(t0)); // no data
|
||||
|
||||
r.record_recv(1, 100, 500, false, t0);
|
||||
assert!(r.should_send_report(t0)); // first time, has data
|
||||
|
||||
let _ = r.build_report(t0);
|
||||
r.record_recv(2, 200, 500, false, t0);
|
||||
assert!(!r.should_send_report(t0)); // just reported
|
||||
|
||||
let t1 = t0 + r.report_interval() + Duration::from_millis(1);
|
||||
assert!(r.should_send_report(t1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_report_interval_cold_start() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
// During cold-start, floor is 200ms (DEFAULT_COLD_START_INTERVAL_MS)
|
||||
// 50ms SRTT → 50ms receiver interval (1× SRTT), clamped to cold-start floor 200ms
|
||||
r.update_report_interval_from_srtt(50_000);
|
||||
assert_eq!(r.report_interval(), Duration::from_millis(200));
|
||||
|
||||
// 500ms SRTT → 500ms (above cold-start floor)
|
||||
r.update_report_interval_from_srtt(500_000);
|
||||
assert_eq!(r.report_interval(), Duration::from_millis(500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_report_interval_after_cold_start() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
// Burn through cold-start samples
|
||||
for _ in 0..COLD_START_SAMPLES {
|
||||
r.update_report_interval_from_srtt(500_000);
|
||||
}
|
||||
|
||||
// 6th sample: steady state, floor is MIN_REPORT_INTERVAL_MS (1000ms)
|
||||
// 50ms SRTT → 50ms receiver interval (1× SRTT), clamped to 1000ms
|
||||
r.update_report_interval_from_srtt(50_000);
|
||||
assert_eq!(
|
||||
r.report_interval(),
|
||||
Duration::from_millis(MIN_REPORT_INTERVAL_MS)
|
||||
);
|
||||
|
||||
// 3s SRTT → 3000ms, within [1000, 5000]
|
||||
r.update_report_interval_from_srtt(3_000_000);
|
||||
assert_eq!(r.report_interval(), Duration::from_millis(3000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rekey_jitter_grace_suppresses_spikes() {
|
||||
let mut r = ReceiverState::new(32);
|
||||
let t0 = Instant::now();
|
||||
|
||||
// Establish baseline with two frames so jitter starts updating
|
||||
r.record_recv(1, 1000, 100, false, t0);
|
||||
r.record_recv(2, 2000, 100, false, t0 + Duration::from_secs(1));
|
||||
assert_eq!(r.jitter_us(), 0); // perfect 1s spacing → 0 jitter
|
||||
|
||||
// Simulate rekey: reset, then send a frame with a large old-session
|
||||
// timestamp followed by a new-session timestamp near zero.
|
||||
// Without grace, this would produce a huge jitter spike.
|
||||
r.reset_for_rekey(t0 + Duration::from_secs(2));
|
||||
|
||||
// Frame arrives during grace period with old-session timestamp
|
||||
r.record_recv(0, 120_000, 100, false, t0 + Duration::from_secs(3));
|
||||
// Next frame with new-session timestamp near zero
|
||||
r.record_recv(1, 100, 100, false, t0 + Duration::from_secs(4));
|
||||
// Jitter should still be zero — updates suppressed during grace
|
||||
assert_eq!(r.jitter_us(), 0);
|
||||
|
||||
// After grace expires, jitter updates resume
|
||||
let after_grace =
|
||||
t0 + Duration::from_secs(2) + Duration::from_secs(REKEY_JITTER_GRACE_SECS + 1);
|
||||
r.record_recv(2, 200, 100, false, after_grace);
|
||||
r.record_recv(3, 300, 100, false, after_grace + Duration::from_millis(100));
|
||||
// Now jitter should be updating (non-zero or zero depending on timing)
|
||||
// The key assertion is that it's not a multi-second spike
|
||||
assert!(r.jitter_us() < 1_000_000); // less than 1 second
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
//! MMP report wire format: SenderReport and ReceiverReport.
|
||||
//!
|
||||
//! Serialization and deserialization for the two report types exchanged
|
||||
//! between link-layer peers. Wire format follows the MMP design doc.
|
||||
|
||||
use crate::protocol::ProtocolError;
|
||||
|
||||
// ============================================================================
|
||||
// SenderReport (msg_type 0x01, 48-byte body including type byte)
|
||||
// ============================================================================
|
||||
|
||||
/// Link-layer sender report.
|
||||
///
|
||||
/// Wire layout (48 bytes total, sent as link message):
|
||||
/// ```text
|
||||
/// [0] msg_type = 0x01
|
||||
/// [1-3] reserved (zero)
|
||||
/// [4-11] interval_start_counter: u64 LE
|
||||
/// [12-19] interval_end_counter: u64 LE
|
||||
/// [20-23] interval_start_timestamp: u32 LE
|
||||
/// [24-27] interval_end_timestamp: u32 LE
|
||||
/// [28-31] interval_bytes_sent: u32 LE
|
||||
/// [32-39] cumulative_packets_sent: u64 LE
|
||||
/// [40-47] cumulative_bytes_sent: u64 LE
|
||||
/// ```
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SenderReport {
|
||||
pub interval_start_counter: u64,
|
||||
pub interval_end_counter: u64,
|
||||
pub interval_start_timestamp: u32,
|
||||
pub interval_end_timestamp: u32,
|
||||
pub interval_bytes_sent: u32,
|
||||
pub cumulative_packets_sent: u64,
|
||||
pub cumulative_bytes_sent: u64,
|
||||
}
|
||||
|
||||
/// ReceiverReport (msg_type 0x02, 68-byte body including type byte)
|
||||
///
|
||||
/// Wire layout (68 bytes total, sent as link message):
|
||||
/// ```text
|
||||
/// [0] msg_type = 0x02
|
||||
/// [1-3] reserved (zero)
|
||||
/// [4-11] highest_counter: u64 LE
|
||||
/// [12-19] cumulative_packets_recv: u64 LE
|
||||
/// [20-27] cumulative_bytes_recv: u64 LE
|
||||
/// [28-31] timestamp_echo: u32 LE
|
||||
/// [32-33] dwell_time: u16 LE
|
||||
/// [34-35] max_burst_loss: u16 LE
|
||||
/// [36-37] mean_burst_loss: u16 LE (u8.8 fixed-point)
|
||||
/// [38-39] reserved: u16 LE
|
||||
/// [40-43] jitter: u32 LE (microseconds)
|
||||
/// [44-47] ecn_ce_count: u32 LE
|
||||
/// [48-51] owd_trend: i32 LE (µs/s)
|
||||
/// [52-55] burst_loss_count: u32 LE
|
||||
/// [56-59] cumulative_reorder_count: u32 LE
|
||||
/// [60-63] interval_packets_recv: u32 LE
|
||||
/// [64-67] interval_bytes_recv: u32 LE
|
||||
/// ```
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ReceiverReport {
|
||||
pub highest_counter: u64,
|
||||
pub cumulative_packets_recv: u64,
|
||||
pub cumulative_bytes_recv: u64,
|
||||
pub timestamp_echo: u32,
|
||||
pub dwell_time: u16,
|
||||
pub max_burst_loss: u16,
|
||||
pub mean_burst_loss: u16,
|
||||
pub jitter: u32,
|
||||
pub ecn_ce_count: u32,
|
||||
pub owd_trend: i32,
|
||||
pub burst_loss_count: u32,
|
||||
pub cumulative_reorder_count: u32,
|
||||
pub interval_packets_recv: u32,
|
||||
pub interval_bytes_recv: u32,
|
||||
}
|
||||
|
||||
// Encode/decode will be implemented in Step 2.
|
||||
|
||||
impl SenderReport {
|
||||
/// Encode to wire format (48 bytes: msg_type + 3 reserved + 44 payload).
|
||||
pub fn encode(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::with_capacity(48);
|
||||
buf.push(0x01); // msg_type
|
||||
buf.extend_from_slice(&[0u8; 3]); // reserved
|
||||
buf.extend_from_slice(&self.interval_start_counter.to_le_bytes());
|
||||
buf.extend_from_slice(&self.interval_end_counter.to_le_bytes());
|
||||
buf.extend_from_slice(&self.interval_start_timestamp.to_le_bytes());
|
||||
buf.extend_from_slice(&self.interval_end_timestamp.to_le_bytes());
|
||||
buf.extend_from_slice(&self.interval_bytes_sent.to_le_bytes());
|
||||
buf.extend_from_slice(&self.cumulative_packets_sent.to_le_bytes());
|
||||
buf.extend_from_slice(&self.cumulative_bytes_sent.to_le_bytes());
|
||||
buf
|
||||
}
|
||||
|
||||
/// Decode from payload after msg_type byte has been consumed.
|
||||
///
|
||||
/// `payload` starts at the reserved bytes (offset 1 in the wire format).
|
||||
pub fn decode(payload: &[u8]) -> Result<Self, ProtocolError> {
|
||||
if payload.len() < 47 {
|
||||
return Err(ProtocolError::MessageTooShort {
|
||||
expected: 47,
|
||||
got: payload.len(),
|
||||
});
|
||||
}
|
||||
// Skip 3 reserved bytes
|
||||
let p = &payload[3..];
|
||||
Ok(Self {
|
||||
interval_start_counter: u64::from_le_bytes(p[0..8].try_into().unwrap()),
|
||||
interval_end_counter: u64::from_le_bytes(p[8..16].try_into().unwrap()),
|
||||
interval_start_timestamp: u32::from_le_bytes(p[16..20].try_into().unwrap()),
|
||||
interval_end_timestamp: u32::from_le_bytes(p[20..24].try_into().unwrap()),
|
||||
interval_bytes_sent: u32::from_le_bytes(p[24..28].try_into().unwrap()),
|
||||
cumulative_packets_sent: u64::from_le_bytes(p[28..36].try_into().unwrap()),
|
||||
cumulative_bytes_sent: u64::from_le_bytes(p[36..44].try_into().unwrap()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ReceiverReport {
|
||||
/// Encode to wire format (68 bytes: msg_type + 3 reserved + 64 payload).
|
||||
pub fn encode(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::with_capacity(68);
|
||||
buf.push(0x02); // msg_type
|
||||
buf.extend_from_slice(&[0u8; 3]); // reserved
|
||||
buf.extend_from_slice(&self.highest_counter.to_le_bytes());
|
||||
buf.extend_from_slice(&self.cumulative_packets_recv.to_le_bytes());
|
||||
buf.extend_from_slice(&self.cumulative_bytes_recv.to_le_bytes());
|
||||
buf.extend_from_slice(&self.timestamp_echo.to_le_bytes());
|
||||
buf.extend_from_slice(&self.dwell_time.to_le_bytes());
|
||||
buf.extend_from_slice(&self.max_burst_loss.to_le_bytes());
|
||||
buf.extend_from_slice(&self.mean_burst_loss.to_le_bytes());
|
||||
buf.extend_from_slice(&[0u8; 2]); // reserved
|
||||
buf.extend_from_slice(&self.jitter.to_le_bytes());
|
||||
buf.extend_from_slice(&self.ecn_ce_count.to_le_bytes());
|
||||
buf.extend_from_slice(&self.owd_trend.to_le_bytes());
|
||||
buf.extend_from_slice(&self.burst_loss_count.to_le_bytes());
|
||||
buf.extend_from_slice(&self.cumulative_reorder_count.to_le_bytes());
|
||||
buf.extend_from_slice(&self.interval_packets_recv.to_le_bytes());
|
||||
buf.extend_from_slice(&self.interval_bytes_recv.to_le_bytes());
|
||||
buf
|
||||
}
|
||||
|
||||
/// Decode from payload after msg_type byte has been consumed.
|
||||
///
|
||||
/// `payload` starts at the reserved bytes (offset 1 in the wire format).
|
||||
pub fn decode(payload: &[u8]) -> Result<Self, ProtocolError> {
|
||||
if payload.len() < 67 {
|
||||
return Err(ProtocolError::MessageTooShort {
|
||||
expected: 67,
|
||||
got: payload.len(),
|
||||
});
|
||||
}
|
||||
// Skip 3 reserved bytes
|
||||
let p = &payload[3..];
|
||||
Ok(Self {
|
||||
highest_counter: u64::from_le_bytes(p[0..8].try_into().unwrap()),
|
||||
cumulative_packets_recv: u64::from_le_bytes(p[8..16].try_into().unwrap()),
|
||||
cumulative_bytes_recv: u64::from_le_bytes(p[16..24].try_into().unwrap()),
|
||||
timestamp_echo: u32::from_le_bytes(p[24..28].try_into().unwrap()),
|
||||
dwell_time: u16::from_le_bytes(p[28..30].try_into().unwrap()),
|
||||
max_burst_loss: u16::from_le_bytes(p[30..32].try_into().unwrap()),
|
||||
mean_burst_loss: u16::from_le_bytes(p[32..34].try_into().unwrap()),
|
||||
// skip 2 reserved bytes at p[34..36]
|
||||
jitter: u32::from_le_bytes(p[36..40].try_into().unwrap()),
|
||||
ecn_ce_count: u32::from_le_bytes(p[40..44].try_into().unwrap()),
|
||||
owd_trend: i32::from_le_bytes(p[44..48].try_into().unwrap()),
|
||||
burst_loss_count: u32::from_le_bytes(p[48..52].try_into().unwrap()),
|
||||
cumulative_reorder_count: u32::from_le_bytes(p[52..56].try_into().unwrap()),
|
||||
interval_packets_recv: u32::from_le_bytes(p[56..60].try_into().unwrap()),
|
||||
interval_bytes_recv: u32::from_le_bytes(p[60..64].try_into().unwrap()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Conversions between link-layer and session-layer report types
|
||||
// ============================================================================
|
||||
|
||||
use crate::protocol::{SessionReceiverReport, SessionSenderReport};
|
||||
|
||||
impl From<&SenderReport> for SessionSenderReport {
|
||||
fn from(r: &SenderReport) -> Self {
|
||||
Self {
|
||||
interval_start_counter: r.interval_start_counter,
|
||||
interval_end_counter: r.interval_end_counter,
|
||||
interval_start_timestamp: r.interval_start_timestamp,
|
||||
interval_end_timestamp: r.interval_end_timestamp,
|
||||
interval_bytes_sent: r.interval_bytes_sent,
|
||||
cumulative_packets_sent: r.cumulative_packets_sent,
|
||||
cumulative_bytes_sent: r.cumulative_bytes_sent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&SessionSenderReport> for SenderReport {
|
||||
fn from(r: &SessionSenderReport) -> Self {
|
||||
Self {
|
||||
interval_start_counter: r.interval_start_counter,
|
||||
interval_end_counter: r.interval_end_counter,
|
||||
interval_start_timestamp: r.interval_start_timestamp,
|
||||
interval_end_timestamp: r.interval_end_timestamp,
|
||||
interval_bytes_sent: r.interval_bytes_sent,
|
||||
cumulative_packets_sent: r.cumulative_packets_sent,
|
||||
cumulative_bytes_sent: r.cumulative_bytes_sent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&ReceiverReport> for SessionReceiverReport {
|
||||
fn from(r: &ReceiverReport) -> Self {
|
||||
Self {
|
||||
highest_counter: r.highest_counter,
|
||||
cumulative_packets_recv: r.cumulative_packets_recv,
|
||||
cumulative_bytes_recv: r.cumulative_bytes_recv,
|
||||
timestamp_echo: r.timestamp_echo,
|
||||
dwell_time: r.dwell_time,
|
||||
max_burst_loss: r.max_burst_loss,
|
||||
mean_burst_loss: r.mean_burst_loss,
|
||||
jitter: r.jitter,
|
||||
ecn_ce_count: r.ecn_ce_count,
|
||||
owd_trend: r.owd_trend,
|
||||
burst_loss_count: r.burst_loss_count,
|
||||
cumulative_reorder_count: r.cumulative_reorder_count,
|
||||
interval_packets_recv: r.interval_packets_recv,
|
||||
interval_bytes_recv: r.interval_bytes_recv,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&SessionReceiverReport> for ReceiverReport {
|
||||
fn from(r: &SessionReceiverReport) -> Self {
|
||||
Self {
|
||||
highest_counter: r.highest_counter,
|
||||
cumulative_packets_recv: r.cumulative_packets_recv,
|
||||
cumulative_bytes_recv: r.cumulative_bytes_recv,
|
||||
timestamp_echo: r.timestamp_echo,
|
||||
dwell_time: r.dwell_time,
|
||||
max_burst_loss: r.max_burst_loss,
|
||||
mean_burst_loss: r.mean_burst_loss,
|
||||
jitter: r.jitter,
|
||||
ecn_ce_count: r.ecn_ce_count,
|
||||
owd_trend: r.owd_trend,
|
||||
burst_loss_count: r.burst_loss_count,
|
||||
cumulative_reorder_count: r.cumulative_reorder_count,
|
||||
interval_packets_recv: r.interval_packets_recv,
|
||||
interval_bytes_recv: r.interval_bytes_recv,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample_sender_report() -> SenderReport {
|
||||
SenderReport {
|
||||
interval_start_counter: 100,
|
||||
interval_end_counter: 200,
|
||||
interval_start_timestamp: 5000,
|
||||
interval_end_timestamp: 6000,
|
||||
interval_bytes_sent: 50_000,
|
||||
cumulative_packets_sent: 10_000,
|
||||
cumulative_bytes_sent: 5_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_receiver_report() -> ReceiverReport {
|
||||
ReceiverReport {
|
||||
highest_counter: 195,
|
||||
cumulative_packets_recv: 9_500,
|
||||
cumulative_bytes_recv: 4_750_000,
|
||||
timestamp_echo: 5900,
|
||||
dwell_time: 5,
|
||||
max_burst_loss: 3,
|
||||
mean_burst_loss: 384, // 1.5 in u8.8
|
||||
jitter: 1200,
|
||||
ecn_ce_count: 0,
|
||||
owd_trend: -50,
|
||||
burst_loss_count: 2,
|
||||
cumulative_reorder_count: 10,
|
||||
interval_packets_recv: 95,
|
||||
interval_bytes_recv: 47_500,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sender_report_encode_size() {
|
||||
let sr = sample_sender_report();
|
||||
let encoded = sr.encode();
|
||||
assert_eq!(encoded.len(), 48);
|
||||
assert_eq!(encoded[0], 0x01); // msg_type
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sender_report_roundtrip() {
|
||||
let sr = sample_sender_report();
|
||||
let encoded = sr.encode();
|
||||
// decode expects payload after msg_type
|
||||
let decoded = SenderReport::decode(&encoded[1..]).unwrap();
|
||||
assert_eq!(sr, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sender_report_too_short() {
|
||||
let result = SenderReport::decode(&[0u8; 10]);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_receiver_report_encode_size() {
|
||||
let rr = sample_receiver_report();
|
||||
let encoded = rr.encode();
|
||||
assert_eq!(encoded.len(), 68);
|
||||
assert_eq!(encoded[0], 0x02); // msg_type
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_receiver_report_roundtrip() {
|
||||
let rr = sample_receiver_report();
|
||||
let encoded = rr.encode();
|
||||
// decode expects payload after msg_type
|
||||
let decoded = ReceiverReport::decode(&encoded[1..]).unwrap();
|
||||
assert_eq!(rr, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_receiver_report_too_short() {
|
||||
let result = ReceiverReport::decode(&[0u8; 10]);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sender_report_zero_values() {
|
||||
let sr = SenderReport {
|
||||
interval_start_counter: 0,
|
||||
interval_end_counter: 0,
|
||||
interval_start_timestamp: 0,
|
||||
interval_end_timestamp: 0,
|
||||
interval_bytes_sent: 0,
|
||||
cumulative_packets_sent: 0,
|
||||
cumulative_bytes_sent: 0,
|
||||
};
|
||||
let encoded = sr.encode();
|
||||
let decoded = SenderReport::decode(&encoded[1..]).unwrap();
|
||||
assert_eq!(sr, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_receiver_report_max_values() {
|
||||
let rr = ReceiverReport {
|
||||
highest_counter: u64::MAX,
|
||||
cumulative_packets_recv: u64::MAX,
|
||||
cumulative_bytes_recv: u64::MAX,
|
||||
timestamp_echo: u32::MAX,
|
||||
dwell_time: u16::MAX,
|
||||
max_burst_loss: u16::MAX,
|
||||
mean_burst_loss: u16::MAX,
|
||||
jitter: u32::MAX,
|
||||
ecn_ce_count: u32::MAX,
|
||||
owd_trend: i32::MAX,
|
||||
burst_loss_count: u32::MAX,
|
||||
cumulative_reorder_count: u32::MAX,
|
||||
interval_packets_recv: u32::MAX,
|
||||
interval_bytes_recv: u32::MAX,
|
||||
};
|
||||
let encoded = rr.encode();
|
||||
let decoded = ReceiverReport::decode(&encoded[1..]).unwrap();
|
||||
assert_eq!(rr, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_receiver_report_negative_owd_trend() {
|
||||
let rr = ReceiverReport {
|
||||
owd_trend: -12345,
|
||||
..sample_receiver_report()
|
||||
};
|
||||
let encoded = rr.encode();
|
||||
let decoded = ReceiverReport::decode(&encoded[1..]).unwrap();
|
||||
assert_eq!(decoded.owd_trend, -12345);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
//! MMP sender state machine.
|
||||
//!
|
||||
//! Tracks what this node has sent to a specific peer and produces
|
||||
//! SenderReport messages on demand. One `SenderState` per active peer.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::mmp::report::SenderReport;
|
||||
use crate::mmp::{
|
||||
COLD_START_SAMPLES, DEFAULT_COLD_START_INTERVAL_MS, MAX_REPORT_INTERVAL_MS,
|
||||
MIN_REPORT_INTERVAL_MS,
|
||||
};
|
||||
|
||||
/// Per-peer sender-side MMP state.
|
||||
///
|
||||
/// Records cumulative and interval counters for every frame transmitted
|
||||
/// to this peer. Produces `SenderReport` snapshots on demand.
|
||||
pub struct SenderState {
|
||||
// --- Cumulative (lifetime) ---
|
||||
cumulative_packets_sent: u64,
|
||||
cumulative_bytes_sent: u64,
|
||||
|
||||
// --- Current interval ---
|
||||
interval_start_counter: u64,
|
||||
interval_start_timestamp: u32,
|
||||
interval_bytes_sent: u32,
|
||||
/// Counter of the most recently sent frame.
|
||||
last_counter: u64,
|
||||
/// Timestamp of the most recently sent frame.
|
||||
last_timestamp: u32,
|
||||
/// Whether any frames have been sent in the current interval.
|
||||
interval_has_data: bool,
|
||||
|
||||
// --- Report timing ---
|
||||
last_report_time: Option<Instant>,
|
||||
report_interval: Duration,
|
||||
|
||||
// --- Send failure backoff ---
|
||||
/// Consecutive send failure count for backoff calculation.
|
||||
consecutive_send_failures: u32,
|
||||
|
||||
// --- Cold-start tracking ---
|
||||
/// Number of SRTT-based interval updates received.
|
||||
srtt_sample_count: u32,
|
||||
}
|
||||
|
||||
impl SenderState {
|
||||
pub fn new() -> Self {
|
||||
Self::new_with_cold_start(DEFAULT_COLD_START_INTERVAL_MS)
|
||||
}
|
||||
|
||||
/// Create with a custom cold-start interval (ms).
|
||||
///
|
||||
/// Used by session-layer MMP which needs a longer initial interval
|
||||
/// since reports consume bandwidth on every transit link.
|
||||
pub fn new_with_cold_start(cold_start_ms: u64) -> Self {
|
||||
Self {
|
||||
cumulative_packets_sent: 0,
|
||||
cumulative_bytes_sent: 0,
|
||||
interval_start_counter: 0,
|
||||
interval_start_timestamp: 0,
|
||||
interval_bytes_sent: 0,
|
||||
last_counter: 0,
|
||||
last_timestamp: 0,
|
||||
interval_has_data: false,
|
||||
last_report_time: None,
|
||||
report_interval: Duration::from_millis(cold_start_ms),
|
||||
consecutive_send_failures: 0,
|
||||
srtt_sample_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a frame sent to this peer.
|
||||
///
|
||||
/// Called on the TX path for every encrypted link message.
|
||||
/// `counter` is the AEAD nonce/counter, `timestamp` is the inner header
|
||||
/// session-relative timestamp (ms), `bytes` is the wire payload size.
|
||||
pub fn record_sent(&mut self, counter: u64, timestamp: u32, bytes: usize) {
|
||||
if !self.interval_has_data {
|
||||
self.interval_start_counter = counter;
|
||||
self.interval_start_timestamp = timestamp;
|
||||
self.interval_has_data = true;
|
||||
}
|
||||
self.last_counter = counter;
|
||||
self.last_timestamp = timestamp;
|
||||
self.interval_bytes_sent = self.interval_bytes_sent.saturating_add(bytes as u32);
|
||||
self.cumulative_packets_sent += 1;
|
||||
self.cumulative_bytes_sent += bytes as u64;
|
||||
}
|
||||
|
||||
/// Build a SenderReport from current state and reset the interval.
|
||||
///
|
||||
/// Returns `None` if no frames have been sent since the last report.
|
||||
pub fn build_report(&mut self, now: Instant) -> Option<SenderReport> {
|
||||
if !self.interval_has_data {
|
||||
return None;
|
||||
}
|
||||
|
||||
let report = SenderReport {
|
||||
interval_start_counter: self.interval_start_counter,
|
||||
interval_end_counter: self.last_counter,
|
||||
interval_start_timestamp: self.interval_start_timestamp,
|
||||
interval_end_timestamp: self.last_timestamp,
|
||||
interval_bytes_sent: self.interval_bytes_sent,
|
||||
cumulative_packets_sent: self.cumulative_packets_sent,
|
||||
cumulative_bytes_sent: self.cumulative_bytes_sent,
|
||||
};
|
||||
|
||||
// Reset interval
|
||||
self.interval_has_data = false;
|
||||
self.interval_bytes_sent = 0;
|
||||
self.last_report_time = Some(now);
|
||||
|
||||
Some(report)
|
||||
}
|
||||
|
||||
/// Check if it's time to send a report.
|
||||
///
|
||||
/// When consecutive send failures have occurred, the effective interval
|
||||
/// is multiplied by an exponential backoff factor (2^failures, capped at 32×).
|
||||
pub fn should_send_report(&self, now: Instant) -> bool {
|
||||
if !self.interval_has_data {
|
||||
return false;
|
||||
}
|
||||
match self.last_report_time {
|
||||
None => true, // Never sent a report — send immediately
|
||||
Some(last) => {
|
||||
let effective = self
|
||||
.report_interval
|
||||
.mul_f64(self.send_failure_backoff_multiplier());
|
||||
now.duration_since(last) >= effective
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a send failure. Returns the new consecutive failure count.
|
||||
pub fn record_send_failure(&mut self) -> u32 {
|
||||
self.consecutive_send_failures += 1;
|
||||
self.consecutive_send_failures
|
||||
}
|
||||
|
||||
/// Record a successful send. Returns the previous failure count (for summary logging).
|
||||
pub fn record_send_success(&mut self) -> u32 {
|
||||
let prev = self.consecutive_send_failures;
|
||||
self.consecutive_send_failures = 0;
|
||||
prev
|
||||
}
|
||||
|
||||
/// Get the backoff multiplier based on consecutive failures.
|
||||
///
|
||||
/// Returns 1.0 for no failures, 2.0 for 1 failure, 4.0 for 2, ...
|
||||
/// capped at 32.0 (5 failures).
|
||||
pub fn send_failure_backoff_multiplier(&self) -> f64 {
|
||||
if self.consecutive_send_failures == 0 {
|
||||
1.0
|
||||
} else {
|
||||
2.0_f64.powi(self.consecutive_send_failures.min(5) as i32)
|
||||
}
|
||||
}
|
||||
|
||||
/// Update the report interval based on SRTT (link-layer defaults).
|
||||
///
|
||||
/// Sender reports at 2× SRTT clamped to [floor, MAX]. During cold-start
|
||||
/// (first `COLD_START_SAMPLES` updates), the floor is the cold-start
|
||||
/// interval (200ms) for fast SRTT convergence. After that, it rises to
|
||||
/// `MIN_REPORT_INTERVAL_MS` (1000ms) for steady-state efficiency.
|
||||
pub fn update_report_interval_from_srtt(&mut self, srtt_us: i64) {
|
||||
self.srtt_sample_count = self.srtt_sample_count.saturating_add(1);
|
||||
let floor = if self.srtt_sample_count <= COLD_START_SAMPLES {
|
||||
DEFAULT_COLD_START_INTERVAL_MS
|
||||
} else {
|
||||
MIN_REPORT_INTERVAL_MS
|
||||
};
|
||||
self.update_report_interval_with_bounds(srtt_us, floor, MAX_REPORT_INTERVAL_MS);
|
||||
}
|
||||
|
||||
/// Update the report interval based on SRTT with custom bounds.
|
||||
///
|
||||
/// Used by session-layer MMP which needs higher clamp values since
|
||||
/// each report consumes bandwidth on every transit link.
|
||||
pub fn update_report_interval_with_bounds(&mut self, srtt_us: i64, min_ms: u64, max_ms: u64) {
|
||||
if srtt_us <= 0 {
|
||||
return;
|
||||
}
|
||||
let interval_us = (srtt_us * 2) as u64;
|
||||
let interval_ms = (interval_us / 1000).clamp(min_ms, max_ms);
|
||||
self.report_interval = Duration::from_millis(interval_ms);
|
||||
}
|
||||
|
||||
// --- Accessors ---
|
||||
|
||||
pub fn cumulative_packets_sent(&self) -> u64 {
|
||||
self.cumulative_packets_sent
|
||||
}
|
||||
|
||||
pub fn cumulative_bytes_sent(&self) -> u64 {
|
||||
self.cumulative_bytes_sent
|
||||
}
|
||||
|
||||
pub fn report_interval(&self) -> Duration {
|
||||
self.report_interval
|
||||
}
|
||||
|
||||
pub fn consecutive_send_failures(&self) -> u32 {
|
||||
self.consecutive_send_failures
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SenderState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_new_sender_state() {
|
||||
let s = SenderState::new();
|
||||
assert_eq!(s.cumulative_packets_sent(), 0);
|
||||
assert_eq!(s.cumulative_bytes_sent(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_sent() {
|
||||
let mut s = SenderState::new();
|
||||
s.record_sent(1, 100, 500);
|
||||
s.record_sent(2, 200, 600);
|
||||
assert_eq!(s.cumulative_packets_sent(), 2);
|
||||
assert_eq!(s.cumulative_bytes_sent(), 1100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_report_empty() {
|
||||
let mut s = SenderState::new();
|
||||
assert!(s.build_report(Instant::now()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_report() {
|
||||
let mut s = SenderState::new();
|
||||
s.record_sent(10, 1000, 500);
|
||||
s.record_sent(11, 1100, 600);
|
||||
s.record_sent(12, 1200, 400);
|
||||
|
||||
let report = s.build_report(Instant::now()).unwrap();
|
||||
assert_eq!(report.interval_start_counter, 10);
|
||||
assert_eq!(report.interval_end_counter, 12);
|
||||
assert_eq!(report.interval_start_timestamp, 1000);
|
||||
assert_eq!(report.interval_end_timestamp, 1200);
|
||||
assert_eq!(report.interval_bytes_sent, 1500);
|
||||
assert_eq!(report.cumulative_packets_sent, 3);
|
||||
assert_eq!(report.cumulative_bytes_sent, 1500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_report_resets_interval() {
|
||||
let mut s = SenderState::new();
|
||||
s.record_sent(1, 100, 500);
|
||||
let _ = s.build_report(Instant::now());
|
||||
|
||||
// Second report with no new data returns None
|
||||
assert!(s.build_report(Instant::now()).is_none());
|
||||
|
||||
// New data starts a fresh interval
|
||||
s.record_sent(2, 200, 300);
|
||||
let report = s.build_report(Instant::now()).unwrap();
|
||||
assert_eq!(report.interval_start_counter, 2);
|
||||
assert_eq!(report.interval_bytes_sent, 300);
|
||||
// Cumulative continues
|
||||
assert_eq!(report.cumulative_packets_sent, 2);
|
||||
assert_eq!(report.cumulative_bytes_sent, 800);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_send_report_no_data() {
|
||||
let s = SenderState::new();
|
||||
assert!(!s.should_send_report(Instant::now()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_send_report_first_time() {
|
||||
let mut s = SenderState::new();
|
||||
s.record_sent(1, 100, 500);
|
||||
assert!(s.should_send_report(Instant::now()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_send_report_respects_interval() {
|
||||
let mut s = SenderState::new();
|
||||
let t0 = Instant::now();
|
||||
s.record_sent(1, 100, 500);
|
||||
let _ = s.build_report(t0);
|
||||
|
||||
s.record_sent(2, 200, 500);
|
||||
// Immediately after report — should not send
|
||||
assert!(!s.should_send_report(t0));
|
||||
|
||||
// After interval elapses
|
||||
let t1 = t0 + s.report_interval() + Duration::from_millis(1);
|
||||
assert!(s.should_send_report(t1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_report_interval_cold_start() {
|
||||
let mut s = SenderState::new();
|
||||
// During cold-start, floor is 200ms (DEFAULT_COLD_START_INTERVAL_MS)
|
||||
// 50ms RTT → 100ms sender interval (2× SRTT), clamped to cold-start floor 200ms
|
||||
s.update_report_interval_from_srtt(50_000);
|
||||
assert_eq!(s.report_interval(), Duration::from_millis(200));
|
||||
|
||||
// 500ms RTT → 1000ms sender interval (above cold-start floor)
|
||||
s.update_report_interval_from_srtt(500_000);
|
||||
assert_eq!(s.report_interval(), Duration::from_millis(1000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_report_interval_after_cold_start() {
|
||||
let mut s = SenderState::new();
|
||||
// Burn through cold-start samples (COLD_START_SAMPLES = 5)
|
||||
for _ in 0..COLD_START_SAMPLES {
|
||||
s.update_report_interval_from_srtt(500_000);
|
||||
}
|
||||
|
||||
// 6th sample: now in steady state, floor is MIN_REPORT_INTERVAL_MS (1000ms)
|
||||
// 50ms RTT → 100ms sender interval (2× SRTT), clamped to 1000ms
|
||||
s.update_report_interval_from_srtt(50_000);
|
||||
assert_eq!(
|
||||
s.report_interval(),
|
||||
Duration::from_millis(MIN_REPORT_INTERVAL_MS)
|
||||
);
|
||||
|
||||
// 3s RTT → 6s, clamped to max 5s
|
||||
s.update_report_interval_from_srtt(3_000_000);
|
||||
assert_eq!(
|
||||
s.report_interval(),
|
||||
Duration::from_millis(MAX_REPORT_INTERVAL_MS)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_multiplier_progression() {
|
||||
let mut s = SenderState::new();
|
||||
|
||||
// No failures → multiplier 1.0
|
||||
assert_eq!(s.send_failure_backoff_multiplier(), 1.0);
|
||||
assert_eq!(s.consecutive_send_failures(), 0);
|
||||
|
||||
// Progressive failures: 2^1, 2^2, 2^3, 2^4, 2^5
|
||||
let expected = [2.0, 4.0, 8.0, 16.0, 32.0];
|
||||
for (i, &exp) in expected.iter().enumerate() {
|
||||
let count = s.record_send_failure();
|
||||
assert_eq!(count, (i + 1) as u32);
|
||||
assert_eq!(s.send_failure_backoff_multiplier(), exp);
|
||||
}
|
||||
|
||||
// Beyond 5 failures: stays capped at 32.0
|
||||
s.record_send_failure(); // 6th
|
||||
assert_eq!(s.send_failure_backoff_multiplier(), 32.0);
|
||||
s.record_send_failure(); // 7th
|
||||
assert_eq!(s.send_failure_backoff_multiplier(), 32.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_reset_on_success() {
|
||||
let mut s = SenderState::new();
|
||||
|
||||
// Accumulate failures
|
||||
s.record_send_failure();
|
||||
s.record_send_failure();
|
||||
s.record_send_failure();
|
||||
assert_eq!(s.consecutive_send_failures(), 3);
|
||||
assert_eq!(s.send_failure_backoff_multiplier(), 8.0);
|
||||
|
||||
// Success resets and returns previous count
|
||||
let prev = s.record_send_success();
|
||||
assert_eq!(prev, 3);
|
||||
assert_eq!(s.consecutive_send_failures(), 0);
|
||||
assert_eq!(s.send_failure_backoff_multiplier(), 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_success_with_no_prior_failures() {
|
||||
let mut s = SenderState::new();
|
||||
|
||||
// Success with no failures returns 0
|
||||
let prev = s.record_send_success();
|
||||
assert_eq!(prev, 0);
|
||||
assert_eq!(s.consecutive_send_failures(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_send_report_respects_backoff() {
|
||||
let mut s = SenderState::new();
|
||||
let t0 = Instant::now();
|
||||
s.record_sent(1, 100, 500);
|
||||
let _ = s.build_report(t0);
|
||||
|
||||
// Record a failure: multiplier becomes 2.0
|
||||
s.record_send_failure();
|
||||
|
||||
s.record_sent(2, 200, 500);
|
||||
|
||||
// At 1× interval: should NOT send (backoff requires 2×)
|
||||
let t1 = t0 + s.report_interval() + Duration::from_millis(1);
|
||||
assert!(!s.should_send_report(t1));
|
||||
|
||||
// At 2× interval: should send
|
||||
let t2 = t0 + s.report_interval() * 2 + Duration::from_millis(1);
|
||||
assert!(s.should_send_report(t2));
|
||||
}
|
||||
}
|
||||
+23
-31
@@ -4,12 +4,12 @@
|
||||
//! including debounced propagation to peers.
|
||||
|
||||
use crate::NodeAddr;
|
||||
use crate::proto::bloom::BloomFilter;
|
||||
use crate::proto::bloom::FilterAnnounce;
|
||||
use crate::bloom::BloomFilter;
|
||||
use crate::protocol::FilterAnnounce;
|
||||
|
||||
use super::reject::BloomReject;
|
||||
use super::{Node, NodeError};
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::HashMap;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
impl Node {
|
||||
@@ -17,8 +17,8 @@ impl Node {
|
||||
///
|
||||
/// Returns a map of (peer_node_addr -> filter) for peers that
|
||||
/// have sent us a FilterAnnounce.
|
||||
pub(super) fn peer_inbound_filters(&self) -> BTreeMap<NodeAddr, BloomFilter> {
|
||||
let mut filters = BTreeMap::new();
|
||||
pub(super) fn peer_inbound_filters(&self) -> HashMap<NodeAddr, BloomFilter> {
|
||||
let mut filters = HashMap::new();
|
||||
for (addr, peer) in &self.peers {
|
||||
if self.is_tree_peer(addr)
|
||||
&& let Some(filter) = peer.inbound_filter()
|
||||
@@ -29,19 +29,27 @@ impl Node {
|
||||
filters
|
||||
}
|
||||
|
||||
/// Send a FilterAnnounce to a specific peer, respecting debounce.
|
||||
/// Build a FilterAnnounce for a specific peer.
|
||||
///
|
||||
/// `filter` is the outgoing filter for this peer, already computed
|
||||
/// with the destination peer's own contribution excluded to prevent
|
||||
/// routing loops (don't tell a peer about destinations reachable
|
||||
/// only through them).
|
||||
/// The outgoing filter excludes the destination peer's own filter
|
||||
/// to prevent routing loops (don't tell a peer about destinations
|
||||
/// reachable only through them).
|
||||
fn build_filter_announce(&mut self, exclude_peer: &NodeAddr) -> FilterAnnounce {
|
||||
let peer_filters = self.peer_inbound_filters();
|
||||
let filter = self
|
||||
.bloom_state
|
||||
.compute_outgoing_filter(exclude_peer, &peer_filters);
|
||||
let sequence = self.bloom_state.next_sequence();
|
||||
FilterAnnounce::new(filter, sequence)
|
||||
}
|
||||
|
||||
/// Send a FilterAnnounce to a specific peer, respecting debounce.
|
||||
///
|
||||
/// If the peer is rate-limited, the update stays pending for
|
||||
/// delivery on the next tick cycle.
|
||||
pub(super) async fn send_filter_announce_to_peer(
|
||||
&mut self,
|
||||
peer_addr: &NodeAddr,
|
||||
filter: BloomFilter,
|
||||
) -> Result<(), NodeError> {
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
@@ -56,7 +64,7 @@ impl Node {
|
||||
}
|
||||
|
||||
// Build and encode
|
||||
let announce = FilterAnnounce::new(filter, self.bloom_state.next_sequence());
|
||||
let announce = self.build_filter_announce(peer_addr);
|
||||
let sent_filter = announce.filter.clone();
|
||||
let encoded = announce.encode().map_err(|e| NodeError::SendFailed {
|
||||
node_addr: *peer_addr,
|
||||
@@ -79,7 +87,7 @@ impl Node {
|
||||
// operator to see one clear message, not spam.
|
||||
let max_fpr = self.config().node.bloom.max_inbound_fpr;
|
||||
let out_fill = sent_filter.fill_ratio();
|
||||
let out_fpr = sent_filter.fpr();
|
||||
let out_fpr = out_fill.powi(sent_filter.hash_count() as i32);
|
||||
if out_fpr > max_fpr {
|
||||
let now = std::time::Instant::now();
|
||||
let should_warn = self
|
||||
@@ -134,24 +142,8 @@ impl Node {
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
if ready.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// One snapshot and one union pass for the whole ready set. The
|
||||
// send path never mutates peer inbound filters or the tree state,
|
||||
// and the rx loop holds `&mut self` across the awaits, so the
|
||||
// snapshot cannot go stale mid-loop.
|
||||
let peer_filters = self.peer_inbound_filters();
|
||||
let mut outgoing = self
|
||||
.bloom_state
|
||||
.compute_outgoing_filters(&ready, &peer_filters);
|
||||
|
||||
for peer_addr in ready {
|
||||
let Some(filter) = outgoing.remove(&peer_addr) else {
|
||||
continue;
|
||||
};
|
||||
if let Err(e) = self.send_filter_announce_to_peer(&peer_addr, filter).await {
|
||||
if let Err(e) = self.send_filter_announce_to_peer(&peer_addr).await {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&peer_addr),
|
||||
error = %e,
|
||||
@@ -221,7 +213,7 @@ impl Node {
|
||||
// to wipe a victim's contribution to aggregation.
|
||||
let max_fpr = self.config().node.bloom.max_inbound_fpr;
|
||||
let fill = announce.filter.fill_ratio();
|
||||
let fpr = announce.filter.fpr();
|
||||
let fpr = fill.powi(announce.filter.hash_count() as i32);
|
||||
if fpr > max_fpr {
|
||||
self.metrics()
|
||||
.bloom
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
//! Data plane: the RX `select!` loop and the per-packet forwarding path.
|
||||
//!
|
||||
//! Holds the whole hot path in one home: the `select!` run loop
|
||||
//! (`rx_loop`), transit/local datagram forwarding (`forwarding`), the
|
||||
//! link-message router (`dispatch`), the RX decrypt path including responder
|
||||
//! K-bit cutover and address-roam writes (`encrypted`), and the per-peer
|
||||
//! connected-UDP fast-path socket activation (`connected_udp`). Each module
|
||||
//! contributes `impl Node` methods driven by the run loop.
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) mod connected_udp;
|
||||
mod dispatch;
|
||||
mod encrypted;
|
||||
mod forwarding;
|
||||
mod peer_actions;
|
||||
mod rx_loop;
|
||||
|
||||
pub(in crate::node) use peer_actions::PeerActionCtx;
|
||||
@@ -1,557 +0,0 @@
|
||||
//! Executor for the per-peer control machine's [`PeerAction`]s.
|
||||
//!
|
||||
//! The per-peer FSM in [`crate::peer::machine`] is a sans-IO reducer: it decides
|
||||
//! *what* must happen and returns a `Vec<PeerAction>`; this module is the *doing*
|
||||
//! half — the thin driver that maps each action onto the exact shell call it
|
||||
//! stands for (`build_msg2` + `transport.send`, `promote_connection`,
|
||||
//! `remove_active_peer`, `index_allocator.free`, `note_link_dead`, …).
|
||||
//!
|
||||
//! ## Progressive cutover
|
||||
//!
|
||||
//! The executor is wired incrementally. Live today: the inbound establish
|
||||
//! (`handle_msg1` → `step(InboundMsg1)`), the outbound msg2 promote
|
||||
//! (`handle_msg2` looks up the dial-persisted machine), the connectionless
|
||||
//! outbound msg1 send (`SendHandshake` with `their_index == None` →
|
||||
//! `send_stored_msg1`, driven from `initiate_connection`), the
|
||||
//! connection-oriented dial (`OpenTransport` performs the non-blocking
|
||||
//! `transport.connect`; `TransportConnected` drives the connect-resolution msg1
|
||||
//! send from `poll_pending_connects`), the rekey cadence (`check_rekey` →
|
||||
//! `route_rekey_cadence` → `RekeyConsume`, driving the `SwapSendState` and
|
||||
//! `CompleteDrain` arms), and the liveness reap (`route_link_dead` →
|
||||
//! `LinkDeadSuspected`, driving `InvalidateSendState` → `remove_active_peer`).
|
||||
//!
|
||||
//! The genuine inert stubs remaining are `SendRekey`, `SendLinkMessage`, and
|
||||
//! the connected-UDP arms. `RegisterDecryptSession` is a deliberate no-op —
|
||||
//! see its arm for the note.
|
||||
//!
|
||||
//! The timer arms (`SetTimer`/`CancelTimer`) populate/clear the per-peer timer
|
||||
//! store (`peer_timers`). The `HandshakeRetransmit` and `HandshakeTimeout`
|
||||
//! deadlines are read and fired by `drive_peer_timers` (the handshake resend +
|
||||
//! reap home). The rekey/liveness kinds are still SHADOW — driven by their own
|
||||
//! shell drivers — so populating them stays behavior-neutral.
|
||||
|
||||
use crate::PeerIdentity;
|
||||
use crate::node::Node;
|
||||
use crate::node::reject::{HandshakeReject, RejectReason};
|
||||
use crate::peer::machine::{LostKind, PeerAction, PeerEvent};
|
||||
use crate::proto::fmp::PromotionResult;
|
||||
use crate::proto::fmp::wire::build_msg2;
|
||||
use crate::transport::{LinkId, TransportAddr, TransportId};
|
||||
use crate::utils::index::SessionIndex;
|
||||
use std::collections::VecDeque;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
/// Ambient shell facts a [`PeerAction`] executor needs that the machine's
|
||||
/// runtime-agnostic action payloads deliberately omit (verified identity,
|
||||
/// transport target, the msg2 framing indices, the promotion timestamp).
|
||||
///
|
||||
/// Unlike a machine event/action payload this is **executor-side**, so it may
|
||||
/// hold real values resolved from the wire context (cf. `handle_msg1`'s
|
||||
/// `wire`/`packet` locals and `promote_connection`'s ambient args). It is
|
||||
/// built fresh per driven step by the caller at cutover time.
|
||||
#[allow(dead_code)]
|
||||
pub(in crate::node) struct PeerActionCtx {
|
||||
/// The authenticated peer identity: `PromoteToActive` /
|
||||
/// `InvalidateSendState` resolve their `NodeAddr` from this.
|
||||
pub(in crate::node) verified_identity: PeerIdentity,
|
||||
/// The transport the exchange is happening over (msg2 send target, decrypt
|
||||
/// cache-key transport half).
|
||||
pub(in crate::node) transport_id: TransportId,
|
||||
/// The peer's wire address (msg2 send target).
|
||||
pub(in crate::node) remote_addr: TransportAddr,
|
||||
/// Our session index for this exchange (msg2 framing sender_idx).
|
||||
pub(in crate::node) our_index: Option<SessionIndex>,
|
||||
/// The peer's session index for this exchange (msg2 framing
|
||||
/// receiver_idx).
|
||||
pub(in crate::node) their_index: Option<SessionIndex>,
|
||||
/// The wire timestamp driving this step (promotion ts / loss-report clock).
|
||||
pub(in crate::node) now_ms: u64,
|
||||
/// Establish direction for this exchange. Discriminates the
|
||||
/// `PromoteToActive` failure cleanup: the pre-refactor inbound
|
||||
/// (`handle_msg1`) and outbound (`handle_msg2`) promote-Err arms were NOT
|
||||
/// byte-identical, so the executor must reproduce each. `false` = inbound
|
||||
/// (drop link + reverse map + free index), `true` = outbound (record the
|
||||
/// reject only; leave the dead link/`addr_to_link` for the stale-connection
|
||||
/// reaper, matching old `handle_msg2`).
|
||||
pub(in crate::node) is_outbound: bool,
|
||||
}
|
||||
|
||||
impl Node {
|
||||
/// Advance the machine for `link` by one event and execute the resulting
|
||||
/// actions.
|
||||
///
|
||||
/// The borrow structure the whole seam turns on: the machine
|
||||
/// needs `&mut IndexAllocator` as a synchronous capability *while it is
|
||||
/// itself borrowed mutably out of `peer_machines`*. `peer_machines` and
|
||||
/// `index_allocator` are **distinct `Node` fields**, so the collect below is
|
||||
/// a disjoint two-field borrow the checker accepts; once the actions are
|
||||
/// collected both borrows drop and the executor runs against `&mut self`.
|
||||
pub(in crate::node) async fn advance_peer_machine(
|
||||
&mut self,
|
||||
link: LinkId,
|
||||
event: PeerEvent,
|
||||
now: u64,
|
||||
ambient: &PeerActionCtx,
|
||||
) {
|
||||
let actions = match self.peer_machines.get_mut(&link) {
|
||||
// Disjoint field borrow: `self.peer_machines` (the map entry) and
|
||||
// `self.index_allocator` (the capability) are separate fields.
|
||||
Some(machine) => machine.step(event, now, &mut self.index_allocator),
|
||||
None => return,
|
||||
};
|
||||
self.execute_peer_actions(link, ambient, actions).await;
|
||||
}
|
||||
|
||||
/// Map each [`PeerAction`] onto its shell call.
|
||||
///
|
||||
/// `PromoteToActive` feeds its [`PromotionResult`](crate::proto::fmp::PromotionResult)
|
||||
/// back into the machine and appends the follow-up actions to the same
|
||||
/// worklist — a queue rather than self-recursion so the async executor stays a
|
||||
/// single flat future (no boxing) and the emitted order is preserved (the
|
||||
/// establish sequences always end in `PromoteToActive`, so its follow-ups run
|
||||
/// after any siblings).
|
||||
pub(in crate::node) async fn execute_peer_actions(
|
||||
&mut self,
|
||||
link: LinkId,
|
||||
ambient: &PeerActionCtx,
|
||||
actions: Vec<PeerAction>,
|
||||
) {
|
||||
let mut queue: VecDeque<PeerAction> = actions.into();
|
||||
while let Some(action) = queue.pop_front() {
|
||||
match action {
|
||||
PeerAction::OpenTransport {
|
||||
transport_id,
|
||||
remote_addr,
|
||||
} => {
|
||||
// Outbound connection-oriented dial. `initiate_connection`'s
|
||||
// oriented branch drove the machine to `Connecting`, which
|
||||
// emitted this action. Perform the non-blocking
|
||||
// `transport.connect` and, on success, push the
|
||||
// `PendingConnect` for `poll_pending_connects` to resolve. On
|
||||
// connect error, tear down the dial-window state (link,
|
||||
// reverse map, control machine) and abort the queue — the
|
||||
// executor-local mirror of the old inline
|
||||
// `initiate_connection` connect+push.
|
||||
if let Some(transport) = self.transports.get(&transport_id) {
|
||||
match transport.connect(&remote_addr).await {
|
||||
Ok(()) => {
|
||||
debug!(
|
||||
transport_id = %transport_id,
|
||||
remote_addr = %remote_addr,
|
||||
link_id = %link,
|
||||
"Transport connect initiated (non-blocking)"
|
||||
);
|
||||
self.peering
|
||||
.pending_connects
|
||||
.push(crate::node::PendingConnect {
|
||||
link_id: link,
|
||||
transport_id,
|
||||
remote_addr,
|
||||
peer_identity: ambient.verified_identity,
|
||||
});
|
||||
}
|
||||
Err(_e) => {
|
||||
self.links.remove(&link);
|
||||
self.addr_to_link.remove(&(transport_id, remote_addr));
|
||||
self.remove_peer_machine(link);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PeerAction::SendHandshake { bytes } => {
|
||||
// Two outbound directions share this action, discriminated by
|
||||
// `their_index`:
|
||||
// msg2 (`their_index == Some`): the machine payload is the
|
||||
// UNFRAMED Noise msg2; frame it with our/their index
|
||||
// (`build_msg2`) and send.
|
||||
// msg1 (`their_index == None`): a fresh outbound handshake;
|
||||
// the machine's empty payload is ignored — the shell already
|
||||
// allocated the index, ran the Noise leaf, and armed the
|
||||
// wire at dial (`prepare_outbound_msg1`); this just sends the
|
||||
// stored wire (see `send_stored_msg1`).
|
||||
if let (Some(sender_idx), Some(receiver_idx)) =
|
||||
(ambient.our_index, ambient.their_index)
|
||||
{
|
||||
let frame = build_msg2(sender_idx, receiver_idx, &bytes);
|
||||
// Surface the send Result. A missing transport skips
|
||||
// the send and continues (mirrors `handle_msg1`'s
|
||||
// `if let Some(transport)` guard); a send *error* runs the
|
||||
// pre-refactor msg2-send-failure cleanup (`handle_msg1`
|
||||
// L494-503) and ABORTS the remaining queue so the queued
|
||||
// `PromoteToActive` never runs.
|
||||
let send_err = match self.transports.get(&ambient.transport_id) {
|
||||
Some(transport) => {
|
||||
transport.send(&ambient.remote_addr, &frame).await.err()
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
if let Some(e) = send_err {
|
||||
// Restored pre-refactor msg2-send-failure warn!
|
||||
// (`handle_msg1` L665): the send error text is surfaced
|
||||
// at the executor point where the failure is now handled.
|
||||
warn!(link_id = %link, error = %e, "Failed to send msg2");
|
||||
self.links.remove(&link);
|
||||
self.addr_to_link
|
||||
.remove(&(ambient.transport_id, ambient.remote_addr.clone()));
|
||||
if let Some(idx) = ambient.our_index {
|
||||
let _ = self.index_allocator.free(idx);
|
||||
}
|
||||
self.remove_peer_machine(link);
|
||||
self.stats_mut()
|
||||
.record_reject(RejectReason::Handshake(HandshakeReject::BadState));
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// msg1: the shell already allocated the index, ran the
|
||||
// Noise leaf, and armed the wire on the connection at dial
|
||||
// (`prepare_outbound_msg1`); send the stored wire. The
|
||||
// machine's empty payload is ignored.
|
||||
let _ = bytes;
|
||||
self.send_stored_msg1(
|
||||
link,
|
||||
ambient.transport_id,
|
||||
&ambient.remote_addr,
|
||||
ambient.now_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
PeerAction::SendRekey { .. } => {
|
||||
// Rekey msg2 framing (`build_msg2(our_new_index, …)`,
|
||||
// `handshake.rs:365`) + send. Rekey fold is not yet wired.
|
||||
}
|
||||
PeerAction::SendLinkMessage { .. } => {
|
||||
// Encrypt + send a link-control frame (heartbeat / filter
|
||||
// / tree / disconnect). Data-plane-owned; not yet wired.
|
||||
}
|
||||
PeerAction::PromoteToActive { link: promote_link } => {
|
||||
// Ambient supplies the verified identity + promotion ts
|
||||
// that `promote_connection` needs (resolved from the wire ctx).
|
||||
match self.promote_connection(
|
||||
promote_link,
|
||||
ambient.verified_identity,
|
||||
ambient.now_ms,
|
||||
) {
|
||||
Ok(result) => {
|
||||
// The decrypt-worker registration relocated
|
||||
// OUT of `promote_connection` into THIS single executor
|
||||
// arm — the one live caller of `promote_connection` (both
|
||||
// the inbound `handle_msg1` and outbound `handle_msg2`
|
||||
// net-new establish paths reach it here). Register iff the
|
||||
// promotion actually created or replaced a peer
|
||||
// (`Promoted | CrossConnectionWon`), NEVER on
|
||||
// `CrossConnectionLost`. Run synchronously right after
|
||||
// `promote_connection` returns, before feeding
|
||||
// `PromotionResolved` and before any await — the exact
|
||||
// synchronous point (and Promoted/Won gating) of the
|
||||
// pre-refactor in-`promote_connection` call. No-op when
|
||||
// the worker pool isn't spawned (`register_...` early-
|
||||
// returns), so the direct `promote_connection` test
|
||||
// callers (which bypass this executor) are unaffected.
|
||||
#[cfg(unix)]
|
||||
match result {
|
||||
PromotionResult::Promoted(node_addr)
|
||||
| PromotionResult::CrossConnectionWon { node_addr, .. } => {
|
||||
self.register_decrypt_worker_session(&node_addr);
|
||||
}
|
||||
PromotionResult::CrossConnectionLost { .. } => {}
|
||||
}
|
||||
|
||||
// Feed the outcome back into the machine and fold the
|
||||
// follow-up actions (RegisterDecryptSession — now a
|
||||
// redundant no-op, see its arm — and the cross-conn index
|
||||
// frees) into the worklist. Disjoint field borrow again.
|
||||
let follow = match self.peer_machines.get_mut(&promote_link) {
|
||||
Some(machine) => machine.step(
|
||||
PeerEvent::PromotionResolved { result },
|
||||
ambient.now_ms,
|
||||
&mut self.index_allocator,
|
||||
),
|
||||
None => Vec::new(),
|
||||
};
|
||||
queue.extend(follow);
|
||||
|
||||
// Defensive cross-connection loser-link surgery.
|
||||
// LINK-ONLY: close the losing transport connection, drop
|
||||
// its link, and re-point `addr_to_link`, reproducing the
|
||||
// pre-refactor inline `handle_msg2`/`handle_msg1` per-arm
|
||||
// order EXACTLY. The index-plane frees/unregisters are
|
||||
// owned by the machine's `PromotionResolved{Won/Lost}`
|
||||
// follow-up (queued just above), so NOTHING here touches
|
||||
// an index — no double-free.
|
||||
//
|
||||
// UNREACHABLE on every current driven path: the inbound
|
||||
// and outbound net-new establish arms only route to the
|
||||
// machine when no promoted peer exists for the node_addr
|
||||
// (and `RestartThenPromote` removes the old peer first),
|
||||
// so `promote_connection` always returns `Promoted`. The
|
||||
// `debug_assert!(false, ..)` catches any future path that
|
||||
// drives a cross-connection through the executor without
|
||||
// the matching send-state handling.
|
||||
match result {
|
||||
PromotionResult::CrossConnectionWon { loser_link_id, .. } => {
|
||||
debug_assert!(
|
||||
false,
|
||||
"executor CrossConnectionWon is unreachable on \
|
||||
driven net-new establish paths"
|
||||
);
|
||||
// Close the losing transport connection (no-op for
|
||||
// connectionless) via the LOSER link's own
|
||||
// transport/addr, then drop the losing link.
|
||||
if let Some(loser_link) = self.links.get(&loser_link_id) {
|
||||
let loser_tid = loser_link.transport_id();
|
||||
let loser_addr = loser_link.remote_addr().clone();
|
||||
if let Some(transport) = self.transports.get(&loser_tid) {
|
||||
transport.close_connection(&loser_addr).await;
|
||||
}
|
||||
}
|
||||
self.remove_link(&loser_link_id);
|
||||
// Point `addr_to_link` at the winning (current)
|
||||
// link.
|
||||
self.addr_to_link.insert(
|
||||
(ambient.transport_id, ambient.remote_addr.clone()),
|
||||
promote_link,
|
||||
);
|
||||
}
|
||||
PromotionResult::CrossConnectionLost { winner_link_id } => {
|
||||
debug_assert!(
|
||||
false,
|
||||
"executor CrossConnectionLost is unreachable on \
|
||||
driven net-new establish paths"
|
||||
);
|
||||
// Close this (losing) connection, drop its link,
|
||||
// and restore `addr_to_link` to the winner.
|
||||
if let Some(transport) =
|
||||
self.transports.get(&ambient.transport_id)
|
||||
{
|
||||
transport.close_connection(&ambient.remote_addr).await;
|
||||
}
|
||||
self.remove_link(&promote_link);
|
||||
self.addr_to_link.insert(
|
||||
(ambient.transport_id, ambient.remote_addr.clone()),
|
||||
winner_link_id,
|
||||
);
|
||||
}
|
||||
PromotionResult::Promoted(_) => {}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
// Promotion failed. `promote_connection` already
|
||||
// removed `connections[link]` and (on error) handled its
|
||||
// own index internally. The pre-refactor inbound and
|
||||
// outbound promote-Err arms were NOT byte-identical, so
|
||||
// discriminate on `ambient.is_outbound`. The queue is
|
||||
// drained (PromoteToActive is the last establish action),
|
||||
// so no explicit abort.
|
||||
if ambient.is_outbound {
|
||||
// OLD outbound (`handle_msg2` promote-Err): warn +
|
||||
// record_reject ONLY. NO `remove_link`, NO
|
||||
// `index_allocator.free`, NO `addr_to_link` removal —
|
||||
// the dead link/addr_to_link/pending_outbound were
|
||||
// left for the 30s stale-connection reaper
|
||||
// (`promote_connection` already handled
|
||||
// `connections[link]`/its index on error). Restored
|
||||
// pre-refactor outbound warn! ("Failed to promote
|
||||
// connection").
|
||||
//
|
||||
// The outbound machine was persisted at dial; it is
|
||||
// additive state that did not exist pre-refactor, so
|
||||
// removing it on promote failure is neutral vs old and
|
||||
// prevents a leak.
|
||||
warn!(
|
||||
target: "fips::node::handlers::handshake",
|
||||
link_id = %promote_link,
|
||||
error = %e,
|
||||
"Failed to promote connection"
|
||||
);
|
||||
self.stats_mut().record_reject(RejectReason::Handshake(
|
||||
HandshakeReject::BadState,
|
||||
));
|
||||
self.remove_peer_machine(promote_link);
|
||||
} else {
|
||||
// OLD inbound (`handle_msg1` L587-591): drop the link
|
||||
// + reverse map, free our index, discard the machine,
|
||||
// and record the reject. Restored pre-refactor inbound
|
||||
// promote-failure warn! (`handle_msg1` L757).
|
||||
warn!(
|
||||
target: "fips::node::handlers::handshake",
|
||||
link_id = %promote_link,
|
||||
error = %e,
|
||||
"Failed to promote inbound connection"
|
||||
);
|
||||
self.remove_link(&promote_link);
|
||||
if let Some(idx) = ambient.our_index {
|
||||
let _ = self.index_allocator.free(idx);
|
||||
}
|
||||
self.remove_peer_machine(promote_link);
|
||||
self.stats_mut().record_reject(RejectReason::Handshake(
|
||||
HandshakeReject::BadState,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PeerAction::ResolveCrossConnection { .. } => {
|
||||
// A decision token, not an effect: the outbound msg2
|
||||
// handler intercepts it and runs the inline swap/keep
|
||||
// resolution itself, so it must never reach the executor.
|
||||
debug_assert!(
|
||||
false,
|
||||
"ResolveCrossConnection is intercepted by the msg2 \
|
||||
handler and must never reach the executor"
|
||||
);
|
||||
}
|
||||
PeerAction::SwapSendState { .. } => {
|
||||
// Initiator cutover: the live authoritative rekey-cadence
|
||||
// path, routed here from `check_rekey` via
|
||||
// `route_rekey_cadence` → `PeerEvent::RekeyConsume`; the
|
||||
// inline body survives only as `cutover_peer_inline`, a
|
||||
// debug-assert release fallback. `addr` is resolved
|
||||
// from the ambient verified identity (as `InvalidateSendState`
|
||||
// does). The decrypt re-register folds HERE, gated on
|
||||
// `did_cutover` — the generic `RegisterDecryptSession` arm stays a
|
||||
// no-op so a promote never double-registers.
|
||||
let node_addr = *ambient.verified_identity.node_addr();
|
||||
let did_cutover = if let Some(peer) = self.peers.get_mut(&node_addr) {
|
||||
if let Some(_old_our_index) = peer.cutover_to_new_session() {
|
||||
// New index was pre-registered in peers_by_index
|
||||
// during msg2 handling (handshake.rs).
|
||||
debug_assert!(
|
||||
peer.transport_id().is_some()
|
||||
&& peer.our_index().is_some()
|
||||
&& self.peers_by_index.contains_key(&(
|
||||
peer.transport_id().unwrap(),
|
||||
peer.our_index().unwrap().as_u32()
|
||||
)),
|
||||
"peers_by_index should contain pre-registered new index after cutover"
|
||||
);
|
||||
debug!(
|
||||
// Pin the target to the pre-refactor module: this
|
||||
// cutover log relocated from handlers/rekey.rs into
|
||||
// the executor, but operators (and the test harness)
|
||||
// filter it under fips::node::handlers::rekey. Keeping
|
||||
// the target preserves the observable log contract.
|
||||
target: "fips::node::handlers::rekey",
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
"Rekey cutover complete (initiator), K-bit flipped"
|
||||
);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
false
|
||||
};
|
||||
// Re-register the new session with the decrypt worker — the
|
||||
// cache_key (transport_id, our_index) just changed, so the
|
||||
// old worker entry is stale and every packet on the new
|
||||
// session would miss the worker's HashMap lookup.
|
||||
#[cfg(unix)]
|
||||
if did_cutover {
|
||||
self.register_decrypt_worker_session(&node_addr);
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
let _ = did_cutover;
|
||||
}
|
||||
PeerAction::CompleteDrain { peer: node_addr } => {
|
||||
// Initiator drain completion: the live authoritative
|
||||
// rekey-cadence path, routed here from `check_rekey` via
|
||||
// `route_rekey_cadence` → `PeerEvent::RekeyConsume`; the
|
||||
// inline body survives only as `drain_peer_inline`, a
|
||||
// debug-assert release fallback. Extract the real previous
|
||||
// index + transport_id under the peer borrow, drop the
|
||||
// borrow, then run the cache_key cleanup (which takes
|
||||
// &mut self for unregister_decrypt_worker_session).
|
||||
let drained = self.peers.get_mut(&node_addr).and_then(|peer| {
|
||||
peer.complete_drain().map(|idx| (idx, peer.transport_id()))
|
||||
});
|
||||
if let Some((old_our_index, transport_id)) = drained {
|
||||
if let Some(tid) = transport_id {
|
||||
let cache_key = (tid, old_our_index.as_u32());
|
||||
self.peers_by_index.remove(&cache_key);
|
||||
#[cfg(unix)]
|
||||
self.unregister_decrypt_worker_session(cache_key);
|
||||
}
|
||||
let _ = self.index_allocator.free(old_our_index);
|
||||
trace!(
|
||||
// Pin to the pre-refactor module (see the cutover log
|
||||
// above) so the relocated drain log stays visible under
|
||||
// the operator's fips::node::handlers::rekey filter.
|
||||
target: "fips::node::handlers::rekey",
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
old_index = %old_our_index,
|
||||
"Drain complete, previous session erased"
|
||||
);
|
||||
}
|
||||
}
|
||||
PeerAction::InvalidateSendState => {
|
||||
// The FULL teardown. `remove_active_peer`
|
||||
// (`dispatch.rs:107`) frees the four index slots
|
||||
// (current/rekey/pending/previous), drops `peers_by_index`,
|
||||
// unregisters the decrypt worker, removes the FSP `sessions`
|
||||
// entry and `pending_tun_packets`. The machine emits NO
|
||||
// `FreeIndex` for those slots, so there is no double-free.
|
||||
self.remove_active_peer(ambient.verified_identity.node_addr());
|
||||
}
|
||||
PeerAction::RegisterDecryptSession { index } => {
|
||||
let _ = index;
|
||||
// No-op by design. The decrypt-worker
|
||||
// registration relocated into the `PromoteToActive` Ok arm above, gated on
|
||||
// the returned `PromotionResult`, so it runs once per live
|
||||
// promote (Promoted/Won) at the pre-refactor synchronous point.
|
||||
// This machine-emitted action is now redundant with that arm;
|
||||
// kept as an inert no-op (rather than removing the emission) so
|
||||
// the machine's action sequence and its unit tests stay
|
||||
// unchanged. The keyed-by-NodeAddr register does not need the
|
||||
// machine's `index` payload.
|
||||
}
|
||||
PeerAction::UnregisterDecryptSession { index } => {
|
||||
// Executor supplies `transport_id` from ambient; keyed by
|
||||
// (tid, index) like `remove_active_peer` / the rekey drain path.
|
||||
#[cfg(unix)]
|
||||
self.unregister_decrypt_worker_session((ambient.transport_id, index.as_u32()));
|
||||
#[cfg(not(unix))]
|
||||
let _ = index;
|
||||
}
|
||||
PeerAction::FreeIndex { index } => {
|
||||
let _ = self.index_allocator.free(index);
|
||||
}
|
||||
PeerAction::ActivateConnectedUdp | PeerAction::TeardownConnectedUdp => {
|
||||
// Connected-UDP plane ownership (`connected_udp.rs`).
|
||||
}
|
||||
PeerAction::SetTimer { kind, at_ms } => {
|
||||
// Populate the per-peer timer store (overwrite = reschedule).
|
||||
// The `HandshakeRetransmit` and `HandshakeTimeout` deadlines
|
||||
// are read + fired by `drive_peer_timers`. Rekey/liveness kinds
|
||||
// are still SHADOW here — they keep their own shell drivers —
|
||||
// so populating them stays behavior-neutral.
|
||||
self.peer_timers
|
||||
.entry(link)
|
||||
.or_default()
|
||||
.insert(kind, at_ms);
|
||||
}
|
||||
PeerAction::CancelTimer { kind } => {
|
||||
if let Some(timers) = self.peer_timers.get_mut(&link) {
|
||||
timers.remove(&kind);
|
||||
}
|
||||
}
|
||||
PeerAction::ReportLost { peer, kind } => {
|
||||
// The single loss token, routed to the reconciler reflex the
|
||||
// `kind` names: an un-promoted handshake attempt takes the
|
||||
// connected-guarded `note_handshake_timeout` (`driver.rs:28`),
|
||||
// an established peer's link-death takes the unconditional
|
||||
// `note_link_dead` (`driver.rs:48`).
|
||||
match kind {
|
||||
LostKind::HandshakeTimeout => {
|
||||
self.note_handshake_timeout(peer, ambient.now_ms);
|
||||
}
|
||||
LostKind::LinkDead => {
|
||||
self.note_link_dead(peer, ambient.now_ms);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -561,7 +561,7 @@ mod tests {
|
||||
let open_cipher = LessSafeKey::new(unbound2);
|
||||
|
||||
let counter: u64 = 7;
|
||||
const HDR: usize = crate::proto::fmp::wire::ESTABLISHED_HEADER_SIZE;
|
||||
const HDR: usize = crate::node::wire::ESTABLISHED_HEADER_SIZE;
|
||||
// Build a wire packet `[16-byte header][4-byte inner ts][1 byte link msg]`
|
||||
// with capacity for the trailing AEAD tag. Header bytes
|
||||
// double as AAD and as the on-wire prefix.
|
||||
@@ -569,7 +569,7 @@ mod tests {
|
||||
// Header: fill the flags byte (the second byte) with both
|
||||
// FLAG_CE and FLAG_SP set; the rest is uninterpreted by the
|
||||
// worker (it just AADs the whole 16 bytes).
|
||||
let flags_byte = crate::proto::fmp::wire::FLAG_CE | crate::proto::fmp::wire::FLAG_SP;
|
||||
let flags_byte = crate::node::wire::FLAG_CE | crate::node::wire::FLAG_SP;
|
||||
let mut header = [0u8; HDR];
|
||||
header[1] = flags_byte;
|
||||
wire.extend_from_slice(&header);
|
||||
@@ -626,11 +626,11 @@ mod tests {
|
||||
"fmp_flags must round-trip from DecryptJob to DecryptFallback"
|
||||
);
|
||||
assert!(
|
||||
fallback.fmp_flags & crate::proto::fmp::wire::FLAG_CE != 0,
|
||||
fallback.fmp_flags & crate::node::wire::FLAG_CE != 0,
|
||||
"FLAG_CE bit lost on worker path"
|
||||
);
|
||||
assert!(
|
||||
fallback.fmp_flags & crate::proto::fmp::wire::FLAG_SP != 0,
|
||||
fallback.fmp_flags & crate::node::wire::FLAG_SP != 0,
|
||||
"FLAG_SP bit lost on worker path"
|
||||
);
|
||||
}
|
||||
@@ -724,7 +724,7 @@ mod tests {
|
||||
let open_cipher = LessSafeKey::new(unbound);
|
||||
|
||||
let counter: u64 = 11;
|
||||
const HDR: usize = crate::proto::fmp::wire::ESTABLISHED_HEADER_SIZE;
|
||||
const HDR: usize = crate::node::wire::ESTABLISHED_HEADER_SIZE;
|
||||
let header = [0u8; HDR];
|
||||
let mut wire = Vec::with_capacity(HDR + 4 + 1 + 16);
|
||||
wire.extend_from_slice(&header);
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
//! Discovery protocol rate limiting and backoff.
|
||||
//!
|
||||
//! Two complementary mechanisms:
|
||||
//!
|
||||
//! - **`DiscoveryBackoff`** (originator-side, optional): Exponential
|
||||
//! suppression of fresh lookups after the per-attempt sequence in
|
||||
//! `node.discovery.attempt_timeouts_secs` has been exhausted.
|
||||
//! **Disabled by default** (base/cap = 0); the per-attempt sequence
|
||||
//! is the only retry pacing in the standard configuration. Reset on
|
||||
//! topology changes (parent change, new peer, first RTT, reconnection).
|
||||
//!
|
||||
//! - **`DiscoveryForwardRateLimiter`** (transit-side): Per-target minimum
|
||||
//! interval for forwarded requests. Defense-in-depth against misbehaving
|
||||
//! nodes generating fresh request_ids at high rate.
|
||||
|
||||
use crate::NodeAddr;
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
// ============================================================================
|
||||
// Originator-side: Discovery Backoff
|
||||
// ============================================================================
|
||||
|
||||
/// Default base backoff after first lookup failure. `0` = disabled.
|
||||
const DEFAULT_BACKOFF_BASE_SECS: u64 = 0;
|
||||
|
||||
/// Default maximum backoff cap. `0` = disabled.
|
||||
const DEFAULT_BACKOFF_MAX_SECS: u64 = 0;
|
||||
|
||||
/// Backoff multiplier per consecutive failure.
|
||||
const BACKOFF_MULTIPLIER: u64 = 2;
|
||||
|
||||
/// Exponential backoff for failed discovery lookups.
|
||||
///
|
||||
/// Tracks targets whose lookups have timed out and suppresses
|
||||
/// re-initiation with increasing delays. Cleared on topology changes.
|
||||
pub struct DiscoveryBackoff {
|
||||
/// Maps target → (suppress_until, consecutive_failures).
|
||||
entries: HashMap<NodeAddr, BackoffEntry>,
|
||||
/// Base backoff duration (first failure).
|
||||
base: Duration,
|
||||
/// Maximum backoff cap.
|
||||
max: Duration,
|
||||
}
|
||||
|
||||
struct BackoffEntry {
|
||||
/// Don't re-initiate until this instant.
|
||||
suppress_until: Instant,
|
||||
/// Consecutive failures (drives exponential backoff).
|
||||
failures: u32,
|
||||
}
|
||||
|
||||
impl DiscoveryBackoff {
|
||||
/// Create with default parameters (disabled — base/cap = 0).
|
||||
pub fn new() -> Self {
|
||||
Self::with_params(DEFAULT_BACKOFF_BASE_SECS, DEFAULT_BACKOFF_MAX_SECS)
|
||||
}
|
||||
|
||||
/// Create with custom base and max backoff in seconds.
|
||||
pub fn with_params(base_secs: u64, max_secs: u64) -> Self {
|
||||
Self {
|
||||
entries: HashMap::new(),
|
||||
base: Duration::from_secs(base_secs),
|
||||
max: Duration::from_secs(max_secs),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a lookup for this target is suppressed.
|
||||
///
|
||||
/// Returns true if the target is in backoff and should not be
|
||||
/// looked up yet.
|
||||
pub fn is_suppressed(&self, target: &NodeAddr) -> bool {
|
||||
if let Some(entry) = self.entries.get(target) {
|
||||
Instant::now() < entry.suppress_until
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a lookup failure (timeout) for a target.
|
||||
///
|
||||
/// Increments the failure count and sets the next suppression
|
||||
/// window using exponential backoff.
|
||||
pub fn record_failure(&mut self, target: &NodeAddr) {
|
||||
let now = Instant::now();
|
||||
let failures = self.entries.get(target).map_or(0, |e| e.failures) + 1;
|
||||
|
||||
let backoff_secs = self
|
||||
.base
|
||||
.as_secs()
|
||||
.saturating_mul(BACKOFF_MULTIPLIER.saturating_pow(failures.saturating_sub(1)));
|
||||
let backoff = Duration::from_secs(backoff_secs.min(self.max.as_secs()));
|
||||
|
||||
self.entries.insert(
|
||||
*target,
|
||||
BackoffEntry {
|
||||
suppress_until: now + backoff,
|
||||
failures,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Record a successful lookup — remove backoff for this target.
|
||||
pub fn record_success(&mut self, target: &NodeAddr) {
|
||||
self.entries.remove(target);
|
||||
}
|
||||
|
||||
/// Clear all backoff entries.
|
||||
///
|
||||
/// Called on topology changes that might make previously-unreachable
|
||||
/// targets reachable (parent change, new peer, first RTT, reconnection).
|
||||
pub fn reset_all(&mut self) {
|
||||
self.entries.clear();
|
||||
}
|
||||
|
||||
/// Whether any entries exist.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
|
||||
/// Current number of entries.
|
||||
pub fn entry_count(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
/// Get the failure count for a target (for logging).
|
||||
pub fn failure_count(&self, target: &NodeAddr) -> u32 {
|
||||
self.entries.get(target).map_or(0, |e| e.failures)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DiscoveryBackoff {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Transit-side: Discovery Forward Rate Limiter
|
||||
// ============================================================================
|
||||
|
||||
/// Default minimum interval between forwarded lookups for the same target.
|
||||
const DEFAULT_FORWARD_MIN_INTERVAL: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Maximum age of entries before cleanup.
|
||||
const FORWARD_MAX_AGE: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Rate limiter for forwarded discovery requests.
|
||||
///
|
||||
/// Tracks the last time a LookupRequest was forwarded for each target
|
||||
/// and enforces a minimum interval to prevent floods from misbehaving
|
||||
/// nodes generating fresh request_ids.
|
||||
pub struct DiscoveryForwardRateLimiter {
|
||||
last_forwarded: HashMap<NodeAddr, Instant>,
|
||||
min_interval: Duration,
|
||||
max_age: Duration,
|
||||
}
|
||||
|
||||
impl DiscoveryForwardRateLimiter {
|
||||
/// Create with default parameters (2s interval).
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
last_forwarded: HashMap::new(),
|
||||
min_interval: DEFAULT_FORWARD_MIN_INTERVAL,
|
||||
max_age: FORWARD_MAX_AGE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create with a custom minimum interval.
|
||||
pub fn with_interval(min_interval: Duration) -> Self {
|
||||
Self {
|
||||
last_forwarded: HashMap::new(),
|
||||
min_interval,
|
||||
max_age: FORWARD_MAX_AGE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if we should forward a lookup for this target.
|
||||
///
|
||||
/// Returns true if enough time has passed since the last forward
|
||||
/// for this target. Updates internal state when returning true.
|
||||
pub fn should_forward(&mut self, target: &NodeAddr) -> bool {
|
||||
let now = Instant::now();
|
||||
|
||||
if let Some(&last) = self.last_forwarded.get(target)
|
||||
&& now.duration_since(last) < self.min_interval
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
self.last_forwarded.insert(*target, now);
|
||||
self.cleanup(now);
|
||||
true
|
||||
}
|
||||
|
||||
/// Replace the minimum interval (e.g., set to zero to disable).
|
||||
#[cfg(test)]
|
||||
pub fn set_interval(&mut self, interval: Duration) {
|
||||
self.min_interval = interval;
|
||||
}
|
||||
|
||||
/// Remove entries older than max_age.
|
||||
fn cleanup(&mut self, now: Instant) {
|
||||
self.last_forwarded
|
||||
.retain(|_, &mut last| now.duration_since(last) < self.max_age);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn len(&self) -> usize {
|
||||
self.last_forwarded.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DiscoveryForwardRateLimiter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::thread;
|
||||
|
||||
fn addr(val: u8) -> NodeAddr {
|
||||
let mut bytes = [0u8; 16];
|
||||
bytes[0] = val;
|
||||
NodeAddr::from_bytes(bytes)
|
||||
}
|
||||
|
||||
// --- DiscoveryBackoff tests ---
|
||||
|
||||
#[test]
|
||||
fn test_backoff_not_suppressed_initially() {
|
||||
let backoff = DiscoveryBackoff::new();
|
||||
assert!(!backoff.is_suppressed(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_suppressed_after_failure() {
|
||||
// Backoff is opt-in; exercise the suppression path with explicit params.
|
||||
let mut backoff = DiscoveryBackoff::with_params(30, 300);
|
||||
backoff.record_failure(&addr(1));
|
||||
assert!(backoff.is_suppressed(&addr(1)));
|
||||
// Different target not affected
|
||||
assert!(!backoff.is_suppressed(&addr(2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_cleared_on_success() {
|
||||
let mut backoff = DiscoveryBackoff::with_params(30, 300);
|
||||
backoff.record_failure(&addr(1));
|
||||
assert!(backoff.is_suppressed(&addr(1)));
|
||||
|
||||
backoff.record_success(&addr(1));
|
||||
assert!(!backoff.is_suppressed(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_reset_all() {
|
||||
let mut backoff = DiscoveryBackoff::new();
|
||||
backoff.record_failure(&addr(1));
|
||||
backoff.record_failure(&addr(2));
|
||||
assert_eq!(backoff.len(), 2);
|
||||
|
||||
backoff.reset_all();
|
||||
assert_eq!(backoff.len(), 0);
|
||||
assert!(!backoff.is_suppressed(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_exponential() {
|
||||
let mut backoff = DiscoveryBackoff::with_params(1, 300);
|
||||
|
||||
// First failure: 1s backoff
|
||||
backoff.record_failure(&addr(1));
|
||||
assert_eq!(backoff.failure_count(&addr(1)), 1);
|
||||
|
||||
// Second failure: 2s backoff
|
||||
backoff.record_failure(&addr(1));
|
||||
assert_eq!(backoff.failure_count(&addr(1)), 2);
|
||||
|
||||
// Third failure: 4s backoff
|
||||
backoff.record_failure(&addr(1));
|
||||
assert_eq!(backoff.failure_count(&addr(1)), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_expires() {
|
||||
let mut backoff = DiscoveryBackoff::with_params(0, 0);
|
||||
backoff.record_failure(&addr(1));
|
||||
// With 0s backoff, should not be suppressed
|
||||
assert!(!backoff.is_suppressed(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backoff_capped() {
|
||||
let mut backoff = DiscoveryBackoff::with_params(1, 10);
|
||||
|
||||
// Record many failures
|
||||
for _ in 0..20 {
|
||||
backoff.record_failure(&addr(1));
|
||||
}
|
||||
|
||||
// Backoff should be capped at max (10s), not overflow
|
||||
let entry = backoff.entries.get(&addr(1)).unwrap();
|
||||
let remaining = entry.suppress_until.duration_since(Instant::now());
|
||||
assert!(remaining <= Duration::from_secs(11));
|
||||
}
|
||||
|
||||
// --- DiscoveryForwardRateLimiter tests ---
|
||||
|
||||
#[test]
|
||||
fn test_forward_first_allowed() {
|
||||
let mut limiter = DiscoveryForwardRateLimiter::new();
|
||||
assert!(limiter.should_forward(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_forward_rapid_rate_limited() {
|
||||
let mut limiter = DiscoveryForwardRateLimiter::new();
|
||||
assert!(limiter.should_forward(&addr(1)));
|
||||
assert!(!limiter.should_forward(&addr(1)));
|
||||
assert!(!limiter.should_forward(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_forward_different_targets_independent() {
|
||||
let mut limiter = DiscoveryForwardRateLimiter::new();
|
||||
assert!(limiter.should_forward(&addr(1)));
|
||||
assert!(limiter.should_forward(&addr(2)));
|
||||
assert!(!limiter.should_forward(&addr(1)));
|
||||
assert!(!limiter.should_forward(&addr(2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_forward_allowed_after_interval() {
|
||||
let mut limiter = DiscoveryForwardRateLimiter::with_interval(Duration::from_millis(100));
|
||||
assert!(limiter.should_forward(&addr(1)));
|
||||
|
||||
thread::sleep(Duration::from_millis(110));
|
||||
|
||||
assert!(limiter.should_forward(&addr(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_forward_cleanup_removes_old() {
|
||||
let mut limiter = DiscoveryForwardRateLimiter::new();
|
||||
assert!(limiter.should_forward(&addr(1)));
|
||||
assert!(limiter.should_forward(&addr(2)));
|
||||
assert_eq!(limiter.len(), 2);
|
||||
|
||||
let future = Instant::now() + Duration::from_secs(61);
|
||||
limiter.cleanup(future);
|
||||
assert_eq!(limiter.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_forward_cleanup_preserves_recent() {
|
||||
let mut limiter = DiscoveryForwardRateLimiter::new();
|
||||
assert!(limiter.should_forward(&addr(1)));
|
||||
assert_eq!(limiter.len(), 1);
|
||||
|
||||
limiter.cleanup(Instant::now());
|
||||
assert_eq!(limiter.len(), 1);
|
||||
}
|
||||
}
|
||||
+19
-16
@@ -50,9 +50,9 @@
|
||||
// warnings rather than gate every function individually.
|
||||
#![cfg_attr(not(unix), allow(dead_code))]
|
||||
|
||||
use crate::proto::fmp::wire::ESTABLISHED_HEADER_SIZE;
|
||||
use crate::proto::fsp::wire::FSP_HEADER_SIZE;
|
||||
use crate::transport::udp::io::AsyncUdpSocket;
|
||||
use crate::node::session_wire::FSP_HEADER_SIZE;
|
||||
use crate::node::wire::ESTABLISHED_HEADER_SIZE;
|
||||
use crate::transport::udp::socket::AsyncUdpSocket;
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
use crossbeam_channel::{Receiver, SendError, Sender, TrySendError, bounded};
|
||||
use ring::aead::{Aad, LessSafeKey, Nonce};
|
||||
@@ -132,7 +132,8 @@ pub(crate) struct FmpSendJob {
|
||||
/// the job completes and the worker drops it, only the peer's
|
||||
/// strong ref remains.
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
pub connected_socket: Option<std::sync::Arc<crate::peer::connected_udp::ConnectedPeerSocket>>,
|
||||
pub connected_socket:
|
||||
Option<std::sync::Arc<crate::transport::udp::connected_peer::ConnectedPeerSocket>>,
|
||||
/// Bulk endpoint data may be dropped when the kernel reports UDP
|
||||
/// send-queue exhaustion. Control/rekey frames keep retrying so
|
||||
/// congestion cannot strand the session.
|
||||
@@ -715,7 +716,8 @@ fn mac_now_ms() -> u64 {
|
||||
struct MacSequencedSendFlow {
|
||||
key: MacSendFlowKey,
|
||||
socket: AsyncUdpSocket,
|
||||
connected_socket: Option<std::sync::Arc<crate::peer::connected_udp::ConnectedPeerSocket>>,
|
||||
connected_socket:
|
||||
Option<std::sync::Arc<crate::transport::udp::connected_peer::ConnectedPeerSocket>>,
|
||||
dest_addr: SocketAddr,
|
||||
next_seq: std::sync::atomic::AtomicU64,
|
||||
last_used_ms: std::sync::atomic::AtomicU64,
|
||||
@@ -752,7 +754,9 @@ impl MacSequencedSendFlow {
|
||||
fn spawn(
|
||||
key: MacSendFlowKey,
|
||||
socket: AsyncUdpSocket,
|
||||
connected_socket: Option<std::sync::Arc<crate::peer::connected_udp::ConnectedPeerSocket>>,
|
||||
connected_socket: Option<
|
||||
std::sync::Arc<crate::transport::udp::connected_peer::ConnectedPeerSocket>,
|
||||
>,
|
||||
dest_addr: SocketAddr,
|
||||
now_ms: u64,
|
||||
) -> Arc<Self> {
|
||||
@@ -1020,7 +1024,8 @@ fn flush_batch_sync(
|
||||
struct EncryptedGroup {
|
||||
socket: AsyncUdpSocket,
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
connected_socket: Option<std::sync::Arc<crate::peer::connected_udp::ConnectedPeerSocket>>,
|
||||
connected_socket:
|
||||
Option<std::sync::Arc<crate::transport::udp::connected_peer::ConnectedPeerSocket>>,
|
||||
dest_addr: SocketAddr,
|
||||
wire_packets: Vec<Vec<u8>>,
|
||||
drop_on_backpressure: bool,
|
||||
@@ -1705,7 +1710,7 @@ fn send_batch_gso(
|
||||
}
|
||||
|
||||
/// Direct `sendmmsg(2)` wrapper for the sync worker. The
|
||||
/// `transport::udp::io` module's existing `send_batch` is
|
||||
/// `transport::udp::socket` module's existing `send_batch` is
|
||||
/// pub(crate) on `UdpRawSocket`, but we don't have a handle to the
|
||||
/// raw socket from here — we just have the FD. Re-implementing
|
||||
/// inline is ~15 lines and avoids tunnelling the inner socket
|
||||
@@ -1775,7 +1780,7 @@ fn send_batch_raw(
|
||||
#[cfg(all(test, unix))]
|
||||
mod unix_tests {
|
||||
use super::*;
|
||||
use crate::transport::udp::io::UdpRawSocket;
|
||||
use crate::transport::udp::socket::UdpRawSocket;
|
||||
use ring::aead::{LessSafeKey, UnboundKey};
|
||||
use std::net::UdpSocket;
|
||||
|
||||
@@ -1899,12 +1904,10 @@ mod unix_tests {
|
||||
#[test]
|
||||
fn pipelined_send_wire_layout_roundtrips_canonical_decoders() {
|
||||
use crate::NodeAddr;
|
||||
use crate::node::session_wire::build_fsp_header;
|
||||
use crate::node::wire::{EncryptedHeader, FLAG_KEY_EPOCH, build_established_header};
|
||||
use crate::noise::TAG_SIZE;
|
||||
use crate::proto::fmp::wire::{EncryptedHeader, FLAG_KEY_EPOCH, build_established_header};
|
||||
use crate::proto::fsp::wire::build_fsp_header;
|
||||
use crate::proto::link::{
|
||||
LinkMessageType, SESSION_DATAGRAM_HEADER_SIZE, SessionDatagramRef,
|
||||
};
|
||||
use crate::protocol::{LinkMessageType, SESSION_DATAGRAM_HEADER_SIZE, SessionDatagramRef};
|
||||
use crate::utils::index::SessionIndex;
|
||||
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -2215,7 +2218,7 @@ mod tests {
|
||||
/// AsRawFd impl.
|
||||
#[test]
|
||||
fn flush_batch_routes_each_target_separately() {
|
||||
use crate::transport::udp::io::UdpRawSocket;
|
||||
use crate::transport::udp::socket::UdpRawSocket;
|
||||
use ring::aead::{LessSafeKey, UnboundKey};
|
||||
use std::net::UdpSocket;
|
||||
|
||||
@@ -2261,7 +2264,7 @@ mod tests {
|
||||
const B_WIRE: usize = 16 + B_PLAINTEXT + 16; // 96
|
||||
|
||||
fn make_job(
|
||||
socket: crate::transport::udp::io::AsyncUdpSocket,
|
||||
socket: crate::transport::udp::socket::AsyncUdpSocket,
|
||||
cipher: &LessSafeKey,
|
||||
counter: u64,
|
||||
dest: SocketAddr,
|
||||
|
||||
@@ -45,10 +45,12 @@ impl Node {
|
||||
/// (e.g. only non-UDP transports). Enabled on Linux and macOS:
|
||||
/// both kernels route a matching peer 5-tuple to the connected
|
||||
/// socket when it shares the wildcard listen port via SO_REUSEPORT.
|
||||
/// Only compiled on Linux/macOS — the sole caller (the rx_loop tick) is
|
||||
/// gated the same way, so on other targets (android) there is nothing to do.
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
pub(in crate::node) async fn activate_connected_udp_sessions(&mut self) {
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
{
|
||||
// No-op on platforms without the connected-UDP fast path.
|
||||
}
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
{
|
||||
if !connected_udp_enabled() {
|
||||
return;
|
||||
@@ -140,24 +142,20 @@ impl Node {
|
||||
(peer_sa, local, recv_buf, send_buf, tx)
|
||||
};
|
||||
|
||||
// Open the connected socket on the kernel side, then adopt the
|
||||
// fd into the owning handle.
|
||||
let owned = crate::transport::udp::open_connected_fd(
|
||||
local_addr,
|
||||
peer_socket_addr,
|
||||
recv_buf,
|
||||
send_buf,
|
||||
)
|
||||
.map_err(|e| format!("open_connected_fd: {e}"))?;
|
||||
let socket = std::sync::Arc::new(crate::peer::connected_udp::ConnectedPeerSocket::from_fd(
|
||||
owned,
|
||||
peer_socket_addr,
|
||||
local_addr,
|
||||
));
|
||||
// Open the connected socket on the kernel side.
|
||||
let socket = std::sync::Arc::new(
|
||||
crate::transport::udp::connected_peer::ConnectedPeerSocket::open(
|
||||
local_addr,
|
||||
peer_socket_addr,
|
||||
recv_buf,
|
||||
send_buf,
|
||||
)
|
||||
.map_err(|e| format!("ConnectedPeerSocket::open: {e}"))?,
|
||||
);
|
||||
|
||||
// Spawn the drain thread. It feeds `packet_tx` exactly like
|
||||
// the wildcard listen socket — rx_loop dispatches identically.
|
||||
let drain = crate::peer::connected_udp::PeerRecvDrain::spawn(
|
||||
let drain = crate::transport::udp::peer_drain::PeerRecvDrain::spawn(
|
||||
socket.clone(),
|
||||
transport_id,
|
||||
peer_socket_addr,
|
||||
@@ -0,0 +1,738 @@
|
||||
//! LookupRequest/LookupResponse discovery protocol handlers.
|
||||
//!
|
||||
//! Handles coordinate discovery via bloom-filter-guided tree routing.
|
||||
//! Requests are forwarded only to tree peers (parent + children) whose
|
||||
//! bloom filter contains the target. TTL and request_id dedup provide
|
||||
//! safety bounds.
|
||||
|
||||
use crate::node::reject::DiscoveryReject;
|
||||
use crate::node::{Node, RecentRequest};
|
||||
use crate::protocol::{LookupRequest, LookupResponse};
|
||||
use crate::transport::{TransportAddr, TransportId};
|
||||
use crate::{NodeAddr, PeerIdentity};
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
const MAX_RECENT_DISCOVERY_REQUESTS: usize = 4096;
|
||||
|
||||
impl Node {
|
||||
/// Handle an incoming LookupRequest from a peer.
|
||||
///
|
||||
/// Processing steps:
|
||||
/// 1. Decode and validate
|
||||
/// 2. Check request_id for duplicates (dedup / reverse-path routing)
|
||||
/// 3. Record request for reverse-path forwarding
|
||||
/// 4. Lazy purge expired entries
|
||||
/// 5. If we're the target, generate and send response
|
||||
/// 6. If TTL > 0, forward to tree peers whose bloom filter matches
|
||||
pub(in crate::node) async fn handle_lookup_request(&mut self, from: &NodeAddr, payload: &[u8]) {
|
||||
self.metrics().discovery.req_received.inc();
|
||||
|
||||
let request = match LookupRequest::decode(payload) {
|
||||
Ok(req) => req,
|
||||
Err(e) => {
|
||||
self.metrics()
|
||||
.discovery
|
||||
.record_reject(DiscoveryReject::ReqDecodeError);
|
||||
debug!(from = %self.peer_display_name(from), error = %e, "Malformed LookupRequest");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let now_ms = Self::now_ms();
|
||||
self.purge_expired_requests(now_ms);
|
||||
|
||||
// Dedup: drop if we've already seen this request_id.
|
||||
// Also serves as loop protection — tree routing is loop-free,
|
||||
// but request_id dedup catches edge cases during tree restructuring.
|
||||
if self.recent_requests.contains_key(&request.request_id) {
|
||||
self.metrics()
|
||||
.discovery
|
||||
.record_reject(DiscoveryReject::ReqDuplicate);
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
from = %self.peer_display_name(from),
|
||||
"Duplicate LookupRequest, dropping"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if self.recent_requests.len() >= MAX_RECENT_DISCOVERY_REQUESTS {
|
||||
self.metrics()
|
||||
.discovery
|
||||
.record_reject(DiscoveryReject::ReqDedupCacheFull);
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
from = %self.peer_display_name(from),
|
||||
recent_requests = self.recent_requests.len(),
|
||||
max_recent_requests = MAX_RECENT_DISCOVERY_REQUESTS,
|
||||
"Discovery request dedup cache full, dropping LookupRequest"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Record for reverse-path forwarding and dedup
|
||||
self.recent_requests
|
||||
.insert(request.request_id, RecentRequest::new(*from, now_ms));
|
||||
|
||||
// Are we the target?
|
||||
if request.target == *self.node_addr() {
|
||||
self.metrics().discovery.req_target_is_us.inc();
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
origin = %self.peer_display_name(&request.origin),
|
||||
"We are the lookup target, generating response"
|
||||
);
|
||||
self.send_lookup_response(&request).await;
|
||||
return;
|
||||
}
|
||||
|
||||
// Forward if TTL permits
|
||||
if request.can_forward() {
|
||||
// Transit-side rate limit: collapse rapid-fire lookups for the
|
||||
// same target from misbehaving nodes generating fresh request_ids.
|
||||
if !self
|
||||
.discovery_forward_limiter
|
||||
.should_forward(&request.target)
|
||||
{
|
||||
self.metrics().discovery.req_forward_rate_limited.inc();
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(&request.target),
|
||||
"Forward rate limited, suppressing LookupRequest"
|
||||
);
|
||||
return;
|
||||
}
|
||||
self.metrics().discovery.req_forwarded.inc();
|
||||
self.forward_lookup_request(request).await;
|
||||
} else {
|
||||
self.metrics()
|
||||
.discovery
|
||||
.record_reject(DiscoveryReject::ReqTtlExhausted);
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(&request.target),
|
||||
"LookupRequest TTL exhausted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle an incoming LookupResponse from a peer.
|
||||
///
|
||||
/// Processing steps:
|
||||
/// 1. Decode and validate
|
||||
/// 2. Check recent_requests to determine if we originated or are forwarding
|
||||
/// 3. If originator: verify proof signature, then cache target_coords and path_mtu in coord_cache
|
||||
/// 4. If transit: apply path_mtu min(outgoing_link_mtu), reverse-path forward to from_peer
|
||||
pub(in crate::node) async fn handle_lookup_response(
|
||||
&mut self,
|
||||
from: &NodeAddr,
|
||||
payload: &[u8],
|
||||
) {
|
||||
self.metrics().discovery.resp_received.inc();
|
||||
|
||||
let mut response = match LookupResponse::decode(payload) {
|
||||
Ok(resp) => resp,
|
||||
Err(e) => {
|
||||
self.metrics()
|
||||
.discovery
|
||||
.record_reject(DiscoveryReject::RespDecodeError);
|
||||
debug!(from = %self.peer_display_name(from), error = %e, "Malformed LookupResponse");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let now_ms = Self::now_ms();
|
||||
|
||||
// Check if we forwarded this request (transit node) or originated it
|
||||
if let Some(recent) = self.recent_requests.get_mut(&response.request_id) {
|
||||
// Already forwarded a response for this request — drop to
|
||||
// prevent response routing loops.
|
||||
if recent.response_forwarded {
|
||||
debug!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&response.target),
|
||||
"Response already forwarded for this request, dropping"
|
||||
);
|
||||
return;
|
||||
}
|
||||
recent.response_forwarded = true;
|
||||
|
||||
// Transit node: reverse-path forward
|
||||
let from_peer = recent.from_peer;
|
||||
self.metrics().discovery.resp_forwarded.inc();
|
||||
|
||||
// Apply path_mtu min() from the outgoing link's transport MTU
|
||||
self.apply_outgoing_link_mtu_to_response(&mut response, &from_peer);
|
||||
|
||||
debug!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&response.target),
|
||||
next_hop = %self.peer_display_name(&from_peer),
|
||||
path_mtu = response.path_mtu,
|
||||
"Reverse-path forwarding LookupResponse"
|
||||
);
|
||||
|
||||
let encoded = response.encode();
|
||||
if let Err(e) = self.send_encrypted_link_message(&from_peer, &encoded).await {
|
||||
debug!(
|
||||
next_hop = %self.peer_display_name(&from_peer),
|
||||
error = %e,
|
||||
"Failed to forward LookupResponse"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// We originated this request — verify proof before caching
|
||||
let target = response.target;
|
||||
let path_mtu = response.path_mtu;
|
||||
|
||||
// Look up the target's public key from identity_cache
|
||||
let mut prefix = [0u8; 15];
|
||||
prefix.copy_from_slice(&target.as_bytes()[0..15]);
|
||||
let target_pubkey = match self.lookup_by_fips_prefix(&prefix) {
|
||||
Some((_addr, pubkey)) => pubkey,
|
||||
None => {
|
||||
self.metrics()
|
||||
.discovery
|
||||
.record_reject(DiscoveryReject::RespIdentityMiss);
|
||||
warn!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&target),
|
||||
"identity_cache miss for lookup target, cannot verify proof"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Verify the proof signature
|
||||
let (xonly, _parity) = target_pubkey.x_only_public_key();
|
||||
let peer_id = PeerIdentity::from_pubkey(xonly);
|
||||
let proof_data =
|
||||
LookupResponse::proof_bytes(response.request_id, &target, &response.target_coords);
|
||||
if !peer_id.verify(&proof_data, &response.proof) {
|
||||
self.metrics()
|
||||
.discovery
|
||||
.record_reject(DiscoveryReject::RespProofFailed);
|
||||
warn!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&target),
|
||||
"LookupResponse proof verification failed, discarding"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
self.metrics().discovery.resp_accepted.inc();
|
||||
|
||||
// Clear backoff on success — target is reachable
|
||||
self.discovery_backoff.record_success(&target);
|
||||
|
||||
info!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&target),
|
||||
depth = response.target_coords.depth(),
|
||||
path_mtu = path_mtu,
|
||||
"Discovery succeeded, proof verified, route cached"
|
||||
);
|
||||
|
||||
self.coord_cache
|
||||
.insert_with_path_mtu(target, response.target_coords, now_ms, path_mtu);
|
||||
|
||||
// Mirror path_mtu into the FipsAddress-keyed read-only lookup
|
||||
// map used by the TUN reader/writer at TCP MSS clamp time.
|
||||
let fips_addr = crate::FipsAddress::from_node_addr(&target);
|
||||
match self.path_mtu_lookup.write() {
|
||||
Ok(mut map) => {
|
||||
let prior = map.insert(fips_addr, path_mtu);
|
||||
debug!(
|
||||
target = %self.peer_display_name(&target),
|
||||
fips_addr = %fips_addr,
|
||||
path_mtu = path_mtu,
|
||||
prior = ?prior,
|
||||
map_len = map.len(),
|
||||
"Wrote path_mtu_lookup from discovery LookupResponse"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target = %self.peer_display_name(&target),
|
||||
fips_addr = %fips_addr,
|
||||
path_mtu = path_mtu,
|
||||
error = %e,
|
||||
"path_mtu_lookup write lock poisoned; clamp will not see this update"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up pending lookup tracking
|
||||
self.pending_lookups.remove(&target);
|
||||
|
||||
// If an established session exists, reset the warmup counter.
|
||||
let n = self.config().node.session.coords_warmup_packets;
|
||||
if let Some(entry) = self.sessions.get_mut(&target)
|
||||
&& entry.is_established()
|
||||
{
|
||||
entry.set_coords_warmup_remaining(n);
|
||||
debug!(
|
||||
dest = %self.peer_display_name(&target),
|
||||
warmup_packets = n,
|
||||
"Reset coords warmup after discovery for existing session"
|
||||
);
|
||||
}
|
||||
|
||||
// If we have pending TUN packets for this target, retry session
|
||||
// initiation. The coord_cache now has coords, so find_next_hop()
|
||||
// should succeed.
|
||||
if let Some(packets) = self.pending_tun_packets.get(&target) {
|
||||
debug!(
|
||||
dest = %self.peer_display_name(&target),
|
||||
queued_packets = packets.len(),
|
||||
"Retrying queued packets after discovery"
|
||||
);
|
||||
self.retry_session_after_discovery(target).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate and send a LookupResponse when we are the target.
|
||||
async fn send_lookup_response(&mut self, request: &LookupRequest) {
|
||||
let our_coords = self.tree_state().my_coords().clone();
|
||||
|
||||
// Sign proof: Identity::sign hashes with SHA-256 internally
|
||||
let proof_data =
|
||||
LookupResponse::proof_bytes(request.request_id, &request.target, &our_coords);
|
||||
let proof = self.identity().sign(&proof_data);
|
||||
|
||||
let mut response =
|
||||
LookupResponse::new(request.request_id, request.target, our_coords, proof);
|
||||
|
||||
// Route toward origin via reverse path.
|
||||
let next_hop_addr = if let Some(recent) = self.recent_requests.get(&request.request_id) {
|
||||
recent.from_peer
|
||||
} else {
|
||||
// Fallback: try greedy tree routing toward origin
|
||||
match self.find_next_hop(&request.origin) {
|
||||
Some(peer) => *peer.node_addr(),
|
||||
None => {
|
||||
debug!(
|
||||
origin = %self.peer_display_name(&request.origin),
|
||||
"Cannot route LookupResponse: no reverse path or tree route to origin"
|
||||
);
|
||||
self.metrics()
|
||||
.discovery
|
||||
.record_reject(DiscoveryReject::RespNoRoute);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Fold our outgoing-link MTU into path_mtu so the target-edge link
|
||||
// appears in the bottleneck calculation. Without this, the response
|
||||
// leaves the target with path_mtu = u16::MAX and only intermediate
|
||||
// transits min-fold; the target's first reverse-path hop is missed.
|
||||
self.apply_outgoing_link_mtu_to_response(&mut response, &next_hop_addr);
|
||||
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
origin = %self.peer_display_name(&request.origin),
|
||||
next_hop = %self.peer_display_name(&next_hop_addr),
|
||||
path_mtu = response.path_mtu,
|
||||
"Sending LookupResponse"
|
||||
);
|
||||
|
||||
let encoded = response.encode();
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message(&next_hop_addr, &encoded)
|
||||
.await
|
||||
{
|
||||
debug!(
|
||||
next_hop = %self.peer_display_name(&next_hop_addr),
|
||||
error = %e,
|
||||
"Failed to send LookupResponse"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Forward a LookupRequest to eligible peers.
|
||||
///
|
||||
/// Primary path: tree peers (parent + children) whose bloom filter
|
||||
/// contains the target. Restricting to tree peers follows the spanning
|
||||
/// tree partition, producing a single directed path.
|
||||
///
|
||||
/// Fallback: if no tree peer's bloom matches, try non-tree peers whose
|
||||
/// bloom contains the target. This recovers from dead ends caused by
|
||||
/// stale bloom filters, tree restructuring, or transit node failures.
|
||||
async fn forward_lookup_request(&mut self, mut request: LookupRequest) {
|
||||
if !request.forward() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Collect tree peers whose bloom filter contains the target
|
||||
let forward_to: Vec<NodeAddr> = self
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|(addr, peer)| self.is_tree_peer(addr) && peer.may_reach(&request.target))
|
||||
.map(|(addr, _)| *addr)
|
||||
.collect();
|
||||
|
||||
// Fallback: if no tree peer matches, try non-tree bloom-matching peers
|
||||
let (forward_to, used_fallback) = if forward_to.is_empty() {
|
||||
let fallback: Vec<NodeAddr> = self
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|(addr, peer)| !self.is_tree_peer(addr) && peer.may_reach(&request.target))
|
||||
.map(|(addr, _)| *addr)
|
||||
.collect();
|
||||
if fallback.is_empty() {
|
||||
self.metrics().discovery.req_no_tree_peer.inc();
|
||||
trace!(
|
||||
request_id = request.request_id,
|
||||
"No eligible peers to forward LookupRequest"
|
||||
);
|
||||
return;
|
||||
}
|
||||
(fallback, true)
|
||||
} else {
|
||||
(forward_to, false)
|
||||
};
|
||||
|
||||
if used_fallback {
|
||||
self.metrics().discovery.req_fallback_forwarded.inc();
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(&request.target),
|
||||
ttl = request.ttl,
|
||||
peer_count = forward_to.len(),
|
||||
"Forwarding LookupRequest via non-tree fallback"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(&request.target),
|
||||
ttl = request.ttl,
|
||||
peer_count = forward_to.len(),
|
||||
"Forwarding LookupRequest"
|
||||
);
|
||||
}
|
||||
|
||||
let encoded = request.encode();
|
||||
|
||||
for peer_addr in forward_to {
|
||||
if let Err(e) = self.send_encrypted_link_message(&peer_addr, &encoded).await {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&peer_addr),
|
||||
error = %e,
|
||||
"Failed to forward LookupRequest to peer"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Initiate a discovery lookup for a target node.
|
||||
///
|
||||
/// Creates a LookupRequest and sends it to tree peers whose bloom
|
||||
/// filters contain the target. Returns the number of peers sent to.
|
||||
/// The originator does NOT record the request_id in recent_requests,
|
||||
/// so when the response arrives, it's recognized as "our request".
|
||||
pub(in crate::node) async fn initiate_lookup(&mut self, target: &NodeAddr, ttl: u8) -> usize {
|
||||
self.metrics().discovery.req_initiated.inc();
|
||||
|
||||
let origin = *self.node_addr();
|
||||
let origin_coords = self.tree_state().my_coords().clone();
|
||||
let request = LookupRequest::generate(*target, origin, origin_coords, ttl, 0);
|
||||
|
||||
// Send only to tree peers whose bloom filter contains the target
|
||||
let peer_addrs: Vec<NodeAddr> = self
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|(addr, peer)| self.is_tree_peer(addr) && peer.may_reach(target))
|
||||
.map(|(addr, _)| *addr)
|
||||
.collect();
|
||||
|
||||
let peer_count = peer_addrs.len();
|
||||
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(target),
|
||||
ttl = ttl,
|
||||
peer_count = peer_count,
|
||||
total_peers = self.peers.len(),
|
||||
"Discovery lookup initiated"
|
||||
);
|
||||
|
||||
if peer_count == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let encoded = request.encode();
|
||||
|
||||
for peer_addr in peer_addrs {
|
||||
if let Err(e) = self.send_encrypted_link_message(&peer_addr, &encoded).await {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&peer_addr),
|
||||
error = %e,
|
||||
"Failed to send LookupRequest to peer"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
peer_count
|
||||
}
|
||||
|
||||
/// Initiate a discovery lookup if one is not already pending for this target.
|
||||
///
|
||||
/// Checks: pending dedup, post-failure backoff (off by default), bloom
|
||||
/// filter pre-check. If all pass, sends the first attempt's LookupRequest.
|
||||
/// Subsequent attempts (with fresh request_ids) are scheduled by
|
||||
/// [`Self::check_pending_lookups`] when each attempt's per-attempt timeout
|
||||
/// expires, using the sequence in `node.discovery.attempt_timeouts_secs`.
|
||||
pub(in crate::node) async fn maybe_initiate_lookup(&mut self, dest: &NodeAddr) {
|
||||
let now_ms = Self::now_ms();
|
||||
|
||||
// Dedup: any pending lookup means we are already trying.
|
||||
if self.pending_lookups.contains_key(dest) {
|
||||
self.metrics().discovery.req_deduplicated.inc();
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
"Discovery lookup deduplicated, already pending"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Optional post-failure suppression. Defaults are 0/0 (inert);
|
||||
// operators can opt in by setting `node.discovery.backoff_*_secs`.
|
||||
if self.discovery_backoff.is_suppressed(dest) {
|
||||
self.metrics().discovery.req_backoff_suppressed.inc();
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
failures = self.discovery_backoff.failure_count(dest),
|
||||
"Discovery lookup suppressed by backoff"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Bloom filter pre-check: if no peer's filter contains the target,
|
||||
// it's not in the mesh — skip the lookup and record as failure.
|
||||
let reachable = self.peers.values().any(|peer| peer.may_reach(dest));
|
||||
if !reachable {
|
||||
self.metrics().discovery.req_bloom_miss.inc();
|
||||
self.discovery_backoff.record_failure(dest);
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
"Discovery skipped, target not in any peer bloom filter"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
self.pending_lookups
|
||||
.insert(*dest, PendingLookup::new(now_ms));
|
||||
let ttl = self.config().node.discovery.ttl;
|
||||
let sent = self.initiate_lookup(dest, ttl).await;
|
||||
|
||||
// If no tree peers had the target, fail immediately
|
||||
if sent == 0 {
|
||||
self.pending_lookups.remove(dest);
|
||||
self.discovery_backoff.record_failure(dest);
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
"Discovery failed, no tree peers with bloom match"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Check pending lookups for next-attempt or final timeout.
|
||||
///
|
||||
/// Called periodically from the tick handler. The lookup state machine
|
||||
/// runs through `node.discovery.attempt_timeouts_secs` (default
|
||||
/// `[1, 2, 4, 8]`): each entry is the deadline for one attempt. When the
|
||||
/// current attempt's deadline elapses:
|
||||
/// - If more entries remain: send the next attempt with a fresh
|
||||
/// `request_id`.
|
||||
/// - Otherwise: declare the destination unreachable, drop queued packets,
|
||||
/// and emit ICMPv6 destination-unreachable for each.
|
||||
pub(in crate::node) async fn check_pending_lookups(&mut self, now_ms: u64) {
|
||||
let timeouts = self.config().node.discovery.attempt_timeouts_secs.clone();
|
||||
let max_attempts = timeouts.len() as u8;
|
||||
|
||||
// Collect targets needing action
|
||||
let mut to_retry: Vec<NodeAddr> = Vec::new();
|
||||
let mut to_timeout: Vec<NodeAddr> = Vec::new();
|
||||
|
||||
for (&target, entry) in &self.pending_lookups {
|
||||
let attempt_idx = (entry.attempt as usize).saturating_sub(1);
|
||||
let attempt_timeout_ms = timeouts.get(attempt_idx).copied().unwrap_or(0) * 1000;
|
||||
if now_ms.saturating_sub(entry.last_sent_ms) >= attempt_timeout_ms {
|
||||
if entry.attempt >= max_attempts {
|
||||
to_timeout.push(target);
|
||||
} else {
|
||||
to_retry.push(target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process retries
|
||||
for target in to_retry {
|
||||
if let Some(entry) = self.pending_lookups.get_mut(&target) {
|
||||
entry.attempt += 1;
|
||||
entry.last_sent_ms = now_ms;
|
||||
let attempt = entry.attempt;
|
||||
|
||||
let ttl = self.config().node.discovery.ttl;
|
||||
let sent = self.initiate_lookup(&target, ttl).await;
|
||||
if sent > 0 {
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(&target),
|
||||
attempt = attempt,
|
||||
"Discovery retry sent"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process timeouts
|
||||
for addr in to_timeout {
|
||||
self.metrics().discovery.resp_timed_out.inc();
|
||||
self.pending_lookups.remove(&addr);
|
||||
|
||||
// Record failure for optional backoff
|
||||
self.discovery_backoff.record_failure(&addr);
|
||||
let failures = self.discovery_backoff.failure_count(&addr);
|
||||
|
||||
let queued = self.pending_tun_packets.remove(&addr);
|
||||
let pkt_count = queued.as_ref().map_or(0, |p| p.len());
|
||||
info!(
|
||||
target_node = %self.peer_display_name(&addr),
|
||||
queued_packets = pkt_count,
|
||||
failures = failures,
|
||||
"Discovery lookup timed out, destination unreachable"
|
||||
);
|
||||
if let Some(packets) = queued {
|
||||
for pkt in &packets {
|
||||
self.send_icmpv6_dest_unreachable(pkt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset discovery backoff on topology changes.
|
||||
pub(in crate::node) fn reset_discovery_backoff(&mut self) {
|
||||
if !self.discovery_backoff.is_empty() {
|
||||
debug!(
|
||||
entries = self.discovery_backoff.entry_count(),
|
||||
"Resetting discovery backoff on topology change"
|
||||
);
|
||||
self.discovery_backoff.reset_all();
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove expired entries from the recent_requests cache.
|
||||
fn purge_expired_requests(&mut self, current_time_ms: u64) {
|
||||
let expiry_ms = self.config().node.discovery.recent_expiry_secs * 1000;
|
||||
self.recent_requests
|
||||
.retain(|_, entry| !entry.is_expired(current_time_ms, expiry_ms));
|
||||
}
|
||||
|
||||
/// Min-fold our outgoing-link MTU into a LookupResponse's `path_mtu`.
|
||||
///
|
||||
/// Used at both transit-side reverse-path forward and at the target's
|
||||
/// own send_lookup_response. The link MTU we apply is the MTU of the
|
||||
/// transport+addr we'll use to deliver the response toward `next_hop`.
|
||||
/// No-op when `next_hop` is not a directly-connected peer or its
|
||||
/// transport is not registered.
|
||||
pub(in crate::node) fn apply_outgoing_link_mtu_to_response(
|
||||
&self,
|
||||
response: &mut LookupResponse,
|
||||
next_hop: &NodeAddr,
|
||||
) {
|
||||
if let Some(peer) = self.peers.get(next_hop)
|
||||
&& let Some(tid) = peer.transport_id()
|
||||
&& let Some(transport) = self.transports.get(&tid)
|
||||
{
|
||||
let link_mtu = if let Some(addr) = peer.current_addr() {
|
||||
transport.link_mtu(addr)
|
||||
} else {
|
||||
transport.mtu()
|
||||
};
|
||||
response.path_mtu = response.path_mtu.min(link_mtu);
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed `path_mtu_lookup` for a directly-connected peer.
|
||||
///
|
||||
/// Called when an FMP link-layer peer is promoted to active. The seed
|
||||
/// value is the local outgoing-link MTU on the peer's transport, which
|
||||
/// is the actual link constraint for direct-link traffic. Stored only
|
||||
/// when no tighter value exists: discovery's reverse-path bottleneck
|
||||
/// or MMP `MtuExceeded` reactive learning take precedence when smaller.
|
||||
///
|
||||
/// Without this seed, configured/auto-connect peers (which establish
|
||||
/// sessions without going through the discovery Lookup flow) leave
|
||||
/// `path_mtu_lookup` empty for their FipsAddress, causing
|
||||
/// `per_flow_max_mss` to fall back to the global ceiling and the
|
||||
/// SYN-time TCP MSS clamp to over-estimate the effective path.
|
||||
pub(in crate::node) fn seed_path_mtu_for_link_peer(
|
||||
&self,
|
||||
peer_addr: &NodeAddr,
|
||||
transport_id: TransportId,
|
||||
addr: &TransportAddr,
|
||||
) {
|
||||
let Some(transport) = self.transports.get(&transport_id) else {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(peer_addr),
|
||||
transport_id = %transport_id,
|
||||
"seed_path_mtu_for_link_peer: transport not registered, skipping seed"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let link_mtu = transport.link_mtu(addr);
|
||||
let fips_addr = crate::FipsAddress::from_node_addr(peer_addr);
|
||||
let Ok(mut map) = self.path_mtu_lookup.write() else {
|
||||
warn!(
|
||||
peer = %self.peer_display_name(peer_addr),
|
||||
"seed_path_mtu_for_link_peer: path_mtu_lookup write lock poisoned"
|
||||
);
|
||||
return;
|
||||
};
|
||||
match map.get(&fips_addr).copied() {
|
||||
Some(existing) if existing <= link_mtu => {
|
||||
// Keep the tighter learned value; never loosen the clamp.
|
||||
debug!(
|
||||
peer = %self.peer_display_name(peer_addr),
|
||||
fips_addr = %fips_addr,
|
||||
link_mtu = link_mtu,
|
||||
existing = existing,
|
||||
"seed_path_mtu_for_link_peer: keeping tighter existing value"
|
||||
);
|
||||
}
|
||||
other => {
|
||||
map.insert(fips_addr, link_mtu);
|
||||
debug!(
|
||||
peer = %self.peer_display_name(peer_addr),
|
||||
fips_addr = %fips_addr,
|
||||
link_mtu = link_mtu,
|
||||
prior = ?other,
|
||||
map_len = map.len(),
|
||||
"seed_path_mtu_for_link_peer: wrote link MTU"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks a pending discovery lookup with retry state.
|
||||
pub struct PendingLookup {
|
||||
/// When the lookup was first initiated.
|
||||
pub initiated_ms: u64,
|
||||
/// When the last attempt was sent.
|
||||
pub last_sent_ms: u64,
|
||||
/// Current attempt number (1 = initial, 2 = first retry, ...).
|
||||
pub attempt: u8,
|
||||
}
|
||||
|
||||
impl PendingLookup {
|
||||
pub fn new(now_ms: u64) -> Self {
|
||||
Self {
|
||||
initiated_ms: now_ms,
|
||||
last_sent_ms: now_ms,
|
||||
attempt: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -73,7 +73,7 @@ impl Node {
|
||||
/// entries — other removal paths (link-dead, decrypt failure, peer
|
||||
/// restart) all schedule reconnect.
|
||||
pub(in crate::node) fn handle_disconnect(&mut self, from: &NodeAddr, payload: &[u8]) {
|
||||
let disconnect = match crate::proto::fmp::Disconnect::decode(payload) {
|
||||
let disconnect = match crate::protocol::Disconnect::decode(payload) {
|
||||
Ok(msg) => msg,
|
||||
Err(e) => {
|
||||
debug!(from = %self.peer_display_name(from), error = %e, "Malformed disconnect message");
|
||||
@@ -93,7 +93,7 @@ impl Node {
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
self.note_link_dead(addr, now_ms);
|
||||
self.schedule_reconnect(addr, now_ms);
|
||||
}
|
||||
|
||||
/// Remove an active peer and clean up all associated state.
|
||||
@@ -187,15 +187,6 @@ impl Node {
|
||||
|
||||
// Remove link and address mapping
|
||||
self.remove_link(&link_id);
|
||||
// Bound `peer_machines`: drop this peer's machine
|
||||
// entry, keyed by the `link_id` derived above BEFORE the `peers` removal.
|
||||
// This cleans up the OLD peer's machine on an inbound restart and prevents
|
||||
// unbounded growth on the establish success path. NEUTRAL: nothing on the
|
||||
// live path reads `peer_machines` except the establish executor, which only
|
||||
// ever touches the in-flight establish's (distinct) `link_id`; no reader
|
||||
// depends on a stale entry, so removal changes no behavior — it only bounds
|
||||
// the map.
|
||||
self.remove_peer_machine(link_id);
|
||||
if let Some(transport_id) = transport_id {
|
||||
self.cleanup_bootstrap_transport_if_unused(transport_id);
|
||||
}
|
||||
@@ -1,11 +1,10 @@
|
||||
//! Encrypted frame handling (hot path).
|
||||
|
||||
use crate::node::Node;
|
||||
use crate::node::wire::{EncryptedHeader, FLAG_CE, FLAG_KEY_EPOCH, FLAG_SP, strip_inner_header};
|
||||
use crate::noise::NoiseError;
|
||||
use crate::proto::fmp::wire::{
|
||||
EncryptedHeader, FLAG_CE, FLAG_KEY_EPOCH, FLAG_SP, strip_inner_header,
|
||||
};
|
||||
use crate::transport::ReceivedPacket;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
/// Force-remove a peer after this many consecutive decryption failures.
|
||||
@@ -172,7 +171,7 @@ impl Node {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let cache_key = (packet.transport_id, header.receiver_idx.as_u32());
|
||||
if let Some(workers) = self.supervisor.decrypt_workers.as_ref().cloned()
|
||||
if let Some(workers) = self.decrypt_workers.as_ref().cloned()
|
||||
&& self.decrypt_registered_sessions.contains(&cache_key)
|
||||
{
|
||||
let job = crate::node::decrypt_worker::DecryptJob {
|
||||
@@ -260,7 +259,7 @@ impl Node {
|
||||
};
|
||||
|
||||
// MMP per-frame processing and statistics
|
||||
let now_ms = crate::time::mono_ms();
|
||||
let now = Instant::now();
|
||||
let ce_flag = header.flags & FLAG_CE != 0;
|
||||
let sp_flag = header.flags & FLAG_SP != 0;
|
||||
|
||||
@@ -271,9 +270,9 @@ impl Node {
|
||||
timestamp,
|
||||
packet.data.len(),
|
||||
ce_flag,
|
||||
now_ms,
|
||||
now,
|
||||
);
|
||||
let _spin_rtt = mmp.spin_bit.rx_observe(sp_flag, header.counter, now_ms);
|
||||
let _spin_rtt = mmp.spin_bit.rx_observe(sp_flag, header.counter, now);
|
||||
}
|
||||
peer.set_current_addr(packet.transport_id, packet.remote_addr.clone());
|
||||
peer.link_stats_mut()
|
||||
@@ -356,7 +355,7 @@ impl Node {
|
||||
} else {
|
||||
return;
|
||||
};
|
||||
let now_ms = crate::time::mono_ms();
|
||||
let now = Instant::now();
|
||||
let mut address_changed = false;
|
||||
if let Some(peer) = self.peers.get_mut(node_addr) {
|
||||
peer.reset_decrypt_failures();
|
||||
@@ -366,8 +365,8 @@ impl Node {
|
||||
peer.touch(packet_timestamp_ms);
|
||||
if let Some(mmp) = peer.mmp_mut() {
|
||||
mmp.receiver
|
||||
.record_recv(fmp_counter, inner_ts, packet_len, ce_flag, now_ms);
|
||||
let _spin_rtt = mmp.spin_bit.rx_observe(sp_flag, fmp_counter, now_ms);
|
||||
.record_recv(fmp_counter, inner_ts, packet_len, ce_flag, now);
|
||||
let _spin_rtt = mmp.spin_bit.rx_observe(sp_flag, fmp_counter, now);
|
||||
}
|
||||
}
|
||||
// Address rotation invalidates the per-peer connect()-ed UDP
|
||||
@@ -451,7 +450,7 @@ impl Node {
|
||||
/// black-hole the session.
|
||||
#[cfg(unix)]
|
||||
pub(in crate::node) fn register_decrypt_worker_session(&mut self, node_addr: &crate::NodeAddr) {
|
||||
let Some(workers) = self.supervisor.decrypt_workers.as_ref().cloned() else {
|
||||
let Some(workers) = self.decrypt_workers.as_ref().cloned() else {
|
||||
return;
|
||||
};
|
||||
let (cache_key, state) = {
|
||||
@@ -492,7 +491,7 @@ impl Node {
|
||||
&mut self,
|
||||
cache_key: (crate::transport::TransportId, u32),
|
||||
) {
|
||||
if let Some(workers) = self.supervisor.decrypt_workers.as_ref() {
|
||||
if let Some(workers) = self.decrypt_workers.as_ref() {
|
||||
workers.unregister_session(cache_key);
|
||||
}
|
||||
self.decrypt_registered_sessions.remove(&cache_key);
|
||||
@@ -534,7 +533,7 @@ impl Node {
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
self.note_link_dead(addr, now_ms);
|
||||
self.schedule_reconnect(addr, now_ms);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,21 @@
|
||||
//! SessionDatagram forwarding handler.
|
||||
//!
|
||||
//! Handles incoming SessionDatagram (0x00) link messages: decodes the
|
||||
//! envelope, performs coordinate cache warming from plaintext session-layer
|
||||
//! headers, pre-resolves the next hop for forwardable transit datagrams, and
|
||||
//! drives the routing core's outcome — local delivery when the datagram is
|
||||
//! addressed to this node, the transit hop-limit drop, the forward, or the
|
||||
//! error signal generated on routing failure.
|
||||
//! envelope, enforces hop limits, performs coordinate cache warming from
|
||||
//! plaintext session-layer headers, routes to the next hop or delivers
|
||||
//! locally, and generates error signals on routing failure.
|
||||
|
||||
use crate::NodeAddr;
|
||||
use crate::node::reject::ForwardingReject;
|
||||
use crate::node::{Node, NodeError, NodeRoutingView};
|
||||
use crate::proto::fsp::wire::{
|
||||
use crate::node::session_wire::{
|
||||
FSP_COMMON_PREFIX_SIZE, FSP_HEADER_SIZE, FSP_PHASE_ESTABLISHED, FSP_PHASE_MSG1, FSP_PHASE_MSG2,
|
||||
FspCommonPrefix, parse_encrypted_coords,
|
||||
};
|
||||
use crate::proto::fsp::{SessionAck, SessionSetup};
|
||||
use crate::proto::link::{SessionDatagram, SessionDatagramRef};
|
||||
use crate::proto::routing::{DropReason, NextHop, RouteAction, RouteOutcome};
|
||||
use crate::node::{Node, NodeError};
|
||||
use crate::protocol::{
|
||||
CoordsRequired, MtuExceeded, PathBroken, SessionAck, SessionDatagram, SessionDatagramRef,
|
||||
SessionSetup,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
@@ -44,169 +43,123 @@ impl Node {
|
||||
}
|
||||
};
|
||||
|
||||
let my_addr = *self.node_addr();
|
||||
// TTL enforcement: decrement for forwarding and drop only if the
|
||||
// received datagram was already exhausted.
|
||||
if datagram_ref.ttl == 0 {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
|
||||
debug!(
|
||||
src = %datagram_ref.src_addr,
|
||||
dest = %datagram_ref.dest_addr,
|
||||
"SessionDatagram TTL exhausted, dropping"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let forwarded_ttl = datagram_ref.ttl - 1;
|
||||
|
||||
// Coordinate cache warming from plaintext session-layer headers. Runs
|
||||
// ahead of both the delivery and the TTL decisions the core makes: the
|
||||
// coords a peer put on the wire are equally valid whichever way those
|
||||
// go, and the only arrivals this newly warms from are those with an
|
||||
// exhausted TTL, whose every insert is already achievable at TTL 1.
|
||||
// Coordinate cache warming from plaintext session-layer headers
|
||||
self.try_warm_coord_cache_ref(&datagram_ref);
|
||||
|
||||
// Pre-resolve the next hop only for datagrams the core can actually
|
||||
// forward: not locally destined, and carrying a TTL that survives the
|
||||
// decrement (`ttl > 1` — the shell-side mirror of the core's
|
||||
// would-leave-zero drop). This keeps `find_next_hop`'s coord-cache
|
||||
// LRU-touch side effect scoped to genuine forwards, as it was when the
|
||||
// TTL test ran inline ahead of it. Warming above has already run, so
|
||||
// the resolution observes freshly cached coords.
|
||||
let next_hop = if datagram_ref.dest_addr != my_addr && datagram_ref.ttl > 1 {
|
||||
self.resolve_next_hop(&datagram_ref.dest_addr)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Local delivery: dispatch to session layer handlers without
|
||||
// materializing an owned SessionDatagram payload Vec.
|
||||
if datagram_ref.dest_addr == *self.node_addr() {
|
||||
self.metrics().forwarding.record_delivered(payload.len());
|
||||
self.handle_session_payload(
|
||||
&datagram_ref.src_addr,
|
||||
datagram_ref.payload,
|
||||
datagram_ref.path_mtu,
|
||||
incoming_ce,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
// Read local congestion once and reuse it for both the CE decision
|
||||
// (via the view) and the congestion metric/log below, keeping
|
||||
// `detect_congestion` the single source of truth.
|
||||
let congested = next_hop
|
||||
.as_ref()
|
||||
.map(|nh| self.detect_congestion(&nh.addr))
|
||||
.unwrap_or(false);
|
||||
let mut datagram = datagram_ref.into_owned();
|
||||
datagram.ttl = forwarded_ttl;
|
||||
|
||||
// Borrow the routing tables disjointly from `&mut self.routing` for
|
||||
// the pure decision, then release both before driving the outcome.
|
||||
let outcome = {
|
||||
let view = NodeRoutingView {
|
||||
coord_cache: &self.coord_cache,
|
||||
peers: &self.peers,
|
||||
tree_state: &self.tree_state,
|
||||
congested,
|
||||
};
|
||||
self.routing
|
||||
.route(&datagram_ref, &my_addr, incoming_ce, next_hop, &view)
|
||||
};
|
||||
|
||||
match outcome {
|
||||
RouteOutcome::Drop {
|
||||
reason: DropReason::TtlExhausted,
|
||||
} => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
|
||||
debug!(
|
||||
src = %datagram_ref.src_addr,
|
||||
dest = %datagram_ref.dest_addr,
|
||||
ttl = datagram_ref.ttl,
|
||||
"SessionDatagram TTL exhausted, dropping"
|
||||
);
|
||||
}
|
||||
RouteOutcome::DeliverLocal => {
|
||||
// Local delivery: dispatch to session layer handlers without
|
||||
// materializing an owned SessionDatagram payload Vec.
|
||||
self.metrics().forwarding.record_delivered(payload.len());
|
||||
self.handle_session_payload(
|
||||
&datagram_ref.src_addr,
|
||||
datagram_ref.payload,
|
||||
datagram_ref.path_mtu,
|
||||
incoming_ce,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
RouteOutcome::NoRoute => {
|
||||
// Find next hop toward destination
|
||||
let next_hop_addr = match self.find_next_hop(&datagram.dest_addr) {
|
||||
Some(peer) => *peer.node_addr(),
|
||||
None => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::NoRoute, payload.len());
|
||||
let original = datagram_ref.into_owned();
|
||||
debug!(
|
||||
src = %self.peer_display_name(&original.src_addr),
|
||||
dest = %self.peer_display_name(&original.dest_addr),
|
||||
src = %self.peer_display_name(&datagram.src_addr),
|
||||
dest = %self.peer_display_name(&datagram.dest_addr),
|
||||
bytes = payload.len(),
|
||||
"Dropping transit SessionDatagram: no route to destination"
|
||||
);
|
||||
self.send_routing_error(&original).await;
|
||||
self.send_routing_error(&datagram).await;
|
||||
return;
|
||||
}
|
||||
RouteOutcome::Forward {
|
||||
next_hop,
|
||||
bytes,
|
||||
outgoing_ce,
|
||||
} => {
|
||||
let dest = datagram_ref.dest_addr;
|
||||
};
|
||||
|
||||
// ECN CE relay: congestion was detected locally above; emit the
|
||||
// metric and rate-limited log at the transit chokepoint.
|
||||
if congested {
|
||||
self.metrics().congestion.congestion_detected.inc();
|
||||
let now = Instant::now();
|
||||
let should_log = self
|
||||
.last_congestion_log
|
||||
.map(|t| now.duration_since(t) >= Duration::from_secs(5))
|
||||
.unwrap_or(true);
|
||||
if should_log {
|
||||
self.last_congestion_log = Some(now);
|
||||
debug!(next_hop = %next_hop, "Congestion detected, CE flag set on forwarded packet");
|
||||
}
|
||||
}
|
||||
|
||||
match self
|
||||
.send_encrypted_link_message_with_ce(&next_hop, &bytes, outgoing_ce)
|
||||
.await
|
||||
{
|
||||
Err(NodeError::MtuExceeded { mtu, .. }) => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::MtuExceeded, payload.len());
|
||||
self.send_mtu_exceeded_error(dest, datagram_ref.src_addr, mtu)
|
||||
.await;
|
||||
}
|
||||
Err(e) => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::SendError, payload.len());
|
||||
debug!(
|
||||
next_hop = %next_hop,
|
||||
dest = %dest,
|
||||
error = %e,
|
||||
"Failed to forward SessionDatagram"
|
||||
);
|
||||
}
|
||||
Ok(()) => {
|
||||
self.metrics().forwarding.record_forwarded(bytes.len());
|
||||
// Classify this transit forward by route class (partition
|
||||
// of forwarded_packets). Done here, at the data-plane
|
||||
// chokepoint, so the error-signal routing callers of
|
||||
// find_next_hop are excluded.
|
||||
let class = self.classify_forward(&dest, &next_hop);
|
||||
self.metrics().forwarding.record_route_class(class);
|
||||
if outgoing_ce {
|
||||
self.metrics().congestion.ce_forwarded.inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the next hop toward `dest` into its address plus the outgoing
|
||||
/// link's transport MTU. Returns `None` when there is no route.
|
||||
///
|
||||
/// The MTU defaults to `u16::MAX` (a no-op min-fold) when the peer's
|
||||
/// transport is not resolvable, matching the pre-refactor inline behavior
|
||||
/// where the MTU `if let` chain simply did not fire.
|
||||
fn resolve_next_hop(&mut self, dest: &NodeAddr) -> Option<NextHop> {
|
||||
let addr = *self.find_next_hop(dest)?.node_addr();
|
||||
let link_mtu = if let Some(peer) = self.peers.get(&addr)
|
||||
// Apply path_mtu min() from the outgoing link's transport MTU
|
||||
if let Some(peer) = self.peers.get(&next_hop_addr)
|
||||
&& let Some(tid) = peer.transport_id()
|
||||
&& let Some(transport) = self.transports.get(&tid)
|
||||
{
|
||||
match peer.current_addr() {
|
||||
Some(link_addr) => transport.link_mtu(link_addr),
|
||||
None => transport.mtu(),
|
||||
if let Some(addr) = peer.current_addr() {
|
||||
datagram.path_mtu = datagram.path_mtu.min(transport.link_mtu(addr));
|
||||
} else {
|
||||
datagram.path_mtu = datagram.path_mtu.min(transport.mtu());
|
||||
}
|
||||
}
|
||||
|
||||
// ECN CE relay: propagate incoming CE and detect local congestion
|
||||
let local_congestion = self.detect_congestion(&next_hop_addr);
|
||||
let outgoing_ce = incoming_ce || local_congestion;
|
||||
if local_congestion {
|
||||
self.metrics().congestion.congestion_detected.inc();
|
||||
let now = Instant::now();
|
||||
let should_log = self
|
||||
.last_congestion_log
|
||||
.map(|t| now.duration_since(t) >= Duration::from_secs(5))
|
||||
.unwrap_or(true);
|
||||
if should_log {
|
||||
self.last_congestion_log = Some(now);
|
||||
debug!(next_hop = %next_hop_addr, "Congestion detected, CE flag set on forwarded packet");
|
||||
}
|
||||
}
|
||||
|
||||
// Forward: re-encode (includes 0x00 type byte) and send
|
||||
let encoded = datagram.encode();
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message_with_ce(&next_hop_addr, &encoded, outgoing_ce)
|
||||
.await
|
||||
{
|
||||
match e {
|
||||
NodeError::MtuExceeded { mtu, .. } => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::MtuExceeded, payload.len());
|
||||
self.send_mtu_exceeded_error(&datagram, mtu).await;
|
||||
}
|
||||
_ => {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::SendError, payload.len());
|
||||
debug!(
|
||||
next_hop = %next_hop_addr,
|
||||
dest = %datagram.dest_addr,
|
||||
error = %e,
|
||||
"Failed to forward SessionDatagram"
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
u16::MAX
|
||||
};
|
||||
Some(NextHop { addr, link_mtu })
|
||||
self.metrics().forwarding.record_forwarded(encoded.len());
|
||||
// Classify this transit forward by route class (partition of
|
||||
// forwarded_packets). Done here, at the data-plane chokepoint, so
|
||||
// the error-signal routing callers of find_next_hop are excluded.
|
||||
let class = self.classify_forward(&datagram.dest_addr, &next_hop_addr);
|
||||
self.metrics().forwarding.record_route_class(class);
|
||||
if outgoing_ce {
|
||||
self.metrics().congestion.ce_forwarded.inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Attempt to warm the coordinate cache from session-layer payload headers.
|
||||
@@ -307,41 +260,35 @@ impl Node {
|
||||
/// If we can't route the error back to the source either, drop silently.
|
||||
/// No cascading errors.
|
||||
async fn send_routing_error(&mut self, original: &SessionDatagram) {
|
||||
// Rate limit: one error signal per destination per 100ms
|
||||
if !self
|
||||
.routing_error_rate_limiter
|
||||
.should_send(&original.dest_addr)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let my_addr = *self.node_addr();
|
||||
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
let default_ttl = self.config().node.session.default_ttl;
|
||||
|
||||
// Pure decision: rate-limit gate + PathBroken/CoordsRequired choice +
|
||||
// error-PDU encode. Borrow the routing tables disjointly from
|
||||
// `&mut self.routing`, then release them before the reverse-hop lookup.
|
||||
let action = {
|
||||
let view = NodeRoutingView {
|
||||
coord_cache: &self.coord_cache,
|
||||
peers: &self.peers,
|
||||
tree_state: &self.tree_state,
|
||||
congested: false,
|
||||
let error_payload =
|
||||
if let Some(coords) = self.coord_cache().get(&original.dest_addr, now_ms) {
|
||||
let coords = coords.clone();
|
||||
PathBroken::new(original.dest_addr, my_addr)
|
||||
.with_last_coords(coords)
|
||||
.encode()
|
||||
} else {
|
||||
CoordsRequired::new(original.dest_addr, my_addr).encode()
|
||||
};
|
||||
self.routing.synth_routing_error(
|
||||
&original.dest_addr,
|
||||
&original.src_addr,
|
||||
&my_addr,
|
||||
&view,
|
||||
now_ms,
|
||||
default_ttl,
|
||||
)
|
||||
};
|
||||
let RouteAction::SendError { toward, bytes } = match action {
|
||||
Some(action) => action,
|
||||
// Rate limited: drop silently. No cascading errors.
|
||||
None => return,
|
||||
};
|
||||
|
||||
// Resolve the reverse link hop only now, after the gate passed, so
|
||||
// `find_next_hop`'s coord-cache touch keeps its pre-refactor scope.
|
||||
let next_hop_addr = match self.find_next_hop(&toward) {
|
||||
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
|
||||
.with_ttl(self.config().node.session.default_ttl);
|
||||
|
||||
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
|
||||
Some(peer) => *peer.node_addr(),
|
||||
None => {
|
||||
debug!(
|
||||
@@ -353,8 +300,9 @@ impl Node {
|
||||
}
|
||||
};
|
||||
|
||||
let encoded = error_dg.encode();
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message(&next_hop_addr, &bytes)
|
||||
.send_encrypted_link_message(&next_hop_addr, &encoded)
|
||||
.await
|
||||
{
|
||||
debug!(
|
||||
@@ -376,50 +324,37 @@ impl Node {
|
||||
/// Called when `send_encrypted_link_message()` fails with
|
||||
/// `NodeError::MtuExceeded` during forwarding. The signal tells the
|
||||
/// source the bottleneck MTU so it can immediately reduce its path MTU.
|
||||
///
|
||||
/// `dest` is the failed datagram's destination (rate-limit key); `toward`
|
||||
/// is its source, where the signal is routed back.
|
||||
async fn send_mtu_exceeded_error(
|
||||
&mut self,
|
||||
dest: NodeAddr,
|
||||
toward: NodeAddr,
|
||||
bottleneck_mtu: u16,
|
||||
) {
|
||||
async fn send_mtu_exceeded_error(&mut self, original: &SessionDatagram, bottleneck_mtu: u16) {
|
||||
// Rate limit: reuse routing_error_rate_limiter keyed on dest_addr
|
||||
if !self
|
||||
.routing_error_rate_limiter
|
||||
.should_send(&original.dest_addr)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let my_addr = *self.node_addr();
|
||||
let now_ms = Self::now_ms();
|
||||
let default_ttl = self.config().node.session.default_ttl;
|
||||
|
||||
// Pure decision: rate-limit gate + MtuExceeded PDU + encode.
|
||||
let action = self.routing.synth_mtu_exceeded(
|
||||
&dest,
|
||||
&toward,
|
||||
&my_addr,
|
||||
bottleneck_mtu,
|
||||
now_ms,
|
||||
default_ttl,
|
||||
);
|
||||
let RouteAction::SendError { toward, bytes } = match action {
|
||||
Some(action) => action,
|
||||
// Rate limited: drop silently. No cascading errors.
|
||||
None => return,
|
||||
};
|
||||
let error_payload = MtuExceeded::new(original.dest_addr, my_addr, bottleneck_mtu).encode();
|
||||
|
||||
// Resolve the reverse link hop only now, after the gate passed, so
|
||||
// `find_next_hop`'s coord-cache touch keeps its pre-refactor scope.
|
||||
let next_hop_addr = match self.find_next_hop(&toward) {
|
||||
let error_dg = SessionDatagram::new(my_addr, original.src_addr, error_payload)
|
||||
.with_ttl(self.config().node.session.default_ttl);
|
||||
|
||||
let next_hop_addr = match self.find_next_hop(&original.src_addr) {
|
||||
Some(peer) => *peer.node_addr(),
|
||||
None => {
|
||||
debug!(
|
||||
src = %toward,
|
||||
dest = %dest,
|
||||
src = %original.src_addr,
|
||||
dest = %original.dest_addr,
|
||||
"Cannot route MtuExceeded signal back to source, dropping"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let encoded = error_dg.encode();
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message(&next_hop_addr, &bytes)
|
||||
.send_encrypted_link_message(&next_hop_addr, &encoded)
|
||||
.await
|
||||
{
|
||||
debug!(
|
||||
@@ -429,8 +364,8 @@ impl Node {
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
original_dest = %dest,
|
||||
error_dest = %toward,
|
||||
original_dest = %original.dest_addr,
|
||||
error_dest = %original.src_addr,
|
||||
bottleneck_mtu,
|
||||
"Sent MtuExceeded error signal"
|
||||
);
|
||||
+565
-890
File diff suppressed because it is too large
Load Diff
@@ -1,739 +0,0 @@
|
||||
//! LookupRequest/LookupResponse mesh lookup protocol handlers.
|
||||
//!
|
||||
//! Handles coordinate lookup via bloom-filter-guided tree routing.
|
||||
//! Requests are forwarded only to tree peers (parent + children) whose
|
||||
//! bloom filter contains the target. TTL and request_id dedup provide
|
||||
//! safety bounds.
|
||||
|
||||
use crate::node::Node;
|
||||
use crate::node::reject::DiscoveryReject;
|
||||
use crate::proto::lookup::{
|
||||
LookupAction, LookupRequest, LookupResponse, MAX_RECENT_LOOKUP_REQUESTS,
|
||||
};
|
||||
use crate::transport::{TransportAddr, TransportId};
|
||||
use crate::{NodeAddr, PeerIdentity};
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
/// Shell adapter exposing the live routing tables to the sans-IO discovery
|
||||
/// core's `RoutingView` read seam. Lives in `node` so it can read `Node`'s
|
||||
/// private `peers` map and call the crate-private tree/bloom predicates.
|
||||
///
|
||||
/// Holding `&Node` whole is fine for the forward path because it does not
|
||||
/// also need `&mut self.lookup` concurrently. A later commit whose core
|
||||
/// step needs `&mut discovery` while reading routing state should narrow this
|
||||
/// to borrow only `peers` + `tree_state` instead of the whole node.
|
||||
struct NodeRoutingView<'a> {
|
||||
node: &'a Node,
|
||||
}
|
||||
|
||||
impl crate::proto::lookup::RoutingView for NodeRoutingView<'_> {
|
||||
fn is_tree_peer(&self, addr: &NodeAddr) -> bool {
|
||||
self.node.is_tree_peer(addr)
|
||||
}
|
||||
fn peers_reaching(&self, target: &NodeAddr) -> Vec<NodeAddr> {
|
||||
self.node
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|(_, peer)| peer.may_reach(target))
|
||||
.map(|(addr, _)| *addr)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Node {
|
||||
/// Handle an incoming LookupRequest from a peer.
|
||||
///
|
||||
/// Processing steps:
|
||||
/// 1. Decode and validate
|
||||
/// 2. Check request_id for duplicates (dedup / reverse-path routing)
|
||||
/// 3. Record request for reverse-path forwarding
|
||||
/// 4. Lazy purge expired entries
|
||||
/// 5. If we're the target, generate and send response
|
||||
/// 6. If TTL > 0, forward to tree peers whose bloom filter matches
|
||||
pub(in crate::node) async fn handle_lookup_request(&mut self, from: &NodeAddr, payload: &[u8]) {
|
||||
self.metrics().lookup.req_received.inc();
|
||||
|
||||
let request = match LookupRequest::decode(payload) {
|
||||
Ok(req) => req,
|
||||
Err(e) => {
|
||||
self.metrics()
|
||||
.lookup
|
||||
.record_reject(DiscoveryReject::ReqDecodeError);
|
||||
debug!(from = %self.peer_display_name(from), error = %e, "Malformed LookupRequest");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let now_ms = Self::now_ms();
|
||||
let recent_expiry_ms = self.config().node.lookup.recent_expiry_secs * 1000;
|
||||
let my_addr = *self.node_addr();
|
||||
use crate::proto::lookup::RequestOutcome;
|
||||
match crate::proto::lookup::classify_request(
|
||||
&mut self.lookup,
|
||||
&request,
|
||||
from,
|
||||
&my_addr,
|
||||
now_ms,
|
||||
recent_expiry_ms,
|
||||
MAX_RECENT_LOOKUP_REQUESTS,
|
||||
) {
|
||||
RequestOutcome::Duplicate => {
|
||||
self.metrics()
|
||||
.lookup
|
||||
.record_reject(DiscoveryReject::ReqDuplicate);
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
from = %self.peer_display_name(from),
|
||||
"Duplicate LookupRequest, dropping"
|
||||
);
|
||||
}
|
||||
RequestOutcome::DedupCacheFull { len } => {
|
||||
self.metrics()
|
||||
.lookup
|
||||
.record_reject(DiscoveryReject::ReqDedupCacheFull);
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
from = %self.peer_display_name(from),
|
||||
recent_requests = len,
|
||||
max_recent_requests = MAX_RECENT_LOOKUP_REQUESTS,
|
||||
"Discovery request dedup cache full, dropping LookupRequest"
|
||||
);
|
||||
}
|
||||
RequestOutcome::RespondAsTarget => {
|
||||
self.metrics().lookup.req_target_is_us.inc();
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
origin = %self.peer_display_name(&request.origin),
|
||||
"We are the lookup target, generating response"
|
||||
);
|
||||
self.send_lookup_response(&request).await;
|
||||
}
|
||||
RequestOutcome::Forward => {
|
||||
self.metrics().lookup.req_forwarded.inc();
|
||||
self.forward_lookup_request(request).await;
|
||||
}
|
||||
RequestOutcome::ForwardRateLimited => {
|
||||
self.metrics().lookup.req_forward_rate_limited.inc();
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(&request.target),
|
||||
"Forward rate limited, suppressing LookupRequest"
|
||||
);
|
||||
}
|
||||
RequestOutcome::TtlExhausted => {
|
||||
self.metrics()
|
||||
.lookup
|
||||
.record_reject(DiscoveryReject::ReqTtlExhausted);
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(&request.target),
|
||||
"LookupRequest TTL exhausted"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle an incoming LookupResponse from a peer.
|
||||
///
|
||||
/// Processing steps:
|
||||
/// 1. Decode and validate
|
||||
/// 2. Check recent_requests to determine if we originated or are forwarding
|
||||
/// 3. If originator: verify proof signature, then cache target_coords and path_mtu in coord_cache
|
||||
/// 4. If transit: apply path_mtu min(outgoing_link_mtu), reverse-path forward to from_peer
|
||||
pub(in crate::node) async fn handle_lookup_response(
|
||||
&mut self,
|
||||
from: &NodeAddr,
|
||||
payload: &[u8],
|
||||
) {
|
||||
self.metrics().lookup.resp_received.inc();
|
||||
|
||||
let mut response = match LookupResponse::decode(payload) {
|
||||
Ok(resp) => resp,
|
||||
Err(e) => {
|
||||
self.metrics()
|
||||
.lookup
|
||||
.record_reject(DiscoveryReject::RespDecodeError);
|
||||
debug!(from = %self.peer_display_name(from), error = %e, "Malformed LookupResponse");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let now_ms = Self::now_ms();
|
||||
|
||||
// Check if we forwarded this request (transit node) or originated it
|
||||
match crate::proto::lookup::classify_response(&mut self.lookup, response.request_id) {
|
||||
crate::proto::lookup::ResponseRoute::AlreadyForwarded => {
|
||||
// Already forwarded a response for this request — drop to
|
||||
// prevent response routing loops.
|
||||
debug!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&response.target),
|
||||
"Response already forwarded for this request, dropping"
|
||||
);
|
||||
}
|
||||
crate::proto::lookup::ResponseRoute::Transit { from_peer } => {
|
||||
// Transit node: reverse-path forward
|
||||
self.metrics().lookup.resp_forwarded.inc();
|
||||
|
||||
// Apply path_mtu min() from the outgoing link's transport MTU
|
||||
self.apply_outgoing_link_mtu_to_response(&mut response, &from_peer);
|
||||
|
||||
debug!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&response.target),
|
||||
next_hop = %self.peer_display_name(&from_peer),
|
||||
path_mtu = response.path_mtu,
|
||||
"Reverse-path forwarding LookupResponse"
|
||||
);
|
||||
|
||||
let encoded = response.encode();
|
||||
if let Err(e) = self.send_encrypted_link_message(&from_peer, &encoded).await {
|
||||
debug!(
|
||||
next_hop = %self.peer_display_name(&from_peer),
|
||||
error = %e,
|
||||
"Failed to forward LookupResponse"
|
||||
);
|
||||
}
|
||||
}
|
||||
crate::proto::lookup::ResponseRoute::Originator => {
|
||||
// We originated this request — verify proof before caching
|
||||
let target = response.target;
|
||||
let path_mtu = response.path_mtu;
|
||||
|
||||
// Look up the target's public key from identity_cache
|
||||
let mut prefix = [0u8; 15];
|
||||
prefix.copy_from_slice(&target.as_bytes()[0..15]);
|
||||
let target_pubkey = match self.lookup_by_fips_prefix(&prefix) {
|
||||
Some((_addr, pubkey)) => pubkey,
|
||||
None => {
|
||||
self.metrics()
|
||||
.lookup
|
||||
.record_reject(DiscoveryReject::RespIdentityMiss);
|
||||
warn!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&target),
|
||||
"identity_cache miss for lookup target, cannot verify proof"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Verify the proof signature
|
||||
let (xonly, _parity) = target_pubkey.x_only_public_key();
|
||||
let peer_id = PeerIdentity::from_pubkey(xonly);
|
||||
let proof_data = LookupResponse::proof_bytes(
|
||||
response.request_id,
|
||||
&target,
|
||||
&response.target_coords,
|
||||
);
|
||||
if !peer_id.verify(&proof_data, &response.proof) {
|
||||
self.metrics()
|
||||
.lookup
|
||||
.record_reject(DiscoveryReject::RespProofFailed);
|
||||
warn!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&target),
|
||||
"LookupResponse proof verification failed, discarding"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
self.metrics().lookup.resp_accepted.inc();
|
||||
|
||||
info!(
|
||||
request_id = response.request_id,
|
||||
target = %self.peer_display_name(&target),
|
||||
depth = response.target_coords.depth(),
|
||||
path_mtu = path_mtu,
|
||||
"Discovery succeeded, proof verified, route cached"
|
||||
);
|
||||
|
||||
// Apply the accept-side effects: the core clears the success
|
||||
// state (backoff + pending lookup) and returns the
|
||||
// cross-subsystem effects for us to drive.
|
||||
let actions = crate::proto::lookup::on_response_accepted(
|
||||
&mut self.lookup,
|
||||
&target,
|
||||
response.target_coords,
|
||||
now_ms,
|
||||
path_mtu,
|
||||
);
|
||||
self.drive_response_actions(actions).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drive the cross-subsystem effects returned by the discovery core's
|
||||
/// accept-side planning. Each arm reproduces the original inline effect
|
||||
/// exactly (same metrics/logs/writes, same order).
|
||||
async fn drive_response_actions(&mut self, actions: Vec<LookupAction>) {
|
||||
for action in actions {
|
||||
match action {
|
||||
LookupAction::CacheCoords {
|
||||
target,
|
||||
coords,
|
||||
now_ms,
|
||||
path_mtu,
|
||||
} => {
|
||||
self.coord_cache
|
||||
.insert_with_path_mtu(target, coords, now_ms, path_mtu);
|
||||
}
|
||||
LookupAction::WritePathMtu { target, path_mtu } => {
|
||||
// Mirror path_mtu into the FipsAddress-keyed read-only lookup
|
||||
// map used by the TUN reader/writer at TCP MSS clamp time.
|
||||
let fips_addr = crate::FipsAddress::from_node_addr(&target);
|
||||
match self.path_mtu_lookup.write() {
|
||||
Ok(mut map) => match map.get(&fips_addr).copied() {
|
||||
Some(existing) if existing <= path_mtu => {
|
||||
// Keep the tighter learned value; never loosen
|
||||
// the clamp. A reactive MtuExceeded or
|
||||
// PathMtuNotification tighten takes precedence
|
||||
// over a looser discovery estimate
|
||||
// (cross-carrier keep-tighter).
|
||||
debug!(
|
||||
target = %self.peer_display_name(&target),
|
||||
fips_addr = %fips_addr,
|
||||
path_mtu = path_mtu,
|
||||
existing = existing,
|
||||
"LookupResponse: keeping tighter existing path_mtu_lookup value"
|
||||
);
|
||||
}
|
||||
other => {
|
||||
map.insert(fips_addr, path_mtu);
|
||||
debug!(
|
||||
target = %self.peer_display_name(&target),
|
||||
fips_addr = %fips_addr,
|
||||
path_mtu = path_mtu,
|
||||
prior = ?other,
|
||||
map_len = map.len(),
|
||||
"Wrote path_mtu_lookup from discovery LookupResponse"
|
||||
);
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target = %self.peer_display_name(&target),
|
||||
fips_addr = %fips_addr,
|
||||
path_mtu = path_mtu,
|
||||
error = %e,
|
||||
"path_mtu_lookup write lock poisoned; clamp will not see this update"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
LookupAction::ResetWarmupIfEstablished { target } => {
|
||||
// If an established session exists, reset the warmup counter.
|
||||
let n = self.config().node.session.coords_warmup_packets;
|
||||
if let Some(entry) = self.sessions.get_mut(&target)
|
||||
&& entry.is_established()
|
||||
{
|
||||
entry.set_coords_warmup_remaining(n);
|
||||
debug!(
|
||||
dest = %self.peer_display_name(&target),
|
||||
warmup_packets = n,
|
||||
"Reset coords warmup after discovery for existing session"
|
||||
);
|
||||
}
|
||||
}
|
||||
LookupAction::RetryQueuedPackets { target } => {
|
||||
// If we have pending TUN packets for this target, retry session
|
||||
// initiation. The coord_cache now has coords, so find_next_hop()
|
||||
// should succeed.
|
||||
if let Some(packets) = self.pending_tun_packets.get(&target) {
|
||||
debug!(
|
||||
dest = %self.peer_display_name(&target),
|
||||
queued_packets = packets.len(),
|
||||
"Retrying queued packets after discovery"
|
||||
);
|
||||
self.retry_session_after_discovery(target).await;
|
||||
}
|
||||
}
|
||||
LookupAction::SendLink { peer, bytes } => {
|
||||
if let Err(e) = self.send_encrypted_link_message(&peer, &bytes).await {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&peer),
|
||||
error = %e,
|
||||
"Failed to send discovery link message"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate and send a LookupResponse when we are the target.
|
||||
async fn send_lookup_response(&mut self, request: &LookupRequest) {
|
||||
let our_coords = self.tree_state().my_coords().clone();
|
||||
|
||||
// Sign proof: Identity::sign hashes with SHA-256 internally
|
||||
let proof_data =
|
||||
LookupResponse::proof_bytes(request.request_id, &request.target, &our_coords);
|
||||
let proof = self.identity().sign(&proof_data);
|
||||
|
||||
let mut response =
|
||||
LookupResponse::new(request.request_id, request.target, our_coords, proof);
|
||||
|
||||
// Route toward origin. The reverse-path decision (the peer the request
|
||||
// arrived from, recorded in recent_requests) is the sans-IO core's; the
|
||||
// greedy tree-route fallback is a &mut coord-cache op kept in the shell.
|
||||
use crate::proto::lookup::ResponseRouteDecision;
|
||||
let next_hop_addr = match crate::proto::lookup::plan_response_route(
|
||||
&self.lookup,
|
||||
request.request_id,
|
||||
) {
|
||||
ResponseRouteDecision::ReversePath(peer) => peer,
|
||||
ResponseRouteDecision::NeedsTreeRoute => match self.find_next_hop(&request.origin) {
|
||||
Some(peer) => *peer.node_addr(),
|
||||
None => {
|
||||
debug!(
|
||||
origin = %self.peer_display_name(&request.origin),
|
||||
"Cannot route LookupResponse: no reverse path or tree route to origin"
|
||||
);
|
||||
self.metrics()
|
||||
.lookup
|
||||
.record_reject(DiscoveryReject::RespNoRoute);
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// Fold our outgoing-link MTU into path_mtu so the target-edge link
|
||||
// appears in the bottleneck calculation. Without this, the response
|
||||
// leaves the target with path_mtu = u16::MAX and only intermediate
|
||||
// transits min-fold; the target's first reverse-path hop is missed.
|
||||
self.apply_outgoing_link_mtu_to_response(&mut response, &next_hop_addr);
|
||||
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
origin = %self.peer_display_name(&request.origin),
|
||||
next_hop = %self.peer_display_name(&next_hop_addr),
|
||||
path_mtu = response.path_mtu,
|
||||
"Sending LookupResponse"
|
||||
);
|
||||
|
||||
let encoded = response.encode();
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message(&next_hop_addr, &encoded)
|
||||
.await
|
||||
{
|
||||
debug!(
|
||||
next_hop = %self.peer_display_name(&next_hop_addr),
|
||||
error = %e,
|
||||
"Failed to send LookupResponse"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Forward a LookupRequest to eligible peers.
|
||||
///
|
||||
/// Primary path: tree peers (parent + children) whose bloom filter
|
||||
/// contains the target. Restricting to tree peers follows the spanning
|
||||
/// tree partition, producing a single directed path.
|
||||
///
|
||||
/// Fallback: if no tree peer's bloom matches, try non-tree peers whose
|
||||
/// bloom contains the target. This recovers from dead ends caused by
|
||||
/// stale bloom filters, tree restructuring, or transit node failures.
|
||||
async fn forward_lookup_request(&mut self, mut request: LookupRequest) {
|
||||
// Plan the forward with the sans-IO decision core. The core owns the
|
||||
// TTL decrement, tree/fallback peer selection, and single-encode
|
||||
// fan-out; the shell keeps all metrics/logging and drives the sends.
|
||||
let outcome = {
|
||||
let rv = NodeRoutingView { node: self };
|
||||
crate::proto::lookup::plan_forward(&mut request, &rv)
|
||||
};
|
||||
match outcome {
|
||||
crate::proto::lookup::ForwardOutcome::TtlExhausted => {}
|
||||
crate::proto::lookup::ForwardOutcome::NoPeers => {
|
||||
self.metrics().lookup.req_no_tree_peer.inc();
|
||||
trace!(
|
||||
request_id = request.request_id,
|
||||
"No eligible peers to forward LookupRequest"
|
||||
);
|
||||
}
|
||||
crate::proto::lookup::ForwardOutcome::Forward {
|
||||
actions,
|
||||
used_fallback,
|
||||
} => {
|
||||
let peer_count = actions.len();
|
||||
if used_fallback {
|
||||
self.metrics().lookup.req_fallback_forwarded.inc();
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(&request.target),
|
||||
ttl = request.ttl,
|
||||
peer_count,
|
||||
"Forwarding LookupRequest via non-tree fallback"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(&request.target),
|
||||
ttl = request.ttl,
|
||||
peer_count,
|
||||
"Forwarding LookupRequest"
|
||||
);
|
||||
}
|
||||
for action in actions {
|
||||
if let LookupAction::SendLink { peer, bytes } = action
|
||||
&& let Err(e) = self.send_encrypted_link_message(&peer, &bytes).await
|
||||
{
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&peer),
|
||||
error = %e,
|
||||
"Failed to forward LookupRequest to peer"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Initiate a discovery lookup for a target node.
|
||||
///
|
||||
/// Creates a LookupRequest and sends it to tree peers whose bloom
|
||||
/// filters contain the target. Returns the number of peers sent to.
|
||||
/// The originator does NOT record the request_id in recent_requests,
|
||||
/// so when the response arrives, it's recognized as "our request".
|
||||
pub(in crate::node) async fn initiate_lookup(&mut self, target: &NodeAddr, ttl: u8) -> usize {
|
||||
self.metrics().lookup.req_initiated.inc();
|
||||
|
||||
let origin = *self.node_addr();
|
||||
let origin_coords = self.tree_state().my_coords().clone();
|
||||
let request_id = {
|
||||
use rand::RngExt;
|
||||
rand::rng().random()
|
||||
};
|
||||
let request = LookupRequest::new(request_id, *target, origin, origin_coords, ttl, 0);
|
||||
|
||||
// Tree-peer bloom-match selection + single encode live in the sans-IO
|
||||
// core. The core keeps the tree-only (no non-tree fallback) behavior;
|
||||
// the shell drives the sends and keeps all metrics/logging.
|
||||
let actions = {
|
||||
let rv = NodeRoutingView { node: self };
|
||||
crate::proto::lookup::plan_initiate(&request, &rv)
|
||||
};
|
||||
|
||||
let peer_count = actions.len();
|
||||
|
||||
debug!(
|
||||
request_id = request.request_id,
|
||||
target = %self.peer_display_name(target),
|
||||
ttl = ttl,
|
||||
peer_count = peer_count,
|
||||
total_peers = self.peers.len(),
|
||||
"Discovery lookup initiated"
|
||||
);
|
||||
|
||||
for action in actions {
|
||||
if let LookupAction::SendLink { peer, bytes } = action
|
||||
&& let Err(e) = self.send_encrypted_link_message(&peer, &bytes).await
|
||||
{
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&peer),
|
||||
error = %e,
|
||||
"Failed to send LookupRequest to peer"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
peer_count
|
||||
}
|
||||
|
||||
/// Initiate a discovery lookup if one is not already pending for this target.
|
||||
///
|
||||
/// Checks: pending dedup, post-failure backoff (off by default), bloom
|
||||
/// filter pre-check. If all pass, sends the first attempt's LookupRequest.
|
||||
/// Subsequent attempts (with fresh request_ids) are scheduled by
|
||||
/// [`Self::check_pending_lookups`] when each attempt's per-attempt timeout
|
||||
/// expires, using the sequence in `node.lookup.attempt_timeouts_secs`.
|
||||
pub(in crate::node) async fn maybe_initiate_lookup(&mut self, dest: &NodeAddr) {
|
||||
let now_ms = Self::now_ms();
|
||||
|
||||
// Bloom filter pre-check (view read) BEFORE the core call: if no peer's
|
||||
// filter contains the target, it's not in the mesh. Reading `self.peers`
|
||||
// here keeps the `&mut self.lookup` borrow in `initiate_gate` from
|
||||
// overlapping the immutable peer-table read.
|
||||
let reachable = self.peers.values().any(|peer| peer.may_reach(dest));
|
||||
|
||||
use crate::proto::lookup::InitiateDecision;
|
||||
match crate::proto::lookup::initiate_gate(&mut self.lookup, dest, now_ms, reachable) {
|
||||
InitiateDecision::Deduplicated => {
|
||||
self.metrics().lookup.req_deduplicated.inc();
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
"Discovery lookup deduplicated, already pending"
|
||||
);
|
||||
}
|
||||
InitiateDecision::Suppressed { failures } => {
|
||||
self.metrics().lookup.req_backoff_suppressed.inc();
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
failures = failures,
|
||||
"Discovery lookup suppressed by backoff"
|
||||
);
|
||||
}
|
||||
InitiateDecision::BloomMiss => {
|
||||
self.metrics().lookup.req_bloom_miss.inc();
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
"Discovery skipped, target not in any peer bloom filter"
|
||||
);
|
||||
}
|
||||
InitiateDecision::Proceed => {
|
||||
let ttl = self.config().node.lookup.ttl;
|
||||
let sent = self.initiate_lookup(dest, ttl).await;
|
||||
|
||||
// If no tree peers had the target, fail immediately
|
||||
if sent == 0 {
|
||||
crate::proto::lookup::initiate_failed(&mut self.lookup, dest, now_ms);
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(dest),
|
||||
"Discovery failed, no tree peers with bloom match"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check pending lookups for next-attempt or final timeout.
|
||||
///
|
||||
/// Called periodically from the tick handler. The lookup state machine
|
||||
/// runs through `node.lookup.attempt_timeouts_secs` (default
|
||||
/// `[1, 2, 4, 8]`): each entry is the deadline for one attempt. When the
|
||||
/// current attempt's deadline elapses:
|
||||
/// - If more entries remain: send the next attempt with a fresh
|
||||
/// `request_id`.
|
||||
/// - Otherwise: declare the destination unreachable, drop queued packets,
|
||||
/// and emit ICMPv6 destination-unreachable for each.
|
||||
pub(in crate::node) async fn check_pending_lookups(&mut self, now_ms: u64) {
|
||||
let attempt_timeouts = self.config().node.lookup.attempt_timeouts_secs.clone();
|
||||
let outcome =
|
||||
crate::proto::lookup::poll_pending(&mut self.lookup, now_ms, &attempt_timeouts);
|
||||
|
||||
for (target, attempt) in outcome.retries {
|
||||
let ttl = self.config().node.lookup.ttl;
|
||||
let sent = self.initiate_lookup(&target, ttl).await;
|
||||
if sent > 0 {
|
||||
debug!(
|
||||
target_node = %self.peer_display_name(&target),
|
||||
attempt = attempt,
|
||||
"Discovery retry sent"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (addr, failures) in outcome.timeouts {
|
||||
self.metrics().lookup.resp_timed_out.inc();
|
||||
let queued = self.pending_tun_packets.remove(&addr);
|
||||
let pkt_count = queued.as_ref().map_or(0, |p| p.len());
|
||||
info!(
|
||||
target_node = %self.peer_display_name(&addr),
|
||||
queued_packets = pkt_count,
|
||||
failures = failures,
|
||||
"Discovery lookup timed out, destination unreachable"
|
||||
);
|
||||
if let Some(packets) = queued {
|
||||
for pkt in &packets {
|
||||
self.send_icmpv6_dest_unreachable(pkt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset discovery backoff on topology changes.
|
||||
pub(in crate::node) fn reset_lookup_backoff(&mut self) {
|
||||
let cleared = self.lookup.reset_backoff();
|
||||
if cleared > 0 {
|
||||
debug!(
|
||||
entries = cleared,
|
||||
"Resetting discovery backoff on topology change"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Min-fold our outgoing-link MTU into a LookupResponse's `path_mtu`.
|
||||
///
|
||||
/// Used at both transit-side reverse-path forward and at the target's
|
||||
/// own send_lookup_response. The link MTU we apply is the MTU of the
|
||||
/// transport+addr we'll use to deliver the response toward `next_hop`.
|
||||
/// No-op when `next_hop` is not a directly-connected peer or its
|
||||
/// transport is not registered.
|
||||
pub(in crate::node) fn apply_outgoing_link_mtu_to_response(
|
||||
&self,
|
||||
response: &mut LookupResponse,
|
||||
next_hop: &NodeAddr,
|
||||
) {
|
||||
if let Some(peer) = self.peers.get(next_hop)
|
||||
&& let Some(tid) = peer.transport_id()
|
||||
&& let Some(transport) = self.transports.get(&tid)
|
||||
{
|
||||
let link_mtu = if let Some(addr) = peer.current_addr() {
|
||||
transport.link_mtu(addr)
|
||||
} else {
|
||||
transport.mtu()
|
||||
};
|
||||
response.path_mtu = response.path_mtu.min(link_mtu);
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed `path_mtu_lookup` for a directly-connected peer.
|
||||
///
|
||||
/// Called when an FMP link-layer peer is promoted to active. The seed
|
||||
/// value is the local outgoing-link MTU on the peer's transport, which
|
||||
/// is the actual link constraint for direct-link traffic. Stored only
|
||||
/// when no tighter value exists: discovery's reverse-path bottleneck
|
||||
/// or MMP `MtuExceeded` reactive learning take precedence when smaller.
|
||||
///
|
||||
/// Without this seed, configured/auto-connect peers (which establish
|
||||
/// sessions without going through the discovery Lookup flow) leave
|
||||
/// `path_mtu_lookup` empty for their FipsAddress, causing
|
||||
/// `per_flow_max_mss` to fall back to the global ceiling and the
|
||||
/// SYN-time TCP MSS clamp to over-estimate the effective path.
|
||||
pub(in crate::node) fn seed_path_mtu_for_link_peer(
|
||||
&self,
|
||||
peer_addr: &NodeAddr,
|
||||
transport_id: TransportId,
|
||||
addr: &TransportAddr,
|
||||
) {
|
||||
let Some(transport) = self.transports.get(&transport_id) else {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(peer_addr),
|
||||
transport_id = %transport_id,
|
||||
"seed_path_mtu_for_link_peer: transport not registered, skipping seed"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let link_mtu = transport.link_mtu(addr);
|
||||
let fips_addr = crate::FipsAddress::from_node_addr(peer_addr);
|
||||
let Ok(mut map) = self.path_mtu_lookup.write() else {
|
||||
warn!(
|
||||
peer = %self.peer_display_name(peer_addr),
|
||||
"seed_path_mtu_for_link_peer: path_mtu_lookup write lock poisoned"
|
||||
);
|
||||
return;
|
||||
};
|
||||
match map.get(&fips_addr).copied() {
|
||||
Some(existing) if existing <= link_mtu => {
|
||||
// Keep the tighter learned value; never loosen the clamp.
|
||||
debug!(
|
||||
peer = %self.peer_display_name(peer_addr),
|
||||
fips_addr = %fips_addr,
|
||||
link_mtu = link_mtu,
|
||||
existing = existing,
|
||||
"seed_path_mtu_for_link_peer: keeping tighter existing value"
|
||||
);
|
||||
}
|
||||
other => {
|
||||
map.insert(fips_addr, link_mtu);
|
||||
debug!(
|
||||
peer = %self.peer_display_name(peer_addr),
|
||||
fips_addr = %fips_addr,
|
||||
link_mtu = link_mtu,
|
||||
prior = ?other,
|
||||
map_len = map.len(),
|
||||
"seed_path_mtu_for_link_peer: wrote link MTU"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+337
-306
@@ -5,63 +5,20 @@
|
||||
//! and teardown metric logs.
|
||||
|
||||
use crate::NodeAddr;
|
||||
use crate::mmp::MmpMode;
|
||||
use crate::mmp::MmpSessionState;
|
||||
use crate::mmp::report::{ReceiverReport, SenderReport};
|
||||
use crate::node::Node;
|
||||
use crate::node::dataplane::PeerActionCtx;
|
||||
use crate::node::reject::{MmpReject, RejectReason, TreeReject};
|
||||
use crate::node::tree::sign_declaration;
|
||||
use crate::peer::machine::PeerEvent;
|
||||
use crate::proto::link::LinkMessageType;
|
||||
use crate::proto::mmp::{
|
||||
LinkReportKind, LinkReportSnapshot, MmpAction, PeerLivenessSnapshot, ReceiverReport, RrLog,
|
||||
SenderReport,
|
||||
use crate::protocol::{
|
||||
LinkMessageType, PathMtuNotification, SessionMessageType, SessionReceiverReport,
|
||||
SessionSenderReport,
|
||||
};
|
||||
use crate::proto::stp::ParentEval;
|
||||
use crate::transport::{TransportAddr, TransportId};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info, trace, warn};
|
||||
|
||||
/// Emit the operator `trace!` point for a processed ReceiverReport outcome.
|
||||
///
|
||||
/// These log points used to live inside `MmpMetrics::process_receiver_report`;
|
||||
/// the sans-IO migration returns the outcome as an [`RrLog`] and re-emits it
|
||||
/// here, shell-side, preserving the original field set, content, and (relative
|
||||
/// to the surrounding handler logs) ordering. The original traces carried no
|
||||
/// peer identifier, so none is added here.
|
||||
pub(super) fn log_rr_outcome(rr: &ReceiverReport, our_timestamp_ms: u32, log: RrLog) {
|
||||
match log {
|
||||
RrLog::Stale {
|
||||
prev_highest,
|
||||
prev_packets,
|
||||
prev_bytes,
|
||||
} => trace!(
|
||||
highest_counter = rr.highest_counter,
|
||||
prev_highest_counter = prev_highest,
|
||||
cumulative_packets_recv = rr.cumulative_packets_recv,
|
||||
prev_cumulative_packets_recv = prev_packets,
|
||||
cumulative_bytes_recv = rr.cumulative_bytes_recv,
|
||||
prev_cumulative_bytes_recv = prev_bytes,
|
||||
"Ignoring stale MMP ReceiverReport"
|
||||
),
|
||||
RrLog::RttSample { rtt_ms, srtt_ms } => trace!(
|
||||
our_ts = our_timestamp_ms,
|
||||
echo = rr.timestamp_echo,
|
||||
dwell = u32::from(rr.dwell_time),
|
||||
rtt_ms = rtt_ms,
|
||||
srtt_ms = srtt_ms,
|
||||
"RTT sample from timestamp echo"
|
||||
),
|
||||
RrLog::InvalidRtt => trace!(
|
||||
our_ts = our_timestamp_ms,
|
||||
echo = rr.timestamp_echo,
|
||||
dwell = u32::from(rr.dwell_time),
|
||||
"Ignoring invalid MMP RTT sample"
|
||||
),
|
||||
RrLog::None => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Format bytes/sec as human-readable throughput.
|
||||
pub(in crate::node) fn format_throughput(bps: f64) -> String {
|
||||
fn format_throughput(bps: f64) -> String {
|
||||
if bps == 0.0 {
|
||||
"n/a".to_string()
|
||||
} else if bps >= 1_000_000.0 {
|
||||
@@ -156,12 +113,10 @@ impl Node {
|
||||
|
||||
// Process the report: computes RTT from timestamp echo, updates
|
||||
// loss rate, goodput rate, jitter trend, and ETX.
|
||||
let now_ms = crate::time::mono_ms();
|
||||
let (first_rtt, rr_log) =
|
||||
mmp.metrics
|
||||
.process_receiver_report(&rr, our_timestamp_ms, now_ms);
|
||||
// Re-emit the operator trace the core used to log mid-decision.
|
||||
log_rr_outcome(&rr, our_timestamp_ms, rr_log);
|
||||
let now = Instant::now();
|
||||
let first_rtt = mmp
|
||||
.metrics
|
||||
.process_receiver_report(&rr, our_timestamp_ms, now);
|
||||
|
||||
// Feed SRTT back to sender/receiver report interval tuning
|
||||
if let Some(srtt_ms) = mmp.metrics.srtt_ms() {
|
||||
@@ -189,43 +144,25 @@ impl Node {
|
||||
// Trigger re-evaluation so the node doesn't wait for the next
|
||||
// periodic tick or TreeAnnounce.
|
||||
if first_rtt {
|
||||
let peer_costs: std::collections::BTreeMap<crate::NodeAddr, f64> = self
|
||||
let peer_costs: std::collections::HashMap<crate::NodeAddr, f64> = self
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|(_, p)| p.has_srtt())
|
||||
.map(|(a, p)| (*a, p.link_cost()))
|
||||
.collect();
|
||||
// Wall-clock seconds for the escaping declaration timestamp;
|
||||
// monotonic ms for the flap-dampening / hold-down timers.
|
||||
let now_secs = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
let mono_now_ms = crate::time::mono_ms();
|
||||
// Compute the flap-dampening / hold-down veto at the edge; a mandatory
|
||||
// switch bypasses it, a discretionary one is taken only if not suppressed.
|
||||
let switch_suppressed = self.tree_state.is_switch_suppressed(mono_now_ms);
|
||||
let new_parent = match self
|
||||
.tree_state
|
||||
.evaluate_parent(&peer_costs, &std::collections::BTreeSet::new())
|
||||
{
|
||||
ParentEval::Mandatory(p) => Some(p),
|
||||
ParentEval::Discretionary(p) if !switch_suppressed => Some(p),
|
||||
ParentEval::Discretionary(_) | ParentEval::None => None,
|
||||
};
|
||||
if let Some(new_parent) = new_parent {
|
||||
if let Some(new_parent) = self.tree_state.evaluate_parent(&peer_costs) {
|
||||
let new_seq = self.tree_state.my_declaration().sequence() + 1;
|
||||
let flap_dampened =
|
||||
self.tree_state
|
||||
.set_parent(new_parent, new_seq, now_secs, mono_now_ms);
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
let flap_dampened = self.tree_state.set_parent(new_parent, new_seq, timestamp);
|
||||
self.tree_state.recompute_coords();
|
||||
// Clone identity once: sign_declaration borrows &mut tree_state while
|
||||
// the identity() accessor borrows all of &self, so an owned copy avoids
|
||||
// the split-borrow conflict on this infrequent parent-switch path.
|
||||
let our_identity = self.identity().clone();
|
||||
if let Err(e) =
|
||||
sign_declaration(self.tree_state.my_declaration_mut(), &our_identity)
|
||||
{
|
||||
if let Err(e) = self.tree_state.sign_declaration(&our_identity) {
|
||||
warn!(error = %e, "Failed to sign declaration after first-RTT parent eval");
|
||||
self.metrics()
|
||||
.tree
|
||||
@@ -235,7 +172,8 @@ impl Node {
|
||||
// Surgical invalidation — see CoordCache::invalidate_via_node doc.
|
||||
self.coord_cache
|
||||
.invalidate_via_node(our_identity.node_addr());
|
||||
self.reset_lookup_backoff();
|
||||
self.reset_discovery_backoff();
|
||||
self.metrics().tree.parent_switched.inc();
|
||||
self.metrics().tree.parent_switches.inc();
|
||||
info!(
|
||||
new_parent = %self.peer_display_name(&new_parent),
|
||||
@@ -253,12 +191,10 @@ impl Node {
|
||||
let all_peers: Vec<crate::NodeAddr> = self.peers.keys().copied().collect();
|
||||
self.bloom_state.mark_all_updates_needed(all_peers);
|
||||
} else if !self.tree_state.is_root() && self.tree_state.should_be_root() {
|
||||
self.tree_state.become_root(now_secs);
|
||||
self.tree_state.become_root();
|
||||
// Clone identity once (see the parent-switch branch above for why).
|
||||
let our_identity = self.identity().clone();
|
||||
if let Err(e) =
|
||||
sign_declaration(self.tree_state.my_declaration_mut(), &our_identity)
|
||||
{
|
||||
if let Err(e) = self.tree_state.sign_declaration(&our_identity) {
|
||||
warn!(error = %e, "Failed to sign self-root declaration after first-RTT");
|
||||
self.metrics()
|
||||
.tree
|
||||
@@ -268,7 +204,8 @@ impl Node {
|
||||
// Surgical invalidation — see CoordCache::invalidate_other_roots doc.
|
||||
self.coord_cache
|
||||
.invalidate_other_roots(our_identity.node_addr());
|
||||
self.reset_lookup_backoff();
|
||||
self.reset_discovery_backoff();
|
||||
self.metrics().tree.parent_switched.inc();
|
||||
self.metrics().tree.parent_switches.inc();
|
||||
info!(
|
||||
new_root = %self.tree_state.root(),
|
||||
@@ -286,91 +223,65 @@ impl Node {
|
||||
///
|
||||
/// Called from the tick handler. Also emits periodic operator logs.
|
||||
pub(in crate::node) async fn check_mmp_reports(&mut self) {
|
||||
let now_ms = crate::time::mono_ms();
|
||||
let now = Instant::now();
|
||||
|
||||
// Build one report-gating snapshot per peer, resolving every timing read
|
||||
// shell-side into a `bool`. `send_sr`/`send_rr` are `true` on the master
|
||||
// (IK) line — there is no profile negotiation here; the forward-merge to
|
||||
// `-next` wires them to `peer.send_sr()`/`peer.send_rr()` (plan spot c).
|
||||
// The snapshots own only `NodeAddr`/`MmpMode`/`bool`, so the
|
||||
// peer-iteration borrow is released before the pure decision runs and the
|
||||
// driving loop mutates the reporting state.
|
||||
let snapshots: Vec<LinkReportSnapshot> = self
|
||||
.peers
|
||||
.iter()
|
||||
.filter_map(|(node_addr, peer)| {
|
||||
let mmp = peer.mmp()?;
|
||||
Some(LinkReportSnapshot {
|
||||
peer: *node_addr,
|
||||
mode: mmp.mode(),
|
||||
send_sr: true,
|
||||
send_rr: true,
|
||||
sr_due: mmp.sender.should_send_report(now_ms),
|
||||
rr_due: mmp.receiver.should_send_report(now_ms),
|
||||
log_due: mmp.should_log(now_ms),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
// Collect peers that need reports (can't borrow self mutably while iterating)
|
||||
let mut sender_reports: Vec<(NodeAddr, Vec<u8>)> = Vec::new();
|
||||
let mut receiver_reports: Vec<(NodeAddr, Vec<u8>)> = Vec::new();
|
||||
|
||||
let actions = self.mmp.plan_link_reports(&snapshots);
|
||||
for (node_addr, peer) in self.peers.iter_mut() {
|
||||
// Compute display name before taking mutable MMP borrow
|
||||
let peer_name = self
|
||||
.peer_aliases
|
||||
.get(node_addr)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| peer.identity().short_npub());
|
||||
|
||||
// Drive the planned actions in their phase-grouped order (all logs, then
|
||||
// all SenderReports, then all ReceiverReports). Logs run first because the
|
||||
// operator log reads cumulative_packets_sent, which each report send
|
||||
// advances (send_encrypted_link_message -> sender.record_sent); the
|
||||
// pre-refactor handler logged during its collect pass, before any send.
|
||||
// `build_report` (which advances the interval state) is called only on a
|
||||
// SendLinkReport action, exactly as the pre-refactor gate did.
|
||||
for action in actions {
|
||||
match action {
|
||||
MmpAction::SendLinkReport { peer, kind } => {
|
||||
let encoded = self
|
||||
.peers
|
||||
.get_mut(&peer)
|
||||
.and_then(|p| p.mmp_mut())
|
||||
.and_then(|mmp| match kind {
|
||||
LinkReportKind::Sender => {
|
||||
mmp.sender.build_report(now_ms).map(|sr| sr.encode())
|
||||
}
|
||||
LinkReportKind::Receiver => {
|
||||
mmp.receiver.build_report(now_ms).map(|rr| rr.encode())
|
||||
}
|
||||
});
|
||||
if let Some(encoded) = encoded
|
||||
&& let Err(e) = self.send_encrypted_link_message(&peer, &encoded).await
|
||||
{
|
||||
let label = match kind {
|
||||
LinkReportKind::Sender => "Failed to send SenderReport",
|
||||
LinkReportKind::Receiver => "Failed to send ReceiverReport",
|
||||
};
|
||||
debug!(peer = %self.peer_display_name(&peer), error = %e, "{}", label);
|
||||
}
|
||||
}
|
||||
MmpAction::LogLink { peer } => {
|
||||
// Resolve the display name exactly as the pre-refactor loop
|
||||
// did (alias, else short_npub) — not `peer_display_name`,
|
||||
// which also consults the host map.
|
||||
let peer_name = self.peer_aliases.get(&peer).cloned().unwrap_or_else(|| {
|
||||
self.peers
|
||||
.get(&peer)
|
||||
.map(|p| p.identity().short_npub())
|
||||
.unwrap_or_default()
|
||||
});
|
||||
if let Some(mmp) = self.peers.get_mut(&peer).and_then(|p| p.mmp_mut()) {
|
||||
Self::log_mmp_metrics(&peer_name, mmp);
|
||||
mmp.mark_logged(now_ms);
|
||||
}
|
||||
}
|
||||
MmpAction::ReapPeer { .. }
|
||||
| MmpAction::Heartbeat { .. }
|
||||
| MmpAction::SendSessionReport { .. }
|
||||
| MmpAction::LogSession { .. } => {}
|
||||
let Some(mmp) = peer.mmp_mut() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let mode = mmp.mode();
|
||||
|
||||
// Sender reports: Full mode only
|
||||
if mode == MmpMode::Full
|
||||
&& mmp.sender.should_send_report(now)
|
||||
&& let Some(sr) = mmp.sender.build_report(now)
|
||||
{
|
||||
sender_reports.push((*node_addr, sr.encode()));
|
||||
}
|
||||
|
||||
// Receiver reports: Full and Lightweight modes
|
||||
if mode != MmpMode::Minimal
|
||||
&& mmp.receiver.should_send_report(now)
|
||||
&& let Some(rr) = mmp.receiver.build_report(now)
|
||||
{
|
||||
receiver_reports.push((*node_addr, rr.encode()));
|
||||
}
|
||||
|
||||
// Periodic operator logging
|
||||
if mmp.should_log(now) {
|
||||
Self::log_mmp_metrics(&peer_name, mmp);
|
||||
mmp.mark_logged(now);
|
||||
}
|
||||
}
|
||||
|
||||
// Send collected reports
|
||||
for (node_addr, encoded) in sender_reports {
|
||||
if let Err(e) = self.send_encrypted_link_message(&node_addr, &encoded).await {
|
||||
debug!(peer = %self.peer_display_name(&node_addr), error = %e, "Failed to send SenderReport");
|
||||
}
|
||||
}
|
||||
|
||||
for (node_addr, encoded) in receiver_reports {
|
||||
if let Err(e) = self.send_encrypted_link_message(&node_addr, &encoded).await {
|
||||
debug!(peer = %self.peer_display_name(&node_addr), error = %e, "Failed to send ReceiverReport");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Emit periodic MMP metrics for a peer.
|
||||
fn log_mmp_metrics(peer_name: &str, mmp: &crate::proto::mmp::MmpPeerState) {
|
||||
fn log_mmp_metrics(peer_name: &str, mmp: &crate::mmp::MmpPeerState) {
|
||||
let m = &mmp.metrics;
|
||||
|
||||
let rtt_str = if m.rtt_trend.initialized() {
|
||||
@@ -398,10 +309,7 @@ impl Node {
|
||||
}
|
||||
|
||||
/// Emit a teardown log summarizing lifetime MMP metrics for a removed peer.
|
||||
pub(in crate::node) fn log_mmp_teardown(
|
||||
peer_name: &str,
|
||||
mmp: &crate::proto::mmp::MmpPeerState,
|
||||
) {
|
||||
pub(in crate::node) fn log_mmp_teardown(peer_name: &str, mmp: &crate::mmp::MmpPeerState) {
|
||||
let m = &mmp.metrics;
|
||||
let jitter_ms = mmp.receiver.jitter_us() as f64 / 1000.0;
|
||||
|
||||
@@ -426,6 +334,205 @@ impl Node {
|
||||
);
|
||||
}
|
||||
|
||||
// === Session-layer MMP ===
|
||||
|
||||
/// Check all sessions for pending MMP reports and send them.
|
||||
///
|
||||
/// Called from the tick handler. Also emits periodic session MMP logs.
|
||||
/// Uses the collect-then-send pattern to avoid borrowing conflicts.
|
||||
pub(in crate::node) async fn check_session_mmp_reports(&mut self) {
|
||||
let now = Instant::now();
|
||||
|
||||
// Collect reports to send: (dest_addr, msg_type, encoded_body)
|
||||
let mut reports: Vec<(NodeAddr, u8, Vec<u8>)> = Vec::new();
|
||||
|
||||
for (dest_addr, entry) in self.sessions.iter_mut() {
|
||||
// Compute display name before taking mutable MMP borrow
|
||||
let session_name = self
|
||||
.peer_aliases
|
||||
.get(dest_addr)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| {
|
||||
let (xonly, _) = entry.remote_pubkey().x_only_public_key();
|
||||
crate::PeerIdentity::from_pubkey(xonly).short_npub()
|
||||
});
|
||||
|
||||
let Some(mmp) = entry.mmp_mut() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let mode = mmp.mode();
|
||||
|
||||
// Sender reports: Full mode only
|
||||
if mode == MmpMode::Full
|
||||
&& mmp.sender.should_send_report(now)
|
||||
&& let Some(sr) = mmp.sender.build_report(now)
|
||||
{
|
||||
let session_sr: SessionSenderReport = SessionSenderReport::from(&sr);
|
||||
reports.push((
|
||||
*dest_addr,
|
||||
SessionMessageType::SenderReport.to_byte(),
|
||||
session_sr.encode(),
|
||||
));
|
||||
}
|
||||
|
||||
// Receiver reports: Full and Lightweight modes
|
||||
if mode != MmpMode::Minimal
|
||||
&& mmp.receiver.should_send_report(now)
|
||||
&& let Some(rr) = mmp.receiver.build_report(now)
|
||||
{
|
||||
let session_rr: SessionReceiverReport = SessionReceiverReport::from(&rr);
|
||||
reports.push((
|
||||
*dest_addr,
|
||||
SessionMessageType::ReceiverReport.to_byte(),
|
||||
session_rr.encode(),
|
||||
));
|
||||
}
|
||||
|
||||
// PathMtu notifications (all modes)
|
||||
if mmp.path_mtu.should_send_notification(now)
|
||||
&& let Some(mtu_value) = mmp.path_mtu.build_notification(now)
|
||||
{
|
||||
let notif = PathMtuNotification::new(mtu_value);
|
||||
reports.push((
|
||||
*dest_addr,
|
||||
SessionMessageType::PathMtuNotification.to_byte(),
|
||||
notif.encode(),
|
||||
));
|
||||
}
|
||||
|
||||
// Periodic operator logging
|
||||
if mmp.should_log(now) {
|
||||
Self::log_session_mmp_metrics(&session_name, mmp);
|
||||
mmp.mark_logged(now);
|
||||
}
|
||||
}
|
||||
|
||||
// Send collected reports via session-layer encryption.
|
||||
// Track per-destination success/failure for backoff and log suppression.
|
||||
let mut send_results: Vec<(NodeAddr, bool)> = Vec::new();
|
||||
for (dest_addr, msg_type, body) in reports {
|
||||
match self.send_session_msg(&dest_addr, msg_type, &body).await {
|
||||
Ok(()) => {
|
||||
send_results.push((dest_addr, true));
|
||||
}
|
||||
Err(e) => {
|
||||
// Peek at current failure count for log suppression
|
||||
let failures = self
|
||||
.sessions
|
||||
.get(&dest_addr)
|
||||
.and_then(|entry| entry.mmp())
|
||||
.map(|mmp| mmp.sender.consecutive_send_failures())
|
||||
.unwrap_or(0);
|
||||
|
||||
if failures < 3 {
|
||||
debug!(
|
||||
dest = %self.peer_display_name(&dest_addr),
|
||||
msg_type,
|
||||
error = %e,
|
||||
"Failed to send session MMP report"
|
||||
);
|
||||
} else if failures == 3 {
|
||||
debug!(
|
||||
dest = %self.peer_display_name(&dest_addr),
|
||||
"Suppressing further session MMP send failure logs"
|
||||
);
|
||||
}
|
||||
// failures > 3: silently suppressed
|
||||
|
||||
send_results.push((dest_addr, false));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update backoff state from send results.
|
||||
// Deduplicate: a destination counts as success if ANY report succeeded,
|
||||
// failure only if ALL reports for that destination failed.
|
||||
let mut dest_success: std::collections::HashMap<NodeAddr, bool> =
|
||||
std::collections::HashMap::new();
|
||||
for (dest, ok) in &send_results {
|
||||
let entry = dest_success.entry(*dest).or_insert(false);
|
||||
if *ok {
|
||||
*entry = true;
|
||||
}
|
||||
}
|
||||
for (dest_addr, success) in dest_success {
|
||||
if let Some(entry) = self.sessions.get_mut(&dest_addr)
|
||||
&& let Some(mmp) = entry.mmp_mut()
|
||||
{
|
||||
if success {
|
||||
let prev = mmp.sender.record_send_success();
|
||||
if prev > 3 {
|
||||
debug!(
|
||||
dest = %self.peer_display_name(&dest_addr),
|
||||
consecutive_failures = prev,
|
||||
"Resumed session MMP reporting"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
mmp.sender.record_send_failure();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Emit periodic session MMP metrics.
|
||||
fn log_session_mmp_metrics(session_name: &str, mmp: &MmpSessionState) {
|
||||
let m = &mmp.metrics;
|
||||
|
||||
let rtt_str = if m.rtt_trend.initialized() {
|
||||
format!("{:.1}ms", m.rtt_trend.long() / 1000.0)
|
||||
} else {
|
||||
"n/a".to_string()
|
||||
};
|
||||
let loss_str = if m.loss_trend.initialized() {
|
||||
format!("{:.1}%", m.loss_trend.long() * 100.0)
|
||||
} else {
|
||||
"n/a".to_string()
|
||||
};
|
||||
let jitter_ms = mmp.receiver.jitter_us() as f64 / 1000.0;
|
||||
|
||||
debug!(
|
||||
session = %session_name,
|
||||
rtt = %rtt_str,
|
||||
loss = %loss_str,
|
||||
jitter = format_args!("{:.1}ms", jitter_ms),
|
||||
goodput = %format_throughput(m.goodput_bps()),
|
||||
mtu = mmp.path_mtu.last_observed_mtu(),
|
||||
tx_pkts = mmp.sender.cumulative_packets_sent(),
|
||||
rx_pkts = mmp.receiver.cumulative_packets_recv(),
|
||||
"MMP session metrics"
|
||||
);
|
||||
}
|
||||
|
||||
/// Emit a teardown log summarizing lifetime session MMP metrics.
|
||||
pub(in crate::node) fn log_session_mmp_teardown(session_name: &str, mmp: &MmpSessionState) {
|
||||
let m = &mmp.metrics;
|
||||
let jitter_ms = mmp.receiver.jitter_us() as f64 / 1000.0;
|
||||
|
||||
let rtt_str = match m.srtt_ms() {
|
||||
Some(rtt) => format!("{:.1}ms", rtt),
|
||||
None => "n/a".to_string(),
|
||||
};
|
||||
let loss_str = format!("{:.1}%", m.loss_rate() * 100.0);
|
||||
|
||||
debug!(
|
||||
session = %session_name,
|
||||
rtt = %rtt_str,
|
||||
loss = %loss_str,
|
||||
jitter = format_args!("{:.1}ms", jitter_ms),
|
||||
etx = format_args!("{:.2}", m.etx),
|
||||
goodput = %format_throughput(m.goodput_bps()),
|
||||
send_mtu = mmp.path_mtu.current_mtu(),
|
||||
observed_mtu = mmp.path_mtu.last_observed_mtu(),
|
||||
tx_pkts = mmp.sender.cumulative_packets_sent(),
|
||||
tx_bytes = mmp.sender.cumulative_bytes_sent(),
|
||||
rx_pkts = mmp.receiver.cumulative_packets_recv(),
|
||||
rx_bytes = mmp.receiver.cumulative_bytes_recv(),
|
||||
"MMP session teardown"
|
||||
);
|
||||
}
|
||||
|
||||
/// Send heartbeats and remove dead peers.
|
||||
///
|
||||
/// Called from the tick handler. Sends a 1-byte heartbeat to each peer
|
||||
@@ -433,159 +540,83 @@ impl Node {
|
||||
/// hasn't sent us a frame within the link dead timeout.
|
||||
pub(in crate::node) async fn check_link_heartbeats(&mut self) {
|
||||
let now = Instant::now();
|
||||
// Monotonic ms for the MMP receiver's injected-`u64` liveness clock; the
|
||||
// Instant `now` is still used for the shell-owned heartbeat timing and
|
||||
// the session-start fallback (both `ActivePeer` Instants).
|
||||
let now_ms = crate::time::mono_ms();
|
||||
let heartbeat_interval = Duration::from_secs(self.config().node.heartbeat_interval_secs);
|
||||
let dead_timeout = Duration::from_secs(self.config().node.link_dead_timeout_secs);
|
||||
let dead_timeout_ms = dead_timeout.as_millis() as u64;
|
||||
let max_resends = self.config().node.rate_limit.handshake_max_resends;
|
||||
let heartbeat_msg = [LinkMessageType::Heartbeat.to_byte()];
|
||||
|
||||
// Build one liveness snapshot per peer, resolving every clock read and
|
||||
// the rekey-suppression predicate shell-side. The snapshots own only
|
||||
// `NodeAddr`/`bool`, so the peer-iteration borrow is released before the
|
||||
// pure decision runs and the driving loop mutates the registry.
|
||||
let snapshots: Vec<PeerLivenessSnapshot> = self
|
||||
.peers
|
||||
.iter()
|
||||
.map(|(node_addr, peer)| {
|
||||
// Check liveness via the MMP receiver's last-received monotonic
|
||||
// ms. Fall back to session_start (an `ActivePeer` Instant) for
|
||||
// peers that never sent data, keeping that branch in Instant
|
||||
// space so no monotonic-ms epoch conversion is needed.
|
||||
let time_dead = if let Some(mmp) = peer.mmp() {
|
||||
match mmp.receiver.last_recv_ms() {
|
||||
Some(last_ms) => now_ms.saturating_sub(last_ms) >= dead_timeout_ms,
|
||||
None => now.duration_since(peer.session_start()) >= dead_timeout,
|
||||
}
|
||||
} else {
|
||||
false
|
||||
};
|
||||
// Collect heartbeats to send and dead peers to remove
|
||||
let mut heartbeats: Vec<NodeAddr> = Vec::new();
|
||||
let mut dead_peers: Vec<NodeAddr> = Vec::new();
|
||||
|
||||
// Suppress teardown while an FMP rekey is genuinely in flight
|
||||
// with budget left: a rekey-handshake link is not silent. The
|
||||
// msg1 resend cap guarantees this terminates (abandon on
|
||||
// exhaustion or cutover on completion clears
|
||||
// `rekey_in_progress`), so a truly dead link is reaped on the
|
||||
// next cycle.
|
||||
let rekey_active = peer.rekey_in_progress()
|
||||
&& peer.rekey_msg1_resend_count() < max_resends
|
||||
&& peer.rekey_msg1().is_some();
|
||||
for (node_addr, peer) in self.peers.iter() {
|
||||
// Check liveness via MMP receiver last_recv_time.
|
||||
// Fall back to session_start for peers that never sent data.
|
||||
let time_dead = if let Some(mmp) = peer.mmp() {
|
||||
let reference_time = mmp
|
||||
.receiver
|
||||
.last_recv_time()
|
||||
.unwrap_or(peer.session_start());
|
||||
now.duration_since(reference_time) >= dead_timeout
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
// Check if heartbeat is due.
|
||||
let heartbeat_due = match peer.last_heartbeat_sent() {
|
||||
None => true,
|
||||
Some(last) => now.duration_since(last) >= heartbeat_interval,
|
||||
};
|
||||
// Suppress teardown while an FMP rekey is genuinely in flight with
|
||||
// budget left: a rekey-handshake link is not silent. The msg1
|
||||
// resend cap guarantees this terminates (abandon on exhaustion or
|
||||
// cutover on completion clears `rekey_in_progress`), so a truly
|
||||
// dead link is reaped on the next cycle.
|
||||
let rekey_active = peer.rekey_in_progress()
|
||||
&& peer.rekey_msg1_resend_count() < max_resends
|
||||
&& peer.rekey_msg1().is_some();
|
||||
|
||||
PeerLivenessSnapshot {
|
||||
peer: *node_addr,
|
||||
time_dead,
|
||||
rekey_active,
|
||||
heartbeat_due,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let is_dead = time_dead && !rekey_active;
|
||||
if is_dead {
|
||||
dead_peers.push(*node_addr);
|
||||
continue;
|
||||
}
|
||||
|
||||
let actions = self.mmp.plan_heartbeats(&snapshots);
|
||||
// Check if heartbeat is due
|
||||
let needs_heartbeat = match peer.last_heartbeat_sent() {
|
||||
None => true,
|
||||
Some(last) => now.duration_since(last) >= heartbeat_interval,
|
||||
};
|
||||
if needs_heartbeat {
|
||||
heartbeats.push(*node_addr);
|
||||
}
|
||||
}
|
||||
|
||||
// Wall-clock basis for reconnect scheduling, sourced once (as before).
|
||||
// Remove dead peers and schedule auto-reconnect
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
|
||||
// Drive the planned actions: all reaps first (each removed +
|
||||
// reconnect-scheduled), then all heartbeats (a just-reaped peer is never
|
||||
// heartbeated — the core never emits both for the same peer).
|
||||
for action in actions {
|
||||
match action {
|
||||
MmpAction::ReapPeer { peer } => {
|
||||
// Log SHELL-SIDE before routing so the reap keeps the
|
||||
// `fips::node::handlers::mmp` tracing target (no relocation into
|
||||
// the executor, no target pin needed).
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&peer),
|
||||
timeout_secs = self.config().node.link_dead_timeout_secs,
|
||||
"Removing peer: link dead timeout"
|
||||
);
|
||||
self.route_link_dead(peer, now_ms).await;
|
||||
}
|
||||
MmpAction::Heartbeat { peer } => {
|
||||
if let Some(p) = self.peers.get_mut(&peer) {
|
||||
p.mark_heartbeat_sent(now);
|
||||
}
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message(&peer, &heartbeat_msg)
|
||||
.await
|
||||
{
|
||||
trace!(peer = %self.peer_display_name(&peer), error = %e, "Failed to send heartbeat");
|
||||
}
|
||||
}
|
||||
MmpAction::SendLinkReport { .. }
|
||||
| MmpAction::LogLink { .. }
|
||||
| MmpAction::SendSessionReport { .. }
|
||||
| MmpAction::LogSession { .. } => {}
|
||||
for addr in &dead_peers {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(addr),
|
||||
timeout_secs = self.config().node.link_dead_timeout_secs,
|
||||
"Removing peer: link dead timeout"
|
||||
);
|
||||
self.remove_active_peer(addr);
|
||||
self.schedule_reconnect(*addr, now_ms);
|
||||
}
|
||||
|
||||
// Send heartbeats (skip peers we just removed)
|
||||
for addr in heartbeats {
|
||||
if dead_peers.contains(&addr) {
|
||||
continue;
|
||||
}
|
||||
if let Some(peer) = self.peers.get_mut(&addr) {
|
||||
peer.mark_heartbeat_sent(now);
|
||||
}
|
||||
if let Err(e) = self
|
||||
.send_encrypted_link_message(&addr, &heartbeat_msg)
|
||||
.await
|
||||
{
|
||||
trace!(peer = %self.peer_display_name(&addr), error = %e, "Failed to send heartbeat");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Route a link-dead liveness reap through the peer machine + executor.
|
||||
/// Mirrors [`route_rekey_cadence`](Node::route_rekey_cadence): the
|
||||
/// shell already decided (the tick sweep's `plan_heartbeats` batch emitted
|
||||
/// this `ReapPeer` in phase order), so the machine only CONSUMES the decision
|
||||
/// via [`PeerEvent::LinkDeadSuspected`]. The resulting executor arms
|
||||
/// (`InvalidateSendState` → `remove_active_peer`, `ReportLost` →
|
||||
/// `note_link_dead`) reproduce the pre-refactor inline reap body exactly.
|
||||
///
|
||||
/// An established peer always has a `peer_machine`. If the peer vanished
|
||||
/// between snapshot and effect, the old inline body was already a
|
||||
/// no-op, so we return; if the machine is absent (which should be impossible)
|
||||
/// we fall back to the byte-identical inline body under a `debug_assert`.
|
||||
///
|
||||
/// `now_ms` is the sweep's hoisted wall-clock ms (the same value the old reap
|
||||
/// fed `note_link_dead`); it flows to the executor `ReportLost` arm via
|
||||
/// `ambient.now_ms`.
|
||||
async fn route_link_dead(&mut self, node_addr: NodeAddr, now_ms: u64) {
|
||||
let link = match self.peers.get(&node_addr) {
|
||||
Some(peer) => peer.link_id(),
|
||||
None => return,
|
||||
};
|
||||
if !self.peer_machines.contains_key(&link) {
|
||||
debug_assert!(false, "peer machine present for every established peer");
|
||||
self.remove_active_peer(&node_addr);
|
||||
self.note_link_dead(node_addr, now_ms);
|
||||
return;
|
||||
}
|
||||
let ambient = self.link_dead_ctx(&node_addr, now_ms);
|
||||
self.advance_peer_machine(link, PeerEvent::LinkDeadSuspected, Self::now_ms(), &ambient)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Ambient shell facts for the routed liveness reap. Mirrors
|
||||
/// [`rekey_cadence_ctx`](Node::rekey_cadence_ctx). The executor reads only
|
||||
/// `verified_identity` (`InvalidateSendState` → `remove_active_peer` resolves
|
||||
/// its `NodeAddr` from it, so it must equal `node_addr`) and `now_ms`
|
||||
/// (`ReportLost` → `note_link_dead`, the wall-clock reconnect basis). The
|
||||
/// transport/index/direction fields are unused by these two arms and are
|
||||
/// populated best-effort for coherence. `now_ms` is threaded in (rather than
|
||||
/// re-read) so the value fed to `note_link_dead` is byte-identical to the old
|
||||
/// reap's hoisted wall-clock for every peer in the sweep.
|
||||
fn link_dead_ctx(&self, node_addr: &NodeAddr, now_ms: u64) -> PeerActionCtx {
|
||||
let peer = &self.peers[node_addr];
|
||||
PeerActionCtx {
|
||||
verified_identity: *peer.identity(),
|
||||
transport_id: peer.transport_id().unwrap_or_else(|| TransportId::new(0)),
|
||||
remote_addr: peer
|
||||
.current_addr()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| TransportAddr::new(Vec::new())),
|
||||
our_index: peer.our_index(),
|
||||
their_index: peer.their_index(),
|
||||
now_ms,
|
||||
is_outbound: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
//! Message handlers: per-message-type behavior on `impl Node`.
|
||||
//! RX event loop and message handlers.
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) mod connected_udp;
|
||||
pub(crate) mod discovery;
|
||||
mod dispatch;
|
||||
mod encrypted;
|
||||
mod forwarding;
|
||||
mod handshake;
|
||||
pub(crate) mod lookup;
|
||||
mod mmp;
|
||||
mod rekey;
|
||||
mod rx_loop;
|
||||
pub(in crate::node) mod session;
|
||||
mod timeout;
|
||||
|
||||
+309
-409
@@ -7,28 +7,28 @@
|
||||
|
||||
use crate::NodeAddr;
|
||||
use crate::node::Node;
|
||||
use crate::node::dataplane::PeerActionCtx;
|
||||
use crate::node::wire::build_msg1;
|
||||
use crate::noise::HandshakeState;
|
||||
use crate::peer::machine::PeerEvent;
|
||||
use crate::proto::fmp::wire::build_msg1;
|
||||
use crate::proto::fmp::{ConnAction, LifecycleView, PeerSnapshot, RekeyCfg, RekeyResendSnapshot};
|
||||
use crate::proto::fsp::{
|
||||
FspAction, RekeyMsg3ResendSnapshot, SessionSetup, SessionSnapshot, cutover_timer_elapsed,
|
||||
};
|
||||
use crate::proto::link::SessionDatagram;
|
||||
use crate::transport::{TransportAddr, TransportId};
|
||||
use crate::protocol::{SessionDatagram, SessionSetup};
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
/// Keep previous session alive for this long after cutover.
|
||||
///
|
||||
/// FMP-scoped copy for `check_rekey`; the FSP session-rekey timing bounds live
|
||||
/// in `crate::proto::fsp::limits`.
|
||||
const DRAIN_WINDOW_SECS: u64 = 10;
|
||||
|
||||
/// Suppress local rekey initiation for this long after receiving
|
||||
/// a peer's rekey msg1. FMP-scoped copy for `check_rekey`.
|
||||
/// a peer's rekey msg1.
|
||||
const REKEY_DAMPENING_SECS: u64 = 30;
|
||||
|
||||
/// Liveness bound on how long the FSP rekey initiator holds the
|
||||
/// `current` + `pending` state before cutting over to the new epoch.
|
||||
///
|
||||
/// This is NOT safety-critical: overlapping-epoch trial-decrypt covers
|
||||
/// any skew between the two endpoints' cutovers. The timer only bounds
|
||||
/// how long the initiator advertises the old K-bit. An opportunistic
|
||||
/// early cutover also fires if the initiator authenticates a peer frame
|
||||
/// against its own `pending` session (the responder cut over first).
|
||||
const FSP_CUTOVER_DELAY_MS: u64 = 2000;
|
||||
|
||||
impl Node {
|
||||
/// Periodic rekey check. Called from the tick loop.
|
||||
///
|
||||
@@ -41,225 +41,121 @@ impl Node {
|
||||
return;
|
||||
}
|
||||
|
||||
let cfg = RekeyCfg {
|
||||
after_secs: self.config().node.rekey.after_secs,
|
||||
after_messages: self.config().node.rekey.after_messages,
|
||||
};
|
||||
let rekey_after_secs = self.config().node.rekey.after_secs;
|
||||
let rekey_after_messages = self.config().node.rekey.after_messages;
|
||||
|
||||
// The shell snapshots each healthy peer's rekey ages/flags (every clock
|
||||
// read resolved here); the core decides cutover/drain/trigger with no
|
||||
// clock, phase-grouped to preserve the pre-refactor execution order.
|
||||
// The batch `poll_rekey` + snapshots STAY SHELL-SIDE and BYTE-UNCHANGED:
|
||||
// the cross-peer phase-grouping (all Cutover → all Drain →
|
||||
// all InitiateRekey) governs the shared `index_allocator` free-then-alloc
|
||||
// SEQUENCE that appears on the wire. The machine must NOT re-poll; it
|
||||
// CONSUMES each decided `ConnAction` in the same order the batch returned.
|
||||
let snapshots = self.rekey_peers();
|
||||
for action in self.fmp.poll_rekey(snapshots, &cfg) {
|
||||
match action {
|
||||
// Initiator cutover: route the decided action through the peer
|
||||
// machine + executor. The executor's `SwapSendState` arm
|
||||
// reproduces the pre-refactor cutover body EXACTLY.
|
||||
ConnAction::Cutover { peer: node_addr } => {
|
||||
self.route_rekey_cadence(node_addr, ConnAction::Cutover { peer: node_addr })
|
||||
.await;
|
||||
}
|
||||
// Drain completion: route through the machine + executor. The
|
||||
// executor's `CompleteDrain` arm reads the REAL previous index
|
||||
// from `complete_drain()` and frees it at the same point the old
|
||||
// inline body did (index-order preserving).
|
||||
ConnAction::Drain { peer: node_addr } => {
|
||||
self.route_rekey_cadence(node_addr, ConnAction::Drain { peer: node_addr })
|
||||
.await;
|
||||
}
|
||||
// Initiate a new rekey: STAYS INLINE (the Noise msg1 build +
|
||||
// index allocation are a shell-side leaf, byte-unchanged). Feed
|
||||
// the machine a `RekeyInitiated` observation afterward so its
|
||||
// control state stays coherent for the next tick's Cutover/Drain.
|
||||
ConnAction::InitiateRekey { peer: node_addr } => {
|
||||
self.initiate_rekey(&node_addr).await;
|
||||
self.observe_rekey_initiated(&node_addr);
|
||||
}
|
||||
#[allow(unreachable_patterns)]
|
||||
_ => {}
|
||||
// Collect peers that need action (to avoid borrow conflicts)
|
||||
let mut peers_to_cutover: Vec<NodeAddr> = Vec::new();
|
||||
let mut peers_to_drain: Vec<NodeAddr> = Vec::new();
|
||||
let mut peers_to_rekey: Vec<NodeAddr> = Vec::new();
|
||||
|
||||
for (node_addr, peer) in &self.peers {
|
||||
if !peer.has_session() || !peer.is_healthy() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 1. Initiator-side cutover: we completed a rekey and have
|
||||
// a pending session ready. Cut over on the next tick.
|
||||
if peer.pending_new_session().is_some() && !peer.rekey_in_progress() {
|
||||
peers_to_cutover.push(*node_addr);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Drain window expiry
|
||||
if peer.is_draining() && peer.drain_expired(DRAIN_WINDOW_SECS) {
|
||||
peers_to_drain.push(*node_addr);
|
||||
}
|
||||
|
||||
// 3. Rekey trigger
|
||||
if peer.rekey_in_progress() {
|
||||
continue;
|
||||
}
|
||||
if peer.is_rekey_dampened(REKEY_DAMPENING_SECS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let elapsed = peer.session_established_at().elapsed().as_secs();
|
||||
let counter = peer
|
||||
.noise_session()
|
||||
.map(|s| s.current_send_counter())
|
||||
.unwrap_or(0);
|
||||
|
||||
// Apply per-session symmetric jitter to desynchronize
|
||||
// dual-initiation in symmetric-start meshes.
|
||||
let effective_after_secs =
|
||||
rekey_after_secs.saturating_add_signed(peer.rekey_jitter_secs());
|
||||
if elapsed >= effective_after_secs || counter >= rekey_after_messages {
|
||||
peers_to_rekey.push(*node_addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Route a cadence-decided `Cutover`/`Drain` `ConnAction` through the peer
|
||||
/// machine + executor. The shell already decided (batch `poll_rekey`);
|
||||
/// the machine consumes via [`PeerEvent::RekeyConsume`] WITHOUT re-polling,
|
||||
/// preserving the phase order. The `SwapSendState`/`CompleteDrain` executor
|
||||
/// arms reproduce the pre-refactor inline effect bodies exactly.
|
||||
///
|
||||
/// An established peer always has a `peer_machine`. If the peer vanished
|
||||
/// between snapshot and effect, the old inline body was a no-op, so we do
|
||||
/// nothing; if the machine is absent (which should be impossible) we fall
|
||||
/// back to the byte-identical inline body under a `debug_assert`.
|
||||
async fn route_rekey_cadence(&mut self, node_addr: NodeAddr, action: ConnAction) {
|
||||
let link = match self.peers.get(&node_addr) {
|
||||
Some(peer) => peer.link_id(),
|
||||
None => return,
|
||||
};
|
||||
if !self.peer_machines.contains_key(&link) {
|
||||
debug_assert!(
|
||||
false,
|
||||
"peer machine present for every established rekey peer"
|
||||
);
|
||||
match action {
|
||||
ConnAction::Cutover { peer } => self.cutover_peer_inline(&peer),
|
||||
ConnAction::Drain { peer } => self.drain_peer_inline(&peer),
|
||||
_ => {}
|
||||
}
|
||||
return;
|
||||
}
|
||||
let ambient = self.rekey_cadence_ctx(&node_addr);
|
||||
self.advance_peer_machine(
|
||||
link,
|
||||
PeerEvent::RekeyConsume { action },
|
||||
ambient.now_ms,
|
||||
&ambient,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Feed the machine the `RekeyInitiated` observation after the inline
|
||||
/// `initiate_rekey`. The obs emits no action, so there is no executor
|
||||
/// pass — a bare `step` keeps the machine's control state coherent.
|
||||
fn observe_rekey_initiated(&mut self, node_addr: &NodeAddr) {
|
||||
let link = match self.peers.get(node_addr) {
|
||||
Some(peer) => peer.link_id(),
|
||||
None => return,
|
||||
};
|
||||
if let Some(machine) = self.peer_machines.get_mut(&link) {
|
||||
let acts = machine.step(
|
||||
PeerEvent::RekeyInitiated,
|
||||
Self::now_ms(),
|
||||
&mut self.index_allocator,
|
||||
);
|
||||
debug_assert!(acts.is_empty(), "RekeyInitiated is a pure observation");
|
||||
} else {
|
||||
debug_assert!(
|
||||
false,
|
||||
"peer machine present for every established rekey peer"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Ambient shell facts for the routed cadence Cutover/Drain step. Only
|
||||
/// `verified_identity` is read by the `SwapSendState`/`CompleteDrain`
|
||||
/// executor arms — `SwapSendState` resolves its `NodeAddr` from it (so it must
|
||||
/// equal `node_addr`), and `CompleteDrain` carries its peer in the action
|
||||
/// payload. The transport/index/direction fields are unused by these two arms
|
||||
/// (they matter only to `PromoteToActive`, never emitted on this path) and are
|
||||
/// populated best-effort for coherence.
|
||||
fn rekey_cadence_ctx(&self, node_addr: &NodeAddr) -> PeerActionCtx {
|
||||
let peer = &self.peers[node_addr];
|
||||
PeerActionCtx {
|
||||
verified_identity: *peer.identity(),
|
||||
transport_id: peer.transport_id().unwrap_or_else(|| TransportId::new(0)),
|
||||
remote_addr: peer
|
||||
.current_addr()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| TransportAddr::new(Vec::new())),
|
||||
our_index: peer.our_index(),
|
||||
their_index: peer.their_index(),
|
||||
now_ms: Self::now_ms(),
|
||||
is_outbound: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pre-refactor initiator cutover body, retained as the release fallback for
|
||||
/// the (should-be-impossible) missing-machine case. Byte-identical to the old
|
||||
/// inline `ConnAction::Cutover` arm and to the executor's `SwapSendState` arm.
|
||||
fn cutover_peer_inline(&mut self, node_addr: &NodeAddr) {
|
||||
let did_cutover = if let Some(peer) = self.peers.get_mut(node_addr) {
|
||||
if let Some(_old_our_index) = peer.cutover_to_new_session() {
|
||||
// New index was pre-registered in peers_by_index during msg2
|
||||
// handling (handshake.rs).
|
||||
debug_assert!(
|
||||
peer.transport_id().is_some()
|
||||
&& peer.our_index().is_some()
|
||||
&& self.peers_by_index.contains_key(&(
|
||||
peer.transport_id().unwrap(),
|
||||
peer.our_index().unwrap().as_u32()
|
||||
)),
|
||||
"peers_by_index should contain pre-registered new index after cutover"
|
||||
);
|
||||
debug!(
|
||||
peer = %self.peer_display_name(node_addr),
|
||||
"Rekey cutover complete (initiator), K-bit flipped"
|
||||
);
|
||||
true
|
||||
// Execute cutover for initiator side
|
||||
for node_addr in peers_to_cutover {
|
||||
let did_cutover = if let Some(peer) = self.peers.get_mut(&node_addr) {
|
||||
if let Some(_old_our_index) = peer.cutover_to_new_session() {
|
||||
// New index was pre-registered in peers_by_index
|
||||
// during msg2 handling (handshake.rs).
|
||||
debug_assert!(
|
||||
peer.transport_id().is_some()
|
||||
&& peer.our_index().is_some()
|
||||
&& self.peers_by_index.contains_key(&(
|
||||
peer.transport_id().unwrap(),
|
||||
peer.our_index().unwrap().as_u32()
|
||||
)),
|
||||
"peers_by_index should contain pre-registered new index after cutover"
|
||||
);
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
"Rekey cutover complete (initiator), K-bit flipped"
|
||||
);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
false
|
||||
};
|
||||
// Re-register the new session with the decrypt worker — the
|
||||
// cache_key (transport_id, our_index) just changed, so the
|
||||
// old worker entry is stale and every packet on the new
|
||||
// session would miss the worker's HashMap lookup.
|
||||
#[cfg(unix)]
|
||||
if did_cutover {
|
||||
self.register_decrypt_worker_session(&node_addr);
|
||||
}
|
||||
} else {
|
||||
false
|
||||
};
|
||||
// Re-register the new session with the decrypt worker — the cache_key
|
||||
// (transport_id, our_index) just changed, so the old worker entry is
|
||||
// stale and every packet on the new session would miss the lookup.
|
||||
#[cfg(unix)]
|
||||
if did_cutover {
|
||||
self.register_decrypt_worker_session(node_addr);
|
||||
#[cfg(not(unix))]
|
||||
let _ = did_cutover;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
let _ = did_cutover;
|
||||
}
|
||||
|
||||
/// Pre-refactor drain-completion body, retained as the release fallback for
|
||||
/// the (should-be-impossible) missing-machine case. Byte-identical to the old
|
||||
/// inline `ConnAction::Drain` arm and to the executor's `CompleteDrain` arm.
|
||||
fn drain_peer_inline(&mut self, node_addr: &NodeAddr) {
|
||||
// Extract the old index and transport_id under the peer borrow, then drop
|
||||
// the borrow so the cache_key cleanup below can take &mut self for
|
||||
// unregister_decrypt_worker_session.
|
||||
let drained = self
|
||||
.peers
|
||||
.get_mut(node_addr)
|
||||
.and_then(|peer| peer.complete_drain().map(|idx| (idx, peer.transport_id())));
|
||||
if let Some((old_our_index, transport_id)) = drained {
|
||||
if let Some(tid) = transport_id {
|
||||
let cache_key = (tid, old_our_index.as_u32());
|
||||
self.peers_by_index.remove(&cache_key);
|
||||
#[cfg(unix)]
|
||||
self.unregister_decrypt_worker_session(cache_key);
|
||||
// Execute drain completion
|
||||
for node_addr in peers_to_drain {
|
||||
// Extract the old index and transport_id under the peer
|
||||
// borrow, then drop the borrow so the cache_key cleanup
|
||||
// below can take &mut self for unregister_decrypt_worker_session.
|
||||
let drained = self
|
||||
.peers
|
||||
.get_mut(&node_addr)
|
||||
.and_then(|peer| peer.complete_drain().map(|idx| (idx, peer.transport_id())));
|
||||
if let Some((old_our_index, transport_id)) = drained {
|
||||
if let Some(tid) = transport_id {
|
||||
let cache_key = (tid, old_our_index.as_u32());
|
||||
self.peers_by_index.remove(&cache_key);
|
||||
#[cfg(unix)]
|
||||
self.unregister_decrypt_worker_session(cache_key);
|
||||
}
|
||||
let _ = self.index_allocator.free(old_our_index);
|
||||
trace!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
old_index = %old_our_index,
|
||||
"Drain complete, previous session erased"
|
||||
);
|
||||
}
|
||||
let _ = self.index_allocator.free(old_our_index);
|
||||
trace!(
|
||||
peer = %self.peer_display_name(node_addr),
|
||||
old_index = %old_our_index,
|
||||
"Drain complete, previous session erased"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot every healthy peer with a session for the rekey decision,
|
||||
/// pre-computing its monotonic ages and timer predicates so the pure core
|
||||
/// applies the thresholds without reading a clock (see [`PeerSnapshot`]).
|
||||
///
|
||||
/// Lives here, beside the drain/dampening constants and the FSP analog, so
|
||||
/// the forward-merge onto `next` reconciles rekey timing in one place.
|
||||
pub(in crate::node) fn rekey_peer_snapshots(&self) -> Vec<PeerSnapshot> {
|
||||
self.peers
|
||||
.iter()
|
||||
.filter(|(_, peer)| peer.has_session() && peer.is_healthy())
|
||||
.map(|(node_addr, peer)| PeerSnapshot {
|
||||
addr: *node_addr,
|
||||
has_pending: peer.pending_new_session().is_some(),
|
||||
rekey_in_progress: peer.rekey_in_progress(),
|
||||
is_draining: peer.is_draining(),
|
||||
drain_expired: peer.drain_expired(DRAIN_WINDOW_SECS),
|
||||
is_dampened: peer.is_rekey_dampened(REKEY_DAMPENING_SECS),
|
||||
elapsed_secs: peer.session_established_at().elapsed().as_secs(),
|
||||
counter: peer
|
||||
.noise_session()
|
||||
.map(|s| s.current_send_counter())
|
||||
.unwrap_or(0),
|
||||
jitter_secs: peer.rekey_jitter_secs(),
|
||||
})
|
||||
.collect()
|
||||
// Initiate new rekeys
|
||||
for node_addr in peers_to_rekey {
|
||||
self.initiate_rekey(&node_addr).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Initiate an outbound rekey to a peer.
|
||||
@@ -364,74 +260,60 @@ impl Node {
|
||||
let backoff = self.config().node.rate_limit.handshake_resend_backoff;
|
||||
let max_resends = self.config().node.rate_limit.handshake_max_resends;
|
||||
|
||||
// The shell snapshots each in-flight rekey (resend-due predicate
|
||||
// resolved here); the core classifies abandon-vs-resend and computes
|
||||
// the backoff, abandons first.
|
||||
let candidates = self.rekey_resend_candidates(now_ms);
|
||||
for action in
|
||||
self.fmp
|
||||
.poll_rekey_resends(candidates, now_ms, interval_ms, backoff, max_resends)
|
||||
{
|
||||
match action {
|
||||
// Abandon rekey cycles that exhausted their retransmission budget.
|
||||
ConnAction::AbandonRekey { peer: node_addr } => {
|
||||
if let Some(peer) = self.peers.get_mut(&node_addr) {
|
||||
peer.abandon_rekey();
|
||||
}
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
"FMP rekey aborted: msg1 unconfirmed after max retransmissions, abandoning cycle"
|
||||
);
|
||||
}
|
||||
ConnAction::ResendRekeyMsg1 {
|
||||
peer: node_addr,
|
||||
bytes,
|
||||
next_resend_at_ms,
|
||||
} => {
|
||||
let (transport_id, remote_addr) = match self.peers.get(&node_addr) {
|
||||
Some(p) => match (p.transport_id(), p.current_addr()) {
|
||||
(Some(tid), Some(addr)) => (tid, addr.clone()),
|
||||
_ => continue,
|
||||
},
|
||||
None => continue,
|
||||
};
|
||||
// Collect peers needing action
|
||||
let mut to_resend: Vec<(NodeAddr, Vec<u8>)> = Vec::new();
|
||||
let mut to_abandon: Vec<NodeAddr> = Vec::new();
|
||||
|
||||
let sent = if let Some(transport) = self.transports.get(&transport_id) {
|
||||
transport.send(&remote_addr, &bytes).await.is_ok()
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
if sent && let Some(peer) = self.peers.get_mut(&node_addr) {
|
||||
peer.record_rekey_msg1_resend(next_resend_at_ms);
|
||||
let count = peer.rekey_msg1_resend_count();
|
||||
trace!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
resend = count,
|
||||
"Resent rekey msg1"
|
||||
);
|
||||
}
|
||||
}
|
||||
#[allow(unreachable_patterns)]
|
||||
_ => {}
|
||||
for (node_addr, peer) in &self.peers {
|
||||
if !peer.rekey_in_progress() || peer.rekey_msg1().is_none() {
|
||||
continue;
|
||||
}
|
||||
if peer.rekey_msg1_resend_count() >= max_resends {
|
||||
to_abandon.push(*node_addr);
|
||||
continue;
|
||||
}
|
||||
if peer.needs_msg1_resend(now_ms) {
|
||||
to_resend.push((*node_addr, peer.rekey_msg1().unwrap().to_vec()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot every peer with a rekey handshake in flight (and a stored
|
||||
/// msg1) for the retransmission decision, pre-evaluating the resend-due
|
||||
/// predicate against `now_ms` so the core reads no clock.
|
||||
pub(in crate::node) fn rekey_resend_snapshots(&self, now_ms: u64) -> Vec<RekeyResendSnapshot> {
|
||||
self.peers
|
||||
.iter()
|
||||
.filter(|(_, peer)| peer.rekey_in_progress() && peer.rekey_msg1().is_some())
|
||||
.map(|(node_addr, peer)| RekeyResendSnapshot {
|
||||
peer: *node_addr,
|
||||
resend_count: peer.rekey_msg1_resend_count(),
|
||||
needs_resend: peer.needs_msg1_resend(now_ms),
|
||||
msg1: peer.rekey_msg1().unwrap().to_vec(),
|
||||
})
|
||||
.collect()
|
||||
// Abandon rekey cycles that exhausted their retransmission budget.
|
||||
for node_addr in to_abandon {
|
||||
if let Some(peer) = self.peers.get_mut(&node_addr) {
|
||||
peer.abandon_rekey();
|
||||
}
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
"FMP rekey aborted: msg1 unconfirmed after max retransmissions, abandoning cycle"
|
||||
);
|
||||
}
|
||||
|
||||
for (node_addr, msg1_bytes) in to_resend {
|
||||
let (transport_id, remote_addr) = match self.peers.get(&node_addr) {
|
||||
Some(p) => match (p.transport_id(), p.current_addr()) {
|
||||
(Some(tid), Some(addr)) => (tid, addr.clone()),
|
||||
_ => continue,
|
||||
},
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let sent = if let Some(transport) = self.transports.get(&transport_id) {
|
||||
transport.send(&remote_addr, &msg1_bytes).await.is_ok()
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
if sent && let Some(peer) = self.peers.get_mut(&node_addr) {
|
||||
let count = peer.rekey_msg1_resend_count() + 1;
|
||||
let next = now_ms + (interval_ms as f64 * backoff.powi(count as i32)) as u64;
|
||||
peer.record_rekey_msg1_resend(next);
|
||||
trace!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
resend = count,
|
||||
"Resent rekey msg1"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Retransmit FSP rekey msg3 until the responder is confirmed on the
|
||||
@@ -470,77 +352,66 @@ impl Node {
|
||||
let ttl = self.config().node.session.default_ttl;
|
||||
let my_addr = *self.node_addr();
|
||||
|
||||
// The shell snapshots each session retaining a msg3 payload (resend-due
|
||||
// predicate resolved here); the core classifies abandon-vs-resend,
|
||||
// abandons first.
|
||||
let candidates = self.rekey_msg3_resend_snapshots(now_ms);
|
||||
for action in self.fsp.poll_rekey_msg3_resends(candidates, max_resends) {
|
||||
match action {
|
||||
FspAction::AbandonRekey { addr } => {
|
||||
if let Some(entry) = self.sessions.get_mut(&addr) {
|
||||
entry.abandon_rekey();
|
||||
}
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&addr),
|
||||
"FSP rekey aborted: msg3 unconfirmed after max retransmissions, abandoning cycle"
|
||||
);
|
||||
}
|
||||
FspAction::ResendSessionMsg3 { addr } => {
|
||||
let payload = match self
|
||||
.sessions
|
||||
.get(&addr)
|
||||
.and_then(|e| e.rekey_msg3_payload())
|
||||
{
|
||||
Some(p) => p.to_vec(),
|
||||
None => continue,
|
||||
};
|
||||
let mut datagram = SessionDatagram::new(my_addr, addr, payload).with_ttl(ttl);
|
||||
let sent = match self.send_session_datagram(&mut datagram).await {
|
||||
Ok(_) => true,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&addr),
|
||||
error = %e,
|
||||
"FSP rekey msg3 retransmission failed"
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
// Collect rekey initiators whose msg3 retransmission is due.
|
||||
let mut to_resend: Vec<(NodeAddr, Vec<u8>)> = Vec::new();
|
||||
let mut to_abandon: Vec<NodeAddr> = Vec::new();
|
||||
|
||||
if sent && let Some(entry) = self.sessions.get_mut(&addr) {
|
||||
let count = entry.rekey_msg3_resend_count() + 1;
|
||||
let next =
|
||||
now_ms + (interval_ms as f64 * backoff.powi(count as i32)) as u64;
|
||||
entry.record_rekey_msg3_resend(next);
|
||||
trace!(
|
||||
peer = %self.peer_display_name(&addr),
|
||||
resend = count,
|
||||
"Resent FSP rekey msg3"
|
||||
);
|
||||
}
|
||||
for (node_addr, entry) in &self.sessions {
|
||||
// Only the rekey initiator retains a msg3 payload.
|
||||
let payload = match entry.rekey_msg3_payload() {
|
||||
Some(p) => p,
|
||||
None => continue,
|
||||
};
|
||||
if entry.rekey_msg3_next_resend_ms() == 0 || now_ms < entry.rekey_msg3_next_resend_ms()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if entry.rekey_msg3_resend_count() >= max_resends {
|
||||
to_abandon.push(*node_addr);
|
||||
continue;
|
||||
}
|
||||
to_resend.push((*node_addr, payload.to_vec()));
|
||||
}
|
||||
|
||||
// Abandon rekey cycles that exhausted their retransmission budget.
|
||||
for node_addr in to_abandon {
|
||||
if let Some(entry) = self.sessions.get_mut(&node_addr) {
|
||||
entry.abandon_rekey();
|
||||
}
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
"FSP rekey aborted: msg3 unconfirmed after max retransmissions, abandoning cycle"
|
||||
);
|
||||
}
|
||||
|
||||
// Retransmit msg3 for cycles still within budget.
|
||||
for (node_addr, payload) in to_resend {
|
||||
let mut datagram = SessionDatagram::new(my_addr, node_addr, payload).with_ttl(ttl);
|
||||
let sent = match self.send_session_datagram(&mut datagram).await {
|
||||
Ok(_) => true,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
error = %e,
|
||||
"FSP rekey msg3 retransmission failed"
|
||||
);
|
||||
false
|
||||
}
|
||||
#[allow(unreachable_patterns)]
|
||||
_ => {}
|
||||
};
|
||||
|
||||
if sent && let Some(entry) = self.sessions.get_mut(&node_addr) {
|
||||
let count = entry.rekey_msg3_resend_count() + 1;
|
||||
let next = now_ms + (interval_ms as f64 * backoff.powi(count as i32)) as u64;
|
||||
entry.record_rekey_msg3_resend(next);
|
||||
trace!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
resend = count,
|
||||
"Resent FSP rekey msg3"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot every session retaining a rekey-msg3 payload for the
|
||||
/// retransmission decision, pre-evaluating the resend-due predicate against
|
||||
/// `now_ms` so the core reads no clock.
|
||||
fn rekey_msg3_resend_snapshots(&self, now_ms: u64) -> Vec<RekeyMsg3ResendSnapshot> {
|
||||
self.sessions
|
||||
.iter()
|
||||
.filter(|(_, entry)| entry.rekey_msg3_payload().is_some())
|
||||
.map(|(node_addr, entry)| RekeyMsg3ResendSnapshot {
|
||||
addr: *node_addr,
|
||||
resend_count: entry.rekey_msg3_resend_count(),
|
||||
resend_due: entry.rekey_msg3_next_resend_ms() != 0
|
||||
&& now_ms >= entry.rekey_msg3_next_resend_ms(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Periodic session (FSP) rekey check. Called from the tick loop.
|
||||
///
|
||||
/// For each established session:
|
||||
@@ -558,71 +429,100 @@ impl Node {
|
||||
return;
|
||||
}
|
||||
|
||||
let cfg = crate::proto::fsp::RekeyCfg {
|
||||
after_secs: self.config().node.rekey.after_secs,
|
||||
after_messages: self.config().node.rekey.after_messages,
|
||||
};
|
||||
let rekey_after_secs = self.config().node.rekey.after_secs;
|
||||
let rekey_after_messages = self.config().node.rekey.after_messages;
|
||||
let now_ms = Self::now_ms();
|
||||
let drain_ms = DRAIN_WINDOW_SECS * 1000;
|
||||
let dampening_ms = REKEY_DAMPENING_SECS * 1000;
|
||||
|
||||
// The shell snapshots each established session's rekey ages/flags
|
||||
// (every clock read resolved here); the core decides
|
||||
// cutover/drain/trigger with no clock, phase-grouped to preserve the
|
||||
// pre-refactor execution order.
|
||||
let snapshots = self.session_rekey_snapshots(now_ms);
|
||||
for action in self.fsp.poll_rekey(snapshots, &cfg) {
|
||||
match action {
|
||||
FspAction::CutOver { addr } => {
|
||||
if let Some(entry) = self.sessions.get_mut(&addr)
|
||||
&& entry.cutover_to_new_session(now_ms)
|
||||
{
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&addr),
|
||||
"FSP rekey cutover complete (initiator), K-bit flipped"
|
||||
);
|
||||
}
|
||||
}
|
||||
FspAction::CompleteDrain { addr } => {
|
||||
if let Some(entry) = self.sessions.get_mut(&addr) {
|
||||
entry.complete_drain();
|
||||
trace!(
|
||||
peer = %self.peer_display_name(&addr),
|
||||
"FSP drain complete, previous session erased"
|
||||
);
|
||||
}
|
||||
}
|
||||
FspAction::InitiateRekey { addr } => {
|
||||
self.initiate_session_rekey(&addr).await;
|
||||
}
|
||||
#[allow(unreachable_patterns)]
|
||||
_ => {}
|
||||
let mut sessions_to_cutover: Vec<NodeAddr> = Vec::new();
|
||||
let mut sessions_to_drain: Vec<NodeAddr> = Vec::new();
|
||||
let mut sessions_to_rekey: Vec<NodeAddr> = Vec::new();
|
||||
|
||||
for (node_addr, entry) in &self.sessions {
|
||||
if !entry.is_established() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 1. Initiator-side cutover (option A): completed rekey,
|
||||
// pending session ready, liveness timer elapsed. This is
|
||||
// an unconditional timer, NOT gated on responder progress —
|
||||
// overlapping-epoch trial-decrypt covers the cutover skew,
|
||||
// so flipping the K-bit here is always safe. An
|
||||
// opportunistic early cutover also happens in
|
||||
// `handle_encrypted_session_msg` if the initiator
|
||||
// authenticates a peer frame against its own `pending`.
|
||||
if entry.pending_new_session().is_some()
|
||||
&& !entry.has_rekey_in_progress()
|
||||
&& entry.is_rekey_initiator()
|
||||
&& now_ms.saturating_sub(entry.rekey_completed_ms()) >= FSP_CUTOVER_DELAY_MS
|
||||
{
|
||||
sessions_to_cutover.push(*node_addr);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Drain window expiry
|
||||
if entry.is_draining() && entry.drain_expired(now_ms, drain_ms) {
|
||||
sessions_to_drain.push(*node_addr);
|
||||
}
|
||||
|
||||
// 3. Rekey trigger
|
||||
if entry.has_rekey_in_progress() {
|
||||
continue;
|
||||
}
|
||||
if entry.pending_new_session().is_some() {
|
||||
continue; // Pending session present, awaiting cutover
|
||||
}
|
||||
if entry.rekey_msg3_payload().is_some() {
|
||||
// Initiator already cut over on its liveness timer but is
|
||||
// still retransmitting msg3 to a responder not yet
|
||||
// confirmed on the new epoch. Don't start another rekey
|
||||
// until the current cycle's msg3 is delivered or abandoned.
|
||||
continue;
|
||||
}
|
||||
if entry.is_rekey_dampened(now_ms, dampening_ms) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let elapsed_secs = now_ms.saturating_sub(entry.session_start_ms()) / 1000;
|
||||
let counter = entry.send_counter();
|
||||
|
||||
// Apply per-session symmetric jitter to desynchronize
|
||||
// dual-initiation in symmetric-start meshes.
|
||||
let effective_after_secs =
|
||||
rekey_after_secs.saturating_add_signed(entry.rekey_jitter_secs());
|
||||
if elapsed_secs >= effective_after_secs || counter >= rekey_after_messages {
|
||||
sessions_to_rekey.push(*node_addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot every established session for the FSP rekey decision,
|
||||
/// pre-computing its monotonic age and timer predicates so the pure core
|
||||
/// applies the thresholds without reading a clock (see [`SessionSnapshot`]).
|
||||
fn session_rekey_snapshots(&self, now_ms: u64) -> Vec<SessionSnapshot> {
|
||||
let drain_ms = crate::proto::fsp::limits::DRAIN_WINDOW_SECS * 1000;
|
||||
let dampening_ms = crate::proto::fsp::limits::REKEY_DAMPENING_SECS * 1000;
|
||||
self.sessions
|
||||
.iter()
|
||||
.filter(|(_, entry)| entry.is_established())
|
||||
.map(|(node_addr, entry)| SessionSnapshot {
|
||||
addr: *node_addr,
|
||||
has_pending: entry.pending_new_session().is_some(),
|
||||
rekey_in_progress: entry.has_rekey_in_progress(),
|
||||
is_rekey_initiator: entry.is_rekey_initiator(),
|
||||
cutover_timer_elapsed: cutover_timer_elapsed(now_ms, entry.rekey_completed_ms()),
|
||||
is_draining: entry.is_draining(),
|
||||
drain_expired: entry.drain_expired(now_ms, drain_ms),
|
||||
has_rekey_msg3_payload: entry.rekey_msg3_payload().is_some(),
|
||||
is_dampened: entry.is_rekey_dampened(now_ms, dampening_ms),
|
||||
elapsed_secs: now_ms.saturating_sub(entry.session_start_ms()) / 1000,
|
||||
counter: entry.send_counter(),
|
||||
jitter_secs: entry.rekey_jitter_secs(),
|
||||
})
|
||||
.collect()
|
||||
// Execute cutover for initiator side
|
||||
for node_addr in sessions_to_cutover {
|
||||
if let Some(entry) = self.sessions.get_mut(&node_addr)
|
||||
&& entry.cutover_to_new_session(now_ms)
|
||||
{
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
"FSP rekey cutover complete (initiator), K-bit flipped"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Execute drain completion
|
||||
for node_addr in sessions_to_drain {
|
||||
if let Some(entry) = self.sessions.get_mut(&node_addr) {
|
||||
entry.complete_drain();
|
||||
trace!(
|
||||
peer = %self.peer_display_name(&node_addr),
|
||||
"FSP drain complete, previous session erased"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Initiate new rekeys
|
||||
for node_addr in sessions_to_rekey {
|
||||
self.initiate_session_rekey(&node_addr).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Initiate an FSP session rekey.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
//! RX event loop and packet dispatch.
|
||||
|
||||
use crate::control::{ControlSocket, commands};
|
||||
use crate::node::{Node, NodeError};
|
||||
use crate::proto::fmp::wire::{
|
||||
use crate::node::wire::{
|
||||
COMMON_PREFIX_SIZE, CommonPrefix, FMP_VERSION, PHASE_ESTABLISHED, PHASE_MSG1, PHASE_MSG2,
|
||||
};
|
||||
use crate::node::{Node, NodeError};
|
||||
use crate::transport::ReceivedPacket;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, info, warn};
|
||||
@@ -42,42 +42,12 @@ impl Node {
|
||||
/// This method takes ownership of the packet_rx channel and runs
|
||||
/// until the channel is closed (typically when stop() is called).
|
||||
pub async fn run_rx_loop(&mut self) -> Result<(), NodeError> {
|
||||
// No shutdown observer → today's infinite loop, byte-identical. All
|
||||
// existing callers/tests use this; `pending()` never fires, so the
|
||||
// shutdown/deadline arms below stay permanently disabled.
|
||||
self.run_rx_loop_with_shutdown(std::future::pending()).await
|
||||
}
|
||||
|
||||
/// The rx event loop, which serves until `shutdown` fires and then drains
|
||||
/// **in place** before returning.
|
||||
///
|
||||
/// The channel receivers are moved into this frame's locals and live across
|
||||
/// both serve and drain, so — unlike a `select!`-cancelled loop — they are
|
||||
/// never destructively dropped mid-flight; they are released only on clean
|
||||
/// exit, after which teardown does not need them.
|
||||
///
|
||||
/// - While serving (`drain_deadline == None`) the loop is behaviorally
|
||||
/// identical to before: the shutdown arm, the deadline arm, and the
|
||||
/// peers-empty early-exit are all guarded off, so the hot per-packet path
|
||||
/// and the `biased` order of the real arms are unchanged.
|
||||
/// - When `shutdown` fires, the loop calls [`Node::enter_drain`] once
|
||||
/// (broadcast Disconnect, gate the reconciler off) and arms the bounded
|
||||
/// deadline, then keeps servicing inbound/tick/peer-removal until all
|
||||
/// peers clear or the deadline elapses, then returns. The caller
|
||||
/// ([`Node::finish_shutdown`]) closes the window and tears down.
|
||||
pub async fn run_rx_loop_with_shutdown(
|
||||
&mut self,
|
||||
shutdown: impl std::future::Future<Output = ()>,
|
||||
) -> Result<(), NodeError> {
|
||||
tokio::pin!(shutdown);
|
||||
// `None` = serving; `Some(deadline)` = draining (bounded window).
|
||||
let mut drain_deadline: Option<tokio::time::Instant> = None;
|
||||
let mut packet_rx = self.packet_rx.take().ok_or(NodeError::NotStarted)?;
|
||||
|
||||
// Take the TUN outbound receiver, or create a dummy channel that never
|
||||
// produces messages (when TUN is disabled). Holding the sender prevents
|
||||
// the channel from closing.
|
||||
let (mut tun_outbound_rx, _tun_guard) = match self.supervisor.tun_outbound_rx.take() {
|
||||
let (mut tun_outbound_rx, _tun_guard) = match self.tun_outbound_rx.take() {
|
||||
Some(rx) => (rx, None),
|
||||
None => {
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
@@ -87,7 +57,7 @@ impl Node {
|
||||
|
||||
// Take the DNS identity receiver, or create a dummy channel (when DNS
|
||||
// is disabled). Same pattern as TUN outbound.
|
||||
let (mut dns_identity_rx, _dns_guard) = match self.supervisor.dns_identity_rx.take() {
|
||||
let (mut dns_identity_rx, _dns_guard) = match self.dns_identity_rx.take() {
|
||||
Some(rx) => (rx, None),
|
||||
None => {
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
@@ -95,20 +65,8 @@ impl Node {
|
||||
}
|
||||
};
|
||||
|
||||
// Take the runtime child-liveness receiver, or a dummy channel (when the
|
||||
// node was seeded straight into Running without a start()). Holding the
|
||||
// dummy sender in the guard keeps the channel open. Same pattern as TUN
|
||||
// outbound / DNS identity.
|
||||
let (mut child_exit_rx, _child_exit_guard) = match self.child_exit_rx.take() {
|
||||
Some(rx) => (rx, None),
|
||||
None => {
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
(rx, Some(tx))
|
||||
}
|
||||
};
|
||||
|
||||
let tick_period = Duration::from_secs(self.config().node.tick_interval_secs);
|
||||
let mut tick = tokio::time::interval(tick_period);
|
||||
let mut tick =
|
||||
tokio::time::interval(Duration::from_secs(self.config().node.tick_interval_secs));
|
||||
|
||||
// Set up control socket channel
|
||||
let (control_tx, mut control_rx) =
|
||||
@@ -164,13 +122,6 @@ impl Node {
|
||||
crate::perf_profile::maybe_spawn_reporter();
|
||||
|
||||
loop {
|
||||
// Bounded drain mode: break as soon as all peers have cleared. In
|
||||
// normal mode (`None`) this short-circuits before touching
|
||||
// `self.peers`, so the loop is byte-identical.
|
||||
if drain_deadline.is_some() && self.peers.is_empty() {
|
||||
info!("Drain complete: all peers cleared, ending drain loop");
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
// Decrypt-worker fallback drains FIRST. Under sustained
|
||||
@@ -263,27 +214,6 @@ impl Node {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Runtime child-liveness. Placed AFTER `packet_rx` so the hot
|
||||
// inbound path keeps its `biased` priority. A directly-observable
|
||||
// child (TUN threads, DNS/mDNS/Nostr) exited on its own; feed the
|
||||
// FSM, which republishes health (Degraded here — a Running node
|
||||
// always has ≥1 transport up). `on_child_exited` only ever emits
|
||||
// `PublishState`; other variants are ignored defensively.
|
||||
maybe_child = child_exit_rx.recv() => {
|
||||
if let Some(child) = maybe_child {
|
||||
let actions = self
|
||||
.supervisor
|
||||
.fsm
|
||||
.step(crate::node::lifecycle::supervisor::Event::ChildExited { child });
|
||||
for action in actions {
|
||||
if let crate::node::lifecycle::supervisor::Action::PublishState(ns) =
|
||||
action
|
||||
{
|
||||
self.supervisor.state = ns;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(ipv6_packet) = tun_outbound_rx.recv() => {
|
||||
self.handle_tun_outbound(ipv6_packet).await;
|
||||
let mut drained = 0;
|
||||
@@ -319,114 +249,41 @@ impl Node {
|
||||
).await;
|
||||
let _ = response_tx.send(response);
|
||||
}
|
||||
deadline = tick.tick() => {
|
||||
// Tick-body instrumentation. The gate is read ONCE per tick
|
||||
// into `instr_on`, which is then passed explicitly to every
|
||||
// `instr_step!` invocation — macro hygiene makes a call-site
|
||||
// local invisible inside the macro body. With the
|
||||
// `profiling` feature off, `gate()` is a `const fn`
|
||||
// returning false and the macro is a pure pass-through, so
|
||||
// the whole arm compiles to the uninstrumented sequence.
|
||||
//
|
||||
// `tick_entry` records how late this entry is against the
|
||||
// deadline the interval scheduled it for. That is the
|
||||
// measurement this instrumentation exists for: the arm is
|
||||
// polled LAST under `biased;`, so the
|
||||
// lateness IS the time it spent waiting behind the packet,
|
||||
// TUN and control arms. `tick()` hands back its scheduled
|
||||
// deadline, so this is a subtraction rather than a model.
|
||||
// The whole-tick span below measures the body alone.
|
||||
let instr_on = crate::instr::gate();
|
||||
crate::instr::tick_entry(instr_on, deadline.into_std(), std::time::Instant::now());
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::WholeTick, {
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckTimeouts,
|
||||
self.check_timeouts());
|
||||
let now_ms = Self::now_ms();
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::ReloadPeerAcl,
|
||||
self.reload_peer_acl().await);
|
||||
// The host map hot-reloads on the same tick as the ACL. It
|
||||
// is polled separately from `reload_peer_acl` because the
|
||||
// ACL's embedded alias reloader and this snapshot are
|
||||
// distinct resources; the `path_mtu_lookup` cache and the
|
||||
// `nostr_rendezvous` subsystem are deliberately excluded
|
||||
// from `Reloadable` since neither reloads from a backing
|
||||
// file (see `node::reloadable`).
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::ReloadHostMap,
|
||||
self.reload_host_map().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::PollPendingConnects,
|
||||
self.poll_pending_connects().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::PollNostrRendezvous,
|
||||
self.poll_nostr_rendezvous().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::PollLanRendezvous,
|
||||
self.poll_lan_rendezvous().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::DrivePeerTimers,
|
||||
self.drive_peer_timers(now_ms).await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::ResendPendingRekeys,
|
||||
self.resend_pending_rekeys(now_ms).await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::ResendPendingSessionHandshakes,
|
||||
self.resend_pending_session_handshakes(now_ms).await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::ResendPendingSessionMsg3,
|
||||
self.resend_pending_session_msg3(now_ms).await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::PurgeIdleSessions,
|
||||
self.purge_idle_sessions(now_ms));
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::ProcessPendingRetries,
|
||||
self.process_pending_retries(now_ms).await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckTreeState,
|
||||
self.check_tree_state().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckBloomState,
|
||||
self.check_bloom_state().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::ComputeMeshSize,
|
||||
self.compute_mesh_size());
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::RecordStatsHistory,
|
||||
self.record_stats_history());
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckMmpReports,
|
||||
self.check_mmp_reports().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckSessionMmpReports,
|
||||
self.check_session_mmp_reports().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckLinkHeartbeats,
|
||||
self.check_link_heartbeats().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckRekey,
|
||||
self.check_rekey().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckSessionRekey,
|
||||
self.check_session_rekey().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::CheckPendingLookups,
|
||||
self.check_pending_lookups(now_ms).await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::PollTransportDiscovery,
|
||||
self.poll_transport_discovery().await);
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::SampleTransportCongestion,
|
||||
self.sample_transport_congestion());
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::ActivateConnectedUdpSessions,
|
||||
self.activate_connected_udp_sessions().await);
|
||||
// Debug-build sweep of the peer-lifecycle map invariant
|
||||
// (leaked machines / machine-less legs); two map scans,
|
||||
// compiled out of release builds.
|
||||
#[cfg(debug_assertions)]
|
||||
instr_step!(instr_on, crate::instr::Domain::Tick, crate::instr::Step::DebugAssertPeerMapsCoherent,
|
||||
self.debug_assert_peer_maps_coherent());
|
||||
});
|
||||
crate::instr::tick_gauges(instr_on, self.peers.len() as u64);
|
||||
}
|
||||
// Shutdown signal → enter the bounded drain in place, ONCE.
|
||||
// Gated on `is_none()` so it only fires while serving; after
|
||||
// entering drain the arm is disabled (the completed signal is
|
||||
// never polled again) and the deadline arm below bounds the
|
||||
// window. Placed after the real arms so their `biased` priority
|
||||
// is unchanged, and inert while serving with `pending()`.
|
||||
_ = &mut shutdown, if drain_deadline.is_none() => {
|
||||
self.enter_drain().await;
|
||||
drain_deadline =
|
||||
Some(tokio::time::Instant::now() + self.config().node.drain_timeout());
|
||||
}
|
||||
// Bounded drain deadline (drain mode only). Placed LAST so the
|
||||
// `biased` priority of the normal arms is unchanged, and gated
|
||||
// on `is_some()` so in normal mode the branch is disabled — the
|
||||
// future is created but never polled and never fires.
|
||||
_ = tokio::time::sleep_until(
|
||||
drain_deadline.unwrap_or_else(tokio::time::Instant::now)
|
||||
), if drain_deadline.is_some() => {
|
||||
info!("Drain deadline elapsed, ending drain loop");
|
||||
break;
|
||||
_ = tick.tick() => {
|
||||
self.check_timeouts();
|
||||
let now_ms = Self::now_ms();
|
||||
self.reload_peer_acl().await;
|
||||
// The host map hot-reloads on the same tick as the ACL. It
|
||||
// is polled separately from `reload_peer_acl` because the
|
||||
// ACL's embedded alias reloader and this snapshot are
|
||||
// distinct resources; the `path_mtu_lookup` cache and the
|
||||
// `nostr_discovery` subsystem are deliberately excluded
|
||||
// from `Reloadable` since neither reloads from a backing
|
||||
// file (see `node::reloadable`).
|
||||
self.reload_host_map().await;
|
||||
self.poll_pending_connects().await;
|
||||
self.poll_nostr_discovery().await;
|
||||
self.poll_lan_discovery().await;
|
||||
self.resend_pending_handshakes(now_ms).await;
|
||||
self.resend_pending_rekeys(now_ms).await;
|
||||
self.resend_pending_session_handshakes(now_ms).await;
|
||||
self.resend_pending_session_msg3(now_ms).await;
|
||||
self.purge_idle_sessions(now_ms);
|
||||
self.process_pending_retries(now_ms).await;
|
||||
self.check_tree_state().await;
|
||||
self.check_bloom_state().await;
|
||||
self.compute_mesh_size();
|
||||
self.record_stats_history();
|
||||
self.check_mmp_reports().await;
|
||||
self.check_session_mmp_reports().await;
|
||||
self.check_link_heartbeats().await;
|
||||
self.check_rekey().await;
|
||||
self.check_session_rekey().await;
|
||||
self.check_pending_lookups(now_ms).await;
|
||||
self.poll_transport_discovery().await;
|
||||
self.sample_transport_congestion();
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
self.activate_connected_udp_sessions().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -462,16 +319,12 @@ impl Node {
|
||||
// though no msg1/msg2 exchange can ever succeed. Bump the
|
||||
// discovery-layer cooldown to the long protocol-mismatch
|
||||
// window and emit a single WARN per fresh observation.
|
||||
if self
|
||||
.supervisor
|
||||
.nostr_rendezvous
|
||||
.is_bootstrap_transport(&packet.transport_id)
|
||||
if self.bootstrap_transports.contains(&packet.transport_id)
|
||||
&& let Some(npub) = self
|
||||
.supervisor
|
||||
.nostr_rendezvous
|
||||
.bootstrap_transport_npub(&packet.transport_id)
|
||||
.bootstrap_transport_npubs
|
||||
.get(&packet.transport_id)
|
||||
.cloned()
|
||||
&& let Some(handle) = self.nostr_rendezvous_handle()
|
||||
&& let Some(handle) = self.nostr_discovery_handle()
|
||||
{
|
||||
let now_ms = Self::now_ms();
|
||||
let cooldown_secs = handle.protocol_mismatch_cooldown_secs();
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user