mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Merge branch 'master' into next
This commit is contained in:
@@ -24,6 +24,28 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
SOURCE_DATE_EPOCH: 0 # overridden per-step after checkout
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# CI parity invariant
|
||||
#
|
||||
# This GitHub integration matrix and the local default suite set
|
||||
# (testing/ci-local.sh) MUST run the same integration suites, EXCEPT for the
|
||||
# deliberate local-only entries below. Adding a suite to one runner without
|
||||
# the other means "local green" and "GitHub green" stop being equivalent.
|
||||
# testing/check-ci-parity.sh enforces this and fails on unexpected drift.
|
||||
#
|
||||
# Deliberate local-only (NOT on the GitHub gate), with reason:
|
||||
# tor-socks5 — requires live Tor network; opt-in via --with-tor,
|
||||
# unreliable on GitHub-hosted runners.
|
||||
# tor-directory — same; live Tor dependency.
|
||||
#
|
||||
# Granularity-only differences (same coverage, different matrix shape —
|
||||
# NOT a divergence):
|
||||
# deb-install — split here into per-distro legs (debian12/ubuntu24/
|
||||
# ubuntu26) for parallelism; local runs all distros in one
|
||||
# suite.
|
||||
# dns-resolver — single leg here; runs all scenarios (same as local).
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Job 1 – Build matrix
|
||||
#
|
||||
@@ -35,9 +57,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
components: rustfmt
|
||||
cache: false
|
||||
rustflags: ''
|
||||
- run: cargo fmt --check
|
||||
|
||||
clippy:
|
||||
@@ -47,9 +71,11 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install system dependencies
|
||||
run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
components: clippy
|
||||
cache: false
|
||||
rustflags: ''
|
||||
- name: Cache Cargo registry + build
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
@@ -98,7 +124,10 @@ jobs:
|
||||
run: sudo nft -c -f packaging/common/fips.nft
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache: false
|
||||
rustflags: ''
|
||||
|
||||
- name: Cache Cargo registry + build
|
||||
uses: actions/cache@v4
|
||||
@@ -160,7 +189,10 @@ jobs:
|
||||
run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache: false
|
||||
rustflags: ''
|
||||
|
||||
- name: Cache Cargo registry + build
|
||||
uses: actions/cache@v4
|
||||
@@ -210,7 +242,10 @@ jobs:
|
||||
run: echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache: false
|
||||
rustflags: ''
|
||||
|
||||
- name: Cache Cargo registry + build
|
||||
uses: actions/cache@v4
|
||||
@@ -239,7 +274,10 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache: false
|
||||
rustflags: ''
|
||||
|
||||
- name: Cache Cargo registry + build
|
||||
uses: actions/cache@v4
|
||||
@@ -323,6 +361,10 @@ jobs:
|
||||
- suite: rekey-outbound-only
|
||||
type: rekey-outbound-only
|
||||
topology: rekey-outbound-only
|
||||
# ── Inbound max_peers admission-cap test ───────────────────────
|
||||
- suite: admission-cap
|
||||
type: admission-cap
|
||||
topology: mesh
|
||||
- suite: acl-allowlist
|
||||
type: acl-allowlist
|
||||
# ── Mixed-profile node-role integration test ─────────────────
|
||||
@@ -755,6 +797,42 @@ jobs:
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile gateway down --volumes --remove-orphans
|
||||
|
||||
# ── Inbound max_peers admission-cap integration test ────────────────
|
||||
# Lowers node.max_peers on one mesh node and asserts the inbound cap
|
||||
# holds under sustained retry pressure: denied peers keep retrying but
|
||||
# are never promoted to an active session. The admission-cap-test.sh
|
||||
# assertions are tailored per link-layer handshake variant; the leg
|
||||
# itself is uniform. Static-style harness on the shared mesh profile.
|
||||
- name: Generate configs (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/generate-configs.sh mesh
|
||||
|
||||
- name: Inject admission-cap config (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/admission-cap-test.sh inject-config
|
||||
|
||||
- name: Start containers (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh up -d
|
||||
|
||||
- name: Run admission-cap test
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/admission-cap-test.sh
|
||||
|
||||
- name: Collect logs on failure (admission-cap)
|
||||
if: matrix.type == 'admission-cap' && failure()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh logs --no-color | tail -300
|
||||
|
||||
- name: Stop containers (admission-cap)
|
||||
if: matrix.type == 'admission-cap' && always()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh down --volumes --remove-orphans
|
||||
|
||||
# ── Real-deb install integration ────────────────────────────────────
|
||||
# The deb-install harness builds its own .deb from source in a
|
||||
# cargo-deb builder image; the pre-built Linux binary from the
|
||||
|
||||
@@ -72,7 +72,10 @@ jobs:
|
||||
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libdbus-1-dev llvm
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache: false
|
||||
rustflags: ''
|
||||
|
||||
- name: Cache Cargo registry + build
|
||||
if: ${{ env.ACT != 'true' }}
|
||||
|
||||
@@ -69,10 +69,11 @@ jobs:
|
||||
run: echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Add cross-compile target
|
||||
run: rustup target add ${{ matrix.target }}
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
cache: false
|
||||
rustflags: ''
|
||||
|
||||
- name: Cache Cargo registry + build
|
||||
uses: actions/cache@v4
|
||||
|
||||
@@ -110,9 +110,11 @@ jobs:
|
||||
|
||||
- name: Install Rust toolchain (stable)
|
||||
if: matrix.rust_channel == 'stable'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
targets: ${{ matrix.rust_target }}
|
||||
target: ${{ matrix.rust_target }}
|
||||
cache: false
|
||||
rustflags: ''
|
||||
|
||||
- name: Install Rust toolchain (nightly, Tier 3)
|
||||
if: matrix.rust_channel == 'nightly'
|
||||
|
||||
@@ -63,7 +63,10 @@ jobs:
|
||||
echo "SOURCE_DATE_EPOCH=$epoch" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache: false
|
||||
rustflags: ''
|
||||
|
||||
- name: Cache Cargo registry + build
|
||||
uses: actions/cache@v4
|
||||
|
||||
@@ -154,6 +154,16 @@ with v0.3.x peers.
|
||||
requests" and "Further reading". Running the checklist before
|
||||
opening surfaces problems that would otherwise come back as review
|
||||
comments, saving a round trip.
|
||||
- [`docs/how-to/tune-file-descriptors.md`](docs/how-to/tune-file-descriptors.md)
|
||||
— an operator how-to for raising `RLIMIT_NOFILE`. A busy node opens
|
||||
roughly three file descriptors per established UDP peer (a
|
||||
`connect()`-ed socket plus a 2-FD drain self-pipe), so the default
|
||||
1024 soft limit is exhausted near 320 peers, after which further
|
||||
admission, handshakes, and discovery fail with `EMFILE`. The guide
|
||||
documents the per-peer FD budget and symptom, the systemd
|
||||
(`LimitNOFILE` drop-in) and OpenWrt (procd `nofile`) procedures to
|
||||
raise the limit, and how to verify the per-peer ratio stays bounded.
|
||||
Linked from the how-to index.
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -290,6 +300,39 @@ with v0.3.x peers.
|
||||
RTT +0.11–0.19 ms from the worker queue handoff. Windows
|
||||
continues on the existing tokio-based send/recv path.
|
||||
|
||||
- The Debian package no longer ships `/etc/fips/fips.yaml` as a dpkg
|
||||
conf-file. The default configuration is installed as an example at
|
||||
`/usr/share/fips/fips.yaml.example`, and `postinst` seeds
|
||||
`/etc/fips/fips.yaml` (mode 600) from it only when the file does not
|
||||
already exist — so a configuration-management-rendered or
|
||||
operator-edited config is never prompted for or clobbered on
|
||||
upgrade, removing the need for a `dpkg-divert` workaround.
|
||||
`fips.service` gains `ConditionPathExists=/etc/fips/fips.yaml`. The
|
||||
example is placed under `/usr/share/fips`, deliberately outside
|
||||
`/usr/share/doc`, which minimal and container installs path-exclude
|
||||
(so the install-time seed source is never dropped).
|
||||
- Local and GitHub CI integration coverage brought into parity, and
|
||||
the Rust toolchain selection given a single source of truth:
|
||||
- The `admission-cap` integration suite, previously run only by
|
||||
`ci-local.sh`, now also runs as a GitHub `ci.yml` matrix leg, so a
|
||||
regression in it turns the GitHub gate red rather than depending on
|
||||
a developer remembering to run local CI. A new
|
||||
`testing/check-ci-parity.sh` (wired as `ci-local.sh
|
||||
--check-parity`) diffs the two runners' integration-suite sets and
|
||||
fails on unexpected drift; the deliberate local-only (live-Tor)
|
||||
and granularity-only differences are documented in a comment block
|
||||
atop both runners.
|
||||
- CI and packaging jobs now select the toolchain with
|
||||
`actions-rust-lang/setup-rust-toolchain` (which reads
|
||||
`rust-toolchain.toml`) instead of `dtolnay/rust-toolchain@stable`.
|
||||
The pinned channel already overrode the installed stable, so each
|
||||
job downloaded an unused toolchain and logged a misleading `rustc`
|
||||
version; the single-source action removes the waste and the
|
||||
confusion. Existing cache steps are kept (`cache: false` on the
|
||||
new action) and `RUSTFLAGS` is left untouched so no global
|
||||
`-D warnings` is newly imposed. The OpenWrt nightly Tier-3 leg
|
||||
keeps `@nightly`.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Inbound connections that arrive while a node is already at its
|
||||
@@ -323,6 +366,56 @@ with v0.3.x peers.
|
||||
Previously the old session could linger after the peer came back
|
||||
with a new startup epoch, leaving the session-layer map out of sync
|
||||
with the freshly promoted peer.
|
||||
|
||||
- Two nodes that each `auto_connect` to the other no longer stall their
|
||||
Nostr-mediated NAT-traversal handshake. Each side ran both an
|
||||
initiator and a responder traversal session, binding a separate UDP
|
||||
socket per session, and adopted only the first `Established` event; if
|
||||
the two sides adopted mismatched sessions, each sent its Noise msg1 to
|
||||
a peer port the peer had already stopped draining and both handshakes
|
||||
hung until the adoption budget expired. The responder now elects a
|
||||
single session deterministically — it declines an incoming offer only
|
||||
when it also has an in-flight outbound initiator for the same peer and
|
||||
its own NodeAddr is smaller — so one matching socket pair survives on
|
||||
both ends and the peer's redundant initiator times out harmlessly.
|
||||
One-sided (asymmetric) `auto_connect` has no co-active initiator and is
|
||||
never suppressed, so connectivity is preserved. (Distinct from the
|
||||
cross-init adoption tie-breaker below, which dedups two simultaneous
|
||||
punches but does not stop each node from running redundant
|
||||
initiator + responder sessions.)
|
||||
- FMP link-layer rekey is now reliable under packet loss, bringing it up
|
||||
to the FSP session layer's rekey discipline:
|
||||
- The rekey msg1 retransmission driver was uncapped and never
|
||||
abandoned, so a rekey that never completed resent msg1 forever. It
|
||||
now uses a bounded retransmission budget (`handshake_max_resends`
|
||||
with exponential backoff) and abandons the rekey cycle cleanly once
|
||||
the budget is exhausted, mirroring the FSP rekey msg3 driver. With
|
||||
the cap in place the link-dead heartbeat is rekey-aware:
|
||||
`check_link_heartbeats` no longer reaps a link that is still
|
||||
actively carrying rekey-handshake traffic, while a genuinely dead
|
||||
link is still reaped once the budget abandons.
|
||||
- At the K-bit cutover the receiver now authenticates an inbound frame
|
||||
against the pending session before promoting it, instead of
|
||||
promoting on the bare header K-bit. Under jitter a node could
|
||||
otherwise promote a stale pending session, leaving the two endpoints
|
||||
on different keys and silently dropping traffic until the link died
|
||||
— the same failure class already closed on FSP, now closed on FMP.
|
||||
- Node-level multi-node tests no longer flake under parallel CPU load.
|
||||
They previously delivered handshake packets over real localhost UDP,
|
||||
whose kernel receive buffer could overflow and drop a packet when many
|
||||
tests ran concurrently, panicking the large-network convergence tests.
|
||||
A `cfg(test)`-only loopback `TransportHandle` variant now delivers
|
||||
packets directly between nodes over an unbounded in-process channel, so
|
||||
there is no socket buffer to overflow, and the previously-quarantined
|
||||
large-network tests run in the default suite again. The shipping daemon
|
||||
build is unaffected (the variant is test-gated).
|
||||
- Integration suites that wait for the mesh to converge no longer
|
||||
false-fail under concurrent CI load. The rekey, static-mesh, and
|
||||
sidecar suites replace a fixed wall-clock baseline timeout (and a blind
|
||||
sleep) with a progress-aware wait that polls the suite's own pairwise
|
||||
pings, returns as soon as every pair is reachable, extends its deadline
|
||||
while the reachable-pair count is still climbing, and gives up only
|
||||
when progress stalls.
|
||||
- TCP and Tor `max_inbound_connections` admission cap is now compared
|
||||
against the per-direction inbound count (`pool_inbound`) rather than
|
||||
the combined pool size. Outbound connect-on-send connections share
|
||||
|
||||
+2
-2
@@ -74,7 +74,7 @@ assets = [
|
||||
["target/release/fips", "/usr/bin/", "755"],
|
||||
["target/release/fipsctl", "/usr/bin/", "755"],
|
||||
["target/release/fipstop", "/usr/bin/", "755"],
|
||||
["packaging/common/fips.yaml", "/etc/fips/fips.yaml", "600"],
|
||||
["packaging/common/fips.yaml", "/usr/share/fips/fips.yaml.example", "644"],
|
||||
["packaging/common/hosts", "/etc/fips/hosts", "644"],
|
||||
["packaging/common/fips.nft", "/etc/fips/fips.nft", "644"],
|
||||
["packaging/debian/fips.service", "/lib/systemd/system/fips.service", "644"],
|
||||
@@ -87,7 +87,7 @@ assets = [
|
||||
["packaging/debian/fips-gateway.service", "/lib/systemd/system/fips-gateway.service", "644"],
|
||||
["docs/design/fips-security.md", "/usr/share/doc/fips/fips-security.md", "644"],
|
||||
]
|
||||
conf-files = ["/etc/fips/fips.yaml", "/etc/fips/hosts", "/etc/fips/fips.nft"]
|
||||
conf-files = ["/etc/fips/hosts", "/etc/fips/fips.nft"]
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.15"
|
||||
|
||||
@@ -18,6 +18,7 @@ X" to "X is done".
|
||||
| [enable-nostr-discovery.md](enable-nostr-discovery.md) | Turn on Nostr-mediated discovery (3 capabilities — resolve, advertise, open — across 5 scenarios) |
|
||||
| [deploy-tor-onion.md](deploy-tor-onion.md) | Run a Tor onion service for inbound FIPS connections |
|
||||
| [tune-udp-buffers.md](tune-udp-buffers.md) | Set host sysctls so FIPS UDP sockets don't get clamped |
|
||||
| [tune-file-descriptors.md](tune-file-descriptors.md) | Raise `RLIMIT_NOFILE` so a busy node doesn't exhaust file descriptors (`EMFILE`) as peer count grows |
|
||||
| [run-as-unprivileged-user.md](run-as-unprivileged-user.md) | Run the daemon under a dedicated unprivileged service account (drops the default-root posture) |
|
||||
| [deploy-gateway.md](deploy-gateway.md) | Manually deploy `fips-gateway` on a non-OpenWrt Linux host (LAN-to-mesh outbound + mesh-to-LAN inbound port-forwards). For the OpenWrt path, see the gateway tutorial. |
|
||||
| [troubleshoot-gateway.md](troubleshoot-gateway.md) | Diagnostic recipes for the gateway, organised by half (outbound, inbound, common) |
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
# Tune the File-Descriptor Limit for FIPS
|
||||
|
||||
A busy FIPS node opens many file descriptors, and the count grows with
|
||||
the number of peers it serves. On most systemd distributions the daemon
|
||||
inherits a soft `RLIMIT_NOFILE` of 1024, which a well-connected node can
|
||||
exhaust — at which point peer admission, handshakes, and discovery start
|
||||
failing with `EMFILE` ("Too many open files").
|
||||
|
||||
This guide explains the FD budget, shows how to raise the limit on
|
||||
systemd and on OpenWrt, and how to verify the result.
|
||||
|
||||
## Why FIPS is FD-hungry
|
||||
|
||||
Unlike a service that multiplexes all traffic over one socket, the FIPS
|
||||
data plane allocates descriptors **per peer**. The dominant term is:
|
||||
|
||||
```text
|
||||
fds ≈ 3·P + fixed overhead (~30)
|
||||
```
|
||||
|
||||
where `P` is the number of established UDP peers. Each such peer consumes
|
||||
**3 file descriptors**:
|
||||
|
||||
- one `connect()`-ed UDP socket dedicated to that peer, plus
|
||||
- a 2-FD self-pipe owned by that peer's receive-drain worker (used to
|
||||
wake and stop the worker cleanly).
|
||||
|
||||
The remaining consumers are bounded and do not scale with peer count:
|
||||
|
||||
- the TUN device (one descriptor, process-lifetime),
|
||||
- the wildcard UDP listen socket(s) (one per bound UDP transport),
|
||||
- TCP and Tor transport listeners and the Tor control socket,
|
||||
- Nostr relay websockets (one per configured discovery relay),
|
||||
- the control socket (one `UnixListener`, plus short-lived per-request
|
||||
client connections for `fipsctl` / `fipstop`),
|
||||
- and base runtime descriptors (epoll, eventfd, logs).
|
||||
|
||||
Together these add a roughly flat overhead of about 30 descriptors. The
|
||||
per-peer term is what drives the daemon toward the FD ceiling.
|
||||
|
||||
## The symptom
|
||||
|
||||
The systemd and distro default **soft** `RLIMIT_NOFILE` is **1024**.
|
||||
With the `3·P` budget above, that ceiling is reached near **~320 peers**
|
||||
(3 × 320 ≈ 960, plus the fixed overhead). Once the process is out of
|
||||
descriptors, every syscall that allocates one — `socket()`, `accept()`,
|
||||
`open()`, `pipe()` — fails with `EMFILE`, which surfaces as:
|
||||
|
||||
- failed peer admission (new peers cannot be accepted),
|
||||
- failed handshakes (the daemon cannot open the per-peer socket), and
|
||||
- dropped discovery (relay or probe sockets cannot be created).
|
||||
|
||||
These symptoms appear only under load, once the node has accumulated
|
||||
enough peers to cross the ceiling, so they can be easy to misattribute.
|
||||
|
||||
## Raise the limit on systemd
|
||||
|
||||
Create a drop-in override for the service:
|
||||
|
||||
```sh
|
||||
sudo systemctl edit fips.service
|
||||
```
|
||||
|
||||
Add:
|
||||
|
||||
```ini
|
||||
[Service]
|
||||
LimitNOFILE=65535
|
||||
```
|
||||
|
||||
A single `LimitNOFILE=` value sets **both** the soft and the hard limit,
|
||||
so no separate soft/hard syntax is needed here.
|
||||
|
||||
Reload systemd and restart the daemon so the new limit takes effect:
|
||||
|
||||
```sh
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart fips
|
||||
```
|
||||
|
||||
`65535` (2¹⁶ − 1) is the conventional headroom value for network
|
||||
daemons. With the `3·P` budget, it supports roughly **~21,800 peers**
|
||||
before the FD ceiling binds — well beyond any plausible single-node FIPS
|
||||
mesh degree. Past that point other limits (threads, memory, CPU) bind
|
||||
first, so raising `LimitNOFILE` higher buys nothing.
|
||||
|
||||
## Raise the limit on OpenWrt
|
||||
|
||||
OpenWrt uses procd, not systemd, so `LimitNOFILE` does not apply.
|
||||
Set the equivalent limit in the init script at `/etc/init.d/fips`,
|
||||
inside the block that starts the service:
|
||||
|
||||
```sh
|
||||
procd_set_param limits nofile="65535 65535"
|
||||
```
|
||||
|
||||
The two values are the soft and hard limits respectively; setting them
|
||||
equal mirrors the single-value systemd behaviour above.
|
||||
|
||||
Restart the service to apply:
|
||||
|
||||
```sh
|
||||
/etc/init.d/fips restart
|
||||
```
|
||||
|
||||
## Verify
|
||||
|
||||
Compare the live descriptor count against the established peer count:
|
||||
|
||||
```sh
|
||||
ls /proc/$(pidof fips)/fd | wc -l
|
||||
fipsctl show peers | wc -l
|
||||
```
|
||||
|
||||
At steady state, expect a stable ratio of about **3 descriptors per
|
||||
peer** plus the flat ~30-descriptor overhead. A ratio that holds steady
|
||||
as peers come and go confirms healthy, bounded scaling.
|
||||
|
||||
If the descriptor count climbs steadily while the peer count stays flat,
|
||||
that would indicate a descriptor leak rather than legitimate scaling —
|
||||
the limit bump would only delay the wall. The current data plane has
|
||||
been audited as leak-free (every per-peer descriptor has a guaranteed
|
||||
close on every teardown path), so a climbing ratio at fixed peer count
|
||||
would be a regression worth investigating.
|
||||
|
||||
## A note on deployment lines
|
||||
|
||||
The per-peer connected-UDP socket — the amplifier behind the
|
||||
`3·P` term — is present on the master and next data planes. It is **not
|
||||
yet present on the maintenance line**. On maintenance-only deployments
|
||||
the 3-descriptor-per-peer term does not apply, and FD pressure comes
|
||||
only from the fixed consumers listed above. Raising `LimitNOFILE` there
|
||||
is still worthwhile as forward-looking headroom, and harmless where the
|
||||
amplifier is absent.
|
||||
|
||||
## See also
|
||||
|
||||
- [tune-udp-buffers.md](tune-udp-buffers.md) — host sysctls so FIPS UDP
|
||||
sockets don't get clamped
|
||||
- [run-as-unprivileged-user.md](run-as-unprivileged-user.md) — run the
|
||||
daemon under a dedicated service account
|
||||
- [../reference/configuration.md](../reference/configuration.md) —
|
||||
transport and discovery configuration that influences the fixed FD
|
||||
overhead
|
||||
+12
-2
@@ -33,10 +33,20 @@ packaging/
|
||||
### Debian/Ubuntu (`.deb`)
|
||||
|
||||
Built with [cargo-deb](https://github.com/kornelski/cargo-deb). Installs
|
||||
`fips`, `fipsctl`, and `fipstop` to `/usr/bin/`, places config at
|
||||
`/etc/fips/fips.yaml` (preserved on upgrade), and enables the systemd
|
||||
`fips`, `fipsctl`, and `fipstop` to `/usr/bin/`, and enables the systemd
|
||||
service.
|
||||
|
||||
The default configuration ships as an example at
|
||||
`/usr/share/fips/fips.yaml.example` and is **not** a dpkg conf-file.
|
||||
(It is deliberately **not** under `/usr/share/doc`, which minimal and
|
||||
container installs path-exclude, since the postinst reads it at install
|
||||
time.)
|
||||
On install, `postinst` seeds `/etc/fips/fips.yaml` (mode 600) from the
|
||||
example **only if it does not already exist**, so a configuration that
|
||||
was rendered by configuration management or edited by an operator is
|
||||
never prompted for or clobbered on upgrade. To reset to defaults, remove
|
||||
`/etc/fips/fips.yaml` and reinstall, or copy the example back manually.
|
||||
|
||||
```sh
|
||||
# Build
|
||||
make deb
|
||||
|
||||
@@ -3,6 +3,11 @@ Description=FIPS Mesh Network Daemon
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
# The config file is no longer a packaged conf-file; postinst seeds it
|
||||
# if absent. Skip the unit (inactive, not failed) rather than crash-loop
|
||||
# if it is ever missing.
|
||||
ConditionPathExists=/etc/fips/fips.yaml
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/fips --config /etc/fips/fips.yaml
|
||||
|
||||
@@ -9,6 +9,16 @@ case "$1" in
|
||||
groupadd --system fips
|
||||
fi
|
||||
|
||||
# Seed /etc/fips/fips.yaml from the shipped example only if it
|
||||
# does not already exist. The live config is no longer a dpkg
|
||||
# conf-file; this copy-if-absent yields to any operator- or
|
||||
# configuration-management-rendered file and never clobbers it.
|
||||
if [ ! -e /etc/fips/fips.yaml ]; then
|
||||
install -m 600 -o root -g root \
|
||||
/usr/share/fips/fips.yaml.example \
|
||||
/etc/fips/fips.yaml
|
||||
fi
|
||||
|
||||
# Drop-in directory for operator nftables rules included by
|
||||
# /etc/fips/fips.nft. Empty by default; the include glob matches
|
||||
# nothing cleanly out of the box.
|
||||
|
||||
Executable
+144
@@ -0,0 +1,144 @@
|
||||
#!/bin/bash
|
||||
# ── CI parity invariant guard ───────────────────────────────────────────────
|
||||
# The GitHub integration matrix (.github/workflows/ci.yml) and the local
|
||||
# default suite set (ci-local.sh) MUST run the same integration suites,
|
||||
# EXCEPT for the deliberate local-only entries listed below. Adding a suite
|
||||
# to one runner without the other means "local green" and "GitHub green" stop
|
||||
# being equivalent claims.
|
||||
#
|
||||
# Deliberate local-only (NOT on the GitHub gate), with reason:
|
||||
# tor-socks5 — requires live Tor network; opt-in via --with-tor,
|
||||
# unreliable on GitHub-hosted runners.
|
||||
# tor-directory — same; live Tor dependency.
|
||||
#
|
||||
# Granularity-only differences folded before comparison (same coverage,
|
||||
# different matrix shape — NOT a divergence):
|
||||
# deb-install — GitHub splits into per-distro legs
|
||||
# (deb-install-debian12/ubuntu24/ubuntu26); local runs all
|
||||
# distros in one suite. Folded to "deb-install".
|
||||
# chaos-* — GitHub fans each chaos scenario into its own matrix leg
|
||||
# (type: chaos); local runs them all via the one CHAOS_SUITES
|
||||
# path. The individual scenario names also differ cosmetically
|
||||
# between runners (e.g. chaos-smoke-10 vs churn-mixed-10).
|
||||
# Folded to a single "chaos" token on both sides.
|
||||
# dns-resolver — single leg / single suite both sides; runs all scenarios.
|
||||
#
|
||||
# Exit 0 = parity clean. Exit 1 = unexpected divergence (suite names printed).
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
CI_LOCAL="$SCRIPT_DIR/ci-local.sh"
|
||||
CI_YML="$PROJECT_ROOT/.github/workflows/ci.yml"
|
||||
|
||||
# Deliberate local-only allowlist (suites intentionally absent from GitHub).
|
||||
ALLOWLIST="tor-socks5 tor-directory"
|
||||
|
||||
for f in "$CI_LOCAL" "$CI_YML"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "check-ci-parity: missing file: $f" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Extract and normalize both suite sets in Python (robust YAML parse of the
|
||||
# matrix; regex extraction of the bash suite arrays). Folding rules above are
|
||||
# applied identically to both sides so only genuine divergence surfaces.
|
||||
python3 - "$CI_LOCAL" "$CI_YML" "$ALLOWLIST" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
ci_local_path, ci_yml_path, allowlist_raw = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
allowlist = set(allowlist_raw.split())
|
||||
|
||||
|
||||
def fold(name):
|
||||
"""Collapse granularity-only matrix shape into canonical suite identity."""
|
||||
if name.startswith("chaos-") or name == "chaos":
|
||||
return "chaos"
|
||||
if name.startswith("deb-install"):
|
||||
return "deb-install"
|
||||
return name
|
||||
|
||||
|
||||
# ── Local: parse the suite arrays from ci-local.sh ───────────────────────────
|
||||
with open(ci_local_path, encoding="utf-8") as fh:
|
||||
local_src = fh.read()
|
||||
|
||||
|
||||
def bash_array(var):
|
||||
m = re.search(rf"^{var}=\((.*?)\)", local_src, re.MULTILINE | re.DOTALL)
|
||||
if not m:
|
||||
return []
|
||||
body = m.group(1)
|
||||
# Quoted entries (chaos uses "display scenario flags"): first token is name.
|
||||
quoted = re.findall(r'"([^"]*)"', body)
|
||||
if quoted:
|
||||
return [entry.split()[0] for entry in quoted if entry.strip()]
|
||||
return [tok for tok in body.split() if tok.strip()]
|
||||
|
||||
|
||||
local = set()
|
||||
# Static, rekey, gateway, sidecar, acl, firewall, nostr, stun, dns, deb.
|
||||
for var in ("STATIC_SUITES", "REKEY_SUITES", "ADMISSION_SUITES",
|
||||
"GATEWAY_SUITES", "SIDECAR_SUITES", "ACL_SUITES",
|
||||
"FIREWALL_SUITES", "NOSTR_RELAY_SUITES", "STUN_FAULTS_SUITES",
|
||||
"DNS_RESOLVER_SUITES", "DEB_INSTALL_SUITES"):
|
||||
local.update(bash_array(var))
|
||||
# Chaos display names → fold to "chaos".
|
||||
for _ in bash_array("CHAOS_SUITES"):
|
||||
local.add("chaos")
|
||||
# NAT scenarios are stored bare (cone/symmetric/lan) and prefixed nat- at use.
|
||||
for scen in bash_array("NAT_SUITES"):
|
||||
local.add(f"nat-{scen}")
|
||||
# TOR_SUITES is the deliberate local-only set — excluded from the default path.
|
||||
|
||||
local = {fold(n) for n in local}
|
||||
|
||||
# ── GitHub: parse the integration matrix suite: values from ci.yml ───────────
|
||||
import yaml # noqa: E402
|
||||
|
||||
with open(ci_yml_path, encoding="utf-8") as fh:
|
||||
doc = yaml.safe_load(fh)
|
||||
|
||||
include = doc["jobs"]["integration"]["strategy"]["matrix"]["include"]
|
||||
github = set()
|
||||
for leg in include:
|
||||
if "suite" not in leg:
|
||||
continue
|
||||
# Chaos legs carry inconsistent suite: names (chaos-smoke-10 vs
|
||||
# churn-mixed-10) but a uniform type: chaos — fold via type, not name.
|
||||
if str(leg.get("type", "")) == "chaos":
|
||||
github.add("chaos")
|
||||
else:
|
||||
github.add(fold(str(leg["suite"])))
|
||||
|
||||
# ── Diff (subtract allowlist from local before comparison) ───────────────────
|
||||
local_cmp = {n for n in local if n not in allowlist}
|
||||
|
||||
local_only = sorted(local_cmp - github)
|
||||
github_only = sorted(github - local_cmp)
|
||||
|
||||
if local_only or github_only:
|
||||
print("CI parity FAILED: integration suite sets diverge.\n")
|
||||
if local_only:
|
||||
print(" Local-only (in ci-local.sh, missing from ci.yml, "
|
||||
"not in deliberate allowlist):")
|
||||
for n in local_only:
|
||||
print(f" - {n}")
|
||||
if github_only:
|
||||
print(" GitHub-only (in ci.yml, missing from local default path):")
|
||||
for n in github_only:
|
||||
print(f" - {n}")
|
||||
print("\n Resolve by adding the suite to the other runner, or by adding "
|
||||
"it\n to the deliberate local-only allowlist in "
|
||||
"check-ci-parity.sh with a\n stated reason.")
|
||||
sys.exit(1)
|
||||
|
||||
print("CI parity OK: integration suite sets match "
|
||||
"(allowlist: " + ", ".join(sorted(allowlist)) + ").")
|
||||
print(f" {len(github)} canonical suites compared on each side.")
|
||||
sys.exit(0)
|
||||
PY
|
||||
@@ -12,6 +12,8 @@
|
||||
# --only <suite> Run a single integration suite
|
||||
# -j, --jobs <N> Max parallel chaos scenarios (default: 4)
|
||||
# --list List available integration suites
|
||||
# --check-parity Verify this suite set matches ci.yml's integration
|
||||
# matrix (see testing/check-ci-parity.sh), then exit
|
||||
# -h, --help Show this help
|
||||
#
|
||||
# Integration suites (default coverage):
|
||||
@@ -32,6 +34,25 @@
|
||||
# Exit codes:
|
||||
# 0 — all stages passed
|
||||
# 1 — one or more stages failed
|
||||
#
|
||||
# ── CI parity invariant ─────────────────────────────────────────────────────
|
||||
# This local default suite set and the GitHub integration matrix
|
||||
# (.github/workflows/ci.yml) MUST run the same integration suites, EXCEPT for
|
||||
# the deliberate local-only entries below. Adding a suite to one runner
|
||||
# without the other means "local green" and "GitHub green" stop being
|
||||
# equivalent. testing/check-ci-parity.sh enforces this and fails on drift.
|
||||
#
|
||||
# Deliberate local-only (NOT on the GitHub gate), with reason:
|
||||
# tor-socks5 — requires live Tor network; opt-in via --with-tor,
|
||||
# unreliable on GitHub-hosted runners.
|
||||
# tor-directory — same; live Tor dependency.
|
||||
#
|
||||
# Granularity-only differences (same coverage, different matrix shape —
|
||||
# NOT a divergence):
|
||||
# deb-install — local runs all distros sequentially in one suite; GitHub
|
||||
# splits into per-distro legs (debian12/ubuntu24/ubuntu26).
|
||||
# dns-resolver — single suite both sides; runs all scenarios.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
@@ -170,6 +191,7 @@ while [[ $# -gt 0 ]]; do
|
||||
--only) ONLY_SUITE="$2"; shift 2 ;;
|
||||
-j|--jobs) PARALLEL_JOBS="$2"; shift 2 ;;
|
||||
--list) list_suites ;;
|
||||
--check-parity) exec "$SCRIPT_DIR/check-ci-parity.sh" ;;
|
||||
-h|--help) usage ;;
|
||||
*) echo "Unknown option: $1"; usage ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user