diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09c970a..f891482 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,28 @@ env: RUST_BACKTRACE: 1 SOURCE_DATE_EPOCH: 0 # overridden per-step after checkout +# ───────────────────────────────────────────────────────────────────────────── +# CI parity invariant +# +# This GitHub integration matrix and the local default suite set +# (testing/ci-local.sh) MUST run the same integration suites, EXCEPT for the +# deliberate local-only entries below. Adding a suite to one runner without +# the other means "local green" and "GitHub green" stop being equivalent. +# testing/check-ci-parity.sh enforces this and fails on unexpected drift. +# +# Deliberate local-only (NOT on the GitHub gate), with reason: +# tor-socks5 — requires live Tor network; opt-in via --with-tor, +# unreliable on GitHub-hosted runners. +# tor-directory — same; live Tor dependency. +# +# Granularity-only differences (same coverage, different matrix shape — +# NOT a divergence): +# deb-install — split here into per-distro legs (debian12/ubuntu24/ +# ubuntu26) for parallelism; local runs all distros in one +# suite. +# dns-resolver — single leg here; runs all scenarios (same as local). +# ───────────────────────────────────────────────────────────────────────────── + # ───────────────────────────────────────────────────────────────────────────── # Job 1 – Build matrix # @@ -35,9 +57,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions-rust-lang/setup-rust-toolchain@v1 with: components: rustfmt + cache: false + rustflags: '' - run: cargo fmt --check clippy: @@ -47,9 +71,11 @@ jobs: - uses: actions/checkout@v4 - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev - - uses: dtolnay/rust-toolchain@stable + - uses: actions-rust-lang/setup-rust-toolchain@v1 with: components: clippy + cache: false + rustflags: '' - name: Cache Cargo registry + build uses: actions/cache@v4 with: @@ -98,7 +124,10 @@ jobs: run: sudo nft -c -f packaging/common/fips.nft - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + rustflags: '' - name: Cache Cargo registry + build uses: actions/cache@v4 @@ -160,7 +189,10 @@ jobs: run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + rustflags: '' - name: Cache Cargo registry + build uses: actions/cache@v4 @@ -210,7 +242,10 @@ jobs: run: echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> "$GITHUB_ENV" - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + rustflags: '' - name: Cache Cargo registry + build uses: actions/cache@v4 @@ -239,7 +274,10 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + rustflags: '' - name: Cache Cargo registry + build uses: actions/cache@v4 @@ -323,6 +361,10 @@ jobs: - suite: rekey-outbound-only type: rekey-outbound-only topology: rekey-outbound-only + # ── Inbound max_peers admission-cap test ─────────────────────── + - suite: admission-cap + type: admission-cap + topology: mesh - suite: acl-allowlist type: acl-allowlist # ── Mixed-profile node-role integration test ───────────────── @@ -755,6 +797,42 @@ jobs: docker compose -f testing/static/docker-compose.yml \ --profile gateway down --volumes --remove-orphans + # ── Inbound max_peers admission-cap integration test ──────────────── + # Lowers node.max_peers on one mesh node and asserts the inbound cap + # holds under sustained retry pressure: denied peers keep retrying but + # are never promoted to an active session. The admission-cap-test.sh + # assertions are tailored per link-layer handshake variant; the leg + # itself is uniform. Static-style harness on the shared mesh profile. + - name: Generate configs (admission-cap) + if: matrix.type == 'admission-cap' + run: bash testing/static/scripts/generate-configs.sh mesh + + - name: Inject admission-cap config (admission-cap) + if: matrix.type == 'admission-cap' + run: bash testing/static/scripts/admission-cap-test.sh inject-config + + - name: Start containers (admission-cap) + if: matrix.type == 'admission-cap' + run: | + docker compose -f testing/static/docker-compose.yml \ + --profile mesh up -d + + - name: Run admission-cap test + if: matrix.type == 'admission-cap' + run: bash testing/static/scripts/admission-cap-test.sh + + - name: Collect logs on failure (admission-cap) + if: matrix.type == 'admission-cap' && failure() + run: | + docker compose -f testing/static/docker-compose.yml \ + --profile mesh logs --no-color | tail -300 + + - name: Stop containers (admission-cap) + if: matrix.type == 'admission-cap' && always() + run: | + docker compose -f testing/static/docker-compose.yml \ + --profile mesh down --volumes --remove-orphans + # ── Real-deb install integration ──────────────────────────────────── # The deb-install harness builds its own .deb from source in a # cargo-deb builder image; the pre-built Linux binary from the diff --git a/.github/workflows/package-linux.yml b/.github/workflows/package-linux.yml index f51047e..f2d4d49 100644 --- a/.github/workflows/package-linux.yml +++ b/.github/workflows/package-linux.yml @@ -72,7 +72,10 @@ jobs: run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libdbus-1-dev llvm - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + rustflags: '' - name: Cache Cargo registry + build if: ${{ env.ACT != 'true' }} diff --git a/.github/workflows/package-macos.yml b/.github/workflows/package-macos.yml index f5bb05f..712be11 100644 --- a/.github/workflows/package-macos.yml +++ b/.github/workflows/package-macos.yml @@ -69,10 +69,11 @@ jobs: run: echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> "$GITHUB_ENV" - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Add cross-compile target - run: rustup target add ${{ matrix.target }} + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + target: ${{ matrix.target }} + cache: false + rustflags: '' - name: Cache Cargo registry + build uses: actions/cache@v4 diff --git a/.github/workflows/package-openwrt.yml b/.github/workflows/package-openwrt.yml index 76a31f9..511d3e2 100644 --- a/.github/workflows/package-openwrt.yml +++ b/.github/workflows/package-openwrt.yml @@ -110,9 +110,11 @@ jobs: - name: Install Rust toolchain (stable) if: matrix.rust_channel == 'stable' - uses: dtolnay/rust-toolchain@stable + uses: actions-rust-lang/setup-rust-toolchain@v1 with: - targets: ${{ matrix.rust_target }} + target: ${{ matrix.rust_target }} + cache: false + rustflags: '' - name: Install Rust toolchain (nightly, Tier 3) if: matrix.rust_channel == 'nightly' diff --git a/.github/workflows/package-windows.yml b/.github/workflows/package-windows.yml index 5b98792..09e4424 100644 --- a/.github/workflows/package-windows.yml +++ b/.github/workflows/package-windows.yml @@ -63,7 +63,10 @@ jobs: echo "SOURCE_DATE_EPOCH=$epoch" >> $env:GITHUB_ENV - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + rustflags: '' - name: Cache Cargo registry + build uses: actions/cache@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index ca9f57a..8d6af7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -154,6 +154,16 @@ with v0.3.x peers. requests" and "Further reading". Running the checklist before opening surfaces problems that would otherwise come back as review comments, saving a round trip. +- [`docs/how-to/tune-file-descriptors.md`](docs/how-to/tune-file-descriptors.md) + — an operator how-to for raising `RLIMIT_NOFILE`. A busy node opens + roughly three file descriptors per established UDP peer (a + `connect()`-ed socket plus a 2-FD drain self-pipe), so the default + 1024 soft limit is exhausted near 320 peers, after which further + admission, handshakes, and discovery fail with `EMFILE`. The guide + documents the per-peer FD budget and symptom, the systemd + (`LimitNOFILE` drop-in) and OpenWrt (procd `nofile`) procedures to + raise the limit, and how to verify the per-peer ratio stays bounded. + Linked from the how-to index. ### Changed @@ -290,6 +300,39 @@ with v0.3.x peers. RTT +0.11–0.19 ms from the worker queue handoff. Windows continues on the existing tokio-based send/recv path. +- The Debian package no longer ships `/etc/fips/fips.yaml` as a dpkg + conf-file. The default configuration is installed as an example at + `/usr/share/fips/fips.yaml.example`, and `postinst` seeds + `/etc/fips/fips.yaml` (mode 600) from it only when the file does not + already exist — so a configuration-management-rendered or + operator-edited config is never prompted for or clobbered on + upgrade, removing the need for a `dpkg-divert` workaround. + `fips.service` gains `ConditionPathExists=/etc/fips/fips.yaml`. The + example is placed under `/usr/share/fips`, deliberately outside + `/usr/share/doc`, which minimal and container installs path-exclude + (so the install-time seed source is never dropped). +- Local and GitHub CI integration coverage brought into parity, and + the Rust toolchain selection given a single source of truth: + - The `admission-cap` integration suite, previously run only by + `ci-local.sh`, now also runs as a GitHub `ci.yml` matrix leg, so a + regression in it turns the GitHub gate red rather than depending on + a developer remembering to run local CI. A new + `testing/check-ci-parity.sh` (wired as `ci-local.sh + --check-parity`) diffs the two runners' integration-suite sets and + fails on unexpected drift; the deliberate local-only (live-Tor) + and granularity-only differences are documented in a comment block + atop both runners. + - CI and packaging jobs now select the toolchain with + `actions-rust-lang/setup-rust-toolchain` (which reads + `rust-toolchain.toml`) instead of `dtolnay/rust-toolchain@stable`. + The pinned channel already overrode the installed stable, so each + job downloaded an unused toolchain and logged a misleading `rustc` + version; the single-source action removes the waste and the + confusion. Existing cache steps are kept (`cache: false` on the + new action) and `RUSTFLAGS` is left untouched so no global + `-D warnings` is newly imposed. The OpenWrt nightly Tier-3 leg + keeps `@nightly`. + ### Fixed - Inbound connections that arrive while a node is already at its @@ -323,6 +366,56 @@ with v0.3.x peers. Previously the old session could linger after the peer came back with a new startup epoch, leaving the session-layer map out of sync with the freshly promoted peer. + +- Two nodes that each `auto_connect` to the other no longer stall their + Nostr-mediated NAT-traversal handshake. Each side ran both an + initiator and a responder traversal session, binding a separate UDP + socket per session, and adopted only the first `Established` event; if + the two sides adopted mismatched sessions, each sent its Noise msg1 to + a peer port the peer had already stopped draining and both handshakes + hung until the adoption budget expired. The responder now elects a + single session deterministically — it declines an incoming offer only + when it also has an in-flight outbound initiator for the same peer and + its own NodeAddr is smaller — so one matching socket pair survives on + both ends and the peer's redundant initiator times out harmlessly. + One-sided (asymmetric) `auto_connect` has no co-active initiator and is + never suppressed, so connectivity is preserved. (Distinct from the + cross-init adoption tie-breaker below, which dedups two simultaneous + punches but does not stop each node from running redundant + initiator + responder sessions.) +- FMP link-layer rekey is now reliable under packet loss, bringing it up + to the FSP session layer's rekey discipline: + - The rekey msg1 retransmission driver was uncapped and never + abandoned, so a rekey that never completed resent msg1 forever. It + now uses a bounded retransmission budget (`handshake_max_resends` + with exponential backoff) and abandons the rekey cycle cleanly once + the budget is exhausted, mirroring the FSP rekey msg3 driver. With + the cap in place the link-dead heartbeat is rekey-aware: + `check_link_heartbeats` no longer reaps a link that is still + actively carrying rekey-handshake traffic, while a genuinely dead + link is still reaped once the budget abandons. + - At the K-bit cutover the receiver now authenticates an inbound frame + against the pending session before promoting it, instead of + promoting on the bare header K-bit. Under jitter a node could + otherwise promote a stale pending session, leaving the two endpoints + on different keys and silently dropping traffic until the link died + — the same failure class already closed on FSP, now closed on FMP. +- Node-level multi-node tests no longer flake under parallel CPU load. + They previously delivered handshake packets over real localhost UDP, + whose kernel receive buffer could overflow and drop a packet when many + tests ran concurrently, panicking the large-network convergence tests. + A `cfg(test)`-only loopback `TransportHandle` variant now delivers + packets directly between nodes over an unbounded in-process channel, so + there is no socket buffer to overflow, and the previously-quarantined + large-network tests run in the default suite again. The shipping daemon + build is unaffected (the variant is test-gated). +- Integration suites that wait for the mesh to converge no longer + false-fail under concurrent CI load. The rekey, static-mesh, and + sidecar suites replace a fixed wall-clock baseline timeout (and a blind + sleep) with a progress-aware wait that polls the suite's own pairwise + pings, returns as soon as every pair is reachable, extends its deadline + while the reachable-pair count is still climbing, and gives up only + when progress stalls. - TCP and Tor `max_inbound_connections` admission cap is now compared against the per-direction inbound count (`pool_inbound`) rather than the combined pool size. Outbound connect-on-send connections share diff --git a/Cargo.toml b/Cargo.toml index e0cee59..64f4153 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,7 +74,7 @@ assets = [ ["target/release/fips", "/usr/bin/", "755"], ["target/release/fipsctl", "/usr/bin/", "755"], ["target/release/fipstop", "/usr/bin/", "755"], - ["packaging/common/fips.yaml", "/etc/fips/fips.yaml", "600"], + ["packaging/common/fips.yaml", "/usr/share/fips/fips.yaml.example", "644"], ["packaging/common/hosts", "/etc/fips/hosts", "644"], ["packaging/common/fips.nft", "/etc/fips/fips.nft", "644"], ["packaging/debian/fips.service", "/lib/systemd/system/fips.service", "644"], @@ -87,7 +87,7 @@ assets = [ ["packaging/debian/fips-gateway.service", "/lib/systemd/system/fips-gateway.service", "644"], ["docs/design/fips-security.md", "/usr/share/doc/fips/fips-security.md", "644"], ] -conf-files = ["/etc/fips/fips.yaml", "/etc/fips/hosts", "/etc/fips/fips.nft"] +conf-files = ["/etc/fips/hosts", "/etc/fips/fips.nft"] [dev-dependencies] tempfile = "3.15" diff --git a/docs/how-to/README.md b/docs/how-to/README.md index cbfa300..ab2ac4f 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -18,6 +18,7 @@ X" to "X is done". | [enable-nostr-discovery.md](enable-nostr-discovery.md) | Turn on Nostr-mediated discovery (3 capabilities — resolve, advertise, open — across 5 scenarios) | | [deploy-tor-onion.md](deploy-tor-onion.md) | Run a Tor onion service for inbound FIPS connections | | [tune-udp-buffers.md](tune-udp-buffers.md) | Set host sysctls so FIPS UDP sockets don't get clamped | +| [tune-file-descriptors.md](tune-file-descriptors.md) | Raise `RLIMIT_NOFILE` so a busy node doesn't exhaust file descriptors (`EMFILE`) as peer count grows | | [run-as-unprivileged-user.md](run-as-unprivileged-user.md) | Run the daemon under a dedicated unprivileged service account (drops the default-root posture) | | [deploy-gateway.md](deploy-gateway.md) | Manually deploy `fips-gateway` on a non-OpenWrt Linux host (LAN-to-mesh outbound + mesh-to-LAN inbound port-forwards). For the OpenWrt path, see the gateway tutorial. | | [troubleshoot-gateway.md](troubleshoot-gateway.md) | Diagnostic recipes for the gateway, organised by half (outbound, inbound, common) | diff --git a/docs/how-to/tune-file-descriptors.md b/docs/how-to/tune-file-descriptors.md new file mode 100644 index 0000000..92963f1 --- /dev/null +++ b/docs/how-to/tune-file-descriptors.md @@ -0,0 +1,144 @@ +# Tune the File-Descriptor Limit for FIPS + +A busy FIPS node opens many file descriptors, and the count grows with +the number of peers it serves. On most systemd distributions the daemon +inherits a soft `RLIMIT_NOFILE` of 1024, which a well-connected node can +exhaust — at which point peer admission, handshakes, and discovery start +failing with `EMFILE` ("Too many open files"). + +This guide explains the FD budget, shows how to raise the limit on +systemd and on OpenWrt, and how to verify the result. + +## Why FIPS is FD-hungry + +Unlike a service that multiplexes all traffic over one socket, the FIPS +data plane allocates descriptors **per peer**. The dominant term is: + +```text +fds ≈ 3·P + fixed overhead (~30) +``` + +where `P` is the number of established UDP peers. Each such peer consumes +**3 file descriptors**: + +- one `connect()`-ed UDP socket dedicated to that peer, plus +- a 2-FD self-pipe owned by that peer's receive-drain worker (used to + wake and stop the worker cleanly). + +The remaining consumers are bounded and do not scale with peer count: + +- the TUN device (one descriptor, process-lifetime), +- the wildcard UDP listen socket(s) (one per bound UDP transport), +- TCP and Tor transport listeners and the Tor control socket, +- Nostr relay websockets (one per configured discovery relay), +- the control socket (one `UnixListener`, plus short-lived per-request + client connections for `fipsctl` / `fipstop`), +- and base runtime descriptors (epoll, eventfd, logs). + +Together these add a roughly flat overhead of about 30 descriptors. The +per-peer term is what drives the daemon toward the FD ceiling. + +## The symptom + +The systemd and distro default **soft** `RLIMIT_NOFILE` is **1024**. +With the `3·P` budget above, that ceiling is reached near **~320 peers** +(3 × 320 ≈ 960, plus the fixed overhead). Once the process is out of +descriptors, every syscall that allocates one — `socket()`, `accept()`, +`open()`, `pipe()` — fails with `EMFILE`, which surfaces as: + +- failed peer admission (new peers cannot be accepted), +- failed handshakes (the daemon cannot open the per-peer socket), and +- dropped discovery (relay or probe sockets cannot be created). + +These symptoms appear only under load, once the node has accumulated +enough peers to cross the ceiling, so they can be easy to misattribute. + +## Raise the limit on systemd + +Create a drop-in override for the service: + +```sh +sudo systemctl edit fips.service +``` + +Add: + +```ini +[Service] +LimitNOFILE=65535 +``` + +A single `LimitNOFILE=` value sets **both** the soft and the hard limit, +so no separate soft/hard syntax is needed here. + +Reload systemd and restart the daemon so the new limit takes effect: + +```sh +sudo systemctl daemon-reload +sudo systemctl restart fips +``` + +`65535` (2¹⁶ − 1) is the conventional headroom value for network +daemons. With the `3·P` budget, it supports roughly **~21,800 peers** +before the FD ceiling binds — well beyond any plausible single-node FIPS +mesh degree. Past that point other limits (threads, memory, CPU) bind +first, so raising `LimitNOFILE` higher buys nothing. + +## Raise the limit on OpenWrt + +OpenWrt uses procd, not systemd, so `LimitNOFILE` does not apply. +Set the equivalent limit in the init script at `/etc/init.d/fips`, +inside the block that starts the service: + +```sh +procd_set_param limits nofile="65535 65535" +``` + +The two values are the soft and hard limits respectively; setting them +equal mirrors the single-value systemd behaviour above. + +Restart the service to apply: + +```sh +/etc/init.d/fips restart +``` + +## Verify + +Compare the live descriptor count against the established peer count: + +```sh +ls /proc/$(pidof fips)/fd | wc -l +fipsctl show peers | wc -l +``` + +At steady state, expect a stable ratio of about **3 descriptors per +peer** plus the flat ~30-descriptor overhead. A ratio that holds steady +as peers come and go confirms healthy, bounded scaling. + +If the descriptor count climbs steadily while the peer count stays flat, +that would indicate a descriptor leak rather than legitimate scaling — +the limit bump would only delay the wall. The current data plane has +been audited as leak-free (every per-peer descriptor has a guaranteed +close on every teardown path), so a climbing ratio at fixed peer count +would be a regression worth investigating. + +## A note on deployment lines + +The per-peer connected-UDP socket — the amplifier behind the +`3·P` term — is present on the master and next data planes. It is **not +yet present on the maintenance line**. On maintenance-only deployments +the 3-descriptor-per-peer term does not apply, and FD pressure comes +only from the fixed consumers listed above. Raising `LimitNOFILE` there +is still worthwhile as forward-looking headroom, and harmless where the +amplifier is absent. + +## See also + +- [tune-udp-buffers.md](tune-udp-buffers.md) — host sysctls so FIPS UDP + sockets don't get clamped +- [run-as-unprivileged-user.md](run-as-unprivileged-user.md) — run the + daemon under a dedicated service account +- [../reference/configuration.md](../reference/configuration.md) — + transport and discovery configuration that influences the fixed FD + overhead diff --git a/packaging/README.md b/packaging/README.md index 161207c..7147728 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -33,10 +33,20 @@ packaging/ ### Debian/Ubuntu (`.deb`) Built with [cargo-deb](https://github.com/kornelski/cargo-deb). Installs -`fips`, `fipsctl`, and `fipstop` to `/usr/bin/`, places config at -`/etc/fips/fips.yaml` (preserved on upgrade), and enables the systemd +`fips`, `fipsctl`, and `fipstop` to `/usr/bin/`, and enables the systemd service. +The default configuration ships as an example at +`/usr/share/fips/fips.yaml.example` and is **not** a dpkg conf-file. +(It is deliberately **not** under `/usr/share/doc`, which minimal and +container installs path-exclude, since the postinst reads it at install +time.) +On install, `postinst` seeds `/etc/fips/fips.yaml` (mode 600) from the +example **only if it does not already exist**, so a configuration that +was rendered by configuration management or edited by an operator is +never prompted for or clobbered on upgrade. To reset to defaults, remove +`/etc/fips/fips.yaml` and reinstall, or copy the example back manually. + ```sh # Build make deb diff --git a/packaging/debian/fips.service b/packaging/debian/fips.service index 4460eaa..d1b5616 100644 --- a/packaging/debian/fips.service +++ b/packaging/debian/fips.service @@ -3,6 +3,11 @@ Description=FIPS Mesh Network Daemon After=network-online.target Wants=network-online.target +# The config file is no longer a packaged conf-file; postinst seeds it +# if absent. Skip the unit (inactive, not failed) rather than crash-loop +# if it is ever missing. +ConditionPathExists=/etc/fips/fips.yaml + [Service] Type=simple ExecStart=/usr/bin/fips --config /etc/fips/fips.yaml diff --git a/packaging/debian/postinst b/packaging/debian/postinst index 3e2c200..5c4bf4c 100755 --- a/packaging/debian/postinst +++ b/packaging/debian/postinst @@ -9,6 +9,16 @@ case "$1" in groupadd --system fips fi + # Seed /etc/fips/fips.yaml from the shipped example only if it + # does not already exist. The live config is no longer a dpkg + # conf-file; this copy-if-absent yields to any operator- or + # configuration-management-rendered file and never clobbers it. + if [ ! -e /etc/fips/fips.yaml ]; then + install -m 600 -o root -g root \ + /usr/share/fips/fips.yaml.example \ + /etc/fips/fips.yaml + fi + # Drop-in directory for operator nftables rules included by # /etc/fips/fips.nft. Empty by default; the include glob matches # nothing cleanly out of the box. diff --git a/testing/check-ci-parity.sh b/testing/check-ci-parity.sh new file mode 100755 index 0000000..cb5ac7e --- /dev/null +++ b/testing/check-ci-parity.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# ── CI parity invariant guard ─────────────────────────────────────────────── +# The GitHub integration matrix (.github/workflows/ci.yml) and the local +# default suite set (ci-local.sh) MUST run the same integration suites, +# EXCEPT for the deliberate local-only entries listed below. Adding a suite +# to one runner without the other means "local green" and "GitHub green" stop +# being equivalent claims. +# +# Deliberate local-only (NOT on the GitHub gate), with reason: +# tor-socks5 — requires live Tor network; opt-in via --with-tor, +# unreliable on GitHub-hosted runners. +# tor-directory — same; live Tor dependency. +# +# Granularity-only differences folded before comparison (same coverage, +# different matrix shape — NOT a divergence): +# deb-install — GitHub splits into per-distro legs +# (deb-install-debian12/ubuntu24/ubuntu26); local runs all +# distros in one suite. Folded to "deb-install". +# chaos-* — GitHub fans each chaos scenario into its own matrix leg +# (type: chaos); local runs them all via the one CHAOS_SUITES +# path. The individual scenario names also differ cosmetically +# between runners (e.g. chaos-smoke-10 vs churn-mixed-10). +# Folded to a single "chaos" token on both sides. +# dns-resolver — single leg / single suite both sides; runs all scenarios. +# +# Exit 0 = parity clean. Exit 1 = unexpected divergence (suite names printed). +# ───────────────────────────────────────────────────────────────────────────── +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +CI_LOCAL="$SCRIPT_DIR/ci-local.sh" +CI_YML="$PROJECT_ROOT/.github/workflows/ci.yml" + +# Deliberate local-only allowlist (suites intentionally absent from GitHub). +ALLOWLIST="tor-socks5 tor-directory" + +for f in "$CI_LOCAL" "$CI_YML"; do + if [[ ! -f "$f" ]]; then + echo "check-ci-parity: missing file: $f" >&2 + exit 2 + fi +done + +# Extract and normalize both suite sets in Python (robust YAML parse of the +# matrix; regex extraction of the bash suite arrays). Folding rules above are +# applied identically to both sides so only genuine divergence surfaces. +python3 - "$CI_LOCAL" "$CI_YML" "$ALLOWLIST" <<'PY' +import re +import sys + +ci_local_path, ci_yml_path, allowlist_raw = sys.argv[1], sys.argv[2], sys.argv[3] +allowlist = set(allowlist_raw.split()) + + +def fold(name): + """Collapse granularity-only matrix shape into canonical suite identity.""" + if name.startswith("chaos-") or name == "chaos": + return "chaos" + if name.startswith("deb-install"): + return "deb-install" + return name + + +# ── Local: parse the suite arrays from ci-local.sh ─────────────────────────── +with open(ci_local_path, encoding="utf-8") as fh: + local_src = fh.read() + + +def bash_array(var): + m = re.search(rf"^{var}=\((.*?)\)", local_src, re.MULTILINE | re.DOTALL) + if not m: + return [] + body = m.group(1) + # Quoted entries (chaos uses "display scenario flags"): first token is name. + quoted = re.findall(r'"([^"]*)"', body) + if quoted: + return [entry.split()[0] for entry in quoted if entry.strip()] + return [tok for tok in body.split() if tok.strip()] + + +local = set() +# Static, rekey, gateway, sidecar, acl, firewall, nostr, stun, dns, deb. +for var in ("STATIC_SUITES", "REKEY_SUITES", "ADMISSION_SUITES", + "GATEWAY_SUITES", "SIDECAR_SUITES", "ACL_SUITES", + "FIREWALL_SUITES", "NOSTR_RELAY_SUITES", "STUN_FAULTS_SUITES", + "DNS_RESOLVER_SUITES", "DEB_INSTALL_SUITES"): + local.update(bash_array(var)) +# Chaos display names → fold to "chaos". +for _ in bash_array("CHAOS_SUITES"): + local.add("chaos") +# NAT scenarios are stored bare (cone/symmetric/lan) and prefixed nat- at use. +for scen in bash_array("NAT_SUITES"): + local.add(f"nat-{scen}") +# TOR_SUITES is the deliberate local-only set — excluded from the default path. + +local = {fold(n) for n in local} + +# ── GitHub: parse the integration matrix suite: values from ci.yml ─────────── +import yaml # noqa: E402 + +with open(ci_yml_path, encoding="utf-8") as fh: + doc = yaml.safe_load(fh) + +include = doc["jobs"]["integration"]["strategy"]["matrix"]["include"] +github = set() +for leg in include: + if "suite" not in leg: + continue + # Chaos legs carry inconsistent suite: names (chaos-smoke-10 vs + # churn-mixed-10) but a uniform type: chaos — fold via type, not name. + if str(leg.get("type", "")) == "chaos": + github.add("chaos") + else: + github.add(fold(str(leg["suite"]))) + +# ── Diff (subtract allowlist from local before comparison) ─────────────────── +local_cmp = {n for n in local if n not in allowlist} + +local_only = sorted(local_cmp - github) +github_only = sorted(github - local_cmp) + +if local_only or github_only: + print("CI parity FAILED: integration suite sets diverge.\n") + if local_only: + print(" Local-only (in ci-local.sh, missing from ci.yml, " + "not in deliberate allowlist):") + for n in local_only: + print(f" - {n}") + if github_only: + print(" GitHub-only (in ci.yml, missing from local default path):") + for n in github_only: + print(f" - {n}") + print("\n Resolve by adding the suite to the other runner, or by adding " + "it\n to the deliberate local-only allowlist in " + "check-ci-parity.sh with a\n stated reason.") + sys.exit(1) + +print("CI parity OK: integration suite sets match " + "(allowlist: " + ", ".join(sorted(allowlist)) + ").") +print(f" {len(github)} canonical suites compared on each side.") +sys.exit(0) +PY diff --git a/testing/ci-local.sh b/testing/ci-local.sh index 6e57486..7630bcb 100755 --- a/testing/ci-local.sh +++ b/testing/ci-local.sh @@ -12,6 +12,8 @@ # --only Run a single integration suite # -j, --jobs Max parallel chaos scenarios (default: 4) # --list List available integration suites +# --check-parity Verify this suite set matches ci.yml's integration +# matrix (see testing/check-ci-parity.sh), then exit # -h, --help Show this help # # Integration suites (default coverage): @@ -32,6 +34,25 @@ # Exit codes: # 0 — all stages passed # 1 — one or more stages failed +# +# ── CI parity invariant ───────────────────────────────────────────────────── +# This local default suite set and the GitHub integration matrix +# (.github/workflows/ci.yml) MUST run the same integration suites, EXCEPT for +# the deliberate local-only entries below. Adding a suite to one runner +# without the other means "local green" and "GitHub green" stop being +# equivalent. testing/check-ci-parity.sh enforces this and fails on drift. +# +# Deliberate local-only (NOT on the GitHub gate), with reason: +# tor-socks5 — requires live Tor network; opt-in via --with-tor, +# unreliable on GitHub-hosted runners. +# tor-directory — same; live Tor dependency. +# +# Granularity-only differences (same coverage, different matrix shape — +# NOT a divergence): +# deb-install — local runs all distros sequentially in one suite; GitHub +# splits into per-distro legs (debian12/ubuntu24/ubuntu26). +# dns-resolver — single suite both sides; runs all scenarios. +# ───────────────────────────────────────────────────────────────────────────── set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -170,6 +191,7 @@ while [[ $# -gt 0 ]]; do --only) ONLY_SUITE="$2"; shift 2 ;; -j|--jobs) PARALLEL_JOBS="$2"; shift 2 ;; --list) list_suites ;; + --check-parity) exec "$SCRIPT_DIR/check-ci-parity.sh" ;; -h|--help) usage ;; *) echo "Unknown option: $1"; usage ;; esac