Merge branch 'master' into next

This commit is contained in:
Johnathan Corgan
2026-06-05 21:30:14 +00:00
14 changed files with 534 additions and 18 deletions
+84 -6
View File
@@ -24,6 +24,28 @@ env:
RUST_BACKTRACE: 1
SOURCE_DATE_EPOCH: 0 # overridden per-step after checkout
# ─────────────────────────────────────────────────────────────────────────────
# CI parity invariant
#
# This GitHub integration matrix and the local default suite set
# (testing/ci-local.sh) MUST run the same integration suites, EXCEPT for the
# deliberate local-only entries below. Adding a suite to one runner without
# the other means "local green" and "GitHub green" stop being equivalent.
# testing/check-ci-parity.sh enforces this and fails on unexpected drift.
#
# Deliberate local-only (NOT on the GitHub gate), with reason:
# tor-socks5 — requires live Tor network; opt-in via --with-tor,
# unreliable on GitHub-hosted runners.
# tor-directory — same; live Tor dependency.
#
# Granularity-only differences (same coverage, different matrix shape —
# NOT a divergence):
# deb-install — split here into per-distro legs (debian12/ubuntu24/
# ubuntu26) for parallelism; local runs all distros in one
# suite.
# dns-resolver — single leg here; runs all scenarios (same as local).
# ─────────────────────────────────────────────────────────────────────────────
# ─────────────────────────────────────────────────────────────────────────────
# Job 1 Build matrix
#
@@ -35,9 +57,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: rustfmt
cache: false
rustflags: ''
- run: cargo fmt --check
clippy:
@@ -47,9 +71,11 @@ jobs:
- uses: actions/checkout@v4
- name: Install system dependencies
run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev
- uses: dtolnay/rust-toolchain@stable
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: clippy
cache: false
rustflags: ''
- name: Cache Cargo registry + build
uses: actions/cache@v4
with:
@@ -98,7 +124,10 @@ jobs:
run: sudo nft -c -f packaging/common/fips.nft
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache: false
rustflags: ''
- name: Cache Cargo registry + build
uses: actions/cache@v4
@@ -160,7 +189,10 @@ jobs:
run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache: false
rustflags: ''
- name: Cache Cargo registry + build
uses: actions/cache@v4
@@ -210,7 +242,10 @@ jobs:
run: echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> "$GITHUB_ENV"
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache: false
rustflags: ''
- name: Cache Cargo registry + build
uses: actions/cache@v4
@@ -239,7 +274,10 @@ jobs:
- uses: actions/checkout@v4
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache: false
rustflags: ''
- name: Cache Cargo registry + build
uses: actions/cache@v4
@@ -323,6 +361,10 @@ jobs:
- suite: rekey-outbound-only
type: rekey-outbound-only
topology: rekey-outbound-only
# ── Inbound max_peers admission-cap test ───────────────────────
- suite: admission-cap
type: admission-cap
topology: mesh
- suite: acl-allowlist
type: acl-allowlist
# ── Mixed-profile node-role integration test ─────────────────
@@ -755,6 +797,42 @@ jobs:
docker compose -f testing/static/docker-compose.yml \
--profile gateway down --volumes --remove-orphans
# ── Inbound max_peers admission-cap integration test ────────────────
# Lowers node.max_peers on one mesh node and asserts the inbound cap
# holds under sustained retry pressure: denied peers keep retrying but
# are never promoted to an active session. The admission-cap-test.sh
# assertions are tailored per link-layer handshake variant; the leg
# itself is uniform. Static-style harness on the shared mesh profile.
- name: Generate configs (admission-cap)
if: matrix.type == 'admission-cap'
run: bash testing/static/scripts/generate-configs.sh mesh
- name: Inject admission-cap config (admission-cap)
if: matrix.type == 'admission-cap'
run: bash testing/static/scripts/admission-cap-test.sh inject-config
- name: Start containers (admission-cap)
if: matrix.type == 'admission-cap'
run: |
docker compose -f testing/static/docker-compose.yml \
--profile mesh up -d
- name: Run admission-cap test
if: matrix.type == 'admission-cap'
run: bash testing/static/scripts/admission-cap-test.sh
- name: Collect logs on failure (admission-cap)
if: matrix.type == 'admission-cap' && failure()
run: |
docker compose -f testing/static/docker-compose.yml \
--profile mesh logs --no-color | tail -300
- name: Stop containers (admission-cap)
if: matrix.type == 'admission-cap' && always()
run: |
docker compose -f testing/static/docker-compose.yml \
--profile mesh down --volumes --remove-orphans
# ── Real-deb install integration ────────────────────────────────────
# The deb-install harness builds its own .deb from source in a
# cargo-deb builder image; the pre-built Linux binary from the
+4 -1
View File
@@ -72,7 +72,10 @@ jobs:
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libdbus-1-dev llvm
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache: false
rustflags: ''
- name: Cache Cargo registry + build
if: ${{ env.ACT != 'true' }}
+5 -4
View File
@@ -69,10 +69,11 @@ jobs:
run: echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> "$GITHUB_ENV"
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Add cross-compile target
run: rustup target add ${{ matrix.target }}
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
target: ${{ matrix.target }}
cache: false
rustflags: ''
- name: Cache Cargo registry + build
uses: actions/cache@v4
+4 -2
View File
@@ -110,9 +110,11 @@ jobs:
- name: Install Rust toolchain (stable)
if: matrix.rust_channel == 'stable'
uses: dtolnay/rust-toolchain@stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
targets: ${{ matrix.rust_target }}
target: ${{ matrix.rust_target }}
cache: false
rustflags: ''
- name: Install Rust toolchain (nightly, Tier 3)
if: matrix.rust_channel == 'nightly'
+4 -1
View File
@@ -63,7 +63,10 @@ jobs:
echo "SOURCE_DATE_EPOCH=$epoch" >> $env:GITHUB_ENV
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache: false
rustflags: ''
- name: Cache Cargo registry + build
uses: actions/cache@v4
+93
View File
@@ -154,6 +154,16 @@ with v0.3.x peers.
requests" and "Further reading". Running the checklist before
opening surfaces problems that would otherwise come back as review
comments, saving a round trip.
- [`docs/how-to/tune-file-descriptors.md`](docs/how-to/tune-file-descriptors.md)
— an operator how-to for raising `RLIMIT_NOFILE`. A busy node opens
roughly three file descriptors per established UDP peer (a
`connect()`-ed socket plus a 2-FD drain self-pipe), so the default
1024 soft limit is exhausted near 320 peers, after which further
admission, handshakes, and discovery fail with `EMFILE`. The guide
documents the per-peer FD budget and symptom, the systemd
(`LimitNOFILE` drop-in) and OpenWrt (procd `nofile`) procedures to
raise the limit, and how to verify the per-peer ratio stays bounded.
Linked from the how-to index.
### Changed
@@ -290,6 +300,39 @@ with v0.3.x peers.
RTT +0.110.19 ms from the worker queue handoff. Windows
continues on the existing tokio-based send/recv path.
- The Debian package no longer ships `/etc/fips/fips.yaml` as a dpkg
conf-file. The default configuration is installed as an example at
`/usr/share/fips/fips.yaml.example`, and `postinst` seeds
`/etc/fips/fips.yaml` (mode 600) from it only when the file does not
already exist — so a configuration-management-rendered or
operator-edited config is never prompted for or clobbered on
upgrade, removing the need for a `dpkg-divert` workaround.
`fips.service` gains `ConditionPathExists=/etc/fips/fips.yaml`. The
example is placed under `/usr/share/fips`, deliberately outside
`/usr/share/doc`, which minimal and container installs path-exclude
(so the install-time seed source is never dropped).
- Local and GitHub CI integration coverage brought into parity, and
the Rust toolchain selection given a single source of truth:
- The `admission-cap` integration suite, previously run only by
`ci-local.sh`, now also runs as a GitHub `ci.yml` matrix leg, so a
regression in it turns the GitHub gate red rather than depending on
a developer remembering to run local CI. A new
`testing/check-ci-parity.sh` (wired as `ci-local.sh
--check-parity`) diffs the two runners' integration-suite sets and
fails on unexpected drift; the deliberate local-only (live-Tor)
and granularity-only differences are documented in a comment block
atop both runners.
- CI and packaging jobs now select the toolchain with
`actions-rust-lang/setup-rust-toolchain` (which reads
`rust-toolchain.toml`) instead of `dtolnay/rust-toolchain@stable`.
The pinned channel already overrode the installed stable, so each
job downloaded an unused toolchain and logged a misleading `rustc`
version; the single-source action removes the waste and the
confusion. Existing cache steps are kept (`cache: false` on the
new action) and `RUSTFLAGS` is left untouched so no global
`-D warnings` is newly imposed. The OpenWrt nightly Tier-3 leg
keeps `@nightly`.
### Fixed
- Inbound connections that arrive while a node is already at its
@@ -323,6 +366,56 @@ with v0.3.x peers.
Previously the old session could linger after the peer came back
with a new startup epoch, leaving the session-layer map out of sync
with the freshly promoted peer.
- Two nodes that each `auto_connect` to the other no longer stall their
Nostr-mediated NAT-traversal handshake. Each side ran both an
initiator and a responder traversal session, binding a separate UDP
socket per session, and adopted only the first `Established` event; if
the two sides adopted mismatched sessions, each sent its Noise msg1 to
a peer port the peer had already stopped draining and both handshakes
hung until the adoption budget expired. The responder now elects a
single session deterministically — it declines an incoming offer only
when it also has an in-flight outbound initiator for the same peer and
its own NodeAddr is smaller — so one matching socket pair survives on
both ends and the peer's redundant initiator times out harmlessly.
One-sided (asymmetric) `auto_connect` has no co-active initiator and is
never suppressed, so connectivity is preserved. (Distinct from the
cross-init adoption tie-breaker below, which dedups two simultaneous
punches but does not stop each node from running redundant
initiator + responder sessions.)
- FMP link-layer rekey is now reliable under packet loss, bringing it up
to the FSP session layer's rekey discipline:
- The rekey msg1 retransmission driver was uncapped and never
abandoned, so a rekey that never completed resent msg1 forever. It
now uses a bounded retransmission budget (`handshake_max_resends`
with exponential backoff) and abandons the rekey cycle cleanly once
the budget is exhausted, mirroring the FSP rekey msg3 driver. With
the cap in place the link-dead heartbeat is rekey-aware:
`check_link_heartbeats` no longer reaps a link that is still
actively carrying rekey-handshake traffic, while a genuinely dead
link is still reaped once the budget abandons.
- At the K-bit cutover the receiver now authenticates an inbound frame
against the pending session before promoting it, instead of
promoting on the bare header K-bit. Under jitter a node could
otherwise promote a stale pending session, leaving the two endpoints
on different keys and silently dropping traffic until the link died
— the same failure class already closed on FSP, now closed on FMP.
- Node-level multi-node tests no longer flake under parallel CPU load.
They previously delivered handshake packets over real localhost UDP,
whose kernel receive buffer could overflow and drop a packet when many
tests ran concurrently, panicking the large-network convergence tests.
A `cfg(test)`-only loopback `TransportHandle` variant now delivers
packets directly between nodes over an unbounded in-process channel, so
there is no socket buffer to overflow, and the previously-quarantined
large-network tests run in the default suite again. The shipping daemon
build is unaffected (the variant is test-gated).
- Integration suites that wait for the mesh to converge no longer
false-fail under concurrent CI load. The rekey, static-mesh, and
sidecar suites replace a fixed wall-clock baseline timeout (and a blind
sleep) with a progress-aware wait that polls the suite's own pairwise
pings, returns as soon as every pair is reachable, extends its deadline
while the reachable-pair count is still climbing, and gives up only
when progress stalls.
- TCP and Tor `max_inbound_connections` admission cap is now compared
against the per-direction inbound count (`pool_inbound`) rather than
the combined pool size. Outbound connect-on-send connections share
+2 -2
View File
@@ -74,7 +74,7 @@ assets = [
["target/release/fips", "/usr/bin/", "755"],
["target/release/fipsctl", "/usr/bin/", "755"],
["target/release/fipstop", "/usr/bin/", "755"],
["packaging/common/fips.yaml", "/etc/fips/fips.yaml", "600"],
["packaging/common/fips.yaml", "/usr/share/fips/fips.yaml.example", "644"],
["packaging/common/hosts", "/etc/fips/hosts", "644"],
["packaging/common/fips.nft", "/etc/fips/fips.nft", "644"],
["packaging/debian/fips.service", "/lib/systemd/system/fips.service", "644"],
@@ -87,7 +87,7 @@ assets = [
["packaging/debian/fips-gateway.service", "/lib/systemd/system/fips-gateway.service", "644"],
["docs/design/fips-security.md", "/usr/share/doc/fips/fips-security.md", "644"],
]
conf-files = ["/etc/fips/fips.yaml", "/etc/fips/hosts", "/etc/fips/fips.nft"]
conf-files = ["/etc/fips/hosts", "/etc/fips/fips.nft"]
[dev-dependencies]
tempfile = "3.15"
+1
View File
@@ -18,6 +18,7 @@ X" to "X is done".
| [enable-nostr-discovery.md](enable-nostr-discovery.md) | Turn on Nostr-mediated discovery (3 capabilities — resolve, advertise, open — across 5 scenarios) |
| [deploy-tor-onion.md](deploy-tor-onion.md) | Run a Tor onion service for inbound FIPS connections |
| [tune-udp-buffers.md](tune-udp-buffers.md) | Set host sysctls so FIPS UDP sockets don't get clamped |
| [tune-file-descriptors.md](tune-file-descriptors.md) | Raise `RLIMIT_NOFILE` so a busy node doesn't exhaust file descriptors (`EMFILE`) as peer count grows |
| [run-as-unprivileged-user.md](run-as-unprivileged-user.md) | Run the daemon under a dedicated unprivileged service account (drops the default-root posture) |
| [deploy-gateway.md](deploy-gateway.md) | Manually deploy `fips-gateway` on a non-OpenWrt Linux host (LAN-to-mesh outbound + mesh-to-LAN inbound port-forwards). For the OpenWrt path, see the gateway tutorial. |
| [troubleshoot-gateway.md](troubleshoot-gateway.md) | Diagnostic recipes for the gateway, organised by half (outbound, inbound, common) |
+144
View File
@@ -0,0 +1,144 @@
# Tune the File-Descriptor Limit for FIPS
A busy FIPS node opens many file descriptors, and the count grows with
the number of peers it serves. On most systemd distributions the daemon
inherits a soft `RLIMIT_NOFILE` of 1024, which a well-connected node can
exhaust — at which point peer admission, handshakes, and discovery start
failing with `EMFILE` ("Too many open files").
This guide explains the FD budget, shows how to raise the limit on
systemd and on OpenWrt, and how to verify the result.
## Why FIPS is FD-hungry
Unlike a service that multiplexes all traffic over one socket, the FIPS
data plane allocates descriptors **per peer**. The dominant term is:
```text
fds ≈ 3·P + fixed overhead (~30)
```
where `P` is the number of established UDP peers. Each such peer consumes
**3 file descriptors**:
- one `connect()`-ed UDP socket dedicated to that peer, plus
- a 2-FD self-pipe owned by that peer's receive-drain worker (used to
wake and stop the worker cleanly).
The remaining consumers are bounded and do not scale with peer count:
- the TUN device (one descriptor, process-lifetime),
- the wildcard UDP listen socket(s) (one per bound UDP transport),
- TCP and Tor transport listeners and the Tor control socket,
- Nostr relay websockets (one per configured discovery relay),
- the control socket (one `UnixListener`, plus short-lived per-request
client connections for `fipsctl` / `fipstop`),
- and base runtime descriptors (epoll, eventfd, logs).
Together these add a roughly flat overhead of about 30 descriptors. The
per-peer term is what drives the daemon toward the FD ceiling.
## The symptom
The systemd and distro default **soft** `RLIMIT_NOFILE` is **1024**.
With the `3·P` budget above, that ceiling is reached near **~320 peers**
(3 × 320 ≈ 960, plus the fixed overhead). Once the process is out of
descriptors, every syscall that allocates one — `socket()`, `accept()`,
`open()`, `pipe()` — fails with `EMFILE`, which surfaces as:
- failed peer admission (new peers cannot be accepted),
- failed handshakes (the daemon cannot open the per-peer socket), and
- dropped discovery (relay or probe sockets cannot be created).
These symptoms appear only under load, once the node has accumulated
enough peers to cross the ceiling, so they can be easy to misattribute.
## Raise the limit on systemd
Create a drop-in override for the service:
```sh
sudo systemctl edit fips.service
```
Add:
```ini
[Service]
LimitNOFILE=65535
```
A single `LimitNOFILE=` value sets **both** the soft and the hard limit,
so no separate soft/hard syntax is needed here.
Reload systemd and restart the daemon so the new limit takes effect:
```sh
sudo systemctl daemon-reload
sudo systemctl restart fips
```
`65535` (2¹⁶ 1) is the conventional headroom value for network
daemons. With the `3·P` budget, it supports roughly **~21,800 peers**
before the FD ceiling binds — well beyond any plausible single-node FIPS
mesh degree. Past that point other limits (threads, memory, CPU) bind
first, so raising `LimitNOFILE` higher buys nothing.
## Raise the limit on OpenWrt
OpenWrt uses procd, not systemd, so `LimitNOFILE` does not apply.
Set the equivalent limit in the init script at `/etc/init.d/fips`,
inside the block that starts the service:
```sh
procd_set_param limits nofile="65535 65535"
```
The two values are the soft and hard limits respectively; setting them
equal mirrors the single-value systemd behaviour above.
Restart the service to apply:
```sh
/etc/init.d/fips restart
```
## Verify
Compare the live descriptor count against the established peer count:
```sh
ls /proc/$(pidof fips)/fd | wc -l
fipsctl show peers | wc -l
```
At steady state, expect a stable ratio of about **3 descriptors per
peer** plus the flat ~30-descriptor overhead. A ratio that holds steady
as peers come and go confirms healthy, bounded scaling.
If the descriptor count climbs steadily while the peer count stays flat,
that would indicate a descriptor leak rather than legitimate scaling —
the limit bump would only delay the wall. The current data plane has
been audited as leak-free (every per-peer descriptor has a guaranteed
close on every teardown path), so a climbing ratio at fixed peer count
would be a regression worth investigating.
## A note on deployment lines
The per-peer connected-UDP socket — the amplifier behind the
`3·P` term — is present on the master and next data planes. It is **not
yet present on the maintenance line**. On maintenance-only deployments
the 3-descriptor-per-peer term does not apply, and FD pressure comes
only from the fixed consumers listed above. Raising `LimitNOFILE` there
is still worthwhile as forward-looking headroom, and harmless where the
amplifier is absent.
## See also
- [tune-udp-buffers.md](tune-udp-buffers.md) — host sysctls so FIPS UDP
sockets don't get clamped
- [run-as-unprivileged-user.md](run-as-unprivileged-user.md) — run the
daemon under a dedicated service account
- [../reference/configuration.md](../reference/configuration.md) —
transport and discovery configuration that influences the fixed FD
overhead
+12 -2
View File
@@ -33,10 +33,20 @@ packaging/
### Debian/Ubuntu (`.deb`)
Built with [cargo-deb](https://github.com/kornelski/cargo-deb). Installs
`fips`, `fipsctl`, and `fipstop` to `/usr/bin/`, places config at
`/etc/fips/fips.yaml` (preserved on upgrade), and enables the systemd
`fips`, `fipsctl`, and `fipstop` to `/usr/bin/`, and enables the systemd
service.
The default configuration ships as an example at
`/usr/share/fips/fips.yaml.example` and is **not** a dpkg conf-file.
(It is deliberately **not** under `/usr/share/doc`, which minimal and
container installs path-exclude, since the postinst reads it at install
time.)
On install, `postinst` seeds `/etc/fips/fips.yaml` (mode 600) from the
example **only if it does not already exist**, so a configuration that
was rendered by configuration management or edited by an operator is
never prompted for or clobbered on upgrade. To reset to defaults, remove
`/etc/fips/fips.yaml` and reinstall, or copy the example back manually.
```sh
# Build
make deb
+5
View File
@@ -3,6 +3,11 @@ Description=FIPS Mesh Network Daemon
After=network-online.target
Wants=network-online.target
# The config file is no longer a packaged conf-file; postinst seeds it
# if absent. Skip the unit (inactive, not failed) rather than crash-loop
# if it is ever missing.
ConditionPathExists=/etc/fips/fips.yaml
[Service]
Type=simple
ExecStart=/usr/bin/fips --config /etc/fips/fips.yaml
+10
View File
@@ -9,6 +9,16 @@ case "$1" in
groupadd --system fips
fi
# Seed /etc/fips/fips.yaml from the shipped example only if it
# does not already exist. The live config is no longer a dpkg
# conf-file; this copy-if-absent yields to any operator- or
# configuration-management-rendered file and never clobbers it.
if [ ! -e /etc/fips/fips.yaml ]; then
install -m 600 -o root -g root \
/usr/share/fips/fips.yaml.example \
/etc/fips/fips.yaml
fi
# Drop-in directory for operator nftables rules included by
# /etc/fips/fips.nft. Empty by default; the include glob matches
# nothing cleanly out of the box.
+144
View File
@@ -0,0 +1,144 @@
#!/bin/bash
# ── CI parity invariant guard ───────────────────────────────────────────────
# The GitHub integration matrix (.github/workflows/ci.yml) and the local
# default suite set (ci-local.sh) MUST run the same integration suites,
# EXCEPT for the deliberate local-only entries listed below. Adding a suite
# to one runner without the other means "local green" and "GitHub green" stop
# being equivalent claims.
#
# Deliberate local-only (NOT on the GitHub gate), with reason:
# tor-socks5 — requires live Tor network; opt-in via --with-tor,
# unreliable on GitHub-hosted runners.
# tor-directory — same; live Tor dependency.
#
# Granularity-only differences folded before comparison (same coverage,
# different matrix shape — NOT a divergence):
# deb-install — GitHub splits into per-distro legs
# (deb-install-debian12/ubuntu24/ubuntu26); local runs all
# distros in one suite. Folded to "deb-install".
# chaos-* — GitHub fans each chaos scenario into its own matrix leg
# (type: chaos); local runs them all via the one CHAOS_SUITES
# path. The individual scenario names also differ cosmetically
# between runners (e.g. chaos-smoke-10 vs churn-mixed-10).
# Folded to a single "chaos" token on both sides.
# dns-resolver — single leg / single suite both sides; runs all scenarios.
#
# Exit 0 = parity clean. Exit 1 = unexpected divergence (suite names printed).
# ─────────────────────────────────────────────────────────────────────────────
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
CI_LOCAL="$SCRIPT_DIR/ci-local.sh"
CI_YML="$PROJECT_ROOT/.github/workflows/ci.yml"
# Deliberate local-only allowlist (suites intentionally absent from GitHub).
ALLOWLIST="tor-socks5 tor-directory"
for f in "$CI_LOCAL" "$CI_YML"; do
if [[ ! -f "$f" ]]; then
echo "check-ci-parity: missing file: $f" >&2
exit 2
fi
done
# Extract and normalize both suite sets in Python (robust YAML parse of the
# matrix; regex extraction of the bash suite arrays). Folding rules above are
# applied identically to both sides so only genuine divergence surfaces.
python3 - "$CI_LOCAL" "$CI_YML" "$ALLOWLIST" <<'PY'
import re
import sys
ci_local_path, ci_yml_path, allowlist_raw = sys.argv[1], sys.argv[2], sys.argv[3]
allowlist = set(allowlist_raw.split())
def fold(name):
"""Collapse granularity-only matrix shape into canonical suite identity."""
if name.startswith("chaos-") or name == "chaos":
return "chaos"
if name.startswith("deb-install"):
return "deb-install"
return name
# ── Local: parse the suite arrays from ci-local.sh ───────────────────────────
with open(ci_local_path, encoding="utf-8") as fh:
local_src = fh.read()
def bash_array(var):
m = re.search(rf"^{var}=\((.*?)\)", local_src, re.MULTILINE | re.DOTALL)
if not m:
return []
body = m.group(1)
# Quoted entries (chaos uses "display scenario flags"): first token is name.
quoted = re.findall(r'"([^"]*)"', body)
if quoted:
return [entry.split()[0] for entry in quoted if entry.strip()]
return [tok for tok in body.split() if tok.strip()]
local = set()
# Static, rekey, gateway, sidecar, acl, firewall, nostr, stun, dns, deb.
for var in ("STATIC_SUITES", "REKEY_SUITES", "ADMISSION_SUITES",
"GATEWAY_SUITES", "SIDECAR_SUITES", "ACL_SUITES",
"FIREWALL_SUITES", "NOSTR_RELAY_SUITES", "STUN_FAULTS_SUITES",
"DNS_RESOLVER_SUITES", "DEB_INSTALL_SUITES"):
local.update(bash_array(var))
# Chaos display names → fold to "chaos".
for _ in bash_array("CHAOS_SUITES"):
local.add("chaos")
# NAT scenarios are stored bare (cone/symmetric/lan) and prefixed nat- at use.
for scen in bash_array("NAT_SUITES"):
local.add(f"nat-{scen}")
# TOR_SUITES is the deliberate local-only set — excluded from the default path.
local = {fold(n) for n in local}
# ── GitHub: parse the integration matrix suite: values from ci.yml ───────────
import yaml # noqa: E402
with open(ci_yml_path, encoding="utf-8") as fh:
doc = yaml.safe_load(fh)
include = doc["jobs"]["integration"]["strategy"]["matrix"]["include"]
github = set()
for leg in include:
if "suite" not in leg:
continue
# Chaos legs carry inconsistent suite: names (chaos-smoke-10 vs
# churn-mixed-10) but a uniform type: chaos — fold via type, not name.
if str(leg.get("type", "")) == "chaos":
github.add("chaos")
else:
github.add(fold(str(leg["suite"])))
# ── Diff (subtract allowlist from local before comparison) ───────────────────
local_cmp = {n for n in local if n not in allowlist}
local_only = sorted(local_cmp - github)
github_only = sorted(github - local_cmp)
if local_only or github_only:
print("CI parity FAILED: integration suite sets diverge.\n")
if local_only:
print(" Local-only (in ci-local.sh, missing from ci.yml, "
"not in deliberate allowlist):")
for n in local_only:
print(f" - {n}")
if github_only:
print(" GitHub-only (in ci.yml, missing from local default path):")
for n in github_only:
print(f" - {n}")
print("\n Resolve by adding the suite to the other runner, or by adding "
"it\n to the deliberate local-only allowlist in "
"check-ci-parity.sh with a\n stated reason.")
sys.exit(1)
print("CI parity OK: integration suite sets match "
"(allowlist: " + ", ".join(sorted(allowlist)) + ").")
print(f" {len(github)} canonical suites compared on each side.")
sys.exit(0)
PY
+22
View File
@@ -12,6 +12,8 @@
# --only <suite> Run a single integration suite
# -j, --jobs <N> Max parallel chaos scenarios (default: 4)
# --list List available integration suites
# --check-parity Verify this suite set matches ci.yml's integration
# matrix (see testing/check-ci-parity.sh), then exit
# -h, --help Show this help
#
# Integration suites (default coverage):
@@ -32,6 +34,25 @@
# Exit codes:
# 0 — all stages passed
# 1 — one or more stages failed
#
# ── CI parity invariant ─────────────────────────────────────────────────────
# This local default suite set and the GitHub integration matrix
# (.github/workflows/ci.yml) MUST run the same integration suites, EXCEPT for
# the deliberate local-only entries below. Adding a suite to one runner
# without the other means "local green" and "GitHub green" stop being
# equivalent. testing/check-ci-parity.sh enforces this and fails on drift.
#
# Deliberate local-only (NOT on the GitHub gate), with reason:
# tor-socks5 — requires live Tor network; opt-in via --with-tor,
# unreliable on GitHub-hosted runners.
# tor-directory — same; live Tor dependency.
#
# Granularity-only differences (same coverage, different matrix shape —
# NOT a divergence):
# deb-install — local runs all distros sequentially in one suite; GitHub
# splits into per-distro legs (debian12/ubuntu24/ubuntu26).
# dns-resolver — single suite both sides; runs all scenarios.
# ─────────────────────────────────────────────────────────────────────────────
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -170,6 +191,7 @@ while [[ $# -gt 0 ]]; do
--only) ONLY_SUITE="$2"; shift 2 ;;
-j|--jobs) PARALLEL_JOBS="$2"; shift 2 ;;
--list) list_suites ;;
--check-parity) exec "$SCRIPT_DIR/check-ci-parity.sh" ;;
-h|--help) usage ;;
*) echo "Unknown option: $1"; usage ;;
esac