mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-01 04:26:14 +00:00
Compare commits
1
Commits
maint
..
v0.4.0-rc2
| Author | SHA1 | Date | |
|---|---|---|---|
 Johnathan Corgan
|
af127409c3 |
@@ -5,23 +5,6 @@ junit = { path = "junit.xml" }
|
||||
# occasional msg1 under burst load even with the per-edge repair loop.
|
||||
# Allow a retry rather than failing the whole CI run on a single
|
||||
# dropped packet.
|
||||
#
|
||||
# Deliberately scoped to [profile.ci] and NOT applied locally, which makes
|
||||
# the two gates disagree: the hosted runner retries a flaky test twice, the
|
||||
# local sweep fails on the first failure. The asymmetry is intended and this
|
||||
# is the record of why, since an undocumented one is indistinguishable from
|
||||
# an oversight.
|
||||
#
|
||||
# It is here for shared-runner packet loss, a property of the hosted
|
||||
# environment and not of the code. Applying it locally would suppress a real
|
||||
# local flake, and a failure that only reproduces under load is a robustness
|
||||
# bug to fix rather than to retry past. Keeping the local sweep strict is
|
||||
# what makes it the sharper of the two gates.
|
||||
#
|
||||
# The cost, stated rather than hidden: a test that fails once and passes on
|
||||
# retry is reported green here with no separate signal, so a genuine
|
||||
# intermittent failure can be absorbed. If that starts mattering, the fix is
|
||||
# to surface retried-but-passed tests, not to drop the retries.
|
||||
retries = 2
|
||||
|
||||
[test-groups]
|
||||
|
||||
+87
-29
@@ -38,12 +38,12 @@ env:
|
||||
# unreliable on GitHub-hosted runners.
|
||||
# tor-directory — same; live Tor dependency.
|
||||
#
|
||||
# The two runners express the same work in different matrix shapes, and the
|
||||
# parity guard compares through that shape rather than around it: chaos legs
|
||||
# are compared per scenario (and per flag) via their `scenario:` field,
|
||||
# deb-install legs per distro. The one leg still compared at leg granularity
|
||||
# is dns-resolver — a single leg here, running all of its scenarios
|
||||
# internally, exactly as the local suite does.
|
||||
# Granularity-only differences (same coverage, different matrix shape —
|
||||
# NOT a divergence):
|
||||
# deb-install — split here into per-distro legs (debian12/debian13/
|
||||
# ubuntu22/ubuntu24/ubuntu26) for parallelism; local runs the
|
||||
# same distro set in one suite.
|
||||
# dns-resolver — single leg here; runs all scenarios (same as local).
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -52,29 +52,6 @@ env:
|
||||
# Builds on Linux x86_64, Linux aarch64, and macOS.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
jobs:
|
||||
ci-parity:
|
||||
name: CI parity
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install Python deps
|
||||
run: pip3 install --quiet pyyaml
|
||||
- name: Check local and GitHub runners cover the same work
|
||||
run: bash testing/check-ci-parity.sh
|
||||
- name: Check test log matchers against the strings src/ emits
|
||||
run: python3 testing/check-log-strings.py
|
||||
- name: Check no tested function's exit status is a log call's
|
||||
run: python3 testing/check-trailing-log.py
|
||||
- name: Check nothing resolves the shared mutable test image
|
||||
run: bash testing/check-image-scoping.sh
|
||||
# Hermetic: synthetic ping functions, no containers, ~45s. Lives beside
|
||||
# the other two so both runners gate on it identically — putting it in
|
||||
# only one would create exactly the drift check-ci-parity.sh exists to
|
||||
# catch, and it is invisible to that checker either way since it is not
|
||||
# a matrix suite.
|
||||
- name: Run convergence-gate unit tests
|
||||
run: bash testing/lib/wait-converge-test.sh
|
||||
|
||||
fmt:
|
||||
name: Format check
|
||||
runs-on: ubuntu-latest
|
||||
@@ -384,6 +361,12 @@ jobs:
|
||||
- suite: rekey-outbound-only
|
||||
type: rekey-outbound-only
|
||||
topology: rekey-outbound-only
|
||||
# ── Inbound max_peers admission-cap test ───────────────────────
|
||||
- suite: admission-cap
|
||||
type: admission-cap
|
||||
topology: mesh
|
||||
- suite: acl-allowlist
|
||||
type: acl-allowlist
|
||||
# ── Firewall baseline (fips0 nftables default-deny) ────────────
|
||||
- suite: firewall
|
||||
type: firewall
|
||||
@@ -392,6 +375,9 @@ jobs:
|
||||
type: gateway
|
||||
topology: gateway
|
||||
# ── Chaos / stochastic scenarios ───────────────────────────────────
|
||||
- suite: chaos-smoke-10
|
||||
type: chaos
|
||||
scenario: smoke-10
|
||||
- suite: churn-mixed-10
|
||||
type: chaos
|
||||
scenario: churn-mixed
|
||||
@@ -405,9 +391,30 @@ jobs:
|
||||
- suite: tcp-mesh
|
||||
type: chaos
|
||||
scenario: tcp-mesh
|
||||
- suite: bottleneck-parent
|
||||
type: chaos
|
||||
scenario: bottleneck-parent
|
||||
- suite: cost-avoidance
|
||||
type: chaos
|
||||
scenario: cost-avoidance
|
||||
- suite: cost-reeval
|
||||
type: chaos
|
||||
scenario: cost-reeval
|
||||
- suite: cost-stability
|
||||
type: chaos
|
||||
scenario: cost-stability
|
||||
- suite: depth-vs-cost
|
||||
type: chaos
|
||||
scenario: depth-vs-cost
|
||||
- suite: mixed-technology
|
||||
type: chaos
|
||||
scenario: mixed-technology
|
||||
- suite: congestion-stress
|
||||
type: chaos
|
||||
scenario: congestion-stress
|
||||
- suite: bloom-storm
|
||||
type: chaos
|
||||
scenario: bloom-storm
|
||||
# ── Sidecar deployment ──────────────────────────────────────────
|
||||
- suite: sidecar
|
||||
type: sidecar
|
||||
@@ -619,6 +626,21 @@ jobs:
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile rekey-outbound-only down --volumes --remove-orphans
|
||||
|
||||
# ── ACL allowlist integration test ─────────────────────────────────────
|
||||
- name: Run ACL allowlist integration test
|
||||
if: matrix.type == 'acl-allowlist'
|
||||
run: bash testing/acl-allowlist/test.sh --skip-build --keep-up
|
||||
|
||||
- name: Collect logs on failure (acl-allowlist)
|
||||
if: matrix.type == 'acl-allowlist' && failure()
|
||||
run: |
|
||||
docker compose -f testing/acl-allowlist/docker-compose.yml logs --no-color
|
||||
|
||||
- name: Stop containers (acl-allowlist)
|
||||
if: matrix.type == 'acl-allowlist' && always()
|
||||
run: |
|
||||
docker compose -f testing/acl-allowlist/docker-compose.yml down --volumes --remove-orphans
|
||||
|
||||
# ── Firewall baseline integration test ─────────────────────────────────
|
||||
- name: Run firewall baseline integration test
|
||||
if: matrix.type == 'firewall'
|
||||
@@ -749,6 +771,42 @@ jobs:
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile gateway down --volumes --remove-orphans
|
||||
|
||||
# ── Inbound max_peers admission-cap integration test ────────────────
|
||||
# Lowers node.max_peers on one mesh node and asserts the inbound cap
|
||||
# holds under sustained retry pressure: denied peers keep retrying but
|
||||
# are never promoted to an active session. The admission-cap-test.sh
|
||||
# assertions are tailored per link-layer handshake variant; the leg
|
||||
# itself is uniform. Static-style harness on the shared mesh profile.
|
||||
- name: Generate configs (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/generate-configs.sh mesh
|
||||
|
||||
- name: Inject admission-cap config (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/admission-cap-test.sh inject-config
|
||||
|
||||
- name: Start containers (admission-cap)
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh up -d
|
||||
|
||||
- name: Run admission-cap test
|
||||
if: matrix.type == 'admission-cap'
|
||||
run: bash testing/static/scripts/admission-cap-test.sh
|
||||
|
||||
- name: Collect logs on failure (admission-cap)
|
||||
if: matrix.type == 'admission-cap' && failure()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh logs --no-color | tail -300
|
||||
|
||||
- name: Stop containers (admission-cap)
|
||||
if: matrix.type == 'admission-cap' && always()
|
||||
run: |
|
||||
docker compose -f testing/static/docker-compose.yml \
|
||||
--profile mesh down --volumes --remove-orphans
|
||||
|
||||
# ── Real-deb install integration ────────────────────────────────────
|
||||
# The deb-install harness builds its own .deb from source in a
|
||||
# cargo-deb builder image; the pre-built Linux binary from the
|
||||
|
||||
@@ -33,11 +33,6 @@ __pycache__/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
|
||||
# Per-run build contexts created by testing/ci-local.sh. Its teardown normally
|
||||
# removes them, but the CI worker's SIGKILL runs no trap, so one can survive a
|
||||
# preempted run; ci-cleanup.sh sweeps the survivors.
|
||||
/testing/docker-*/
|
||||
|
||||
# Runtime artifacts from running fips in-tree during local testing.
|
||||
# Root-anchored so legitimately-tracked fips.yaml under packaging/ and
|
||||
# examples/ stays included.
|
||||
|
||||
+1
-91
@@ -11,96 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Changed
|
||||
|
||||
- `SessionDatagram::decrement_ttl` and `SessionDatagram::can_forward` now match
|
||||
the forwarder's IP hop-limit semantics: `decrement_ttl` decrements first and
|
||||
reports false when the result is zero, and `can_forward` is true only at a
|
||||
TTL of 2 or more.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Nostr NAT traversal no longer breaks after the host suspends. The traversal
|
||||
clock cached a Unix timestamp once at startup and advanced it with a
|
||||
monotonic `Instant`, which does not tick while a machine is asleep, so after
|
||||
a suspend the daemon's idea of the time trailed real time by the suspend
|
||||
duration for the rest of the process lifetime. Every NIP-40 expiration it
|
||||
computed was therefore published already in the past: relays dropped the
|
||||
offers as expired, the initiator logged a signal timeout waiting for an
|
||||
answer, and traversal stayed broken until the daemon was restarted. The
|
||||
clock now reads the wall clock on every call. This is not macOS-specific,
|
||||
though a laptop that sleeps is where it is easiest to hit; any host that
|
||||
suspends or hibernates was affected. Reported in
|
||||
[#128](https://github.com/jmcorgan/fips/issues/128).
|
||||
|
||||
- `SessionDatagram` hop-limit handling now follows IP semantics. Delivery to
|
||||
the addressed node is no longer TTL-gated, and a forwarder decrements before
|
||||
deciding rather than after, so a datagram that would leave with a TTL of zero
|
||||
is dropped instead of transmitted. Previously the TTL check ran ahead of the
|
||||
local-delivery test, so a datagram addressed to this node that arrived with
|
||||
TTL 0 was dropped, and a forwarder receiving a transit datagram at TTL 1
|
||||
transmitted it at TTL 0 for the next hop to discard, wasting one transmission
|
||||
per expiring datagram. The reachable radius is unchanged, because the two
|
||||
behaviors compensated exactly: a path of `h` links still delivers for any
|
||||
source TTL of `h` or more. During a rolling upgrade, an unupgraded forwarder
|
||||
feeding an upgraded destination delivers one hop further than either version
|
||||
does on its own; no version mix delivers less far. The `TtlExhausted` reject
|
||||
counter now charges at the node that makes the decision rather than at the
|
||||
hop after it.
|
||||
|
||||
## [0.4.1] - 2026-07-19
|
||||
|
||||
### Changed
|
||||
|
||||
- `node.bloom.max_inbound_fpr` default raised from `0.10` to `0.20`. The
|
||||
cap rejects inbound `FilterAnnounce` whose FPR (`fill^k`) exceeds it. On
|
||||
the fixed 1 KB / k=5 filter, `0.10` corresponds to fill 0.631 (~1,630
|
||||
reachable entries), and the busiest nodes' aggregates had again begun to
|
||||
reach it as the mesh grew. `0.20` (fill 0.7248, ~2,114 entries) restores
|
||||
headroom without materially weakening the antipoison gate: a saturated or
|
||||
poisoned filter is ~100% FPR and still rejected. This is the second raise
|
||||
of this cap in two releases; the fixed 1 KB filter is the underlying
|
||||
constraint, and the structural remedy is the v2 filter work rather than a
|
||||
further raise. A node running this default accepts announcements that a
|
||||
v0.4.0 node drops, so during a rolling upgrade the two versions can
|
||||
disagree about mesh size.
|
||||
- Bloom filter probing computes its SHA-256 digest once per operation
|
||||
rather than once per hash function. All k indices were already derived
|
||||
from a single digest, but the digest was recomputed inside the
|
||||
per-function loop, so every insert and membership test hashed the same
|
||||
bytes `hash_count` times (5x at the default). Output is bit-for-bit
|
||||
identical; this is the hottest path in packet forwarding and mesh-size
|
||||
estimation.
|
||||
- Identity operations reuse one shared `secp256k1` context instead of
|
||||
constructing a fresh one at every sign, verify, and key-derive site.
|
||||
Each construction allocated a context and ran randomization and blinding
|
||||
table setup. Behavior is unchanged: the same API calls are made, only the
|
||||
context lifetime differs, and the shared context still performs the
|
||||
standard construction-time blinding.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Spanning tree: the coordinate cache is now invalidated when the parent
|
||||
link is lost through peer removal. That path reparents or self-roots the
|
||||
node but omitted the invalidation every other position-change path
|
||||
performs, so cached entries for downstream destinations kept the node's
|
||||
now-stale coordinate prefix. Because routing access refreshes an entry's
|
||||
TTL, an actively routed stale entry never self-expired and was corrected
|
||||
only by a fresh insert.
|
||||
- Discovery: applying a `LookupResponse` now keeps the tighter of the
|
||||
cached and received `path_mtu` rather than overwriting unconditionally.
|
||||
A looser estimate arriving in a later response could clobber a tighter
|
||||
value already learned from a reactive `MtuExceeded` or
|
||||
`PathMtuNotification`, loosening a clamp that had been correctly
|
||||
tightened.
|
||||
|
||||
### Removed
|
||||
|
||||
- The `parent_switched` spanning-tree metric counter. It was incremented on
|
||||
the line immediately before `parent_switches` at every site and never
|
||||
independently, so the two were always identical. `parent_switches`
|
||||
remains as the sole counter. Consumers reading `parent_switched` from the
|
||||
control socket or `fipstop` should use `parent_switches`.
|
||||
|
||||
## [0.4.0] - 2026-06-27
|
||||
## [0.4.0] - 2026-06-21
|
||||
|
||||
### Added
|
||||
|
||||
@@ -414,9 +327,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
example is placed under `/usr/share/fips`, deliberately outside
|
||||
`/usr/share/doc`, which minimal and container installs path-exclude
|
||||
(so the install-time seed source is never dropped).
|
||||
- openwrt: the `.apk` package now defaults `ethernet.wan` to the
|
||||
OpenWrt 25 DSA port name `wan`; the `.ipk` package keeps `eth0` for
|
||||
OpenWrt 24 and earlier.
|
||||
|
||||
#### CI & test-harness reliability
|
||||
|
||||
|
||||
Generated
+1
-1
@@ -1074,7 +1074,7 @@ checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5"
|
||||
|
||||
[[package]]
|
||||
name = "fips"
|
||||
version = "0.4.2-dev"
|
||||
version = "0.4.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bech32",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "fips"
|
||||
version = "0.4.2-dev"
|
||||
version = "0.4.0-rc.2"
|
||||
edition = "2024"
|
||||
description = "A distributed, decentralized network routing protocol for mesh nodes connecting over arbitrary transports"
|
||||
license = "MIT"
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
[](LICENSE)
|
||||
[](https://www.rust-lang.org/)
|
||||
[](#status--roadmap)
|
||||
[](#status--roadmap)
|
||||
|
||||
A self-organizing encrypted mesh network built on Nostr identities,
|
||||
capable of operating over arbitrary transports without central
|
||||
@@ -210,14 +210,12 @@ testing/ Docker-based integration test harnesses + chaos simulation
|
||||
|
||||
## Status & roadmap
|
||||
|
||||
FIPS is at **v0.4.2-dev** on the `maint` branch.
|
||||
[v0.4.1](https://github.com/jmcorgan/fips/releases/tag/v0.4.1) has
|
||||
shipped; this line carries patch-level fixes for the 0.4.x series. The
|
||||
core protocol works end-to-end over
|
||||
FIPS is at **v0.4.0**. The core protocol works end-to-end over
|
||||
UDP, TCP, Ethernet, Tor, Nym, and Bluetooth on a global, public test
|
||||
mesh of thousands of nodes. v0.4.0 added the Nym mixnet transport and
|
||||
mDNS LAN discovery alongside the existing Nostr-mediated peer discovery,
|
||||
UDP NAT traversal, peer ACL, and packaging hardening. New wire-format work
|
||||
mesh of thousands of nodes. v0.4.0 builds on the v0.3.0 testing-and-polishing
|
||||
track, adding the Nym mixnet transport and mDNS LAN discovery
|
||||
alongside the existing Nostr-mediated peer discovery, UDP NAT
|
||||
traversal, peer ACL, and packaging hardening. New wire-format work
|
||||
continues to be staged on the `next` branch for the subsequent
|
||||
release line.
|
||||
|
||||
|
||||
+284
-109
@@ -1,134 +1,302 @@
|
||||
# FIPS v0.4.1
|
||||
# FIPS v0.4.0
|
||||
|
||||
**Released**: 2026-07-19
|
||||
**Released**: 2026-06-21 (provisional)
|
||||
|
||||
v0.4.1 is a maintenance release on the v0.4.x line. It raises the default
|
||||
antipoison cap on inbound bloom filter announcements, removes a redundant
|
||||
spanning-tree metric counter, fixes two convergence and path-MTU bugs, and
|
||||
cuts per-packet CPU in the bloom and identity paths. There is no wire
|
||||
format change and no new feature surface.
|
||||
v0.4.0 is the throughput-and-observability release on the v0.3.x wire
|
||||
format. It adds two new ways for nodes to find and reach each other (the
|
||||
Nym mixnet transport and opt-in mDNS LAN discovery), overhauls the data
|
||||
plane for higher single-node throughput and lower per-packet CPU, moves
|
||||
the entire operator read surface off the data-plane hot path so
|
||||
observability stays responsive under load, ships a reworked `fipstop`
|
||||
TUI, and hardens FMP and FSP rekey to be hitless under packet loss in
|
||||
both directions. It also folds in the accumulated mesh-convergence,
|
||||
admission-control, and packaging fixes from the maintenance line.
|
||||
|
||||
v0.4.1 is wire-compatible with v0.4.0. Nodes can be upgraded one at a time
|
||||
with no coordinated restart, though one behavior change below is worth
|
||||
reading before you start a rolling upgrade.
|
||||
v0.4.0 is wire-compatible with v0.3.0. Mixed meshes interoperate; there
|
||||
is no flag-day upgrade. A deployed v0.3.0 node and an upgraded v0.4.0
|
||||
node peer, rekey, and route normally, so you can roll the upgrade out
|
||||
across a mesh in any order.
|
||||
|
||||
## At a glance
|
||||
|
||||
- `node.bloom.max_inbound_fpr` default moves from `0.10` to `0.20`.
|
||||
- The `parent_switched` metric counter is gone. Use `parent_switches`.
|
||||
- Spanning tree no longer serves stale coordinates after a parent link is
|
||||
lost through peer removal.
|
||||
- Discovery no longer loosens a path MTU clamp it had correctly tightened.
|
||||
- Bloom probing and identity operations do measurably less work per call,
|
||||
with identical results.
|
||||
- New outbound Nym mixnet transport with a single-container demo and a
|
||||
new mixnet-relay example.
|
||||
- Opt-in mDNS / DNS-SD discovery on the local link.
|
||||
- Data-plane overhaul: off-task encrypt and decrypt worker pools, GSO,
|
||||
connected-UDP send path, copy-avoidance on receive, batched macOS
|
||||
receive.
|
||||
- The full `show_*` read surface now serves off the receive loop, so
|
||||
`fipsctl` and `fipstop` stay responsive on loaded nodes; a new
|
||||
counter-only `show_metrics` query enables a Prometheus scraper at no
|
||||
hot-path cost.
|
||||
- Reworked `fipstop` TUI on a machine-verified render-snapshot base.
|
||||
- Rekey is now hitless under loss and reordering in both directions.
|
||||
- New packaging targets: an OpenWrt `.apk` for OpenWrt 25+ and a Nix
|
||||
flake for reproducible from-source builds on Nix/NixOS.
|
||||
- Six route-class transit counters partition forwarded traffic by its
|
||||
tree relationship to the next hop, visible via `show_routing` and
|
||||
`show_status`.
|
||||
|
||||
## What's new
|
||||
|
||||
### Nym mixnet transport
|
||||
|
||||
FIPS can now peer over the [Nym](https://nymtech.net/) mixnet for
|
||||
metadata-resistant connectivity. The new `transports.nym` transport
|
||||
makes outbound connections through a `nym-socks5-client` SOCKS5 proxy
|
||||
that you run alongside the daemon (for example as a service running
|
||||
alongside the fips daemon, or as a sidecar container). The transport
|
||||
waits at startup for the nym-socks5-client to become ready before giving
|
||||
up.
|
||||
|
||||
This is a privacy and anonymity deployment mode chosen for its own
|
||||
properties. It mixes your FIPS traffic into the Nym cover-traffic
|
||||
network so that link-level observers cannot correlate which mesh peers
|
||||
are talking. A new `examples/sidecar-nostr-mixnet-relay/` demonstrates a
|
||||
FIPS-reachable Nostr relay peered across the mixnet end to end, and a
|
||||
single-container demo ships with the transport.
|
||||
|
||||
Enable it by adding a `transports.nym` instance and pointing it at your
|
||||
running nym-socks5-client. See the transports reference for the field
|
||||
set.
|
||||
|
||||
### mDNS LAN discovery
|
||||
|
||||
Nodes on a shared local link can now find each other with zero address
|
||||
configuration. The opt-in `node.discovery.lan` path runs an mDNS /
|
||||
DNS-SD responder and browser: each node advertises a FIPS service record
|
||||
on the link and adopts the peers it discovers. This complements the
|
||||
existing Nostr-mediated overlay discovery for the common case where the
|
||||
peers are simply on the same LAN.
|
||||
|
||||
Turn it on with `node.discovery.lan.enabled: true`. `service_type` and
|
||||
`scope` tune the advertised service record and which interfaces
|
||||
participate. Discovery on the local link needs no relay and no STUN.
|
||||
|
||||
### Data-plane throughput overhaul
|
||||
|
||||
The receive and send paths were reworked for higher single-node
|
||||
throughput and lower per-packet CPU, building on the v0.3.0
|
||||
crypto-backend swap:
|
||||
|
||||
- **Off-task encrypt and decrypt.** Per-peer encrypt and decrypt now run
|
||||
on dedicated worker tasks rather than inline on the receive loop, so a
|
||||
single busy peer no longer serializes the whole node's crypto.
|
||||
- **GSO and connected-UDP send.** The Linux send path uses generic
|
||||
segmentation offload and a connected-UDP socket where available,
|
||||
cutting syscall overhead on bulk flows.
|
||||
- **Copy-avoidance on receive.** The receive hot path avoids buffer
|
||||
copies it previously made per packet.
|
||||
- **Batched macOS receive.** macOS gains a `recvmsg_x` batched receive,
|
||||
mirroring the Linux `recvmmsg` batching from v0.3.0.
|
||||
- **Shared immutable-state context and an atomic metric registry.**
|
||||
Immutable per-node state moved into a single shared context, and
|
||||
counters live in an atomic metric registry that the new `show_metrics`
|
||||
query reads without touching the hot path.
|
||||
|
||||
These are all internal to the data plane and require no operator action.
|
||||
|
||||
### Observability off the hot path
|
||||
|
||||
Every read-only control query now renders from a snapshot published once
|
||||
per tick into a lock-free `ArcSwap`, served from the control accept task
|
||||
instead of round-tripping the data-plane receive loop. This covers
|
||||
`show_status`, `show_stats_*`, `show_peers`, `show_sessions`,
|
||||
`show_links`, `show_connections`, `show_transports`, `show_mmp`,
|
||||
`show_tree`, `show_bloom`, `show_cache`, `show_routing`,
|
||||
`show_identity_cache`, `show_acl`, `show_listening_sockets`, and the new
|
||||
`show_metrics`. Only the mutating `connect` and `disconnect` commands
|
||||
still reach the loop.
|
||||
|
||||
The practical effect: on a loaded node where the receive loop was busy,
|
||||
`fipsctl` and `fipstop` queries previously stalled or timed out (the
|
||||
five-second query pattern operators saw). They now answer promptly
|
||||
regardless of data-plane load. Per-entity snapshots reuse unchanged rows
|
||||
by pointer, so the per-tick publish cost stays bounded as peer and
|
||||
session counts grow.
|
||||
|
||||
A new **`show_metrics`** query (surfaced as `fipsctl stats metrics`)
|
||||
returns a counter-only snapshot of every metric family. It is the
|
||||
enabler for a Prometheus scraper that pulls node counters at no hot-path
|
||||
cost.
|
||||
|
||||
Six **route-class transit counters** partition transit-forwarded packets
|
||||
by their tree relationship to the chosen next hop — tree-up, tree-down,
|
||||
tree-down-cross, cross-link descend, cross-link ascend, and direct-peer
|
||||
— and the six classes sum to `forwarded_packets`. They surface through
|
||||
`show_routing` and `show_status`, and the `fipstop` routing tab is
|
||||
reorganized so its two columns separate own/endpoint traffic from
|
||||
forwarded/transit traffic with the tree-down-cross line visually flagged.
|
||||
|
||||
### Reworked fipstop TUI
|
||||
|
||||
`fipstop` gets a rendering, navigation, and read-surface overhaul on a
|
||||
machine-verified base: a render-snapshot harness asserts the exact text
|
||||
grid and per-cell style of every view against canned control-socket
|
||||
output. New daemon-resolved fields surface through the snapshots,
|
||||
including effective persistence, root and is-root state, a
|
||||
per-transport-type peer-count map, per-peer effective depth, the root
|
||||
npub, and the last-sent uptree filter fill ratio with the subtree size
|
||||
estimate.
|
||||
|
||||
A separate fix clears a garbled-screen problem on startup and stray
|
||||
bytes on quit, most visible over SSH and inside tmux: startup now forces
|
||||
a full repaint before the first draw, and quit stops and joins the
|
||||
stdin-poll thread before restoring the terminal, so post-raw-mode
|
||||
keystrokes no longer echo onto the restored screen.
|
||||
|
||||
### Rekey reliability
|
||||
|
||||
FMP and FSP session rekey are now hitless under packet loss and
|
||||
reordering in both directions:
|
||||
|
||||
- Inbound frames are authenticated against the pending session before
|
||||
the K-bit cutover promotes it, so a spoofed or stale frame cannot
|
||||
derail a rekey in progress.
|
||||
- Rekey message-1 retransmission is bounded, and the link-dead heartbeat
|
||||
is rekey-aware so an in-flight rekey is not mistaken for a dead link.
|
||||
- FSP session rekey holds connectivity across the rekey window under
|
||||
loss and reordering.
|
||||
- Dual-initiation races (both peers starting a rekey at once on a
|
||||
high-latency link) are desynchronized with symmetric jitter so the two
|
||||
sides converge on one session rather than fighting.
|
||||
- An exhausted retransmission-budget abort, an expected and self-limiting
|
||||
outcome on lossy or high-latency links, is logged at debug rather than
|
||||
warn.
|
||||
|
||||
The net operator takeaway: rekey completes cleanly without dropping
|
||||
traffic, even on lossy or high-latency links, and the log no longer
|
||||
cries wolf when a rekey gives up and retries.
|
||||
|
||||
### New packaging targets
|
||||
|
||||
- **OpenWrt `.apk`.** A new `.apk` package targets OpenWrt 25+, where
|
||||
apk-tools is the mandatory package manager; the existing `.ipk`
|
||||
continues to cover OpenWrt 24.x and earlier. It is built SDK-free,
|
||||
reusing the `.ipk` cross-compile and installed-filesystem payload, and
|
||||
releases publish `.apk` artifacts and checksums alongside `.ipk`. Like
|
||||
the `.ipk`, the package is unsigned and installed with
|
||||
`apk add --allow-untrusted`.
|
||||
- **Nix flake.** A `flake.nix` at the project root builds all four
|
||||
binaries (`fips`, `fipsctl`, `fips-gateway`, `fipstop`) from source on
|
||||
Nix/NixOS, pinning the exact toolchain and wiring the native build
|
||||
dependencies so no host setup is needed beyond Nix with flakes
|
||||
enabled. It exposes `nix build`, `nix run`, a `nix develop` dev shell,
|
||||
and `nix flake check`, with `flake.lock` committed for reproducibility.
|
||||
|
||||
## Behavior changes worth flagging
|
||||
|
||||
### The inbound filter FPR cap default doubles again
|
||||
These affect operators on upgrade.
|
||||
|
||||
`node.bloom.max_inbound_fpr` goes from `0.10` to `0.20`. The cap rejects
|
||||
inbound `FilterAnnounce` frames whose advertised false positive rate
|
||||
exceeds it. On the fixed 1 KB, k=5 filter, `0.10` corresponds to a fill of
|
||||
0.631 and roughly 1,630 reachable entries, and the busiest nodes'
|
||||
aggregates had started reaching that ceiling as the mesh grew. `0.20`
|
||||
corresponds to a fill of 0.7248 and roughly 2,114 entries.
|
||||
|
||||
Be aware that this is the second time in two releases that this default
|
||||
has doubled, for the same reason both times. That is worth stating plainly
|
||||
rather than repeating the previous release's framing: raising the cap buys
|
||||
headroom, it does not fix anything. The real constraint is the fixed 1 KB
|
||||
filter size, which is a protocol constant. The structural remedy is the v2
|
||||
filter work, where filter capacity scales with the mesh instead of being
|
||||
pinned. This release is an interim step to keep legitimate aggregates from
|
||||
being rejected until that lands. It is not the start of a pattern of
|
||||
raising the cap once per release, and if you are sizing capacity planning
|
||||
around this number, plan against the v2 work rather than against a third
|
||||
raise.
|
||||
|
||||
The antipoison property the cap exists for is preserved. A saturated or
|
||||
deliberately poisoned filter still presents an FPR near 100% and is still
|
||||
rejected.
|
||||
|
||||
**This matters during a rolling upgrade.** A v0.4.1 node accepts a
|
||||
`FilterAnnounce` with a derived FPR between 0.10 and 0.20; a v0.4.0 node
|
||||
drops the same frame, and the drop is silent on the wire with no NACK. The
|
||||
cap also gates the mesh size estimator, which declines to produce a value
|
||||
when any contributing filter is over the cap. So while a mesh is partly
|
||||
upgraded, upgraded and not-yet-upgraded nodes can legitimately report
|
||||
different mesh sizes, or one can report a size while the other reports
|
||||
unknown. This resolves once every node is on v0.4.1. If you want to avoid
|
||||
the window entirely, set `node.bloom.max_inbound_fpr: 0.10` explicitly in
|
||||
your config before upgrading and remove it after the last node is done.
|
||||
|
||||
### The `parent_switched` counter is removed
|
||||
|
||||
`parent_switched` was incremented on the line immediately before
|
||||
`parent_switches` at every site and never independently, so the two
|
||||
counters always held the same value. `parent_switched` is now gone from
|
||||
the tree metrics, the control socket snapshot, and the `fipstop` tree
|
||||
view. `parent_switches` remains and is unchanged.
|
||||
|
||||
If you scrape the control socket, or have dashboards or alerts referencing
|
||||
`parent_switched`, point them at `parent_switches`. Anything still asking
|
||||
for `parent_switched` will find nothing rather than a zero.
|
||||
- **Bloom filter antipoison cap raised.** `node.bloom.max_inbound_fpr`
|
||||
moves from 0.05 to 0.10, accepting filters with a higher derived
|
||||
false-positive rate before rejecting them. This reduces spurious
|
||||
filter rejections on larger meshes while keeping the antipoison
|
||||
protection in place.
|
||||
- **TCP inbound cap honors `max_connections`.** The TCP inbound accept
|
||||
ceiling now resolves from explicit per-transport
|
||||
`max_inbound_connections`, then node-wide
|
||||
`node.limits.max_connections`, then the built-in default of 256.
|
||||
Previously the TCP inbound ceiling was hardwired to 256 and ignored
|
||||
`max_connections`, so raising it had no effect on inbound TCP.
|
||||
- **Static host aliases hot-reload.** `/etc/fips/hosts` now reloads on
|
||||
mtime change once per tick rather than only at startup, so display
|
||||
names in `fipsctl` and `fipstop` reflect edits without a daemon
|
||||
restart. The peer ACL reloads through the same lock-free snapshot
|
||||
mechanism.
|
||||
- **Quieter logs on busy public-mesh nodes.** Routine per-peer
|
||||
connection-lifecycle and capacity-cap events, no-route session-datagram
|
||||
drops, and exhausted rekey-budget aborts are demoted to debug, so
|
||||
genuinely notable info and warn lines are no longer drowned out.
|
||||
- **More visible drops.** Receive-path silent rejections now flow
|
||||
through typed reject-reason counters, and discovery counts requests
|
||||
dropped when the dedup cache is full (`req_dedup_cache_full`, visible
|
||||
via `show_routing`). Drops that were previously silent are now
|
||||
countable.
|
||||
- **Tor connect-refused accounting.** The Tor transport increments its
|
||||
`connect_refused` statistic (the "Refused" line in `fipstop`) on an
|
||||
actively-refused SOCKS5 connect, instead of recording every connect
|
||||
failure as a generic SOCKS5 error.
|
||||
|
||||
## Notable bug fixes
|
||||
|
||||
### Stale coordinates after losing a parent through peer removal
|
||||
The CHANGELOG has the exhaustive list. This is the operator-relevant
|
||||
subset of fixes for behavior that shipped in v0.3.0.
|
||||
|
||||
When a node's parent link dropped via peer removal, the node correctly
|
||||
reparented or self-rooted, but skipped the coordinate cache invalidation
|
||||
that every other position-change path performs. Cached entries for
|
||||
downstream destinations kept the node's old coordinate prefix. This did
|
||||
not self-correct the way a stale cache entry normally would: routing
|
||||
access refreshes an entry's TTL, so an entry that was actively being
|
||||
routed through never expired, and was only fixed by an unrelated fresh
|
||||
insert. Both invalidation classes now run on this path, matching the
|
||||
loop-detection branch.
|
||||
|
||||
### Discovery could loosen a tightened path MTU clamp
|
||||
|
||||
An originator handling a `LookupResponse` overwrote its cached path MTU
|
||||
unconditionally. If a reactive `MtuExceeded` or `PathMtuNotification` had
|
||||
already taught it a tighter value, a later, looser discovery estimate
|
||||
would clobber that and re-loosen the clamp, risking a return to dropped
|
||||
oversized packets. The cached and received values are now compared and the
|
||||
tighter one is kept.
|
||||
- **Symmetric peer teardown on manual disconnect.** A manual
|
||||
`fipsctl disconnect` now sends the peer a scoped Disconnect so both
|
||||
ends tear down and re-handshake cleanly. Previously a manual
|
||||
disconnect tore down only the local side, leaving the peer with a
|
||||
stale session that was never re-adopted as a child and whose bloom
|
||||
filter was never re-recorded.
|
||||
- **Gateway holds long-lived and DNS-cached mappings.** `fips-gateway`
|
||||
no longer drops a virtual-IP mapping while traffic is still flowing.
|
||||
The mapping TTL clock previously advanced only on DNS re-query, so a
|
||||
busy long-lived or DNS-cached client could have its mapping reclaimed
|
||||
mid-flow. The tick now refreshes the mapping whenever conntrack reports
|
||||
active sessions and recovers a draining mapping to active when traffic
|
||||
resumes; only genuinely idle mappings drain.
|
||||
- **Accurate mesh-size estimate under filter overlap.** The mesh-size
|
||||
estimator now estimates the cardinality of the OR-union of self plus
|
||||
every connected peer's inbound filter, instead of summing per-filter
|
||||
cardinalities of tree peers. Summing assumed the filters were disjoint,
|
||||
so a stale or oversized parent filter or a routing loop inflated the
|
||||
reported mesh size and a tree rebalance flapped the count. OR-union
|
||||
deduplicates overlap, equals the old result in the disjoint case, and
|
||||
removes the estimate's dependence on tree-declaration cache freshness.
|
||||
- **Single-uplink node reattaches within a round-trip.** A node with one
|
||||
tree peer, which has periodic parent re-evaluation disabled, was left
|
||||
self-rooted and unreachable if its one-shot attaching TreeAnnounce was
|
||||
lost, until the next periodic re-broadcast. Tree-position exchange is
|
||||
now self-healing on the receive path: a node that hears an announce
|
||||
advertising a strictly worse root echoes its own declaration back,
|
||||
provoking the better-rooted peer to re-push its real position
|
||||
immediately.
|
||||
- **macOS self-connections work end to end (#117).** Traffic a macOS
|
||||
node sends to its own `<npub>.fips` address is now delivered locally
|
||||
for full TCP/UDP, not just `ping6`. The point-to-point `utun` egresses
|
||||
self-addressed packets into the daemon with an unfinished transport
|
||||
checksum (macOS offloads it on the `lo0` loopback route), so
|
||||
re-injecting them verbatim made the local stack drop every segment the
|
||||
MSS-clamp rewrite did not happen to fix and self-connections
|
||||
half-opened and hung. The hairpin path now recomputes the TCP/UDP
|
||||
checksum before re-injection. Linux was unaffected.
|
||||
|
||||
## Upgrade notes
|
||||
|
||||
This is a drop-in upgrade from v0.4.0 with no wire format change, no
|
||||
config migration, and no coordinated restart. Upgrade nodes in whatever
|
||||
order you like.
|
||||
Operator-actionable items moving from v0.3.0 to v0.4.0:
|
||||
|
||||
Two things to do rather than assume:
|
||||
- **Wire-compatible, no flag day.** v0.4.0 peers with v0.3.0. Upgrade
|
||||
nodes in any order. During a rolling upgrade you may see some log lines
|
||||
on the upgraded side as it interacts with not-yet-upgraded peers;
|
||||
behavior is correct, log noise only.
|
||||
- **Bloom antipoison cap default changed.** `node.bloom.max_inbound_fpr`
|
||||
now defaults to 0.10 (was 0.05). If you set this explicitly, review
|
||||
whether you still want the old value.
|
||||
- **New optional config surfaces.** `transports.nym` (outbound Nym
|
||||
mixnet) and `node.discovery.lan` (mDNS LAN discovery) are both opt-in
|
||||
and off by default. Adding them is the only way to turn the new paths
|
||||
on.
|
||||
- **TCP inbound cap.** If you relied on the old hardwired 256 inbound-TCP
|
||||
ceiling, note it now honors `max_inbound_connections` then
|
||||
`node.limits.max_connections` then 256.
|
||||
- **New observability query.** `fipsctl stats metrics` (the
|
||||
`show_metrics` control query) returns a counter-only snapshot suitable
|
||||
for a scraper.
|
||||
|
||||
1. If you monitor `parent_switched`, move to `parent_switches` before
|
||||
upgrading, or your dashboards will go blank rather than error.
|
||||
2. During the rolling window, expect upgraded and not-yet-upgraded nodes
|
||||
to potentially disagree about mesh size, per the FPR cap section above.
|
||||
This is expected and self-resolves. Do not chase it as a bug unless it
|
||||
persists after every node reports `0.4.1`.
|
||||
|
||||
If you have pinned `node.bloom.max_inbound_fpr` explicitly in your config,
|
||||
your setting is honored and nothing changes for you. The change only
|
||||
affects nodes taking the default.
|
||||
|
||||
Downgrading to v0.4.0 is supported and needs no special handling.
|
||||
|
||||
## Getting v0.4.1
|
||||
## Getting v0.4.0
|
||||
|
||||
- **Linux x86_64 / aarch64**: `.deb` and tarball at the
|
||||
[v0.4.1 release page](https://github.com/jmcorgan/fips/releases/tag/v0.4.1).
|
||||
[v0.4.0 release page](https://github.com/jmcorgan/fips/releases/tag/v0.4.0).
|
||||
- **Arch Linux**: `fips` from the AUR.
|
||||
- **macOS**: `.pkg` at the v0.4.1 release page.
|
||||
- **Windows**: ZIP at the v0.4.1 release page.
|
||||
- **macOS**: `.pkg` at the v0.4.0 release page.
|
||||
- **Windows**: ZIP at the v0.4.0 release page.
|
||||
- **OpenWrt**: `.ipk` (OpenWrt 24.x and earlier) or `.apk` (OpenWrt 25+)
|
||||
at the v0.4.1 release page.
|
||||
- **From source**: `cargo build --release` from a checkout of the v0.4.1
|
||||
at the v0.4.0 release page.
|
||||
- **From source**: `cargo build --release` from a checkout of the v0.4.0
|
||||
tag (Rust 1.94.1 per `rust-toolchain.toml`; `libclang-dev` is a
|
||||
required Linux build prerequisite).
|
||||
- **Nix / NixOS**: `nix build .#fips` from a checkout of the v0.4.1 tag
|
||||
- **Nix / NixOS**: `nix build .#fips` from a checkout of the v0.4.0 tag
|
||||
builds the binaries from source with the pinned toolchain and no manual
|
||||
prerequisites (see the Nix section of `packaging/README.md`).
|
||||
|
||||
@@ -141,6 +309,13 @@ The full per-commit changelog lives in
|
||||
Thanks to everyone who contributed code, packaging work, bug reports, or
|
||||
reviews to this release.
|
||||
|
||||
- [@jcorgan](https://github.com/jmcorgan): release shepherd, spanning-tree
|
||||
and discovery fixes, bloom and identity performance work, antipoison cap
|
||||
change, and testing.
|
||||
- [@jcorgan](https://github.com/jmcorgan): release shepherd, high-level
|
||||
design, control read plane, rekey hardening, admission, bug fixes,
|
||||
testing, packaging, PR coordination, and issue resolution.
|
||||
- [@mmalmi](https://github.com/mmalmi): opt-in mDNS LAN discovery and
|
||||
data-plane performance work.
|
||||
- [@Origami74](https://github.com/Origami74): macOS packaging and
|
||||
website coordination.
|
||||
- [@dskvr](https://github.com/dskvr): AUR packaging.
|
||||
- [@oleksky](https://github.com/oleksky): Nym mixnet transport and the
|
||||
single-container mixnet demo.
|
||||
|
||||
@@ -360,14 +360,14 @@ control socket and `fipstop` dashboard. (See `compute_mesh_size()` in
|
||||
|
||||
The estimator refuses to produce a value when any contributing filter
|
||||
is above the antipoison FPR cap (`node.bloom.max_inbound_fpr`,
|
||||
default `0.20`); a partial aggregate would silently underestimate.
|
||||
default `0.10`); a partial aggregate would silently underestimate.
|
||||
Consumers handle the resulting `None` by displaying an "unknown"
|
||||
state rather than a misleading number.
|
||||
|
||||
## Antipoison: Inbound FPR Cap
|
||||
|
||||
Inbound `FilterAnnounce` payloads are checked against
|
||||
`node.bloom.max_inbound_fpr` (default `0.20`). Filters whose
|
||||
`node.bloom.max_inbound_fpr` (default `0.10`). Filters whose
|
||||
estimated false positive rate exceeds the cap are dropped silently
|
||||
(no NACK on the wire) — they would otherwise inflate downstream
|
||||
candidate evaluation cost without contributing useful discrimination.
|
||||
|
||||
@@ -277,7 +277,7 @@ Controls tree construction and parent selection.
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `node.bloom.update_debounce_ms` | u64 | `500` | Debounce interval for filter update propagation |
|
||||
| `node.bloom.max_inbound_fpr` | f64 | `0.20` | Antipoison cap: reject inbound `FilterAnnounce` frames whose advertised false-positive rate exceeds this value. Valid range `(0.0, 1.0)`. The default `0.20` corresponds to fill 0.7248 at k=5 (≈2,114 entries on the 1 KB filter); a saturated/poisoned filter is still ~100% FPR and rejected |
|
||||
| `node.bloom.max_inbound_fpr` | f64 | `0.10` | Antipoison cap: reject inbound `FilterAnnounce` frames whose advertised false-positive rate exceeds this value. Valid range `(0.0, 1.0)`. The default `0.10` corresponds to fill 0.631 at k=5 (≈1,630 entries on the 1 KB filter); a saturated/poisoned filter is still ~100% FPR and rejected |
|
||||
|
||||
Bloom filter size (1 KB), hash count (5), and size classes are protocol
|
||||
constants and not configurable.
|
||||
@@ -944,7 +944,7 @@ node:
|
||||
flap_dampening_secs: 120 # extended hold-down on flap
|
||||
bloom:
|
||||
update_debounce_ms: 500
|
||||
max_inbound_fpr: 0.20 # antipoison cap on inbound FilterAnnounce FPR
|
||||
max_inbound_fpr: 0.10 # antipoison cap on inbound FilterAnnounce FPR
|
||||
session:
|
||||
default_ttl: 64
|
||||
pending_packets_per_dest: 16
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# FIPS v0.4.0
|
||||
|
||||
**Released**: 2026-06-27
|
||||
**Released**: 2026-06-21 (provisional)
|
||||
|
||||
v0.4.0 is the throughput-and-observability release on the v0.3.x wire
|
||||
format. It adds two new ways for nodes to find and reach each other (the
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
# FIPS v0.4.1
|
||||
|
||||
**Released**: 2026-07-19
|
||||
|
||||
v0.4.1 is a maintenance release on the v0.4.x line. It raises the default
|
||||
antipoison cap on inbound bloom filter announcements, removes a redundant
|
||||
spanning-tree metric counter, fixes two convergence and path-MTU bugs, and
|
||||
cuts per-packet CPU in the bloom and identity paths. There is no wire
|
||||
format change and no new feature surface.
|
||||
|
||||
v0.4.1 is wire-compatible with v0.4.0. Nodes can be upgraded one at a time
|
||||
with no coordinated restart, though one behavior change below is worth
|
||||
reading before you start a rolling upgrade.
|
||||
|
||||
## At a glance
|
||||
|
||||
- `node.bloom.max_inbound_fpr` default moves from `0.10` to `0.20`.
|
||||
- The `parent_switched` metric counter is gone. Use `parent_switches`.
|
||||
- Spanning tree no longer serves stale coordinates after a parent link is
|
||||
lost through peer removal.
|
||||
- Discovery no longer loosens a path MTU clamp it had correctly tightened.
|
||||
- Bloom probing and identity operations do measurably less work per call,
|
||||
with identical results.
|
||||
|
||||
## Behavior changes worth flagging
|
||||
|
||||
### The inbound filter FPR cap default doubles again
|
||||
|
||||
`node.bloom.max_inbound_fpr` goes from `0.10` to `0.20`. The cap rejects
|
||||
inbound `FilterAnnounce` frames whose advertised false positive rate
|
||||
exceeds it. On the fixed 1 KB, k=5 filter, `0.10` corresponds to a fill of
|
||||
0.631 and roughly 1,630 reachable entries, and the busiest nodes'
|
||||
aggregates had started reaching that ceiling as the mesh grew. `0.20`
|
||||
corresponds to a fill of 0.7248 and roughly 2,114 entries.
|
||||
|
||||
Be aware that this is the second time in two releases that this default
|
||||
has doubled, for the same reason both times. That is worth stating plainly
|
||||
rather than repeating the previous release's framing: raising the cap buys
|
||||
headroom, it does not fix anything. The real constraint is the fixed 1 KB
|
||||
filter size, which is a protocol constant. The structural remedy is the v2
|
||||
filter work, where filter capacity scales with the mesh instead of being
|
||||
pinned. This release is an interim step to keep legitimate aggregates from
|
||||
being rejected until that lands. It is not the start of a pattern of
|
||||
raising the cap once per release, and if you are sizing capacity planning
|
||||
around this number, plan against the v2 work rather than against a third
|
||||
raise.
|
||||
|
||||
The antipoison property the cap exists for is preserved. A saturated or
|
||||
deliberately poisoned filter still presents an FPR near 100% and is still
|
||||
rejected.
|
||||
|
||||
**This matters during a rolling upgrade.** A v0.4.1 node accepts a
|
||||
`FilterAnnounce` with a derived FPR between 0.10 and 0.20; a v0.4.0 node
|
||||
drops the same frame, and the drop is silent on the wire with no NACK. The
|
||||
cap also gates the mesh size estimator, which declines to produce a value
|
||||
when any contributing filter is over the cap. So while a mesh is partly
|
||||
upgraded, upgraded and not-yet-upgraded nodes can legitimately report
|
||||
different mesh sizes, or one can report a size while the other reports
|
||||
unknown. This resolves once every node is on v0.4.1. If you want to avoid
|
||||
the window entirely, set `node.bloom.max_inbound_fpr: 0.10` explicitly in
|
||||
your config before upgrading and remove it after the last node is done.
|
||||
|
||||
### The `parent_switched` counter is removed
|
||||
|
||||
`parent_switched` was incremented on the line immediately before
|
||||
`parent_switches` at every site and never independently, so the two
|
||||
counters always held the same value. `parent_switched` is now gone from
|
||||
the tree metrics, the control socket snapshot, and the `fipstop` tree
|
||||
view. `parent_switches` remains and is unchanged.
|
||||
|
||||
If you scrape the control socket, or have dashboards or alerts referencing
|
||||
`parent_switched`, point them at `parent_switches`. Anything still asking
|
||||
for `parent_switched` will find nothing rather than a zero.
|
||||
|
||||
## Notable bug fixes
|
||||
|
||||
### Stale coordinates after losing a parent through peer removal
|
||||
|
||||
When a node's parent link dropped via peer removal, the node correctly
|
||||
reparented or self-rooted, but skipped the coordinate cache invalidation
|
||||
that every other position-change path performs. Cached entries for
|
||||
downstream destinations kept the node's old coordinate prefix. This did
|
||||
not self-correct the way a stale cache entry normally would: routing
|
||||
access refreshes an entry's TTL, so an entry that was actively being
|
||||
routed through never expired, and was only fixed by an unrelated fresh
|
||||
insert. Both invalidation classes now run on this path, matching the
|
||||
loop-detection branch.
|
||||
|
||||
### Discovery could loosen a tightened path MTU clamp
|
||||
|
||||
An originator handling a `LookupResponse` overwrote its cached path MTU
|
||||
unconditionally. If a reactive `MtuExceeded` or `PathMtuNotification` had
|
||||
already taught it a tighter value, a later, looser discovery estimate
|
||||
would clobber that and re-loosen the clamp, risking a return to dropped
|
||||
oversized packets. The cached and received values are now compared and the
|
||||
tighter one is kept.
|
||||
|
||||
## Upgrade notes
|
||||
|
||||
This is a drop-in upgrade from v0.4.0 with no wire format change, no
|
||||
config migration, and no coordinated restart. Upgrade nodes in whatever
|
||||
order you like.
|
||||
|
||||
Two things to do rather than assume:
|
||||
|
||||
1. If you monitor `parent_switched`, move to `parent_switches` before
|
||||
upgrading, or your dashboards will go blank rather than error.
|
||||
2. During the rolling window, expect upgraded and not-yet-upgraded nodes
|
||||
to potentially disagree about mesh size, per the FPR cap section above.
|
||||
This is expected and self-resolves. Do not chase it as a bug unless it
|
||||
persists after every node reports `0.4.1`.
|
||||
|
||||
If you have pinned `node.bloom.max_inbound_fpr` explicitly in your config,
|
||||
your setting is honored and nothing changes for you. The change only
|
||||
affects nodes taking the default.
|
||||
|
||||
Downgrading to v0.4.0 is supported and needs no special handling.
|
||||
|
||||
## Getting v0.4.1
|
||||
|
||||
- **Linux x86_64 / aarch64**: `.deb` and tarball at the
|
||||
[v0.4.1 release page](https://github.com/jmcorgan/fips/releases/tag/v0.4.1).
|
||||
- **Arch Linux**: `fips` from the AUR.
|
||||
- **macOS**: `.pkg` at the v0.4.1 release page.
|
||||
- **Windows**: ZIP at the v0.4.1 release page.
|
||||
- **OpenWrt**: `.ipk` (OpenWrt 24.x and earlier) or `.apk` (OpenWrt 25+)
|
||||
at the v0.4.1 release page.
|
||||
- **From source**: `cargo build --release` from a checkout of the v0.4.1
|
||||
tag (Rust 1.94.1 per `rust-toolchain.toml`; `libclang-dev` is a
|
||||
required Linux build prerequisite).
|
||||
- **Nix / NixOS**: `nix build .#fips` from a checkout of the v0.4.1 tag
|
||||
builds the binaries from source with the pinned toolchain and no manual
|
||||
prerequisites (see the Nix section of `packaging/README.md`).
|
||||
|
||||
The full per-commit changelog lives in
|
||||
[`CHANGELOG.md`](../../CHANGELOG.md). Issues and discussion at
|
||||
[github.com/jmcorgan/fips](https://github.com/jmcorgan/fips).
|
||||
|
||||
## Contributors
|
||||
|
||||
Thanks to everyone who contributed code, packaging work, bug reports, or
|
||||
reviews to this release.
|
||||
|
||||
- [@jcorgan](https://github.com/jmcorgan): release shepherd, spanning-tree
|
||||
and discovery fixes, bloom and identity performance work, antipoison cap
|
||||
change, and testing.
|
||||
@@ -191,13 +191,6 @@ install -d "$STAGE_DIR/etc/fips"
|
||||
install -m 0600 "$FILES_DIR/etc/fips/fips.yaml" "$STAGE_DIR/etc/fips/fips.yaml"
|
||||
install -m 0755 "$FILES_DIR/etc/fips/firewall.sh" "$STAGE_DIR/etc/fips/firewall.sh"
|
||||
|
||||
# The shared fips.yaml ships ethernet.wan.interface: "eth0", the OpenWrt 24
|
||||
# default. This .apk package targets OpenWrt 25+ (DSA), where the WAN port is
|
||||
# named "wan", so ship "wan" as the default. Patching the staged copy keeps the
|
||||
# as-installed config correct for the platform without maintaining a second copy
|
||||
# of the file; operators can still edit /etc/fips/fips.yaml for non-standard boards.
|
||||
sed -i 's|interface: "eth0"|interface: "wan"|' "$STAGE_DIR/etc/fips/fips.yaml"
|
||||
|
||||
install -d "$STAGE_DIR/etc/dnsmasq.d"
|
||||
install -m 0644 "$FILES_DIR/etc/dnsmasq.d/fips.conf" "$STAGE_DIR/etc/dnsmasq.d/fips.conf"
|
||||
|
||||
|
||||
@@ -132,10 +132,7 @@ The default config enables:
|
||||
|
||||
For Ethernet transport, uncomment the `ethernet:` section and set the correct
|
||||
physical interface names for your router. **Always use physical port names
|
||||
(`eth0`, `eth1`, or DSA port names like `wan`/`lan1`), never bridge names
|
||||
(`br-lan`).** The shipped default WAN port is `eth0` (OpenWrt 24); on OpenWrt
|
||||
25 (DSA) boards the WAN port is named `wan` — the `.apk` package ships that
|
||||
default. Run `ip link show` to confirm the names on your board. See
|
||||
(`eth0`, `eth1`), never bridge names (`br-lan`).** See
|
||||
[`deploy/native/README.md`](../../deploy/native/README.md) for details.
|
||||
|
||||
## Service management
|
||||
|
||||
@@ -1189,15 +1189,6 @@ fn mmp_focused_pane_indicator() {
|
||||
});
|
||||
// The focused Session MMP title is cyan; the unfocused Link MMP title is not.
|
||||
assert_eq!(testkit::fg_at(&buf, "Session MMP"), Some(Color::Cyan));
|
||||
// Presence before colour. `fg_at` is `find(..)?` mapped to the cell's fg, so
|
||||
// it returns None for a title that was never drawn, and None != Some(Cyan) --
|
||||
// meaning the assertion below passed when the Link MMP pane was missing
|
||||
// entirely. Asserting it is on screen first is what makes the next line read
|
||||
// "not highlighted" rather than "not there".
|
||||
assert!(
|
||||
testkit::find(&buf, "Link MMP").is_some(),
|
||||
"the unfocused Link MMP title should still be rendered"
|
||||
);
|
||||
assert_ne!(testkit::fg_at(&buf, "Link MMP"), Some(Color::Cyan));
|
||||
}
|
||||
|
||||
|
||||
@@ -174,6 +174,10 @@ fn draw_stats(frame: &mut Frame, data: &serde_json::Value, scroll: u16, focused:
|
||||
&helpers::nested_u64(data, "stats", "sig_failed"),
|
||||
),
|
||||
helpers::kv_line("Stale", &helpers::nested_u64(data, "stats", "stale")),
|
||||
helpers::kv_line(
|
||||
"Parent Switched",
|
||||
&helpers::nested_u64(data, "stats", "parent_switched"),
|
||||
),
|
||||
helpers::kv_line(
|
||||
"Loop Detected",
|
||||
&helpers::nested_u64(data, "stats", "loop_detected"),
|
||||
|
||||
+9
-19
@@ -69,18 +69,16 @@ impl BloomFilter {
|
||||
|
||||
/// Insert a NodeAddr into the filter.
|
||||
pub fn insert(&mut self, node_addr: &NodeAddr) {
|
||||
let (h1, h2) = Self::base_hashes(node_addr.as_bytes());
|
||||
for i in 0..self.hash_count {
|
||||
let bit_index = self.bit_index(h1, h2, i);
|
||||
let bit_index = self.hash(node_addr.as_bytes(), i);
|
||||
self.set_bit(bit_index);
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert raw bytes into the filter.
|
||||
pub fn insert_bytes(&mut self, data: &[u8]) {
|
||||
let (h1, h2) = Self::base_hashes(data);
|
||||
for i in 0..self.hash_count {
|
||||
let bit_index = self.bit_index(h1, h2, i);
|
||||
let bit_index = self.hash(data, i);
|
||||
self.set_bit(bit_index);
|
||||
}
|
||||
}
|
||||
@@ -95,9 +93,8 @@ impl BloomFilter {
|
||||
|
||||
/// Check if the filter might contain raw bytes.
|
||||
pub fn contains_bytes(&self, data: &[u8]) -> bool {
|
||||
let (h1, h2) = Self::base_hashes(data);
|
||||
for i in 0..self.hash_count {
|
||||
let bit_index = self.bit_index(h1, h2, i);
|
||||
let bit_index = self.hash(data, i);
|
||||
if !self.get_bit(bit_index) {
|
||||
return false;
|
||||
}
|
||||
@@ -199,28 +196,21 @@ impl BloomFilter {
|
||||
self.hash_count
|
||||
}
|
||||
|
||||
/// Compute the two base hashes for `data` with a single SHA-256 digest.
|
||||
/// Compute a hash index for the given data and hash function number.
|
||||
///
|
||||
/// Double hashing derives the k hash functions from two base hashes:
|
||||
/// h(x,i) = (h1(x) + i*h2(x)) mod m. Computing the digest once here and
|
||||
/// reusing `(h1, h2)` across all k functions avoids re-hashing per k.
|
||||
fn base_hashes(data: &[u8]) -> (u64, u64) {
|
||||
// Use first 16 bytes of SHA-256 for h1 and h2.
|
||||
/// Uses double hashing: h(x,i) = (h1(x) + i*h2(x)) mod m
|
||||
fn hash(&self, data: &[u8], k: u8) -> usize {
|
||||
// Use first 16 bytes of SHA-256 for h1 and h2
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data);
|
||||
let hash = hasher.finalize();
|
||||
|
||||
// h1 from first 8 bytes, h2 from next 8 bytes (little-endian).
|
||||
// h1 from first 8 bytes
|
||||
let h1 = u64::from_le_bytes(hash[0..8].try_into().unwrap());
|
||||
// h2 from next 8 bytes
|
||||
let h2 = u64::from_le_bytes(hash[8..16].try_into().unwrap());
|
||||
(h1, h2)
|
||||
}
|
||||
|
||||
/// Derive the bit index for hash function `k` from the base hashes.
|
||||
///
|
||||
/// Uses double hashing: h(x,k) = (h1(x) + k*h2(x)) mod m.
|
||||
fn bit_index(&self, h1: u64, h2: u64, k: u8) -> usize {
|
||||
let combined = h1.wrapping_add((k as u64).wrapping_mul(h2));
|
||||
(combined as usize) % self.num_bits
|
||||
}
|
||||
|
||||
+7
-65
@@ -172,79 +172,21 @@ impl BloomState {
|
||||
peer_addrs: &[NodeAddr],
|
||||
peer_filters: &HashMap<NodeAddr, BloomFilter>,
|
||||
) {
|
||||
let targets: Vec<NodeAddr> = peer_addrs
|
||||
.iter()
|
||||
.filter(|addr| *addr != exclude_from)
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
for (peer_addr, new_filter) in self.compute_outgoing_filters(&targets, peer_filters) {
|
||||
let changed = match self.last_sent_filters.get(&peer_addr) {
|
||||
for peer_addr in peer_addrs {
|
||||
if peer_addr == exclude_from {
|
||||
continue;
|
||||
}
|
||||
let new_filter = self.compute_outgoing_filter(peer_addr, peer_filters);
|
||||
let changed = match self.last_sent_filters.get(peer_addr) {
|
||||
Some(last) => *last != new_filter,
|
||||
None => true, // never sent → must send
|
||||
};
|
||||
if changed {
|
||||
self.pending_updates.insert(peer_addr);
|
||||
self.pending_updates.insert(*peer_addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the outgoing filter for many peers in one pass.
|
||||
///
|
||||
/// Equivalent to calling [`compute_outgoing_filter`](Self::compute_outgoing_filter)
|
||||
/// once per target, but linear in the number of contributing peer
|
||||
/// filters instead of quadratic. The per-peer call rebuilds the whole
|
||||
/// union from scratch, so computing it for every peer costs
|
||||
/// O(targets × filters) 1 KB merges; announce fan-out on a
|
||||
/// large node does exactly that, once per tick and again on every
|
||||
/// inbound announce.
|
||||
///
|
||||
/// The split-horizon exclusion is the only thing that differs between
|
||||
/// targets, so the union of "everything except peer i" is assembled
|
||||
/// from a running prefix union and a precomputed suffix union. Merging
|
||||
/// is a bytewise OR, which is commutative and associative, so the
|
||||
/// result is bit-identical to the per-peer computation.
|
||||
pub fn compute_outgoing_filters(
|
||||
&self,
|
||||
targets: &[NodeAddr],
|
||||
peer_filters: &HashMap<NodeAddr, BloomFilter>,
|
||||
) -> HashMap<NodeAddr, BloomFilter> {
|
||||
let base = self.base_filter();
|
||||
let keys: Vec<NodeAddr> = peer_filters.keys().copied().collect();
|
||||
let n = keys.len();
|
||||
|
||||
// suffix[i] = union of peer_filters[keys[i..]]; suffix[n] is empty.
|
||||
let mut suffix = vec![BloomFilter::new(); n + 1];
|
||||
for i in (0..n).rev() {
|
||||
let mut acc = suffix[i + 1].clone();
|
||||
// Size mismatches are skipped, exactly as in the per-peer path.
|
||||
let _ = acc.merge(&peer_filters[&keys[i]]);
|
||||
suffix[i] = acc;
|
||||
}
|
||||
|
||||
// Filter for a target that contributes nothing: everything merged.
|
||||
let mut all = base.clone();
|
||||
let _ = all.merge(&suffix[0]);
|
||||
|
||||
let mut per_key: HashMap<NodeAddr, BloomFilter> = HashMap::with_capacity(n);
|
||||
let mut prefix = BloomFilter::new();
|
||||
for i in 0..n {
|
||||
let mut outgoing = base.clone();
|
||||
let _ = outgoing.merge(&prefix);
|
||||
let _ = outgoing.merge(&suffix[i + 1]);
|
||||
per_key.insert(keys[i], outgoing);
|
||||
let _ = prefix.merge(&peer_filters[&keys[i]]);
|
||||
}
|
||||
|
||||
targets
|
||||
.iter()
|
||||
.map(|target| {
|
||||
let filter = per_key.get(target).cloned().unwrap_or_else(|| all.clone());
|
||||
(*target, filter)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Compute the outgoing filter for a specific peer.
|
||||
///
|
||||
/// The filter includes:
|
||||
|
||||
@@ -228,84 +228,6 @@ fn test_bloom_filter_insert_bytes_contains_bytes() {
|
||||
assert!(filter.contains_bytes(data2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_bit_indices_match_double_hashing_formula() {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
// Independently recompute the documented double-hashing bit indices:
|
||||
// one SHA-256 digest of the input, h1 = bytes[0..8] LE, h2 = bytes[8..16]
|
||||
// LE, then for k in 0..hash_count: (h1 + k*h2) mod num_bits. This pins
|
||||
// bit-identical behavior regardless of the internal implementation.
|
||||
fn expected_indices(data: &[u8], num_bits: usize, hash_count: u8) -> Vec<usize> {
|
||||
let digest = Sha256::digest(data);
|
||||
let h1 = u64::from_le_bytes(digest[0..8].try_into().unwrap());
|
||||
let h2 = u64::from_le_bytes(digest[8..16].try_into().unwrap());
|
||||
(0..hash_count)
|
||||
.map(|k| {
|
||||
let combined = h1.wrapping_add((k as u64).wrapping_mul(h2));
|
||||
(combined as usize) % num_bits
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bit_is_set(filter: &BloomFilter, index: usize) -> bool {
|
||||
let byte = filter.as_bytes()[index / 8];
|
||||
(byte >> (index % 8)) & 1 == 1
|
||||
}
|
||||
|
||||
let configs = [(1024usize, 5u8), (8192usize, 7u8)];
|
||||
let inputs: [&[u8]; 4] = [b"", b"alpha", b"the quick brown fox", &[0u8, 1, 2, 3, 255]];
|
||||
|
||||
for (num_bits, hash_count) in configs {
|
||||
for data in inputs {
|
||||
let mut filter = BloomFilter::with_params(num_bits, hash_count).unwrap();
|
||||
let expected = expected_indices(data, num_bits, hash_count);
|
||||
|
||||
filter.insert_bytes(data);
|
||||
|
||||
// Every expected bit is set.
|
||||
for &idx in &expected {
|
||||
assert!(
|
||||
bit_is_set(&filter, idx),
|
||||
"expected bit {} set for input {:?} (num_bits={}, k={})",
|
||||
idx,
|
||||
data,
|
||||
num_bits,
|
||||
hash_count
|
||||
);
|
||||
}
|
||||
|
||||
// No unexpected bits are set: the set-bit count never exceeds the
|
||||
// number of distinct expected indices.
|
||||
use std::collections::HashSet;
|
||||
let distinct: HashSet<usize> = expected.iter().copied().collect();
|
||||
assert_eq!(
|
||||
filter.count_ones(),
|
||||
distinct.len(),
|
||||
"unexpected bits set for input {:?}",
|
||||
data
|
||||
);
|
||||
|
||||
// contains reports the inserted item as present.
|
||||
assert!(filter.contains_bytes(data));
|
||||
}
|
||||
}
|
||||
|
||||
// NodeAddr path uses the same formula over its byte view.
|
||||
let node = make_node_addr(7);
|
||||
let mut filter = BloomFilter::with_params(1024, 5).unwrap();
|
||||
let expected = expected_indices(node.as_bytes(), 1024, 5);
|
||||
filter.insert(&node);
|
||||
for &idx in &expected {
|
||||
assert!(bit_is_set(&filter, idx));
|
||||
}
|
||||
assert!(filter.contains(&node));
|
||||
|
||||
// Spot-check a definitely-absent item is reported absent.
|
||||
let absent = make_node_addr(200);
|
||||
assert!(!filter.contains(&absent));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bloom_filter_estimated_count_saturated() {
|
||||
// Create a small filter with all bits set
|
||||
@@ -684,70 +606,3 @@ fn test_bloom_state_mark_changed_peers_excludes_source() {
|
||||
|
||||
assert!(!state.needs_update(&peer1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_outgoing_filters_matches_per_peer() {
|
||||
let node = make_node_addr(0);
|
||||
let mut state = BloomState::new(node);
|
||||
state.add_leaf_dependent(make_node_addr(200));
|
||||
state.add_leaf_dependent(make_node_addr(201));
|
||||
|
||||
// Six contributing peers with overlapping content, plus one whose
|
||||
// filter is a different size and must be skipped by both paths.
|
||||
let mut peer_filters = HashMap::new();
|
||||
for i in 1u8..=6 {
|
||||
let mut filter = BloomFilter::new();
|
||||
for j in 0..5u8 {
|
||||
filter.insert(&make_node_addr(i.wrapping_mul(7).wrapping_add(j)));
|
||||
}
|
||||
peer_filters.insert(make_node_addr(i), filter);
|
||||
}
|
||||
let odd_peer = make_node_addr(7);
|
||||
let mut odd = BloomFilter::with_params(4096, DEFAULT_HASH_COUNT).unwrap();
|
||||
odd.insert(&make_node_addr(99));
|
||||
peer_filters.insert(odd_peer, odd);
|
||||
|
||||
// Targets: every contributing peer, the odd-sized one, and two peers
|
||||
// that contribute nothing (non-tree peers get announces too).
|
||||
let mut targets: Vec<NodeAddr> = (1u8..=7).map(make_node_addr).collect();
|
||||
targets.push(make_node_addr(120));
|
||||
targets.push(make_node_addr(121));
|
||||
|
||||
let batch = state.compute_outgoing_filters(&targets, &peer_filters);
|
||||
|
||||
assert_eq!(batch.len(), targets.len());
|
||||
for target in &targets {
|
||||
let expected = state.compute_outgoing_filter(target, &peer_filters);
|
||||
assert_eq!(
|
||||
batch.get(target),
|
||||
Some(&expected),
|
||||
"batch filter for {:?} differs from per-peer computation",
|
||||
target
|
||||
);
|
||||
}
|
||||
|
||||
// Split horizon is real, not vacuous: a contributing peer's own
|
||||
// entries must be absent from its own outgoing filter, and present
|
||||
// in another peer's.
|
||||
let peer3 = make_node_addr(3);
|
||||
let own_entry = make_node_addr(3u8.wrapping_mul(7));
|
||||
assert!(!batch[&peer3].contains(&own_entry));
|
||||
assert!(batch[&make_node_addr(1)].contains(&own_entry));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_outgoing_filters_empty_inputs() {
|
||||
let node = make_node_addr(0);
|
||||
let state = BloomState::new(node);
|
||||
let peer_filters = HashMap::new();
|
||||
|
||||
assert!(
|
||||
state
|
||||
.compute_outgoing_filters(&[], &peer_filters)
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let target = make_node_addr(1);
|
||||
let batch = state.compute_outgoing_filters(&[target], &peer_filters);
|
||||
assert_eq!(batch[&target], state.base_filter());
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ mod node;
|
||||
mod peer;
|
||||
mod transport;
|
||||
|
||||
use crate::node::REKEY_JITTER_SECS;
|
||||
use crate::upper::config::{DnsConfig, TunConfig};
|
||||
use crate::{Identity, IdentityError};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -687,32 +686,6 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
// Reject rekey triggers that fire immediately and forever. Both
|
||||
// arms are checked regardless of `node.rekey.enabled` so that
|
||||
// turning rekey on later cannot surface a config error at a
|
||||
// surprising moment. There is deliberately no upper bound:
|
||||
// u64::MAX is the idiom for disabling one arm of the trigger.
|
||||
let rekey = &self.node.rekey;
|
||||
|
||||
if rekey.after_messages == 0 {
|
||||
return Err(ConfigError::Validation(
|
||||
"`node.rekey.after_messages` must be at least 1; 0 fires the message-count trigger on every poll instead of disabling it. \
|
||||
Use a very large value to effectively disable the message-count trigger."
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let jitter_secs = REKEY_JITTER_SECS.unsigned_abs();
|
||||
if rekey.after_secs <= jitter_secs {
|
||||
return Err(ConfigError::Validation(format!(
|
||||
"`node.rekey.after_secs` is {}, but must be greater than the per-session rekey jitter of {jitter_secs}s; \
|
||||
each session offsets the interval by a random value in [-{jitter_secs}, +{jitter_secs}] seconds, so a smaller interval saturates to zero \
|
||||
and rekeys on sight for roughly half of sessions. \
|
||||
Use a very large value to effectively disable the timer trigger.",
|
||||
rekey.after_secs
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1486,86 +1459,6 @@ peers:
|
||||
.expect("outbound_only should be exempt from the loopback check");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_default_rekey_settings_ok() {
|
||||
Config::default()
|
||||
.validate()
|
||||
.expect("shipped default rekey settings must validate");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_after_messages_zero_rejected() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_messages = 0;
|
||||
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("after_messages"), "got: {msg}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_after_messages_one_accepted() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_messages = 1;
|
||||
|
||||
config
|
||||
.validate()
|
||||
.expect("after_messages = 1 rekeys every message, which is wasteful but well defined");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_after_secs_at_or_below_jitter_rejected() {
|
||||
let jitter = REKEY_JITTER_SECS.unsigned_abs();
|
||||
|
||||
for after_secs in [0, 1, jitter - 1, jitter] {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_secs = after_secs;
|
||||
|
||||
match config.validate() {
|
||||
Err(e) => assert!(e.to_string().contains("after_secs"), "got: {e}"),
|
||||
Ok(()) => panic!("after_secs = {after_secs} should be rejected"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_after_secs_just_above_jitter_accepted() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_secs = REKEY_JITTER_SECS.unsigned_abs() + 1;
|
||||
|
||||
config
|
||||
.validate()
|
||||
.expect("one second above the jitter bound leaves a non-zero effective interval");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_unbounded_values_accepted() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.after_secs = u64::MAX;
|
||||
config.node.rekey.after_messages = u64::MAX;
|
||||
|
||||
config
|
||||
.validate()
|
||||
.expect("u64::MAX disables an arm of the trigger and must stay legal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rekey_checked_even_when_disabled() {
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.enabled = false;
|
||||
config.node.rekey.after_messages = 0;
|
||||
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
assert!(err.to_string().contains("after_messages"));
|
||||
|
||||
let mut config = Config::default();
|
||||
config.node.rekey.enabled = false;
|
||||
config.node.rekey.after_secs = REKEY_JITTER_SECS.unsigned_abs();
|
||||
|
||||
let err = config.validate().expect_err("validation should fail");
|
||||
assert!(err.to_string().contains("after_secs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_outbound_only_forces_ephemeral_bind() {
|
||||
let cfg = UdpConfig {
|
||||
|
||||
+5
-5
@@ -635,8 +635,8 @@ pub struct BloomConfig {
|
||||
pub update_debounce_ms: u64,
|
||||
/// Antipoison cap: reject inbound FilterAnnounce whose FPR exceeds
|
||||
/// this value (`node.bloom.max_inbound_fpr`). Valid range `(0.0, 1.0)`.
|
||||
/// Default `0.20` ≈ fill 0.7248 at k=5 ≈ ~2,114 entries on the 1 KB
|
||||
/// filter (Swamidass–Baldi). Raised from 0.10 so aggregates that are
|
||||
/// Default `0.10` ≈ fill 0.631 at k=5 ≈ ~1,630 entries on the 1 KB
|
||||
/// filter (Swamidass–Baldi). Raised from 0.05 so aggregates that are
|
||||
/// legitimately near their operating ceiling are not rejected before
|
||||
/// the network reaches the fixed-filter capacity limit; conceptually
|
||||
/// distinct from future autoscaling hysteresis setpoints — same unit,
|
||||
@@ -648,8 +648,8 @@ pub struct BloomConfig {
|
||||
impl Default for BloomConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
update_debounce_ms: Self::default_update_debounce_ms(),
|
||||
max_inbound_fpr: Self::default_max_inbound_fpr(),
|
||||
update_debounce_ms: 500,
|
||||
max_inbound_fpr: 0.10,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -659,7 +659,7 @@ impl BloomConfig {
|
||||
500
|
||||
}
|
||||
fn default_max_inbound_fpr() -> f64 {
|
||||
0.20
|
||||
0.10
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"loop_detected": 0,
|
||||
"outbound_sign_failed": 0,
|
||||
"parent_losses": 0,
|
||||
"parent_switched": 0,
|
||||
"parent_switches": 0,
|
||||
"rate_limited": 0,
|
||||
"received": 0,
|
||||
|
||||
@@ -1302,16 +1302,6 @@ impl NostrDiscovery {
|
||||
|
||||
self.mark_session_seen(&offer.session_id).await?;
|
||||
|
||||
// Resolve the answer's relays before binding a socket and running STUN.
|
||||
// Nothing in the relay choice depends on what STUN observes, and an offer
|
||||
// from a peer we share no relay with cannot be answered at all — doing it
|
||||
// in this order spends a STUN round trip, and holds an offer slot for its
|
||||
// duration, only to discard the result.
|
||||
let relays = self.preferred_signal_relays(sender, None).await?;
|
||||
if relays.is_empty() {
|
||||
return Err(BootstrapError::MissingRelays(offer.sender_npub.clone()));
|
||||
}
|
||||
|
||||
let base_socket = std::net::UdpSocket::bind(("0.0.0.0", 0))?;
|
||||
base_socket.set_nonblocking(true)?;
|
||||
let (reflexive_address, local_addresses, stun_server) = observe_traversal_addresses(
|
||||
@@ -1346,6 +1336,7 @@ impl NostrDiscovery {
|
||||
(!accepted).then_some("no-usable-addresses".to_string()),
|
||||
Some(offer_received_at),
|
||||
);
|
||||
let relays = self.preferred_signal_relays(sender, None).await?;
|
||||
let answer_event = self.send_signal(&relays, sender, &answer).await?;
|
||||
debug!(
|
||||
peer = %peer_short,
|
||||
@@ -1475,21 +1466,22 @@ impl NostrDiscovery {
|
||||
target_pubkey: PublicKey,
|
||||
advert: Option<&OverlayAdvert>,
|
||||
) -> Result<Vec<String>, BootstrapError> {
|
||||
let inbox = self.find_recipient_inbox_relays(target_pubkey).await?;
|
||||
let pool: HashSet<RelayUrl> = self.client.pool().all_relays().await.into_keys().collect();
|
||||
let usable = signal_relays(
|
||||
&inbox,
|
||||
advert.and_then(|advert| advert.signal_relays.as_deref()),
|
||||
&self.config.dm_relays,
|
||||
&pool,
|
||||
);
|
||||
debug!(
|
||||
peer = %target_pubkey.to_bech32().map(|npub| short_npub(&npub)).unwrap_or_default(),
|
||||
inbox = inbox.len(),
|
||||
usable = usable.len(),
|
||||
"traversal: signal relays resolved against the client pool"
|
||||
);
|
||||
Ok(usable)
|
||||
let mut merged = self.find_recipient_inbox_relays(target_pubkey).await?;
|
||||
if let Some(advert) = advert
|
||||
&& let Some(relays) = advert.signal_relays.as_ref()
|
||||
{
|
||||
for relay in relays {
|
||||
if !merged.contains(relay) {
|
||||
merged.push(relay.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
for relay in &self.config.dm_relays {
|
||||
if !merged.contains(relay) {
|
||||
merged.push(relay.clone());
|
||||
}
|
||||
}
|
||||
Ok(merged)
|
||||
}
|
||||
|
||||
async fn find_recipient_inbox_relays(
|
||||
@@ -1740,56 +1732,6 @@ impl NostrDiscovery {
|
||||
}
|
||||
}
|
||||
|
||||
/// Retain only the candidates the client pool actually holds.
|
||||
///
|
||||
/// `send_event_to` rejects the whole send with `RelayNotFound` if any single URL
|
||||
/// is outside the pool, so a signal addressed to a peer's advertised relays fails
|
||||
/// entirely on one relay we are not configured with. Filtering first turns that
|
||||
/// into a send to the relays we share.
|
||||
///
|
||||
/// Comparison is on the normalized `RelayUrl` rather than the raw string, because
|
||||
/// the pool is keyed that way: a candidate spelled `wss://relay.example/` matches
|
||||
/// a configured `wss://relay.example`. Order is preserved, candidates that fail
|
||||
/// to parse are dropped, and duplicates that normalize alike are collapsed.
|
||||
fn retain_pooled_relays(candidates: &[String], pool: &HashSet<RelayUrl>) -> Vec<String> {
|
||||
let mut seen: HashSet<RelayUrl> = HashSet::new();
|
||||
let mut usable = Vec::with_capacity(candidates.len());
|
||||
for candidate in candidates {
|
||||
let Ok(url) = RelayUrl::parse(candidate) else {
|
||||
continue;
|
||||
};
|
||||
if pool.contains(&url) && seen.insert(url.clone()) {
|
||||
usable.push(url.to_string());
|
||||
}
|
||||
}
|
||||
usable
|
||||
}
|
||||
|
||||
/// Choose the relays a traversal signal for one peer should be sent to.
|
||||
///
|
||||
/// The candidates are the peer's NIP-17 inbox relays, then the relays its advert
|
||||
/// nominates for signaling, then our own DM relays — remote-supplied first, ours
|
||||
/// last, so a peer's preference is honored where we can act on it. The result is
|
||||
/// whatever survives [`retain_pooled_relays`].
|
||||
///
|
||||
/// This is the whole decision, kept synchronous so it can be exercised without a
|
||||
/// relay client: the caller's only job is to supply the fetched inbox list and
|
||||
/// the pool.
|
||||
pub(super) fn signal_relays(
|
||||
inbox: &[String],
|
||||
advert_signal: Option<&[String]>,
|
||||
dm_relays: &[String],
|
||||
pool: &HashSet<RelayUrl>,
|
||||
) -> Vec<String> {
|
||||
let mut merged: Vec<String> = inbox.to_vec();
|
||||
for relay in advert_signal.unwrap_or_default().iter().chain(dm_relays) {
|
||||
if !merged.contains(relay) {
|
||||
merged.push(relay.clone());
|
||||
}
|
||||
}
|
||||
retain_pooled_relays(&merged, pool)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl NostrDiscovery {
|
||||
/// Build a minimal `NostrDiscovery` for unit tests. No relay client is
|
||||
@@ -1861,18 +1803,6 @@ impl NostrDiscovery {
|
||||
}
|
||||
}
|
||||
|
||||
/// Point the test instance's advert relays at explicit URLs. Unit tests
|
||||
/// that exercise `refetch_advert_for_stale_check` use this to replace the
|
||||
/// default public relay list with a local blackhole, so the refetch runs
|
||||
/// its full 2s timeout without touching the network.
|
||||
pub(crate) async fn set_advert_relays_for_test(&mut self, relays: Vec<String>) {
|
||||
for url in &relays {
|
||||
let _ = self.client.add_relay(url.as_str()).await;
|
||||
}
|
||||
self.client.connect().await;
|
||||
self.config.advert_relays = relays;
|
||||
}
|
||||
|
||||
/// Insert a cached advert directly into the in-memory cache. Used by
|
||||
/// unit tests to set up consumer-side state without needing live relays.
|
||||
pub(crate) async fn insert_advert_for_test(&self, npub: String, advert: CachedOverlayAdvert) {
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
use std::collections::HashSet;
|
||||
use nostr::prelude::{EventBuilder, Kind, Tag, Timestamp};
|
||||
|
||||
use nostr::prelude::{EventBuilder, Kind, RelayUrl, Tag, Timestamp};
|
||||
|
||||
use super::runtime::{NostrDiscovery, signal_relays, suppress_responder_for_own_initiator};
|
||||
use super::runtime::{NostrDiscovery, suppress_responder_for_own_initiator};
|
||||
use super::signal::{
|
||||
FreshnessOutcome, build_signal_event, create_traversal_answer, create_traversal_offer,
|
||||
estimate_clock_skew, validate_offer_freshness, validate_traversal_answer_for_offer,
|
||||
};
|
||||
use super::stun::{parse_stun_binding_success, parse_stun_url};
|
||||
use super::traversal::{
|
||||
PunchStrategy, build_punch_packet, now_ms, parse_punch_packet, plan_punch_targets,
|
||||
PunchStrategy, build_punch_packet, parse_punch_packet, plan_punch_targets,
|
||||
planned_remote_endpoints, session_hash,
|
||||
};
|
||||
use super::{
|
||||
@@ -672,187 +670,3 @@ fn responder_suppression_election() {
|
||||
&smaller, &smaller, true
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn now_ms_tracks_the_wall_clock() {
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn wall_ms() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock is after the Unix epoch")
|
||||
.as_millis() as u64
|
||||
}
|
||||
|
||||
// Bracket a sample between two independent wall-clock reads taken either
|
||||
// side of it. This is the property the traversal clock has to hold for the
|
||||
// NIP-40 expiration tags it computes to be in the future when published.
|
||||
//
|
||||
// Read this for what it is: it pins the contract (Unix epoch, milliseconds,
|
||||
// tracking real time) and it fires on a host that has actually suspended,
|
||||
// where the sample falls below `before` by the suspend duration. It is NOT a
|
||||
// regression guard for the anchored-clock defect. Nothing reachable from a
|
||||
// unit test can simulate a suspend, so on a machine that has not slept, an
|
||||
// anchored implementation passes this -- deterministically when this is the
|
||||
// first caller of `now_ms()` in the binary, and otherwise with a probability
|
||||
// set by the fractional millisecond the anchor happened to capture.
|
||||
let before = wall_ms();
|
||||
let sampled = now_ms();
|
||||
let after = wall_ms();
|
||||
|
||||
assert!(
|
||||
sampled >= before,
|
||||
"now_ms() is behind the wall clock: {sampled} < {before}"
|
||||
);
|
||||
assert!(
|
||||
sampled <= after,
|
||||
"now_ms() is ahead of the wall clock: {sampled} > {after}"
|
||||
);
|
||||
}
|
||||
|
||||
fn pool(urls: &[&str]) -> HashSet<RelayUrl> {
|
||||
urls.iter()
|
||||
.map(|url| RelayUrl::parse(url).expect("test pool url parses"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn candidates(urls: &[&str]) -> Vec<String> {
|
||||
urls.iter().map(|url| url.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_pool_relay_does_not_suppress_the_shared_ones() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&[
|
||||
"wss://relay.damus.io",
|
||||
"wss://temp.iris.to",
|
||||
"wss://nos.lol",
|
||||
]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&[
|
||||
"wss://relay.damus.io",
|
||||
"wss://nos.lol",
|
||||
"wss://offchain.pub",
|
||||
]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable,
|
||||
vec![
|
||||
"wss://relay.damus.io".to_string(),
|
||||
"wss://nos.lol".to_string()
|
||||
],
|
||||
"the unknown relay must be dropped without taking the shared ones with it"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_slash_and_host_case_variants_are_retained() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://Relay.Damus.io/", "wss://nos.lol"]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&["wss://relay.damus.io", "wss://nos.lol"]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable.len(),
|
||||
2,
|
||||
"normalized spellings of a configured relay are the same relay: {usable:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicates_that_normalize_alike_are_collapsed() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://nos.lol", "wss://nos.lol/", "wss://NOS.LOL"]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&["wss://nos.lol"]),
|
||||
);
|
||||
assert_eq!(usable, vec!["wss://nos.lol".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unparseable_candidates_are_dropped_rather_than_failing_the_set() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["not a url", "wss://nos.lol"]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&["wss://nos.lol"]),
|
||||
);
|
||||
assert_eq!(usable, vec!["wss://nos.lol".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_shared_relay_yields_an_empty_set_for_the_caller_to_reject() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://temp.iris.to"]),
|
||||
None,
|
||||
&[],
|
||||
&pool(&["wss://nos.lol"]),
|
||||
);
|
||||
assert!(
|
||||
usable.is_empty(),
|
||||
"with no overlap the caller must see nothing to send to, not a doomed send"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_relays_merges_all_three_sources_then_filters() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://temp.iris.to", "wss://nos.lol"]),
|
||||
Some(&candidates(&[
|
||||
"wss://relay.damus.io",
|
||||
"wss://unknown.example",
|
||||
])),
|
||||
&candidates(&["wss://offchain.pub"]),
|
||||
&pool(&[
|
||||
"wss://nos.lol",
|
||||
"wss://relay.damus.io",
|
||||
"wss://offchain.pub",
|
||||
]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable,
|
||||
vec![
|
||||
"wss://nos.lol".to_string(),
|
||||
"wss://relay.damus.io".to_string(),
|
||||
"wss://offchain.pub".to_string(),
|
||||
],
|
||||
"every source must contribute, and only the out-of-pool entries drop out"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_relays_keeps_our_dm_relays_when_the_peer_shares_nothing() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://temp.iris.to"]),
|
||||
Some(&candidates(&["wss://also.unknown"])),
|
||||
&candidates(&["wss://nos.lol"]),
|
||||
&pool(&["wss://nos.lol"]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable,
|
||||
vec!["wss://nos.lol".to_string()],
|
||||
"our own DM relays are always in the pool, so the result is never empty \
|
||||
while any are configured"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_relays_without_an_advert_still_resolves() {
|
||||
let usable = signal_relays(
|
||||
&candidates(&["wss://nos.lol", "wss://temp.iris.to"]),
|
||||
None,
|
||||
&candidates(&["wss://offchain.pub"]),
|
||||
&pool(&["wss://nos.lol", "wss://offchain.pub"]),
|
||||
);
|
||||
assert_eq!(
|
||||
usable,
|
||||
vec![
|
||||
"wss://nos.lol".to_string(),
|
||||
"wss://offchain.pub".to_string()
|
||||
],
|
||||
"the responder path passes no advert and must still produce a target set"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::net::UdpSocket;
|
||||
@@ -197,35 +197,25 @@ pub(super) fn nonce() -> String {
|
||||
format!("{}-{:016x}", now_ms(), rand::random::<u64>())
|
||||
}
|
||||
|
||||
/// Current Unix time in milliseconds, read from the wall clock on every call.
|
||||
///
|
||||
/// This deliberately does not cache a start-of-process anchor and advance it
|
||||
/// with a monotonic `Instant`. A monotonic clock does not advance while the host
|
||||
/// is suspended, so an anchored value trails real time by the suspend duration
|
||||
/// for the remaining life of the process. Every expiry computed from it is then
|
||||
/// published already in the past, the relay drops the event as expired, and
|
||||
/// traversal signalling fails until the daemon is restarted.
|
||||
///
|
||||
/// About half the consumers publish or serialize the value as an absolute
|
||||
/// timestamp: the NIP-40 expiration tags on adverts and traversal signals, and
|
||||
/// the `issuedAt`/`expiresAt` fields of offers and answers. The rest compare it
|
||||
/// against timestamps on the same basis, including the peer-authored, signed
|
||||
/// `created_at` of a received advert, so they need it to track real time too.
|
||||
///
|
||||
/// The interval-shaped consumers survive a step in the wall clock. A forward
|
||||
/// step, which is what a resume produces, saturates the punch start delay to
|
||||
/// zero so punching begins immediately; the attempt's own bounds are monotonic
|
||||
/// `Instant` deadlines, so its length is unaffected. A backward step lengthens
|
||||
/// that delay instead and can cost a single punch attempt, which retries. Early
|
||||
/// eviction from the replay window cannot admit a replay under the shipped
|
||||
/// defaults, because the freshness window a replayed offer would also have to
|
||||
/// satisfy (`signal_ttl_secs` plus `FRESHNESS_SKEW_TOLERANCE_MS`, 180s) is
|
||||
/// strictly narrower than the replay window itself (`replay_window_secs`, 300s).
|
||||
pub(super) fn now_ms() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
struct ClockAnchor {
|
||||
started_at: Instant,
|
||||
started_unix_ms: u64,
|
||||
}
|
||||
|
||||
static ANCHOR: OnceLock<ClockAnchor> = OnceLock::new();
|
||||
|
||||
let anchor = ANCHOR.get_or_init(|| ClockAnchor {
|
||||
started_at: Instant::now(),
|
||||
started_unix_ms: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_millis() as u64)
|
||||
.unwrap_or(0),
|
||||
});
|
||||
|
||||
anchor
|
||||
.started_unix_ms
|
||||
.saturating_add(anchor.started_at.elapsed().as_millis() as u64)
|
||||
}
|
||||
|
||||
pub(super) fn session_hash(session_id: &str) -> [u8; 16] {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Authentication challenge-response protocol.
|
||||
|
||||
use rand::Rng;
|
||||
use secp256k1::XOnlyPublicKey;
|
||||
use secp256k1::{Secp256k1, XOnlyPublicKey};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{IdentityError, NodeAddr};
|
||||
@@ -34,9 +34,9 @@ impl AuthChallenge {
|
||||
/// Verify a response to this challenge.
|
||||
pub fn verify(&self, response: &AuthResponse) -> Result<NodeAddr, IdentityError> {
|
||||
let digest = auth_challenge_digest(&self.0, response.timestamp);
|
||||
let secp = Secp256k1::new();
|
||||
|
||||
super::SECP
|
||||
.verify_schnorr(&response.signature, &digest, &response.pubkey)
|
||||
secp.verify_schnorr(&response.signature, &digest, &response.pubkey)
|
||||
.map_err(|_| IdentityError::SignatureVerificationFailed)?;
|
||||
|
||||
Ok(NodeAddr::from_pubkey(&response.pubkey))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Local node identity with signing capability.
|
||||
|
||||
use secp256k1::{Keypair, PublicKey, SecretKey, XOnlyPublicKey};
|
||||
use secp256k1::{Keypair, PublicKey, Secp256k1, SecretKey, XOnlyPublicKey};
|
||||
use std::fmt;
|
||||
|
||||
use super::auth::{AuthResponse, auth_challenge_digest};
|
||||
@@ -42,7 +42,8 @@ impl Identity {
|
||||
|
||||
/// Create an identity from a secret key.
|
||||
pub fn from_secret_key(secret_key: SecretKey) -> Self {
|
||||
let keypair = Keypair::from_secret_key(&super::SECP, &secret_key);
|
||||
let secp = Secp256k1::new();
|
||||
let keypair = Keypair::from_secret_key(&secp, &secret_key);
|
||||
Self::from_keypair(keypair)
|
||||
}
|
||||
|
||||
@@ -92,8 +93,9 @@ impl Identity {
|
||||
|
||||
/// Sign arbitrary data with this identity's secret key.
|
||||
pub fn sign(&self, data: &[u8]) -> secp256k1::schnorr::Signature {
|
||||
let secp = Secp256k1::new();
|
||||
let digest = sha256(data);
|
||||
super::SECP.sign_schnorr(&digest, &self.keypair)
|
||||
secp.sign_schnorr(&digest, &self.keypair)
|
||||
}
|
||||
|
||||
/// Create an authentication response for a challenge.
|
||||
@@ -101,7 +103,8 @@ impl Identity {
|
||||
/// The response signs: SHA256("fips-auth-v1" || challenge || timestamp)
|
||||
pub fn sign_challenge(&self, challenge: &[u8; 32], timestamp: u64) -> AuthResponse {
|
||||
let digest = auth_challenge_digest(challenge, timestamp);
|
||||
let signature = super::SECP.sign_schnorr(&digest, &self.keypair);
|
||||
let secp = Secp256k1::new();
|
||||
let signature = secp.sign_schnorr(&digest, &self.keypair);
|
||||
AuthResponse {
|
||||
pubkey: self.pubkey(),
|
||||
timestamp,
|
||||
|
||||
@@ -11,9 +11,6 @@ mod local;
|
||||
mod node_addr;
|
||||
mod peer;
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use secp256k1::{All, Secp256k1};
|
||||
use sha2::{Digest, Sha256};
|
||||
use thiserror::Error;
|
||||
|
||||
@@ -24,15 +21,6 @@ pub use local::Identity;
|
||||
pub use node_addr::NodeAddr;
|
||||
pub use peer::PeerIdentity;
|
||||
|
||||
/// Shared secp256k1 context reused across all identity operations.
|
||||
///
|
||||
/// `Secp256k1::new()` allocates a `Secp256k1<All>` and runs randomization /
|
||||
/// blinding table setup; it is designed to be created once and reused rather
|
||||
/// than rebuilt per sign / verify / key-derive call. This single `All` context
|
||||
/// serves both signing and verification across the identity module and still
|
||||
/// performs the standard construction-time blinding.
|
||||
pub(crate) static SECP: LazyLock<Secp256k1<All>> = LazyLock::new(Secp256k1::new);
|
||||
|
||||
/// FIPS address prefix (IPv6 ULA range).
|
||||
pub const FIPS_ADDRESS_PREFIX: u8 = 0xfd;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Remote peer identity (public key only, no signing capability).
|
||||
|
||||
use secp256k1::{Parity, PublicKey, XOnlyPublicKey};
|
||||
use secp256k1::{Parity, PublicKey, Secp256k1, XOnlyPublicKey};
|
||||
use std::fmt;
|
||||
|
||||
use super::encoding::{decode_npub, encode_npub};
|
||||
@@ -107,9 +107,9 @@ impl PeerIdentity {
|
||||
|
||||
/// Verify a signature from this peer.
|
||||
pub fn verify(&self, data: &[u8], signature: &secp256k1::schnorr::Signature) -> bool {
|
||||
let secp = Secp256k1::new();
|
||||
let digest = sha256(data);
|
||||
super::SECP
|
||||
.verify_schnorr(signature, &digest, &self.pubkey)
|
||||
secp.verify_schnorr(signature, &digest, &self.pubkey)
|
||||
.is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::collections::HashSet;
|
||||
use std::net::Ipv6Addr;
|
||||
|
||||
use secp256k1::{Keypair, SecretKey};
|
||||
use secp256k1::{Keypair, Secp256k1, SecretKey};
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -161,10 +161,10 @@ fn test_identity_sign() {
|
||||
let sig = identity.sign(data);
|
||||
|
||||
// Verify the signature manually
|
||||
let secp = secp256k1::Secp256k1::new();
|
||||
let digest = super::sha256(data);
|
||||
assert!(
|
||||
super::SECP
|
||||
.verify_schnorr(&sig, &digest, &identity.pubkey())
|
||||
secp.verify_schnorr(&sig, &digest, &identity.pubkey())
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
@@ -580,12 +580,13 @@ fn test_peer_identity_pubkey_full_even_parity_fallback() {
|
||||
#[test]
|
||||
fn test_peer_identity_pubkey_full_preserved_parity() {
|
||||
// Create two identities and find one with odd parity to make this test meaningful
|
||||
let secp = Secp256k1::new();
|
||||
let secret_bytes: [u8; 32] = [
|
||||
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
|
||||
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e,
|
||||
0x1f, 0x20,
|
||||
];
|
||||
let keypair = Keypair::from_seckey_slice(&super::SECP, &secret_bytes).unwrap();
|
||||
let keypair = Keypair::from_seckey_slice(&secp, &secret_bytes).unwrap();
|
||||
let full_pubkey = keypair.public_key();
|
||||
|
||||
let peer = PeerIdentity::from_pubkey_full(full_pubkey);
|
||||
|
||||
+16
-24
@@ -29,19 +29,27 @@ impl Node {
|
||||
filters
|
||||
}
|
||||
|
||||
/// Send a FilterAnnounce to a specific peer, respecting debounce.
|
||||
/// Build a FilterAnnounce for a specific peer.
|
||||
///
|
||||
/// `filter` is the outgoing filter for this peer, already computed
|
||||
/// with the destination peer's own contribution excluded to prevent
|
||||
/// routing loops (don't tell a peer about destinations reachable
|
||||
/// only through them).
|
||||
/// The outgoing filter excludes the destination peer's own filter
|
||||
/// to prevent routing loops (don't tell a peer about destinations
|
||||
/// reachable only through them).
|
||||
fn build_filter_announce(&mut self, exclude_peer: &NodeAddr) -> FilterAnnounce {
|
||||
let peer_filters = self.peer_inbound_filters();
|
||||
let filter = self
|
||||
.bloom_state
|
||||
.compute_outgoing_filter(exclude_peer, &peer_filters);
|
||||
let sequence = self.bloom_state.next_sequence();
|
||||
FilterAnnounce::new(filter, sequence)
|
||||
}
|
||||
|
||||
/// Send a FilterAnnounce to a specific peer, respecting debounce.
|
||||
///
|
||||
/// If the peer is rate-limited, the update stays pending for
|
||||
/// delivery on the next tick cycle.
|
||||
pub(super) async fn send_filter_announce_to_peer(
|
||||
&mut self,
|
||||
peer_addr: &NodeAddr,
|
||||
filter: BloomFilter,
|
||||
) -> Result<(), NodeError> {
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
@@ -56,7 +64,7 @@ impl Node {
|
||||
}
|
||||
|
||||
// Build and encode
|
||||
let announce = FilterAnnounce::new(filter, self.bloom_state.next_sequence());
|
||||
let announce = self.build_filter_announce(peer_addr);
|
||||
let sent_filter = announce.filter.clone();
|
||||
let encoded = announce.encode().map_err(|e| NodeError::SendFailed {
|
||||
node_addr: *peer_addr,
|
||||
@@ -134,24 +142,8 @@ impl Node {
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
if ready.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// One snapshot and one union pass for the whole ready set. The
|
||||
// send path never mutates peer inbound filters or the tree state,
|
||||
// and the rx loop holds `&mut self` across the awaits, so the
|
||||
// snapshot cannot go stale mid-loop.
|
||||
let peer_filters = self.peer_inbound_filters();
|
||||
let mut outgoing = self
|
||||
.bloom_state
|
||||
.compute_outgoing_filters(&ready, &peer_filters);
|
||||
|
||||
for peer_addr in ready {
|
||||
let Some(filter) = outgoing.remove(&peer_addr) else {
|
||||
continue;
|
||||
};
|
||||
if let Err(e) = self.send_filter_announce_to_peer(&peer_addr, filter).await {
|
||||
if let Err(e) = self.send_filter_announce_to_peer(&peer_addr).await {
|
||||
debug!(
|
||||
peer = %self.peer_display_name(&peer_addr),
|
||||
error = %e,
|
||||
|
||||
@@ -240,32 +240,17 @@ impl Node {
|
||||
// map used by the TUN reader/writer at TCP MSS clamp time.
|
||||
let fips_addr = crate::FipsAddress::from_node_addr(&target);
|
||||
match self.path_mtu_lookup.write() {
|
||||
Ok(mut map) => match map.get(&fips_addr).copied() {
|
||||
Some(existing) if existing <= path_mtu => {
|
||||
// Keep the tighter learned value; never loosen the
|
||||
// clamp. A reactive MtuExceeded or PathMtuNotification
|
||||
// tighten takes precedence over a looser discovery
|
||||
// estimate (cross-carrier keep-tighter).
|
||||
debug!(
|
||||
target = %self.peer_display_name(&target),
|
||||
fips_addr = %fips_addr,
|
||||
path_mtu = path_mtu,
|
||||
existing = existing,
|
||||
"LookupResponse: keeping tighter existing path_mtu_lookup value"
|
||||
);
|
||||
}
|
||||
other => {
|
||||
map.insert(fips_addr, path_mtu);
|
||||
debug!(
|
||||
target = %self.peer_display_name(&target),
|
||||
fips_addr = %fips_addr,
|
||||
path_mtu = path_mtu,
|
||||
prior = ?other,
|
||||
map_len = map.len(),
|
||||
"Wrote path_mtu_lookup from discovery LookupResponse"
|
||||
);
|
||||
}
|
||||
},
|
||||
Ok(mut map) => {
|
||||
let prior = map.insert(fips_addr, path_mtu);
|
||||
debug!(
|
||||
target = %self.peer_display_name(&target),
|
||||
fips_addr = %fips_addr,
|
||||
path_mtu = path_mtu,
|
||||
prior = ?prior,
|
||||
map_len = map.len(),
|
||||
"Wrote path_mtu_lookup from discovery LookupResponse"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target = %self.peer_display_name(&target),
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
//! SessionDatagram forwarding handler.
|
||||
//!
|
||||
//! Handles incoming SessionDatagram (0x00) link messages: decodes the
|
||||
//! envelope, performs coordinate cache warming from plaintext session-layer
|
||||
//! headers, delivers locally when the datagram is addressed to this node,
|
||||
//! otherwise enforces the transit hop limit and routes to the next hop, and
|
||||
//! generates error signals on routing failure.
|
||||
//! envelope, enforces hop limits, performs coordinate cache warming from
|
||||
//! plaintext session-layer headers, routes to the next hop or delivers
|
||||
//! locally, and generates error signals on routing failure.
|
||||
|
||||
use crate::NodeAddr;
|
||||
use crate::node::reject::ForwardingReject;
|
||||
@@ -44,16 +43,26 @@ impl Node {
|
||||
}
|
||||
};
|
||||
|
||||
// Coordinate cache warming from plaintext session-layer headers.
|
||||
// Runs ahead of both the delivery and the TTL decisions: the coords
|
||||
// a peer put on the wire are equally valid whichever way those go.
|
||||
// TTL enforcement: decrement for forwarding and drop only if the
|
||||
// received datagram was already exhausted.
|
||||
if datagram_ref.ttl == 0 {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
|
||||
debug!(
|
||||
src = %datagram_ref.src_addr,
|
||||
dest = %datagram_ref.dest_addr,
|
||||
"SessionDatagram TTL exhausted, dropping"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let forwarded_ttl = datagram_ref.ttl - 1;
|
||||
|
||||
// Coordinate cache warming from plaintext session-layer headers
|
||||
self.try_warm_coord_cache_ref(&datagram_ref);
|
||||
|
||||
// Local delivery: dispatch to session layer handlers without
|
||||
// materializing an owned SessionDatagram payload Vec. Delivery to
|
||||
// the addressed node is *not* TTL-gated — under IP semantics the
|
||||
// TTL governs forwarding, not delivery to the addressed host — so
|
||||
// this test precedes the TTL gate below.
|
||||
// materializing an owned SessionDatagram payload Vec.
|
||||
if datagram_ref.dest_addr == *self.node_addr() {
|
||||
self.metrics().forwarding.record_delivered(payload.len());
|
||||
self.handle_session_payload(
|
||||
@@ -66,24 +75,6 @@ impl Node {
|
||||
return;
|
||||
}
|
||||
|
||||
// TTL enforcement on the transit path: decrement first, then drop if
|
||||
// the datagram would leave with a TTL of zero. `saturating_sub` folds
|
||||
// the already-exhausted arrival (ttl=0) into the same test as the
|
||||
// last-hop arrival (ttl=1); neither is transmitted.
|
||||
let forwarded_ttl = datagram_ref.ttl.saturating_sub(1);
|
||||
if forwarded_ttl == 0 {
|
||||
self.metrics()
|
||||
.forwarding
|
||||
.record_reject_bytes(ForwardingReject::TtlExhausted, payload.len());
|
||||
debug!(
|
||||
src = %datagram_ref.src_addr,
|
||||
dest = %datagram_ref.dest_addr,
|
||||
ttl = datagram_ref.ttl,
|
||||
"SessionDatagram TTL exhausted, dropping"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let mut datagram = datagram_ref.into_owned();
|
||||
datagram.ttl = forwarded_ttl;
|
||||
|
||||
@@ -417,10 +408,13 @@ impl Node {
|
||||
for (&tid, transport) in &self.transports {
|
||||
let congestion = transport.congestion();
|
||||
let state = self.transport_drops.entry(tid).or_default();
|
||||
if let Some(current) = congestion.recv_drops
|
||||
&& state.observe_drops(current)
|
||||
{
|
||||
new_drop_events.push(tid);
|
||||
if let Some(current) = congestion.recv_drops {
|
||||
let new_drops = current > state.prev_drops;
|
||||
if new_drops && !state.dropping {
|
||||
new_drop_events.push(tid);
|
||||
}
|
||||
state.dropping = new_drops;
|
||||
state.prev_drops = current;
|
||||
}
|
||||
}
|
||||
for tid in new_drop_events {
|
||||
|
||||
@@ -173,6 +173,7 @@ impl Node {
|
||||
self.coord_cache
|
||||
.invalidate_via_node(our_identity.node_addr());
|
||||
self.reset_discovery_backoff();
|
||||
self.metrics().tree.parent_switched.inc();
|
||||
self.metrics().tree.parent_switches.inc();
|
||||
info!(
|
||||
new_parent = %self.peer_display_name(&new_parent),
|
||||
@@ -204,6 +205,7 @@ impl Node {
|
||||
self.coord_cache
|
||||
.invalidate_other_roots(our_identity.node_addr());
|
||||
self.reset_discovery_backoff();
|
||||
self.metrics().tree.parent_switched.inc();
|
||||
self.metrics().tree.parent_switches.inc();
|
||||
info!(
|
||||
new_root = %self.tree_state.root(),
|
||||
|
||||
@@ -317,6 +317,7 @@ pub struct TreeMetrics {
|
||||
pub stale: Counter,
|
||||
pub ancestry_invalid: Counter,
|
||||
pub accepted: Counter,
|
||||
pub parent_switched: Counter,
|
||||
pub loop_detected: Counter,
|
||||
pub ancestry_changed: Counter,
|
||||
pub sent: Counter,
|
||||
@@ -350,6 +351,7 @@ impl TreeMetrics {
|
||||
stale: self.stale.get(),
|
||||
ancestry_invalid: self.ancestry_invalid.get(),
|
||||
accepted: self.accepted.get(),
|
||||
parent_switched: self.parent_switched.get(),
|
||||
loop_detected: self.loop_detected.get(),
|
||||
ancestry_changed: self.ancestry_changed.get(),
|
||||
sent: self.sent.get(),
|
||||
|
||||
+3
-44
@@ -266,32 +266,6 @@ struct TransportDropState {
|
||||
dropping: bool,
|
||||
}
|
||||
|
||||
impl TransportDropState {
|
||||
/// Fold a new cumulative `recv_drops` sample into the state and report
|
||||
/// whether it marks the *transition* into a dropping condition.
|
||||
///
|
||||
/// Returns true only on the edge where the cumulative `SO_RXQ_OVFL`
|
||||
/// counter rose since the previous sample **and** the transport was not
|
||||
/// already flagged as dropping. That edge is what `kernel_drop_events`
|
||||
/// counts: a first observation of a new drop burst, not every sample in
|
||||
/// which the counter happens to be non-zero. A sample with no rise
|
||||
/// clears the flag, so a later rise counts as a fresh event.
|
||||
///
|
||||
/// Pure and sans-IO by design: the tick handler reads the kernel
|
||||
/// counter from the socket and does the logging, but the detection
|
||||
/// decision lives here so it can be tested without a socket, a
|
||||
/// transport, or a running node — which is the only way it can be
|
||||
/// tested at all, since the kernel drop itself cannot be provoked
|
||||
/// deterministically.
|
||||
fn observe_drops(&mut self, current: u64) -> bool {
|
||||
let rose = current > self.prev_drops;
|
||||
let new_event = rose && !self.dropping;
|
||||
self.dropping = rose;
|
||||
self.prev_drops = current;
|
||||
new_event
|
||||
}
|
||||
}
|
||||
|
||||
/// State for a link waiting for transport-level connection establishment.
|
||||
///
|
||||
/// For connection-oriented transports (TCP, Tor), the transport connect runs
|
||||
@@ -318,23 +292,8 @@ struct PendingConnect {
|
||||
/// 1. **Connection phase** (`connections`): Handshake in progress, indexed by LinkId
|
||||
/// 2. **Active phase** (`peers`): Authenticated, indexed by NodeAddr
|
||||
///
|
||||
/// The `addr_to_link` map is a reverse lookup from `(transport, address)` to
|
||||
/// link. It is **not** a packet-dispatch path, despite what this comment used
|
||||
/// to say: `find_link_by_addr` has no callers outside its own tests, and
|
||||
/// encrypted frames are dispatched by session index. Its live readers are the
|
||||
/// `should_admit_msg1` fast path and the duplicate-inbound-handshake check in
|
||||
/// `handle_msg1`.
|
||||
///
|
||||
/// **Do not key a peer-identity question on it.** The address form is not
|
||||
/// canonical: an outbound dial registers the literal configured string, which
|
||||
/// may be a hostname (`"node-b:2121"`), while an inbound packet carries the
|
||||
/// resolved form (`"10.128.2.4:2121"`). `TransportAddr` compares byte-wise, so
|
||||
/// those never match, and a lookup keyed on an inbound address silently returns
|
||||
/// "no such link" for every hostname-configured peer rather than failing. The
|
||||
/// entry is also single-valued per key, so an inbound handshake overwrites an
|
||||
/// outbound dial's entry for the same address. The readers above tolerate this
|
||||
/// because each compares a key written in the same form it reads; a new reader
|
||||
/// that does not will be quietly wrong.
|
||||
/// The `addr_to_link` map enables dispatching incoming packets to the right
|
||||
/// connection before authentication completes.
|
||||
// Discovery lookup constants moved to config: node.discovery.attempt_timeouts_secs, node.discovery.ttl
|
||||
pub struct Node {
|
||||
// === Immutable Context ===
|
||||
@@ -1223,7 +1182,7 @@ impl Node {
|
||||
return name.clone();
|
||||
}
|
||||
if let Some(peer) = self.peers.get(addr) {
|
||||
return peer.short_npub().to_string();
|
||||
return peer.identity().short_npub();
|
||||
}
|
||||
if let Some(entry) = self.sessions.get(addr) {
|
||||
let (xonly, _) = entry.remote_pubkey().x_only_public_key();
|
||||
|
||||
+1
-5
@@ -232,11 +232,7 @@ pub enum ForwardingReject {
|
||||
/// `SessionDatagramRef::decode` returned an error. Tracked via
|
||||
/// [`ForwardingStats::decode_error_packets`](crate::node::stats::ForwardingStats).
|
||||
DecodeError,
|
||||
/// Transit datagram whose TTL would reach zero on this hop, so it is
|
||||
/// dropped rather than forwarded. Charged for an arrival at TTL 1 as
|
||||
/// well as an already-exhausted arrival at TTL 0. Never charged for a
|
||||
/// datagram addressed to this node, whose delivery is not TTL-gated and
|
||||
/// is decided ahead of this test.
|
||||
/// Datagram arrived with TTL=0 — already exhausted, no forward.
|
||||
/// Tracked via
|
||||
/// [`ForwardingStats::ttl_exhausted_packets`](crate::node::stats::ForwardingStats).
|
||||
TtlExhausted,
|
||||
|
||||
+7
-15
@@ -279,7 +279,7 @@ impl Node {
|
||||
|
||||
let peer_config = state.peer_config.clone();
|
||||
|
||||
// Kick off a refresh of the peer's overlay advert. The cache is
|
||||
// Refresh the peer's overlay advert before retrying. The cache is
|
||||
// read-only on hit (see fetch_advert), so every retry without a
|
||||
// refetch dials the same cached endpoint — and the most common
|
||||
// reason a peer ended up in retry_pending is that the cached
|
||||
@@ -289,21 +289,13 @@ impl Node {
|
||||
//
|
||||
// refetch_advert_for_stale_check uses the relay's advert as
|
||||
// ground truth: replaces the cache if there's a newer one,
|
||||
// evicts if the relay has nothing, otherwise leaves it.
|
||||
//
|
||||
// Fire-and-forget, NOT awaited: this runs inline on the 1s
|
||||
// rx-loop tick, and the fetch carries a 2s relay timeout that
|
||||
// would stall the tick — and every other rx-loop arm with it —
|
||||
// by up to 2s per due peer, MAX_RETRY_CONNECTIONS_PER_TICK times
|
||||
// over. So the dial below uses whatever advert is cached now and
|
||||
// the refreshed one lands for the *next* retry of this peer.
|
||||
// Retries are backoff-paced, so that defers the benefit by one
|
||||
// backoff interval rather than losing it.
|
||||
// evicts if the relay has nothing, otherwise leaves it. Cheap
|
||||
// (one Filter fetch with 2s timeout) and bounded by the retry
|
||||
// backoff cadence.
|
||||
if let Some(bootstrap) = self.nostr_discovery.clone() {
|
||||
let npub = peer_config.npub.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = bootstrap.refetch_advert_for_stale_check(&npub).await;
|
||||
});
|
||||
let _ = bootstrap
|
||||
.refetch_advert_for_stale_check(&peer_config.npub)
|
||||
.await;
|
||||
}
|
||||
|
||||
match self.initiate_peer_connection(&peer_config).await {
|
||||
|
||||
@@ -273,6 +273,7 @@ pub struct TreeStatsSnapshot {
|
||||
pub stale: u64,
|
||||
pub ancestry_invalid: u64,
|
||||
pub accepted: u64,
|
||||
pub parent_switched: u64,
|
||||
pub loop_detected: u64,
|
||||
pub ancestry_changed: u64,
|
||||
pub sent: u64,
|
||||
|
||||
@@ -912,45 +912,6 @@ async fn test_originator_stores_path_mtu_in_cache() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_originator_lookup_response_keeps_tighter_path_mtu_lookup() {
|
||||
// Regression: a LookupResponse carrying a looser (larger) path_mtu must
|
||||
// NOT clobber a tighter (smaller) value already in path_mtu_lookup that a
|
||||
// reactive MtuExceeded or PathMtuNotification learned. Cross-carrier
|
||||
// keep-tighter: the clamp must never loosen.
|
||||
let mut node = make_node();
|
||||
let from = make_node_addr(0xAA);
|
||||
|
||||
let target_identity = Identity::generate();
|
||||
let target = *target_identity.node_addr();
|
||||
let root = make_node_addr(0xF0);
|
||||
let coords = TreeCoordinate::from_addrs(vec![target, root]).unwrap();
|
||||
|
||||
node.register_identity(target, target_identity.pubkey_full());
|
||||
|
||||
// Pre-seed a tighter value, as if a reactive signal already narrowed it.
|
||||
let target_fips = crate::FipsAddress::from_node_addr(&target);
|
||||
node.path_mtu_lookup_insert(target_fips, 1280);
|
||||
|
||||
let proof_data = LookupResponse::proof_bytes(800, &target, &coords);
|
||||
let proof = target_identity.sign(&proof_data);
|
||||
|
||||
let mut response = LookupResponse::new(800, target, coords.clone(), proof);
|
||||
// Looser discovery estimate that must be rejected in favor of the tighter
|
||||
// existing entry.
|
||||
response.path_mtu = 1500;
|
||||
|
||||
let payload = &response.encode()[1..];
|
||||
|
||||
node.handle_lookup_response(&from, payload).await;
|
||||
|
||||
assert_eq!(
|
||||
node.path_mtu_lookup_get(&target_fips),
|
||||
Some(1280),
|
||||
"LookupResponse must not loosen a tighter existing path_mtu_lookup value"
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Open-Discovery Sweep — cache-injection unit test
|
||||
// ============================================================================
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
//! Ethernet transport integration tests.
|
||||
//!
|
||||
//! Tests that the Ethernet transport works end-to-end using veth pairs.
|
||||
//! All tests require root or CAP_NET_RAW and are marked `#[ignore]`.
|
||||
|
||||
use super::*;
|
||||
use crate::config::EthernetConfig;
|
||||
use crate::transport::ethernet::EthernetTransport;
|
||||
use crate::transport::{TransportAddr, TransportHandle, TransportId, packet_channel};
|
||||
use spanning_tree::{TestNode, cleanup_nodes, drain_all_packets, initiate_handshake};
|
||||
|
||||
use std::process::Command;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
|
||||
/// Atomic counter for unique veth names across tests.
|
||||
static VETH_COUNTER: AtomicU32 = AtomicU32::new(0);
|
||||
|
||||
/// RAII wrapper for a veth pair.
|
||||
///
|
||||
/// Creates a pair of connected virtual Ethernet interfaces. Destroying
|
||||
/// one end automatically destroys the other.
|
||||
struct VethPair {
|
||||
name_a: String,
|
||||
name_b: String,
|
||||
}
|
||||
|
||||
impl VethPair {
|
||||
/// Create a new veth pair with unique interface names.
|
||||
///
|
||||
/// Names are kept under 15 chars (IFNAMSIZ limit). Format: `ftXXa`/`ftXXb`
|
||||
/// where XX is an atomic counter combined with PID for cross-process uniqueness.
|
||||
fn create() -> Self {
|
||||
let id = VETH_COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
let pid = std::process::id() % 10000;
|
||||
let name_a = format!("ft{}{}a", pid, id);
|
||||
let name_b = format!("ft{}{}b", pid, id);
|
||||
|
||||
assert!(name_a.len() <= 15, "veth name too long: {}", name_a);
|
||||
assert!(name_b.len() <= 15, "veth name too long: {}", name_b);
|
||||
|
||||
// Create veth pair
|
||||
let status = Command::new("ip")
|
||||
.args([
|
||||
"link", "add", &name_a, "type", "veth", "peer", "name", &name_b,
|
||||
])
|
||||
.status()
|
||||
.expect("failed to run 'ip link add'");
|
||||
assert!(status.success(), "failed to create veth pair");
|
||||
|
||||
// Bring both ends up
|
||||
let status = Command::new("ip")
|
||||
.args(["link", "set", &name_a, "up"])
|
||||
.status()
|
||||
.expect("failed to run 'ip link set up'");
|
||||
assert!(status.success(), "failed to bring up {}", name_a);
|
||||
|
||||
let status = Command::new("ip")
|
||||
.args(["link", "set", &name_b, "up"])
|
||||
.status()
|
||||
.expect("failed to run 'ip link set up'");
|
||||
assert!(status.success(), "failed to bring up {}", name_b);
|
||||
|
||||
VethPair { name_a, name_b }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for VethPair {
|
||||
fn drop(&mut self) {
|
||||
// Deleting one end destroys both
|
||||
let _ = Command::new("ip")
|
||||
.args(["link", "delete", &self.name_a])
|
||||
.status();
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a test node with a live Ethernet transport on the given interface.
|
||||
///
|
||||
/// Parallel to `make_test_node()` in spanning_tree.rs but uses
|
||||
/// EthernetTransport instead of UDP.
|
||||
async fn make_test_node_ethernet(interface: &str) -> TestNode {
|
||||
let mut node = make_node();
|
||||
let transport_id = TransportId::new(1);
|
||||
|
||||
let config = EthernetConfig {
|
||||
interface: interface.to_string(),
|
||||
discovery: Some(false),
|
||||
announce: Some(false),
|
||||
accept_connections: Some(true),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (packet_tx, packet_rx) = packet_channel(256);
|
||||
let mut transport = EthernetTransport::new(transport_id, None, config, packet_tx);
|
||||
transport.start_async().await.unwrap();
|
||||
|
||||
let mac = transport
|
||||
.local_mac()
|
||||
.expect("transport should have MAC after start");
|
||||
let addr = TransportAddr::from_bytes(&mac);
|
||||
|
||||
node.transports
|
||||
.insert(transport_id, TransportHandle::Ethernet(transport));
|
||||
|
||||
TestNode {
|
||||
node,
|
||||
transport_id,
|
||||
packet_rx: spanning_tree::bridge_to_unbounded(packet_rx),
|
||||
addr,
|
||||
}
|
||||
}
|
||||
|
||||
/// Two nodes on a veth pair complete a Noise handshake and establish peering.
|
||||
#[tokio::test]
|
||||
#[ignore] // Requires root or CAP_NET_RAW
|
||||
async fn test_ethernet_two_node_handshake() {
|
||||
let veth = VethPair::create();
|
||||
|
||||
let mut nodes = vec![
|
||||
make_test_node_ethernet(&veth.name_a).await,
|
||||
make_test_node_ethernet(&veth.name_b).await,
|
||||
];
|
||||
|
||||
// Initiate handshake from node 0 to node 1
|
||||
initiate_handshake(&mut nodes, 0, 1).await;
|
||||
|
||||
// Drain all packets (handshake + tree announce)
|
||||
let total = drain_all_packets(&mut nodes, false).await;
|
||||
assert!(total > 0, "should have processed packets");
|
||||
|
||||
// Verify bidirectional peering
|
||||
let addr_0 = *nodes[0].node.node_addr();
|
||||
let addr_1 = *nodes[1].node.node_addr();
|
||||
assert!(
|
||||
nodes[0].node.get_peer(&addr_1).is_some(),
|
||||
"node 0 should have node 1 as peer"
|
||||
);
|
||||
assert!(
|
||||
nodes[1].node.get_peer(&addr_0).is_some(),
|
||||
"node 1 should have node 0 as peer"
|
||||
);
|
||||
|
||||
cleanup_nodes(&mut nodes).await;
|
||||
}
|
||||
|
||||
/// Two Ethernet nodes converge to a correct spanning tree (2-node tree).
|
||||
#[tokio::test]
|
||||
#[ignore] // Requires root or CAP_NET_RAW
|
||||
async fn test_ethernet_data_exchange() {
|
||||
use spanning_tree::verify_tree_convergence;
|
||||
|
||||
let veth = VethPair::create();
|
||||
|
||||
let mut nodes = vec![
|
||||
make_test_node_ethernet(&veth.name_a).await,
|
||||
make_test_node_ethernet(&veth.name_b).await,
|
||||
];
|
||||
|
||||
initiate_handshake(&mut nodes, 0, 1).await;
|
||||
let total = drain_all_packets(&mut nodes, false).await;
|
||||
assert!(total > 0);
|
||||
|
||||
// Verify spanning tree convergence
|
||||
verify_tree_convergence(&nodes);
|
||||
|
||||
// The root should be the node with the smallest NodeAddr
|
||||
let expected_root = std::cmp::min(*nodes[0].node.node_addr(), *nodes[1].node.node_addr());
|
||||
assert_eq!(*nodes[0].node.tree_state().root(), expected_root);
|
||||
assert_eq!(*nodes[1].node.tree_state().root(), expected_root);
|
||||
|
||||
cleanup_nodes(&mut nodes).await;
|
||||
}
|
||||
|
||||
/// Mixed transport: 2 Ethernet nodes + 2 UDP nodes coexist.
|
||||
///
|
||||
/// Each transport forms its own connected component. Validates that
|
||||
/// `process_available_packets()` handles heterogeneous transport types.
|
||||
#[tokio::test]
|
||||
#[ignore] // Requires root or CAP_NET_RAW
|
||||
async fn test_mixed_transport_coexistence() {
|
||||
use spanning_tree::{make_test_node, verify_tree_convergence_components};
|
||||
|
||||
let veth = VethPair::create();
|
||||
|
||||
// Create 2 Ethernet nodes and 2 UDP nodes
|
||||
let eth_0 = make_test_node_ethernet(&veth.name_a).await;
|
||||
let eth_1 = make_test_node_ethernet(&veth.name_b).await;
|
||||
let udp_0 = make_test_node().await;
|
||||
let udp_1 = make_test_node().await;
|
||||
|
||||
let mut nodes = vec![eth_0, eth_1, udp_0, udp_1];
|
||||
|
||||
// Handshake within each component
|
||||
initiate_handshake(&mut nodes, 0, 1).await; // Ethernet pair
|
||||
initiate_handshake(&mut nodes, 2, 3).await; // UDP pair
|
||||
|
||||
// Drain all packets across both transports
|
||||
let total = drain_all_packets(&mut nodes, false).await;
|
||||
assert!(total > 0);
|
||||
|
||||
// Verify each component converges independently
|
||||
verify_tree_convergence_components(&nodes, &[vec![0, 1], vec![2, 3]]);
|
||||
|
||||
// Ethernet component has its own root
|
||||
let eth_root = std::cmp::min(*nodes[0].node.node_addr(), *nodes[1].node.node_addr());
|
||||
assert_eq!(*nodes[0].node.tree_state().root(), eth_root);
|
||||
assert_eq!(*nodes[1].node.tree_state().root(), eth_root);
|
||||
|
||||
// UDP component has its own root
|
||||
let udp_root = std::cmp::min(*nodes[2].node.node_addr(), *nodes[3].node.node_addr());
|
||||
assert_eq!(*nodes[2].node.tree_state().root(), udp_root);
|
||||
assert_eq!(*nodes[3].node.tree_state().root(), udp_root);
|
||||
|
||||
cleanup_nodes(&mut nodes).await;
|
||||
}
|
||||
@@ -40,116 +40,22 @@ async fn test_forwarding_hop_limit_exhausted() {
|
||||
node.handle_session_datagram(&from, &encoded[1..], false)
|
||||
.await;
|
||||
// No panic, no send (node has no peers)
|
||||
let fwd = &node.metrics().forwarding;
|
||||
assert_eq!(
|
||||
fwd.ttl_exhausted_packets.get(),
|
||||
1,
|
||||
"transit ttl=0 should be charged to TtlExhausted"
|
||||
);
|
||||
assert_eq!(
|
||||
fwd.drop_no_route_packets.get(),
|
||||
0,
|
||||
"transit ttl=0 should never reach the routing step"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_forwarding_ttl_one_local_delivery_is_not_gated() {
|
||||
// dest == self, so this is local delivery, not transit: the TTL gate
|
||||
// does not apply and the datagram is handed to the session layer.
|
||||
async fn test_forwarding_hop_limit_one_drops_at_transit() {
|
||||
// ttl=1 means after decrement it becomes 0 — the datagram can
|
||||
// still be delivered this hop but would be dropped at the next.
|
||||
// decrement_ttl returns true (1 > 0), so the handler proceeds.
|
||||
let mut node = make_node();
|
||||
let from = make_node_addr(0xAA);
|
||||
let my_addr = *node.node_addr();
|
||||
let src = make_node_addr(0x01);
|
||||
let dg = SessionDatagram::new(src, my_addr, vec![0x10, 0x00, 0x00, 0x00]).with_ttl(1);
|
||||
let encoded = dg.encode();
|
||||
// Should succeed — ttl=1 decrements to 0 but packet is still processed
|
||||
node.handle_session_datagram(&from, &encoded[1..], false)
|
||||
.await;
|
||||
let fwd = &node.metrics().forwarding;
|
||||
assert_eq!(fwd.delivered_packets.get(), 1, "ttl=1 should be delivered");
|
||||
assert_eq!(fwd.ttl_exhausted_packets.get(), 0);
|
||||
}
|
||||
|
||||
/// Acceptance: a datagram addressed to this node with ttl=0 is delivered
|
||||
/// locally. The TTL governs forwarding, not delivery to the addressed host,
|
||||
/// so the gate must sit after the local-delivery test — and the
|
||||
/// `TtlExhausted` reject must not be charged for a delivered datagram.
|
||||
#[tokio::test]
|
||||
async fn test_forwarding_ttl_zero_local_delivery_is_not_gated() {
|
||||
let mut node = make_node();
|
||||
let from = make_node_addr(0xAA);
|
||||
let my_addr = *node.node_addr();
|
||||
let src = make_node_addr(0x01);
|
||||
let dg = SessionDatagram::new(src, my_addr, vec![0x10, 0x00, 0x00, 0x00]).with_ttl(0);
|
||||
let encoded = dg.encode();
|
||||
node.handle_session_datagram(&from, &encoded[1..], false)
|
||||
.await;
|
||||
let fwd = &node.metrics().forwarding;
|
||||
assert_eq!(
|
||||
fwd.delivered_packets.get(),
|
||||
1,
|
||||
"ttl=0 addressed to this node must still be delivered locally"
|
||||
);
|
||||
assert_eq!(
|
||||
fwd.ttl_exhausted_packets.get(),
|
||||
0,
|
||||
"local delivery must not be charged to the TtlExhausted reject"
|
||||
);
|
||||
assert_eq!(fwd.drop_no_route_packets.get(), 0);
|
||||
}
|
||||
|
||||
/// Acceptance: a transit datagram arriving with ttl=1 would leave with ttl=0,
|
||||
/// so it is dropped here rather than transmitted. Reaching the routing step at
|
||||
/// all (`drop_no_route`) would mean it had been handed to the forwarder.
|
||||
#[tokio::test]
|
||||
async fn test_forwarding_ttl_one_transit_dropped_before_routing() {
|
||||
let mut node = make_node();
|
||||
let from = make_node_addr(0xAA);
|
||||
let src = make_node_addr(0x01);
|
||||
let dest = make_node_addr(0x02);
|
||||
let dg = SessionDatagram::new(src, dest, vec![0x10, 0x00, 0x00, 0x00]).with_ttl(1);
|
||||
let encoded = dg.encode();
|
||||
node.handle_session_datagram(&from, &encoded[1..], false)
|
||||
.await;
|
||||
let fwd = &node.metrics().forwarding;
|
||||
assert_eq!(
|
||||
fwd.ttl_exhausted_packets.get(),
|
||||
1,
|
||||
"transit ttl=1 must be dropped as TTL-exhausted, not forwarded"
|
||||
);
|
||||
assert_eq!(
|
||||
fwd.drop_no_route_packets.get(),
|
||||
0,
|
||||
"transit ttl=1 must not reach the routing step"
|
||||
);
|
||||
assert_eq!(fwd.forwarded_packets.get(), 0);
|
||||
assert_eq!(fwd.delivered_packets.get(), 0);
|
||||
}
|
||||
|
||||
/// The other side of the same boundary: ttl=2 clears the gate. This node has
|
||||
/// no peers, so it fails at the routing step instead — which is the evidence
|
||||
/// that the TTL gate passed it through.
|
||||
#[tokio::test]
|
||||
async fn test_forwarding_ttl_two_transit_clears_the_gate() {
|
||||
let mut node = make_node();
|
||||
let from = make_node_addr(0xAA);
|
||||
let src = make_node_addr(0x01);
|
||||
let dest = make_node_addr(0x02);
|
||||
let dg = SessionDatagram::new(src, dest, vec![0x10, 0x00, 0x00, 0x00]).with_ttl(2);
|
||||
let encoded = dg.encode();
|
||||
node.handle_session_datagram(&from, &encoded[1..], false)
|
||||
.await;
|
||||
let fwd = &node.metrics().forwarding;
|
||||
assert_eq!(
|
||||
fwd.ttl_exhausted_packets.get(),
|
||||
0,
|
||||
"transit ttl=2 must clear the TTL gate"
|
||||
);
|
||||
assert_eq!(
|
||||
fwd.drop_no_route_packets.get(),
|
||||
1,
|
||||
"transit ttl=2 should have reached the routing step and found no route"
|
||||
);
|
||||
}
|
||||
|
||||
// --- Local delivery ---
|
||||
@@ -486,9 +392,9 @@ async fn test_forwarding_multi_hop() {
|
||||
#[tokio::test]
|
||||
async fn test_forwarding_hop_limit_prevents_infinite_loops() {
|
||||
// 3-node chain: 0 -- 1 -- 2
|
||||
// Send a datagram with ttl=2. Node 1 forwards it as transit (2 -> 1) and
|
||||
// node 2 delivers it locally, which is not TTL-gated. Had node 2 been
|
||||
// transit instead, the arriving ttl=1 would have stopped it there.
|
||||
// Send a datagram with ttl=1. It should be forwarded by node 1
|
||||
// (decrement to 0) and delivered at node 2 (local delivery). If node 2
|
||||
// tried to forward further, the 0 ttl would prevent it.
|
||||
let edges = vec![(0, 1), (1, 2)];
|
||||
let mut nodes = run_tree_test(3, &edges, false).await;
|
||||
verify_tree_convergence(&nodes);
|
||||
@@ -503,7 +409,7 @@ async fn test_forwarding_hop_limit_prevents_infinite_loops() {
|
||||
node2_addr,
|
||||
vec![0x10, 0x00, 0x04, 0x00, 1, 2, 3, 4],
|
||||
)
|
||||
.with_ttl(2); // Node 1 forwards with ttl=1; node 2 is the destination
|
||||
.with_ttl(2); // Enough for 0->1 (decrement to 1) and 1->2 (decrement to 0, local delivery)
|
||||
|
||||
let encoded = dg.encode();
|
||||
|
||||
@@ -522,115 +428,6 @@ async fn test_forwarding_hop_limit_prevents_infinite_loops() {
|
||||
cleanup_nodes(&mut nodes).await;
|
||||
}
|
||||
|
||||
/// Acceptance: a transit datagram arriving with ttl=2 leaves with ttl=1.
|
||||
///
|
||||
/// Pinned on a live 3-node chain (0 -- 1 -- 2) by where the datagram stops,
|
||||
/// since the TTL that leaves node 0 is only observable through what the next
|
||||
/// hop does with it. Both injections are transit at node 0 (external source,
|
||||
/// destined for node 2), so node 1 is a forwarder in both.
|
||||
///
|
||||
/// - ttl=2 in: node 0 must emit ttl=1, which node 1 (transit) drops. If node 0
|
||||
/// emitted ttl=2 unchanged, node 1 would forward and node 2 would deliver.
|
||||
/// - ttl=3 in: node 0 emits 2, node 1 emits 1, node 2 delivers (delivery is
|
||||
/// not TTL-gated). If either hop decremented by more than one, the datagram
|
||||
/// would have died at node 1 instead.
|
||||
///
|
||||
/// Together the two pin the decrement at exactly one per hop and the drop at
|
||||
/// would-leave-zero.
|
||||
#[tokio::test]
|
||||
async fn test_forwarding_ttl_decrement_is_one_per_hop() {
|
||||
let edges = vec![(0, 1), (1, 2)];
|
||||
let mut nodes = run_tree_test(3, &edges, false).await;
|
||||
verify_tree_convergence(&nodes);
|
||||
populate_all_coord_caches(&mut nodes);
|
||||
|
||||
let node0_addr = *nodes[0].node.node_addr();
|
||||
let node2_addr = *nodes[2].node.node_addr();
|
||||
let external_src = make_node_addr(0xEE);
|
||||
|
||||
// --- ttl=2: must die at node 1, one hop short of the destination ---
|
||||
let dg =
|
||||
SessionDatagram::new(external_src, node2_addr, vec![0x10, 0x00, 0x00, 0x00]).with_ttl(2);
|
||||
let encoded = dg.encode();
|
||||
nodes[0]
|
||||
.node
|
||||
.handle_session_datagram(&node0_addr, &encoded[1..], false)
|
||||
.await;
|
||||
|
||||
for _ in 0..3 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
process_available_packets(&mut nodes).await;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
nodes[0].node.metrics().forwarding.forwarded_packets.get(),
|
||||
1,
|
||||
"node 0 should have forwarded the ttl=2 datagram"
|
||||
);
|
||||
assert_eq!(
|
||||
nodes[1]
|
||||
.node
|
||||
.metrics()
|
||||
.forwarding
|
||||
.ttl_exhausted_packets
|
||||
.get(),
|
||||
1,
|
||||
"node 1 should have received ttl=1 and dropped it as TTL-exhausted"
|
||||
);
|
||||
assert_eq!(
|
||||
nodes[1].node.metrics().forwarding.forwarded_packets.get(),
|
||||
0,
|
||||
"node 1 must not forward a datagram that would leave with ttl=0"
|
||||
);
|
||||
assert_eq!(
|
||||
nodes[2].node.metrics().forwarding.delivered_packets.get(),
|
||||
0,
|
||||
"node 2 must never see the ttl=2 datagram"
|
||||
);
|
||||
|
||||
// --- ttl=3: must survive both transit hops and be delivered at node 2 ---
|
||||
let dg =
|
||||
SessionDatagram::new(external_src, node2_addr, vec![0x10, 0x00, 0x00, 0x00]).with_ttl(3);
|
||||
let encoded = dg.encode();
|
||||
nodes[0]
|
||||
.node
|
||||
.handle_session_datagram(&node0_addr, &encoded[1..], false)
|
||||
.await;
|
||||
|
||||
for _ in 0..3 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
process_available_packets(&mut nodes).await;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
nodes[0].node.metrics().forwarding.forwarded_packets.get(),
|
||||
2,
|
||||
"node 0 should have forwarded the ttl=3 datagram too"
|
||||
);
|
||||
assert_eq!(
|
||||
nodes[1].node.metrics().forwarding.forwarded_packets.get(),
|
||||
1,
|
||||
"node 1 should have forwarded the ttl=2 it received"
|
||||
);
|
||||
assert_eq!(
|
||||
nodes[1]
|
||||
.node
|
||||
.metrics()
|
||||
.forwarding
|
||||
.ttl_exhausted_packets
|
||||
.get(),
|
||||
1,
|
||||
"node 1 should not have dropped the second datagram"
|
||||
);
|
||||
assert_eq!(
|
||||
nodes[2].node.metrics().forwarding.delivered_packets.get(),
|
||||
1,
|
||||
"node 2 should have delivered the datagram that arrived with ttl=1"
|
||||
);
|
||||
|
||||
cleanup_nodes(&mut nodes).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_forwarding_no_route_generates_error() {
|
||||
// 2-node network: 0 -- 1
|
||||
|
||||
@@ -13,6 +13,8 @@ mod bootstrap;
|
||||
mod decrypt_failure;
|
||||
mod disconnect;
|
||||
mod discovery;
|
||||
#[cfg(target_os = "linux")]
|
||||
mod ethernet;
|
||||
mod forwarding;
|
||||
mod handshake;
|
||||
mod heartbeat;
|
||||
|
||||
@@ -1333,125 +1333,3 @@ fn test_route_class_partition_sums_to_forwarded() {
|
||||
assert_eq!(snap.route_crosslink_ascend, 3);
|
||||
assert_eq!(snap.route_direct_peer, 1);
|
||||
}
|
||||
|
||||
// === Coord-cache invalidation on parent loss ===
|
||||
//
|
||||
// Parent-lost-via-peer-removal is a genuine position change and must
|
||||
// surgically invalidate the coordinate cache like every other such path
|
||||
// (reparent → invalidate_via_node; self-root → invalidate_other_roots).
|
||||
// `make_node_addr(0)` is the network minimum, so the node's random identity
|
||||
// addr is always greater than it — the reparent/child geometry is deterministic.
|
||||
|
||||
#[test]
|
||||
fn test_parent_loss_reparent_invalidates_coord_cache() {
|
||||
let mut node = make_node();
|
||||
let my_addr = *node.node_addr();
|
||||
|
||||
let root = make_node_addr(0);
|
||||
let parent = make_node_addr(1);
|
||||
let alt = make_node_addr(2);
|
||||
|
||||
// Current parent and an alternative, both rooted at `root`.
|
||||
node.tree_state_mut().update_peer(
|
||||
ParentDeclaration::new(parent, root, 1, 1000),
|
||||
TreeCoordinate::from_addrs(vec![parent, root]).unwrap(),
|
||||
);
|
||||
node.tree_state_mut().update_peer(
|
||||
ParentDeclaration::new(alt, root, 1, 1000),
|
||||
TreeCoordinate::from_addrs(vec![alt, root]).unwrap(),
|
||||
);
|
||||
// Adopt `parent`; our coords become [my_addr, parent, root], root = `root`.
|
||||
node.tree_state_mut().set_parent(parent, 1, 1000);
|
||||
node.tree_state_mut().recompute_coords();
|
||||
assert!(!node.tree_state().is_root());
|
||||
assert_eq!(node.tree_state().root(), &root);
|
||||
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
|
||||
// via-node class: a downstream destination that routes through us.
|
||||
let downstream = make_node_addr(10);
|
||||
node.coord_cache_mut().insert(
|
||||
downstream,
|
||||
TreeCoordinate::from_addrs(vec![downstream, my_addr, root]).unwrap(),
|
||||
now_ms,
|
||||
);
|
||||
// survivor: same root, does not route through us.
|
||||
let sibling_dest = make_node_addr(11);
|
||||
node.coord_cache_mut().insert(
|
||||
sibling_dest,
|
||||
TreeCoordinate::from_addrs(vec![sibling_dest, alt, root]).unwrap(),
|
||||
now_ms,
|
||||
);
|
||||
|
||||
// Parent link drops; node reparents onto `alt` (still rooted at `root`).
|
||||
let changed = node.handle_peer_removal_tree_cleanup(&parent);
|
||||
assert!(changed);
|
||||
assert_eq!(node.tree_state().my_declaration().parent_id(), &alt);
|
||||
assert_eq!(node.tree_state().root(), &root);
|
||||
|
||||
assert!(
|
||||
!node.coord_cache().contains(&downstream, now_ms),
|
||||
"entry routing through us must be invalidated after reparent"
|
||||
);
|
||||
assert!(
|
||||
node.coord_cache().contains(&sibling_dest, now_ms),
|
||||
"same-root entry not routing through us must survive (surgical, not a flush)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_loss_selfroot_invalidates_coord_cache() {
|
||||
let mut node = make_node();
|
||||
let my_addr = *node.node_addr();
|
||||
|
||||
let old_root = make_node_addr(0);
|
||||
let parent = make_node_addr(1);
|
||||
|
||||
// Adopt `parent` (rooted at `old_root`); no alternative peers exist, so a
|
||||
// parent loss self-roots the node.
|
||||
node.tree_state_mut().update_peer(
|
||||
ParentDeclaration::new(parent, old_root, 1, 1000),
|
||||
TreeCoordinate::from_addrs(vec![parent, old_root]).unwrap(),
|
||||
);
|
||||
node.tree_state_mut().set_parent(parent, 1, 1000);
|
||||
node.tree_state_mut().recompute_coords();
|
||||
assert!(!node.tree_state().is_root());
|
||||
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
|
||||
// via-node class: routes through us.
|
||||
let downstream = make_node_addr(10);
|
||||
node.coord_cache_mut().insert(
|
||||
downstream,
|
||||
TreeCoordinate::from_addrs(vec![downstream, my_addr, old_root]).unwrap(),
|
||||
now_ms,
|
||||
);
|
||||
// other-roots class: on the old root, does not route through us.
|
||||
let foreign = make_node_addr(11);
|
||||
node.coord_cache_mut().insert(
|
||||
foreign,
|
||||
TreeCoordinate::from_addrs(vec![foreign, parent, old_root]).unwrap(),
|
||||
now_ms,
|
||||
);
|
||||
|
||||
// Parent link drops; no alternative parent → node self-roots.
|
||||
let changed = node.handle_peer_removal_tree_cleanup(&parent);
|
||||
assert!(changed);
|
||||
assert!(node.tree_state().is_root());
|
||||
assert_eq!(node.tree_state().root(), &my_addr);
|
||||
|
||||
assert!(
|
||||
!node.coord_cache().contains(&downstream, now_ms),
|
||||
"via-node entry must be invalidated after self-root"
|
||||
);
|
||||
assert!(
|
||||
!node.coord_cache().contains(&foreign, now_ms),
|
||||
"stale old-root entry must be invalidated after self-root"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1707,93 +1707,6 @@ async fn process_pending_retries_gated_at_capacity() {
|
||||
);
|
||||
}
|
||||
|
||||
/// A TCP listener that accepts connections and then never speaks. A relay
|
||||
/// URL pointed at it makes the nostr client's websocket handshake hang, so
|
||||
/// `refetch_advert_for_stale_check` burns its full 2s fetch timeout without
|
||||
/// any network egress.
|
||||
fn spawn_blackhole_relay() -> String {
|
||||
use std::net::TcpListener;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind blackhole listener");
|
||||
let port = listener.local_addr().expect("blackhole local addr").port();
|
||||
std::thread::spawn(move || {
|
||||
let mut held = Vec::new();
|
||||
while let Ok((stream, _)) = listener.accept() {
|
||||
held.push(stream);
|
||||
}
|
||||
});
|
||||
format!("ws://127.0.0.1:{port}")
|
||||
}
|
||||
|
||||
/// The per-tick retry loop must not await the pre-dial advert refetch.
|
||||
///
|
||||
/// `process_pending_retries` runs inline on the node's 1s rx-loop tick. Each
|
||||
/// due peer's refetch carries a 2s relay-fetch timeout, so awaiting it stalls
|
||||
/// the whole tick by 2s per peer — up to `MAX_RETRY_CONNECTIONS_PER_TICK`
|
||||
/// times in one tick body. The refresh is fire-and-forget: it exists to make
|
||||
/// the *next* retry dial a fresh endpoint, and retries are backoff-paced.
|
||||
///
|
||||
/// Discriminator: wall-clock duration of one `process_pending_retries` call
|
||||
/// with several due peers whose refetches all hang. Awaited, the call takes
|
||||
/// `2s * peers`; spawned, it returns without waiting on any of them.
|
||||
#[tokio::test]
|
||||
async fn process_pending_retries_does_not_await_advert_refetch() {
|
||||
use std::time::Instant;
|
||||
|
||||
const DUE_PEERS: usize = 4;
|
||||
// Awaited: >= 8s (4 x 2s). Spawned: milliseconds. A 3s bound sits far
|
||||
// from both, so neither machine load nor the 2s timeout's own slack can
|
||||
// flip the verdict.
|
||||
const MAX_TICK_MS: u128 = 3_000;
|
||||
|
||||
let mut node = make_node_with_max_peers(64);
|
||||
|
||||
let mut bootstrap = NostrDiscovery::new_for_test();
|
||||
bootstrap
|
||||
.set_advert_relays_for_test(vec![spawn_blackhole_relay()])
|
||||
.await;
|
||||
node.nostr_discovery = Some(Arc::new(bootstrap));
|
||||
|
||||
let mut queued = Vec::new();
|
||||
for _ in 0..DUE_PEERS {
|
||||
let peer_npub = Identity::generate().npub();
|
||||
let peer_node_addr = *PeerIdentity::from_npub(&peer_npub).unwrap().node_addr();
|
||||
let mut state = super::super::retry::RetryState::new(crate::config::PeerConfig::new(
|
||||
peer_npub,
|
||||
"udp",
|
||||
"127.0.0.1:9",
|
||||
));
|
||||
state.retry_after_ms = 0;
|
||||
state.reconnect = true;
|
||||
node.retry_pending.insert(peer_node_addr, state);
|
||||
queued.push(peer_node_addr);
|
||||
}
|
||||
|
||||
let started = Instant::now();
|
||||
node.process_pending_retries(1_000).await;
|
||||
let elapsed = started.elapsed();
|
||||
|
||||
assert!(
|
||||
elapsed.as_millis() < MAX_TICK_MS,
|
||||
"retry tick must not block on the advert refetch: took {}ms for {} due peers \
|
||||
(a per-peer 2s relay-fetch timeout awaited inline is the fingerprint)",
|
||||
elapsed.as_millis(),
|
||||
DUE_PEERS
|
||||
);
|
||||
|
||||
// The rest of the loop body is unchanged: every due peer was still
|
||||
// attempted, failed for want of a transport, and was rescheduled.
|
||||
for addr in &queued {
|
||||
let state = node
|
||||
.retry_pending
|
||||
.get(addr)
|
||||
.expect("due peer must remain queued after a failed attempt");
|
||||
assert_eq!(
|
||||
state.retry_count, 1,
|
||||
"each due peer must still have been attempted and rescheduled"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn poll_nostr_discovery_established_gated_at_capacity() {
|
||||
use crate::discovery::EstablishedTraversal;
|
||||
@@ -2082,86 +1995,3 @@ async fn handle_msg1_admits_existing_peer_at_cap() {
|
||||
"rate limiter must rebalance after the (bypass-admitted) handler returns"
|
||||
);
|
||||
}
|
||||
|
||||
// ===== Transport kernel-drop detection (sans-IO) =====
|
||||
//
|
||||
// The drop-detection edge-detector, tested directly. It replaces the
|
||||
// congestion-drops docker scenario, which could not provoke SO_RXQ_OVFL
|
||||
// deterministically (a fresh daemon reader keeps up with container-speed
|
||||
// traffic, so the kernel never overflows the socket queue). The kernel
|
||||
// dropping datagrams is not FIPS behaviour to test; the FIPS behaviour is
|
||||
// reading the SO_RXQ_OVFL counter and firing kernel_drop_events on the
|
||||
// transition into a new drop burst, which is exactly this decision.
|
||||
|
||||
#[test]
|
||||
fn test_transport_drop_state_fires_on_edge_and_rearms() {
|
||||
let mut s = TransportDropState::default();
|
||||
// Cumulative counter still 0: no rise, no event.
|
||||
assert!(!s.observe_drops(0));
|
||||
// First rise (0 -> 5): a new drop burst is observed, so it fires.
|
||||
assert!(s.observe_drops(5));
|
||||
// Counter keeps rising (5 -> 9) but we are already dropping: this is
|
||||
// the "first observed" contract, so it must NOT fire again.
|
||||
assert!(!s.observe_drops(9));
|
||||
// A sample with no further rise clears the dropping flag (no event).
|
||||
assert!(!s.observe_drops(9));
|
||||
// A later rise (9 -> 12) is a fresh burst and fires again.
|
||||
assert!(s.observe_drops(12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transport_drop_state_steady_counter_fires_once() {
|
||||
let mut s = TransportDropState::default();
|
||||
// A cumulative counter that jumps once and then holds steady must
|
||||
// register exactly one event, not one per sample — otherwise a single
|
||||
// historical drop burst would report congestion forever.
|
||||
assert!(s.observe_drops(7));
|
||||
assert!(!s.observe_drops(7));
|
||||
assert!(!s.observe_drops(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_display_name_uses_cached_short_npub() {
|
||||
// Path 3 of `peer_display_name` (no host entry, no alias) reads the
|
||||
// per-peer cached short npub; it must still equal the value derived
|
||||
// from the peer's identity.
|
||||
let mut node = make_node();
|
||||
let peer_identity_full = Identity::generate();
|
||||
let peer_addr = *peer_identity_full.node_addr();
|
||||
let peer_identity = PeerIdentity::from_pubkey(peer_identity_full.pubkey());
|
||||
node.peers
|
||||
.insert(peer_addr, ActivePeer::new(peer_identity, LinkId::new(1), 0));
|
||||
|
||||
assert_eq!(
|
||||
node.peer_display_name(&peer_addr),
|
||||
peer_identity.short_npub()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_display_name_tracks_alias_change() {
|
||||
// The display name is NOT cached on the peer: `peer_aliases` is a
|
||||
// runtime-mutable map (`update_peers` inserts and removes entries), so
|
||||
// a cached name would go stale. Caching only the immutable short npub
|
||||
// must leave that tracking intact.
|
||||
let mut node = make_node();
|
||||
let peer_identity_full = Identity::generate();
|
||||
let peer_addr = *peer_identity_full.node_addr();
|
||||
let peer_identity = PeerIdentity::from_pubkey(peer_identity_full.pubkey());
|
||||
node.peers
|
||||
.insert(peer_addr, ActivePeer::new(peer_identity, LinkId::new(1), 0));
|
||||
|
||||
assert_eq!(
|
||||
node.peer_display_name(&peer_addr),
|
||||
peer_identity.short_npub()
|
||||
);
|
||||
|
||||
node.peer_aliases.insert(peer_addr, "gateway".to_string());
|
||||
assert_eq!(node.peer_display_name(&peer_addr), "gateway");
|
||||
|
||||
node.peer_aliases.remove(&peer_addr);
|
||||
assert_eq!(
|
||||
node.peer_display_name(&peer_addr),
|
||||
peer_identity.short_npub()
|
||||
);
|
||||
}
|
||||
|
||||
+4
-10
@@ -300,6 +300,7 @@ impl Node {
|
||||
.invalidate_via_node(our_identity.node_addr());
|
||||
self.reset_discovery_backoff();
|
||||
|
||||
self.metrics().tree.parent_switched.inc();
|
||||
self.metrics().tree.parent_switches.inc();
|
||||
|
||||
info!(
|
||||
@@ -337,6 +338,7 @@ impl Node {
|
||||
self.coord_cache
|
||||
.invalidate_other_roots(our_identity.node_addr());
|
||||
self.reset_discovery_backoff();
|
||||
self.metrics().tree.parent_switched.inc();
|
||||
self.metrics().tree.parent_switches.inc();
|
||||
info!(
|
||||
new_root = %self.tree_state.root(),
|
||||
@@ -522,6 +524,7 @@ impl Node {
|
||||
.invalidate_via_node(our_identity.node_addr());
|
||||
self.reset_discovery_backoff();
|
||||
|
||||
self.metrics().tree.parent_switched.inc();
|
||||
self.metrics().tree.parent_switches.inc();
|
||||
|
||||
info!(
|
||||
@@ -557,6 +560,7 @@ impl Node {
|
||||
self.coord_cache
|
||||
.invalidate_other_roots(our_identity.node_addr());
|
||||
self.reset_discovery_backoff();
|
||||
self.metrics().tree.parent_switched.inc();
|
||||
self.metrics().tree.parent_switches.inc();
|
||||
info!(
|
||||
new_root = %self.tree_state.root(),
|
||||
@@ -616,16 +620,6 @@ impl Node {
|
||||
.tree
|
||||
.record_reject(TreeReject::OutboundSignFailed);
|
||||
}
|
||||
// handle_parent_lost may promote to root OR find new parent;
|
||||
// cover both invalidation classes (same as the loop-detection
|
||||
// branch above). Without this, cached downstream entries keep
|
||||
// our now-stale coordinate prefix until TTL — and get_and_touch
|
||||
// refreshes the TTL on every routing access, so an actively
|
||||
// routed stale entry never self-expires.
|
||||
self.coord_cache
|
||||
.invalidate_via_node(our_identity.node_addr());
|
||||
self.coord_cache
|
||||
.invalidate_other_roots(self.tree_state.root());
|
||||
info!(
|
||||
new_root = %self.tree_state.root(),
|
||||
is_root = self.tree_state.is_root(),
|
||||
|
||||
+1
-82
@@ -81,16 +81,6 @@ pub struct ActivePeer {
|
||||
// === Identity (Verified) ===
|
||||
/// Cryptographic identity (verified via handshake).
|
||||
identity: PeerIdentity,
|
||||
/// Bech32 npub, derived once at construction.
|
||||
///
|
||||
/// The npub is a pure function of `identity`'s public key, and
|
||||
/// `identity` is never mutated after construction, so this can never
|
||||
/// go stale. Deriving it costs a bech32 encode, which the per-tick
|
||||
/// stats snapshot was paying once per peer per tick.
|
||||
npub: String,
|
||||
/// Shortened npub for log/UI display, derived once at construction.
|
||||
/// Immutable for the same reason as [`ActivePeer::npub`].
|
||||
short_npub: String,
|
||||
|
||||
// === Connection ===
|
||||
/// Link used to reach this peer.
|
||||
@@ -234,8 +224,6 @@ impl ActivePeer {
|
||||
pub fn new(identity: PeerIdentity, link_id: LinkId, authenticated_at: u64) -> Self {
|
||||
let now = Instant::now();
|
||||
Self {
|
||||
npub: identity.npub(),
|
||||
short_npub: identity.short_npub(),
|
||||
identity,
|
||||
link_id,
|
||||
connectivity: ConnectivityState::Connected,
|
||||
@@ -322,8 +310,6 @@ impl ActivePeer {
|
||||
) -> Self {
|
||||
let now = Instant::now();
|
||||
Self {
|
||||
npub: identity.npub(),
|
||||
short_npub: identity.short_npub(),
|
||||
identity,
|
||||
link_id,
|
||||
connectivity: ConnectivityState::Connected,
|
||||
@@ -437,21 +423,8 @@ impl ActivePeer {
|
||||
}
|
||||
|
||||
/// Get the peer's npub string.
|
||||
///
|
||||
/// Returns a clone of the value cached at construction; the bech32
|
||||
/// encode is not repeated.
|
||||
pub fn npub(&self) -> String {
|
||||
self.npub.clone()
|
||||
}
|
||||
|
||||
/// Borrow the peer's cached npub without allocating.
|
||||
pub fn npub_str(&self) -> &str {
|
||||
&self.npub
|
||||
}
|
||||
|
||||
/// Borrow the peer's cached shortened npub (e.g. `npub1abcd...wxyz`).
|
||||
pub fn short_npub(&self) -> &str {
|
||||
&self.short_npub
|
||||
self.identity.npub()
|
||||
}
|
||||
|
||||
// === Connection Accessors ===
|
||||
@@ -1246,60 +1219,6 @@ mod tests {
|
||||
assert!(peer.needs_filter_update()); // New peers need filter
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_npub_cache_matches_identity() {
|
||||
let identity = make_peer_identity();
|
||||
let peer = ActivePeer::new(identity, LinkId::new(1), 1000);
|
||||
|
||||
assert_eq!(peer.npub(), identity.npub());
|
||||
assert_eq!(peer.npub_str(), identity.npub());
|
||||
assert_eq!(peer.short_npub(), identity.short_npub());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_npub_cache_matches_identity_with_session() {
|
||||
// `with_session` builds its own struct literal, so it needs its
|
||||
// own check that the cache is populated from the same identity.
|
||||
let identity = make_peer_identity();
|
||||
let (session, _peer_session) = ik_session_pair();
|
||||
|
||||
let peer = ActivePeer::with_session(
|
||||
identity,
|
||||
LinkId::new(1),
|
||||
1000,
|
||||
session,
|
||||
SessionIndex::new(1),
|
||||
SessionIndex::new(2),
|
||||
TransportId::new(1),
|
||||
TransportAddr::from_string("127.0.0.1:9000"),
|
||||
LinkStats::new(),
|
||||
true,
|
||||
&MmpConfig::default(),
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(peer.npub(), identity.npub());
|
||||
assert_eq!(peer.short_npub(), identity.short_npub());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_npub_is_memoized_not_rederived() {
|
||||
// The whole point of the fix: the strings are stored on the peer,
|
||||
// not recomputed per call. A stored string keeps one heap buffer,
|
||||
// so repeated borrows have a stable address. A per-call bech32
|
||||
// encode would hand back a fresh allocation each time.
|
||||
let identity = make_peer_identity();
|
||||
let peer = ActivePeer::new(identity, LinkId::new(1), 1000);
|
||||
|
||||
let first = peer.npub_str().as_ptr();
|
||||
let second = peer.npub_str().as_ptr();
|
||||
assert_eq!(first, second);
|
||||
|
||||
let short_first = peer.short_npub().as_ptr();
|
||||
let short_second = peer.short_npub().as_ptr();
|
||||
assert_eq!(short_first, short_second);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_connectivity_transitions() {
|
||||
let identity = make_peer_identity();
|
||||
|
||||
+9
-57
@@ -337,27 +337,19 @@ impl SessionDatagram {
|
||||
self
|
||||
}
|
||||
|
||||
/// Decrement the TTL for a transit hop, returning whether the result may
|
||||
/// still be transmitted.
|
||||
///
|
||||
/// Follows IP semantics: the decrement happens first, and a datagram that
|
||||
/// would leave with a TTL of zero is not transmitted. `saturating_sub`
|
||||
/// folds an already-exhausted arrival (TTL 0) into the same outcome as a
|
||||
/// last-hop arrival (TTL 1); both leave `ttl` at 0 and return false.
|
||||
///
|
||||
/// This governs forwarding only. Delivery to the addressed node is not
|
||||
/// TTL-gated and must not consult this method.
|
||||
/// Decrement TTL, returning false if exhausted.
|
||||
pub fn decrement_ttl(&mut self) -> bool {
|
||||
self.ttl = self.ttl.saturating_sub(1);
|
||||
self.ttl > 0
|
||||
if self.ttl > 0 {
|
||||
self.ttl -= 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether this datagram would survive a transit hop.
|
||||
///
|
||||
/// True only at TTL 2 or more: at TTL 1 the decrement leaves zero, so the
|
||||
/// datagram is dropped rather than forwarded.
|
||||
/// Check if the datagram can be forwarded.
|
||||
pub fn can_forward(&self) -> bool {
|
||||
self.ttl > 1
|
||||
self.ttl > 0
|
||||
}
|
||||
|
||||
/// Encode as link-layer message (msg_type + ttl + path_mtu + src_addr + dest_addr + payload).
|
||||
@@ -683,44 +675,4 @@ mod tests {
|
||||
assert_eq!(decoded.ttl, hop);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_session_datagram_can_forward() {
|
||||
let dg = SessionDatagram::new(make_node_addr(1), make_node_addr(2), vec![0x42]);
|
||||
|
||||
assert!(!dg.clone().with_ttl(0).can_forward());
|
||||
assert!(
|
||||
!dg.clone().with_ttl(1).can_forward(),
|
||||
"ttl=1 would leave at zero, so it is not forwardable"
|
||||
);
|
||||
assert!(
|
||||
dg.clone().with_ttl(2).can_forward(),
|
||||
"ttl=2 leaves at one, so it is forwardable"
|
||||
);
|
||||
assert!(dg.with_ttl(255).can_forward());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_session_datagram_decrement_ttl() {
|
||||
let base = SessionDatagram::new(make_node_addr(1), make_node_addr(2), vec![0x42]);
|
||||
|
||||
let mut dg = base.clone().with_ttl(0);
|
||||
assert!(!dg.decrement_ttl(), "ttl=0 is already exhausted");
|
||||
assert_eq!(dg.ttl, 0, "decrement must saturate rather than wrap");
|
||||
|
||||
let mut dg = base.clone().with_ttl(1);
|
||||
assert!(
|
||||
!dg.decrement_ttl(),
|
||||
"ttl=1 leaves at zero, so it is dropped"
|
||||
);
|
||||
assert_eq!(dg.ttl, 0);
|
||||
|
||||
let mut dg = base.clone().with_ttl(2);
|
||||
assert!(dg.decrement_ttl());
|
||||
assert_eq!(dg.ttl, 1);
|
||||
|
||||
let mut dg = base.with_ttl(64);
|
||||
assert!(dg.decrement_ttl());
|
||||
assert_eq!(dg.ttl, 63);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -853,7 +853,7 @@ mod tests {
|
||||
/// find N packets already buffered, and one syscall reaps the burst.
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[ignore = "microbenchmark; run explicitly with --ignored --nocapture"]
|
||||
#[ignore]
|
||||
async fn bench_udp_recv_amortization() {
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
@@ -536,172 +536,6 @@ fn test_evaluate_parent_picks_loop_free_over_loopy() {
|
||||
assert_eq!(result, Some(peer2));
|
||||
}
|
||||
|
||||
// ===== Cost-based parent selection =====
|
||||
//
|
||||
// These exercise evaluate_parent's MMP-cost path directly, as a sans-IO
|
||||
// unit test of the exact decision. They replace six Docker chaos
|
||||
// scenarios — cost-reeval, cost-avoidance, cost-stability, depth-vs-cost,
|
||||
// mixed-technology and bottleneck-parent — whose subject was this
|
||||
// decision but which could not test it reliably: the mesh's root is
|
||||
// whichever node holds the smallest NodeAddr, MMP costs take several
|
||||
// measurement windows to settle, and hold-down plus hysteresis timing all
|
||||
// confounded the assertion. Here the peer ancestry, depths and costs are
|
||||
// constructed directly, so the decision is deterministic and each check
|
||||
// can fail on a real regression.
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_cost_prefers_cheaper_link_at_equal_depth() {
|
||||
// mixed-technology / cost-avoidance subject: two candidate parents at
|
||||
// the SAME depth, one over a cheap (fiber) link and one over an
|
||||
// expensive (Bluetooth) link. The cheaper link must win.
|
||||
//
|
||||
// The cheap peer is given the LARGER NodeAddr on purpose: with cost
|
||||
// ignored the two candidates tie on depth and the NodeAddr tiebreak
|
||||
// would pick the expensive, smaller-addr peer. Only a cost-aware
|
||||
// decision picks the cheaper, larger-addr one, so the assertion
|
||||
// discriminates.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
|
||||
let expensive = make_node_addr(2); // smaller addr, high cost (Bluetooth)
|
||||
let cheap = make_node_addr(3); // larger addr, low cost (fiber)
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(expensive, root, 1, 1000),
|
||||
make_coords(&[2, 0]), // depth 1
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(cheap, root, 1, 1000),
|
||||
make_coords(&[3, 0]), // depth 1
|
||||
);
|
||||
|
||||
// eff_depth(expensive) = 1 + 4.0 = 5.0; eff_depth(cheap) = 1 + 1.0 = 2.0
|
||||
let costs = HashMap::from([(expensive, 4.0_f64), (cheap, 1.0_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), Some(cheap));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_cost_switches_when_link_to_parent_degrades() {
|
||||
// cost-reeval subject: the node is parented to A over a cheap link;
|
||||
// that link then degrades so the alternative B is strictly cheaper.
|
||||
// Re-evaluation must switch to B. This is the periodic-reeval decision,
|
||||
// taken here without any timer, netem or MMP-measurement latency.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
|
||||
let peer_a = make_node_addr(2);
|
||||
let peer_b = make_node_addr(3);
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_a, root, 1, 1000),
|
||||
make_coords(&[2, 0]), // depth 1
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_b, root, 1, 1000),
|
||||
make_coords(&[3, 0]), // depth 1
|
||||
);
|
||||
|
||||
// Adopt A as parent (both links cheap at first).
|
||||
state.set_parent(peer_a, 1, 1000);
|
||||
state.recompute_coords();
|
||||
assert!(!state.is_root());
|
||||
|
||||
// A's link degrades: eff(A) = 1 + 5.0 = 6.0, eff(B) = 1 + 1.0 = 2.0.
|
||||
// Default hysteresis is zero, so the strictly-cheaper B wins.
|
||||
let costs = HashMap::from([(peer_a, 5.0_f64), (peer_b, 1.0_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), Some(peer_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_hysteresis_suppresses_marginal_cost_change() {
|
||||
// cost-stability subject: a cost change smaller than the hysteresis
|
||||
// band must NOT trigger a reparent. This is the property the scenario
|
||||
// was named for and could only approximate with a switch-count ceiling.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
state.set_parent_hysteresis(0.2);
|
||||
|
||||
let peer_a = make_node_addr(2);
|
||||
let peer_b = make_node_addr(3);
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_a, root, 1, 1000),
|
||||
make_coords(&[2, 0]),
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_b, root, 1, 1000),
|
||||
make_coords(&[3, 0]),
|
||||
);
|
||||
state.set_parent(peer_a, 1, 1000);
|
||||
state.recompute_coords();
|
||||
|
||||
// eff(A) = 1 + 1.0 = 2.0; eff(B) = 1 + 0.9 = 1.9. B is cheaper, but
|
||||
// 1.9 is not below 2.0 * (1 - 0.2) = 1.6, so hysteresis holds the parent.
|
||||
let costs = HashMap::from([(peer_a, 1.0_f64), (peer_b, 0.9_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_hysteresis_allows_significant_cost_change() {
|
||||
// cost-stability healthy-path companion: a change LARGER than the band
|
||||
// must still switch, so the hysteresis test above is not passing merely
|
||||
// because the node never reparents.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
state.set_parent_hysteresis(0.2);
|
||||
|
||||
let peer_a = make_node_addr(2);
|
||||
let peer_b = make_node_addr(3);
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_a, root, 1, 1000),
|
||||
make_coords(&[2, 0]),
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(peer_b, root, 1, 1000),
|
||||
make_coords(&[3, 0]),
|
||||
);
|
||||
state.set_parent(peer_a, 1, 1000);
|
||||
state.recompute_coords();
|
||||
|
||||
// eff(A) = 2.0; eff(B) = 1 + 0.3 = 1.3 < 1.6 threshold → switch to B.
|
||||
let costs = HashMap::from([(peer_a, 1.0_f64), (peer_b, 0.3_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), Some(peer_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_parent_effective_depth_weighs_depth_against_cost() {
|
||||
// depth-vs-cost / bottleneck-parent subject: a shallow parent reached
|
||||
// over an expensive (bottleneck) link versus a deeper parent over a
|
||||
// cheap link. effective_depth = depth + link_cost decides, and here the
|
||||
// deeper-but-cheaper path wins — the outcome the depth-vs-cost scenario
|
||||
// named no falsifiable answer for.
|
||||
let my_node = make_node_addr(5);
|
||||
let mut state = TreeState::new(my_node);
|
||||
|
||||
let shallow = make_node_addr(2); // depth 1, bottleneck link
|
||||
let deep = make_node_addr(3); // depth 3, cheap link
|
||||
let root = make_node_addr(0);
|
||||
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(shallow, root, 1, 1000),
|
||||
make_coords(&[2, 0]), // depth 1
|
||||
);
|
||||
state.update_peer(
|
||||
ParentDeclaration::new(deep, make_node_addr(6), 1, 1000),
|
||||
make_coords(&[3, 6, 7, 0]), // depth 3
|
||||
);
|
||||
|
||||
// eff(shallow) = 1 + 3.0 = 4.0; eff(deep) = 3 + 0.5 = 3.5 → pick deep.
|
||||
// A depth-only decision would take the shallow bottleneck instead.
|
||||
let costs = HashMap::from([(shallow, 3.0_f64), (deep, 0.5_f64)]);
|
||||
assert_eq!(state.evaluate_parent(&costs), Some(deep));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_handle_parent_lost_finds_alternative() {
|
||||
let my_node = make_node_addr(5);
|
||||
|
||||
+2
-107
@@ -16,6 +16,8 @@ configurations.
|
||||
| ----------- | ----- | --------- | -------------------------------- |
|
||||
| mesh | 5 | UDP | Sparse mesh, 6 links, multi-hop |
|
||||
| chain | 5 | UDP | Linear chain, max 4-hop paths |
|
||||
| mesh-public | 5+1 | UDP | Mesh with external public node |
|
||||
| tcp-chain | 3 | TCP | Linear chain over TCP (port 8443) |
|
||||
| rekey | 5 | UDP | Rekey integration test topology |
|
||||
|
||||
### [tor/](tor/) -- Tor Transport Integration
|
||||
@@ -70,110 +72,3 @@ and optional trace-level RUST_LOG, capturing per-rep diagnostics and a
|
||||
mechanism-match summary across the run. Used for statistical reliability
|
||||
characterization of known flake classes under calibrated stress, not as
|
||||
a per-commit gate; not part of `ci-local.sh`.
|
||||
|
||||
## Running CI locally (`ci-local.sh`)
|
||||
|
||||
[`ci-local.sh`](ci-local.sh) runs the full local CI pipeline — build,
|
||||
clippy, unit tests, and the integration suites (including the chaos
|
||||
scenarios) — mirroring the GitHub `ci.yml` integration matrix. Run
|
||||
`./ci-local.sh --help` for the full option list and `--list` for the
|
||||
available suites. Every run starts with a parity check that verifies the
|
||||
local suite set covers the same work as the GitHub matrix, per scenario for
|
||||
chaos and per distro for deb-install; a divergence fails the run. GitHub
|
||||
runs the same check as its own `ci-parity` job. `--check-parity` runs it
|
||||
alone (see [check-ci-parity.sh](check-ci-parity.sh)).
|
||||
|
||||
### Per-run isolation and the `FIPS_CI_RUN_ID` override
|
||||
|
||||
Every invocation derives a **run id** and scopes all of its Docker
|
||||
resources to it, so two simultaneous runs on the same host (for example,
|
||||
one per git worktree, or an operator testing by hand while CI is in
|
||||
flight) never collide:
|
||||
|
||||
- **Compose projects** are named `fipsci_<run-id>_<suite>`, so
|
||||
container, network, and volume names are all prefixed per run.
|
||||
- **Build images** are tagged `fips-test:<run-id>` and
|
||||
`fips-test-app:<run-id>`, exported as `FIPS_TEST_IMAGE` /
|
||||
`FIPS_TEST_APP_IMAGE`, and **every** compose file and suite script reads
|
||||
those. The run does not write `fips-test:latest` at all: a bridge back to
|
||||
that shared mutable name would let a consumer that had been missed keep
|
||||
working while resolving whichever concurrent run wrote the tag last.
|
||||
`:latest` stays the hand-build name, produced by
|
||||
`testing/scripts/build.sh`, and remains the default every consumer falls
|
||||
back to when the variables are unset.
|
||||
- **The build context** is a per-run copy at `testing/docker-<run-id>/`,
|
||||
exported as `FIPS_BUILD_CONTEXT`. It is absolute because compose resolves
|
||||
a relative build context against the compose file's own directory rather
|
||||
than the working directory. `testing/docker/` is the hand-run context and
|
||||
a CI run does not write to it. Without this, two runs race on the contents
|
||||
of one directory and either can build a correctly-per-run-tagged image
|
||||
from the other's binaries.
|
||||
- Each parallel chaos child gets a unique, non-overlapping `/24` in
|
||||
`10.30.x` (via the sim `--subnet` override). `10.30.x` sits outside
|
||||
Docker's default address pool and the fixed-subnet suites' `172.x`
|
||||
ranges, so neither a sibling chaos child nor an auto-assigned network
|
||||
can swallow a pinned subnet.
|
||||
|
||||
By default the run id is `<short-git-sha>-<random>` — the SHA portion
|
||||
records *what code* a container is testing, the random suffix keeps
|
||||
simultaneous runs of the same SHA disjoint. Override it for a
|
||||
reproducible, attach-by-name debug session:
|
||||
|
||||
```sh
|
||||
FIPS_CI_RUN_ID=mydebug ./ci-local.sh --only static-mesh
|
||||
# containers are named fipsci_mydebug_static_fips-node-a, etc.
|
||||
```
|
||||
|
||||
### Preemption-safety and exit codes
|
||||
|
||||
`ci-local.sh` is safe to cancel mid-run. A signal trap tears down *every*
|
||||
compose project the run started (not just the current suite) and reaps
|
||||
any in-flight parallel chaos children, bounded by a `timeout` so a stuck
|
||||
`compose down` cannot wedge the trap. Exit codes distinguish a cancelled
|
||||
run from a failing one:
|
||||
|
||||
| Code | Meaning |
|
||||
| ---- | ------- |
|
||||
| `0` | all stages passed |
|
||||
| `1` | one or more stages failed |
|
||||
| `130` | interrupted by SIGINT — cancelled, not a failure |
|
||||
| `143` | terminated by SIGTERM — cancelled, not a failure |
|
||||
|
||||
A preempting CI worker (the push-triggered, CI-gated build pipeline that
|
||||
kills an in-flight run when a newer same-branch tip arrives) maps
|
||||
`130`/`143` → *cancelled* (discard, do not record a failing commit), `0`
|
||||
→ green, any other non-zero → red.
|
||||
|
||||
### Cleaning up leftover resources
|
||||
|
||||
Every CI-created container, network, and volume carries the label
|
||||
`com.corganlabs.fips-ci=1`. If a run is hard-killed (SIGKILL, OOM, crash)
|
||||
and leaves resources behind, reap them with:
|
||||
|
||||
```sh
|
||||
./ci-local.sh --reap # or: ./ci-cleanup.sh
|
||||
```
|
||||
|
||||
[`ci-cleanup.sh`](ci-cleanup.sh) force-removes everything bearing the CI
|
||||
label or a `fipsci_` compose-project prefix; it is safe to run when there
|
||||
is nothing to reap and safe to run repeatedly. Pass `--project-prefix` to
|
||||
scope the sweep to a single run.
|
||||
|
||||
It also removes the chaos simulation's leftover host-namespace veth
|
||||
interfaces (`vh…a`/`vh…b`), the one resource it touches that is neither a
|
||||
docker object nor labelled — a host interface can carry neither a label
|
||||
nor a compose project, so it is matched by name shape alone. That makes
|
||||
the reach here asymmetric with everything above, and worth stating
|
||||
plainly:
|
||||
|
||||
- A bare `chaos.sh` run's **containers** survive a broad reap. Its
|
||||
compose project is not `fipsci_`, and the simulation labels only the
|
||||
network, not the services.
|
||||
- A bare `chaos.sh` run's **veth interfaces do not.** An unscoped reap
|
||||
deletes them while they are in use, severing the Ethernet links of a
|
||||
live simulation and leaving its containers running.
|
||||
|
||||
So do not run a broad `--reap` while a bare simulation is up. Scope the
|
||||
interface sweep with `--veth-suffixes` (which is what `ci-local.sh`'s own
|
||||
teardown passes) or wait for the simulation to finish. `--project-prefix`
|
||||
does not help here: it scopes only the compose-project sweep.
|
||||
|
||||
@@ -1 +1 @@
|
||||
generated-configs*
|
||||
generated-configs
|
||||
|
||||
@@ -79,12 +79,6 @@ Docker service/container/hostname identifiers in this harness intentionally use
|
||||
`node-a` through `node-f`. For data-plane checks and operator examples, use the
|
||||
explicit FIPS names such as `node-a.fips` and `node-d.fips`.
|
||||
|
||||
The bridge network requests no subnet, so docker assigns one from its own
|
||||
address pool and two concurrent runs of this harness never contend for a fixed
|
||||
range. Consequently no node's IPv4 address is known before startup, and the
|
||||
generated peer stanzas address each other by docker hostname (`host-a` …
|
||||
`host-f`), resolved through the container's dnsmasq to docker's embedded DNS.
|
||||
|
||||
ACL paths are fixed in this branch:
|
||||
|
||||
- `/etc/fips/peers.allow`
|
||||
@@ -99,9 +93,7 @@ Mounted ACL files in this harness:
|
||||
|
||||
Generated fixture location:
|
||||
|
||||
- `testing/acl-allowlist/generated-configs/`, or
|
||||
`generated-configs<suffix>/` when `FIPS_CI_NAME_SUFFIX` is set, which is how
|
||||
concurrent runs keep their fixtures apart
|
||||
- `testing/acl-allowlist/generated-configs/`
|
||||
|
||||
Inspect peer state:
|
||||
|
||||
|
||||
@@ -1,30 +1,14 @@
|
||||
networks:
|
||||
# No subnet is requested: docker assigns one from the daemon's address pool.
|
||||
# A fixed request is honoured verbatim, so two runs asking for the same range
|
||||
# collide on "Pool overlaps" — which is why the generated peer addresses are
|
||||
# docker hostnames (host-a … host-f) rather than literal IPs. Docker's
|
||||
# embedded DNS is per-network, so the same hostname in two concurrent runs
|
||||
# resolves inside each run's own subnet.
|
||||
#
|
||||
# That removes one of the two obstacles to running this suite twice at once,
|
||||
# not both. The compose project name is still fixed, so two runs that do not
|
||||
# set COMPOSE_PROJECT_NAME share a project, and the second `up` recreates the
|
||||
# first's containers while either `down` removes both. Nothing scopes it for
|
||||
# this suite: it is run by hand only, so the caller has to. Do not read the
|
||||
# floating subnet as making concurrent bare runs safe.
|
||||
acl-net:
|
||||
driver: bridge
|
||||
labels:
|
||||
- "com.corganlabs.fips-ci=1"
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.31.0.0/24
|
||||
|
||||
x-fips-common: &fips-common
|
||||
build:
|
||||
# The harness scopes its build context per run and passes it here; the
|
||||
# shared directory is the hand-run default. Compose resolves a relative
|
||||
# value against THIS file's directory, so the harness must export an
|
||||
# absolute path.
|
||||
context: ${FIPS_BUILD_CONTEXT:-../docker}
|
||||
image: ${FIPS_TEST_IMAGE:-fips-test:latest}
|
||||
context: ../docker
|
||||
image: fips-test:latest
|
||||
entrypoint: ["/usr/local/bin/entrypoint.sh"]
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
@@ -42,80 +26,86 @@ x-fips-common: &fips-common
|
||||
services:
|
||||
service-a:
|
||||
<<: *fips-common
|
||||
container_name: fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: fips-acl-container-a
|
||||
hostname: host-a
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/peers.allow:/etc/fips/peers.allow:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/peers.deny:/etc/fips/peers.deny:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/fips.key:/etc/fips/fips.key:ro
|
||||
- ./generated-configs/node-a/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs/node-a/peers.allow:/etc/fips/peers.allow:ro
|
||||
- ./generated-configs/node-a/peers.deny:/etc/fips/peers.deny:ro
|
||||
- ./generated-configs/node-a/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs/node-a/fips.key:/etc/fips/fips.key:ro
|
||||
networks:
|
||||
- acl-net
|
||||
acl-net:
|
||||
ipv4_address: 172.31.0.10
|
||||
|
||||
service-b:
|
||||
<<: *fips-common
|
||||
container_name: fips-acl-container-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: fips-acl-container-b
|
||||
hostname: host-b
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/peers.allow:/etc/fips/peers.allow:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/peers.deny:/etc/fips/peers.deny:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.key:/etc/fips/fips.key:ro
|
||||
- ./generated-configs/node-b/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs/node-b/peers.allow:/etc/fips/peers.allow:ro
|
||||
- ./generated-configs/node-b/peers.deny:/etc/fips/peers.deny:ro
|
||||
- ./generated-configs/node-b/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs/node-b/fips.key:/etc/fips/fips.key:ro
|
||||
networks:
|
||||
- acl-net
|
||||
acl-net:
|
||||
ipv4_address: 172.31.0.11
|
||||
|
||||
service-c:
|
||||
<<: *fips-common
|
||||
container_name: fips-acl-container-c${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: fips-acl-container-c
|
||||
hostname: host-c
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-c/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-c/peers.allow:/etc/fips/peers.allow:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-c/peers.deny:/etc/fips/peers.deny:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-c/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-c/fips.key:/etc/fips/fips.key:ro
|
||||
- ./generated-configs/node-c/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs/node-c/peers.allow:/etc/fips/peers.allow:ro
|
||||
- ./generated-configs/node-c/peers.deny:/etc/fips/peers.deny:ro
|
||||
- ./generated-configs/node-c/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs/node-c/fips.key:/etc/fips/fips.key:ro
|
||||
networks:
|
||||
- acl-net
|
||||
acl-net:
|
||||
ipv4_address: 172.31.0.12
|
||||
|
||||
service-d:
|
||||
<<: *fips-common
|
||||
container_name: fips-acl-container-d${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: fips-acl-container-d
|
||||
hostname: host-d
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-d/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-d/peers.allow:/etc/fips/peers.allow:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-d/peers.deny:/etc/fips/peers.deny:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-d/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-d/fips.key:/etc/fips/fips.key:ro
|
||||
- ./generated-configs/node-d/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs/node-d/peers.allow:/etc/fips/peers.allow:ro
|
||||
- ./generated-configs/node-d/peers.deny:/etc/fips/peers.deny:ro
|
||||
- ./generated-configs/node-d/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs/node-d/fips.key:/etc/fips/fips.key:ro
|
||||
networks:
|
||||
- acl-net
|
||||
acl-net:
|
||||
ipv4_address: 172.31.0.13
|
||||
|
||||
service-e:
|
||||
<<: *fips-common
|
||||
container_name: fips-acl-container-e${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: fips-acl-container-e
|
||||
hostname: host-e
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-e/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-e/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-e/fips.key:/etc/fips/fips.key:ro
|
||||
- ./generated-configs/node-e/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs/node-e/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs/node-e/fips.key:/etc/fips/fips.key:ro
|
||||
networks:
|
||||
- acl-net
|
||||
acl-net:
|
||||
ipv4_address: 172.31.0.14
|
||||
|
||||
service-f:
|
||||
<<: *fips-common
|
||||
container_name: fips-acl-container-f${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: fips-acl-container-f
|
||||
hostname: host-f
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-f/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-f/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-f/fips.key:/etc/fips/fips.key:ro
|
||||
- ./generated-configs/node-f/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs/node-f/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs/node-f/fips.key:/etc/fips/fips.key:ro
|
||||
networks:
|
||||
- acl-net
|
||||
acl-net:
|
||||
ipv4_address: 172.31.0.15
|
||||
|
||||
@@ -3,12 +3,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
# Scoped by the per-run suffix because this directory is wiped and rewritten
|
||||
# below: two runs sharing one output directory would delete each other's
|
||||
# fixtures out from under running containers. Unset (a bare hand run, or the
|
||||
# GitHub-hosted path) it collapses to the historical "generated-configs".
|
||||
GENERATED_DIR="$SCRIPT_DIR/generated-configs${FIPS_CI_NAME_SUFFIX:-}"
|
||||
GENERATED_DIR="$SCRIPT_DIR/generated-configs"
|
||||
|
||||
write_file() {
|
||||
local path="$1"
|
||||
@@ -28,10 +23,6 @@ node-f npub1ytrut7gjncn2zfnhn56c0zgftf0w6p99gf6fu8j73hzw5603zglqc9av6c
|
||||
EOF
|
||||
}
|
||||
|
||||
# Peers are addressed by the docker hostname the compose file assigns
|
||||
# (host-a … host-f), not by IP. The network requests no subnet so that two
|
||||
# concurrent runs cannot collide on one address range, which means no node's
|
||||
# address is knowable before `docker compose up`.
|
||||
echo "Generating ACL allowlist fixtures..."
|
||||
rm -rf "$GENERATED_DIR"
|
||||
|
||||
@@ -57,31 +48,31 @@ peers:
|
||||
alias: "node-b"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-b:2121"
|
||||
addr: "172.31.0.11:2121"
|
||||
connect_policy: auto_connect
|
||||
- npub: "npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6"
|
||||
alias: "node-c"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-c:2121"
|
||||
addr: "172.31.0.12:2121"
|
||||
connect_policy: auto_connect
|
||||
- npub: "npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl"
|
||||
alias: "node-d"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-d:2121"
|
||||
addr: "172.31.0.13:2121"
|
||||
connect_policy: auto_connect
|
||||
- npub: "npub1x5z9rwzzm26q9verutx4aajhf2zw2pyp34c6whhde2zduxqav40qgq36l6"
|
||||
alias: "node-e"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-e:2121"
|
||||
addr: "172.31.0.14:2121"
|
||||
connect_policy: auto_connect
|
||||
- npub: "npub1ytrut7gjncn2zfnhn56c0zgftf0w6p99gf6fu8j73hzw5603zglqc9av6c"
|
||||
alias: "node-f"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-f:2121"
|
||||
addr: "172.31.0.15:2121"
|
||||
connect_policy: auto_connect
|
||||
EOF
|
||||
|
||||
@@ -122,19 +113,19 @@ peers:
|
||||
alias: "node-a"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-a:2121"
|
||||
addr: "172.31.0.10:2121"
|
||||
connect_policy: auto_connect
|
||||
- npub: "npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6"
|
||||
alias: "node-c"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-c:2121"
|
||||
addr: "172.31.0.12:2121"
|
||||
connect_policy: auto_connect
|
||||
- npub: "npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl"
|
||||
alias: "node-d"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-d:2121"
|
||||
addr: "172.31.0.13:2121"
|
||||
connect_policy: auto_connect
|
||||
EOF
|
||||
|
||||
@@ -175,7 +166,7 @@ peers:
|
||||
alias: "node-a"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-a:2121"
|
||||
addr: "172.31.0.10:2121"
|
||||
connect_policy: auto_connect
|
||||
EOF
|
||||
|
||||
@@ -218,7 +209,7 @@ peers:
|
||||
alias: "node-a"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-a:2121"
|
||||
addr: "172.31.0.10:2121"
|
||||
connect_policy: auto_connect
|
||||
EOF
|
||||
|
||||
@@ -261,7 +252,7 @@ peers:
|
||||
alias: "node-a"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-a:2121"
|
||||
addr: "172.31.0.10:2121"
|
||||
connect_policy: auto_connect
|
||||
EOF
|
||||
|
||||
@@ -291,7 +282,7 @@ peers:
|
||||
alias: "node-a"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-a:2121"
|
||||
addr: "172.31.0.10:2121"
|
||||
connect_policy: auto_connect
|
||||
EOF
|
||||
|
||||
|
||||
+51
-145
@@ -75,103 +75,47 @@ assert_acl_field() {
|
||||
echo "PASS: $container ACL field $field matches expected value"
|
||||
}
|
||||
|
||||
# Connected-peer count for a container, or the empty string if it did not
|
||||
# answer.
|
||||
#
|
||||
# Empty is deliberately distinct from a real 0, and here the distinction is
|
||||
# the whole point: the ACL denial checks below expect exactly 0, so an
|
||||
# `|| echo 0` fallback lets an unreachable container satisfy them on the first
|
||||
# iteration and the security property is never observed. Same shape as
|
||||
# admission-cap-test.sh's read_peer_count.
|
||||
read_connected_peers() {
|
||||
local container="$1"
|
||||
docker exec "$container" fipsctl show peers 2>/dev/null \
|
||||
| python3 -c 'import json,sys; data=json.load(sys.stdin); print(sum(1 for p in data.get("peers", []) if p.get("connectivity") == "connected"))' 2>/dev/null \
|
||||
|| true
|
||||
}
|
||||
|
||||
wait_for_peers_exact() {
|
||||
local container="$1"
|
||||
local expected_count="$2"
|
||||
local timeout="${3:-30}"
|
||||
|
||||
local count="" answered=false
|
||||
for _ in $(seq 1 "$timeout"); do
|
||||
count=$(read_connected_peers "$container")
|
||||
if [ -n "$count" ]; then
|
||||
answered=true
|
||||
if [ "$count" -eq "$expected_count" ]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ "$answered" = false ]; then
|
||||
echo "FAIL: $container never answered a peer query in ${timeout}s, so a count of $expected_count was never actually observed" >&2
|
||||
else
|
||||
echo "FAIL: $container did not reach $expected_count connected peers in ${timeout}s (last answer: $count)" >&2
|
||||
fi
|
||||
docker exec "$container" fipsctl show peers >&2 || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Assert that ONE log line in $container contains every one of the given
|
||||
# fixed strings.
|
||||
#
|
||||
# Single-line matching is the point. Independent whole-log greps for an
|
||||
# npub and for `decision=denylist match` are jointly satisfied by "this
|
||||
# npub appears somewhere" plus "somebody was rejected by denylist", which
|
||||
# is not the property this suite exists to prove. Node-a lists the denied
|
||||
# peers as auto_connect peers, so it logs their npubs on the outbound
|
||||
# connect path whether or not a rejection ever happened; the npub has to
|
||||
# be on the rejection line itself to mean anything.
|
||||
#
|
||||
# Strings match in any order, by chaining fixed-string greps over the
|
||||
# surviving lines, so the assertion does not depend on the order the
|
||||
# tracing formatter emits a message and its fields in. A grep over empty
|
||||
# input yields the empty string rather than a value that could satisfy
|
||||
# the caller, so a container that cannot be read times out and fails
|
||||
# rather than passing.
|
||||
#
|
||||
# Polls rather than reading once: under the XX handshake the
|
||||
# cross-connection tie-breaker decides which side reaches its ACL check
|
||||
# first, so an inbound-handshake rejection may not emit until a later
|
||||
# retry. Same wait-with-timeout shape as wait_for_peers_exact above.
|
||||
#
|
||||
# Deliberately NOT registered in check-log-strings.py's SHELL_HELPERS,
|
||||
# for two reasons, and note that registering it would in fact capture
|
||||
# nothing: that extractor reads only a helper's FIRST argument and only
|
||||
# when it is a quoted literal free of `$` (check-log-strings.py:116),
|
||||
# whereas the first argument here is the unquoted container name
|
||||
# carrying ${FIPS_CI_NAME_SUFFIX}. Verified by running the extractor
|
||||
# with this helper added: zero hits. The same is true of the
|
||||
# `assert_log_contains` this replaced, so nothing left the check's scope.
|
||||
#
|
||||
# It should stay out of scope regardless: that check exists for strings
|
||||
# whose disappearance from src/ would let an assertion silently pass,
|
||||
# and every string here is a positive requirement, so a missing one
|
||||
# exhausts the poll and exits 1, which is loud.
|
||||
assert_log_line_contains_all() {
|
||||
local container="$1"
|
||||
local timeout="$2"
|
||||
shift 2
|
||||
|
||||
local logs surviving pattern
|
||||
for _ in $(seq 1 "$timeout"); do
|
||||
logs="$(docker logs "$container" 2>&1 | python3 -c 'import re,sys; print(re.sub(r"\x1b\[[0-9;]*m", "", sys.stdin.read()), end="")' || true)"
|
||||
surviving="$logs"
|
||||
for pattern in "$@"; do
|
||||
surviving="$(printf '%s' "$surviving" | grep -F -- "$pattern" || true)"
|
||||
done
|
||||
if [ -n "$surviving" ]; then
|
||||
echo "PASS: $container has a log line matching all of: $*"
|
||||
local count
|
||||
count=$(docker exec "$container" fipsctl show peers 2>/dev/null \
|
||||
| python3 -c 'import json,sys; data=json.load(sys.stdin); print(sum(1 for p in data.get("peers", []) if p.get("connectivity") == "connected"))' 2>/dev/null || echo 0)
|
||||
if [ "$count" -eq "$expected_count" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "FAIL: no single log line in $container contains all of: $* (waited ${timeout}s)" >&2
|
||||
echo "FAIL: $container did not reach $expected_count connected peers in ${timeout}s" >&2
|
||||
docker exec "$container" fipsctl show peers >&2 || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
assert_log_contains() {
|
||||
local container="$1"
|
||||
local pattern="$2"
|
||||
local timeout="${3:-15}"
|
||||
local logs
|
||||
|
||||
# Poll docker logs instead of one-shot reading: under XX handshake,
|
||||
# the cross-connection tie-breaker determines which side reaches
|
||||
# its ACL-check point first, so the inbound-handshake-context
|
||||
# rejection may not emit until a later retry. Same wait-with-timeout
|
||||
# shape as wait_for_peers_exact above.
|
||||
for _ in $(seq 1 "$timeout"); do
|
||||
logs="$(docker logs "$container" 2>&1 | python3 -c 'import re,sys; print(re.sub(r"\x1b\[[0-9;]*m", "", sys.stdin.read()), end="")' || true)"
|
||||
if printf '%s' "$logs" | grep -F "$pattern" >/dev/null; then
|
||||
echo "PASS: $container logs contain expected ACL rejection"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "FAIL: missing log pattern in $container: $pattern (waited ${timeout}s)" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -185,72 +129,34 @@ log "Generating ACL allowlist fixtures"
|
||||
|
||||
log "Starting ACL allowlist harness"
|
||||
docker compose -f "$COMPOSE_FILE" down >/dev/null 2>&1 || true
|
||||
# --build only on the hand path. Under a harness, --skip-build means the caller
|
||||
# has already built the image this compose file names, and rebuilding it here
|
||||
# would overwrite that image from whatever the shared build context happens to
|
||||
# hold — which is how a suite ends up certifying binaries it was never given.
|
||||
if [ "$SKIP_BUILD" = false ]; then
|
||||
docker compose -f "$COMPOSE_FILE" up -d --build
|
||||
else
|
||||
docker compose -f "$COMPOSE_FILE" up -d
|
||||
fi
|
||||
docker compose -f "$COMPOSE_FILE" up -d --build
|
||||
|
||||
log "Waiting for expected peer convergence"
|
||||
wait_for_peers_exact fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-} 3 40
|
||||
wait_for_peers_exact fips-acl-container-b${FIPS_CI_NAME_SUFFIX:-} 1 40
|
||||
wait_for_peers_exact fips-acl-container-c${FIPS_CI_NAME_SUFFIX:-} 0 5
|
||||
wait_for_peers_exact fips-acl-container-d${FIPS_CI_NAME_SUFFIX:-} 0 5
|
||||
wait_for_peers_exact fips-acl-container-e${FIPS_CI_NAME_SUFFIX:-} 1 40
|
||||
wait_for_peers_exact fips-acl-container-f${FIPS_CI_NAME_SUFFIX:-} 1 40
|
||||
wait_for_peers_exact fips-acl-container-a 3 40
|
||||
wait_for_peers_exact fips-acl-container-b 1 40
|
||||
wait_for_peers_exact fips-acl-container-c 0 5
|
||||
wait_for_peers_exact fips-acl-container-d 0 5
|
||||
wait_for_peers_exact fips-acl-container-e 1 40
|
||||
wait_for_peers_exact fips-acl-container-f 1 40
|
||||
|
||||
log "Verifying peer sets"
|
||||
assert_peer_set fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-} "npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le npub1x5z9rwzzm26q9verutx4aajhf2zw2pyp34c6whhde2zduxqav40qgq36l6 npub1ytrut7gjncn2zfnhn56c0zgftf0w6p99gf6fu8j73hzw5603zglqc9av6c"
|
||||
assert_peer_set fips-acl-container-b${FIPS_CI_NAME_SUFFIX:-} "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
|
||||
assert_peer_set fips-acl-container-c${FIPS_CI_NAME_SUFFIX:-} ""
|
||||
assert_peer_set fips-acl-container-d${FIPS_CI_NAME_SUFFIX:-} ""
|
||||
assert_peer_set fips-acl-container-e${FIPS_CI_NAME_SUFFIX:-} "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
|
||||
assert_peer_set fips-acl-container-f${FIPS_CI_NAME_SUFFIX:-} "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
|
||||
assert_peer_set fips-acl-container-a "npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le npub1x5z9rwzzm26q9verutx4aajhf2zw2pyp34c6whhde2zduxqav40qgq36l6 npub1ytrut7gjncn2zfnhn56c0zgftf0w6p99gf6fu8j73hzw5603zglqc9av6c"
|
||||
assert_peer_set fips-acl-container-b "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
|
||||
assert_peer_set fips-acl-container-c ""
|
||||
assert_peer_set fips-acl-container-d ""
|
||||
assert_peer_set fips-acl-container-e "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
|
||||
assert_peer_set fips-acl-container-f "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
|
||||
|
||||
log "Checking alias-based ACL resolution"
|
||||
assert_acl_field fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-} allow_file_entries "node-a node-b node-e node-f"
|
||||
assert_acl_field fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-} allow_entries "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le npub1x5z9rwzzm26q9verutx4aajhf2zw2pyp34c6whhde2zduxqav40qgq36l6 npub1ytrut7gjncn2zfnhn56c0zgftf0w6p99gf6fu8j73hzw5603zglqc9av6c"
|
||||
assert_acl_field fips-acl-container-c${FIPS_CI_NAME_SUFFIX:-} allow_file_entries "node-a node-b node-c node-d node-e node-f"
|
||||
assert_acl_field fips-acl-container-c${FIPS_CI_NAME_SUFFIX:-} allow_entries "npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6 npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le npub1x5z9rwzzm26q9verutx4aajhf2zw2pyp34c6whhde2zduxqav40qgq36l6 npub1ytrut7gjncn2zfnhn56c0zgftf0w6p99gf6fu8j73hzw5603zglqc9av6c"
|
||||
assert_acl_field fips-acl-container-a allow_file_entries "node-a node-b node-e node-f"
|
||||
assert_acl_field fips-acl-container-a allow_entries "npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le npub1x5z9rwzzm26q9verutx4aajhf2zw2pyp34c6whhde2zduxqav40qgq36l6 npub1ytrut7gjncn2zfnhn56c0zgftf0w6p99gf6fu8j73hzw5603zglqc9av6c"
|
||||
assert_acl_field fips-acl-container-c allow_file_entries "node-a node-b node-c node-d node-e node-f"
|
||||
assert_acl_field fips-acl-container-c allow_entries "npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6 npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le npub1x5z9rwzzm26q9verutx4aajhf2zw2pyp34c6whhde2zduxqav40qgq36l6 npub1ytrut7gjncn2zfnhn56c0zgftf0w6p99gf6fu8j73hzw5603zglqc9av6c"
|
||||
|
||||
log "Checking ACL rejection logs"
|
||||
# One assertion per denied peer, each requiring the real message, that
|
||||
# peer's npub and the denylist decision on the SAME line. The npub being
|
||||
# on the rejection line is what carries the weight here: node-a rejected
|
||||
# THIS peer. The `decision=` conjunct adds no discrimination, since
|
||||
# PeerAclDecision::allowed() returns early for AllowList and DefaultAllow
|
||||
# (src/node/acl.rs:44-46), so DenyList is the only value that can reach
|
||||
# that warn! and every rejection line carries it. It is kept because the
|
||||
# remediation prescribes it and it documents the expected decision.
|
||||
#
|
||||
# Residual, recorded rather than hidden: node-a has auto_connect stanzas
|
||||
# for both denied peers and authorizes before dialing, so it emits a
|
||||
# fully-formed rejection line for each on the outbound_connect path.
|
||||
# These two assertions are therefore satisfiable without the inbound ACL
|
||||
# check running at all. The suite still catches that, because the denied
|
||||
# peer would then connect and the peer-count assertions above would time
|
||||
# out; but these two lines alone do not prove the inbound path.
|
||||
assert_log_line_contains_all fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-} 15 \
|
||||
"Rejected peer by ACL" \
|
||||
"npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6" \
|
||||
"decision=denylist match"
|
||||
assert_log_line_contains_all fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-} 15 \
|
||||
"Rejected peer by ACL" \
|
||||
"npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl" \
|
||||
"decision=denylist match"
|
||||
# The outsider-initiated path specifically, asserted separately and not
|
||||
# per-npub. Node-a carries static stanzas for both denied peers, so each
|
||||
# can be rejected on the outbound_connect path as well and which context
|
||||
# a given npub lands in is not deterministic (see README). Requiring an
|
||||
# inbound rejection of a *named* peer would red on scheduling rather than
|
||||
# on a regression; requiring that one exists at all does not.
|
||||
assert_log_line_contains_all fips-acl-container-a${FIPS_CI_NAME_SUFFIX:-} 15 \
|
||||
"Rejected peer by ACL" \
|
||||
"context=inbound_handshake" \
|
||||
"decision=denylist match"
|
||||
assert_log_contains fips-acl-container-a "npub1cld9yay0u24davpu6c35l4vldrhzvaq66pcqtg9a0j2cnjrn9rtsxx2pe6"
|
||||
assert_log_contains fips-acl-container-a "npub1n9lpnv0592cc2ps6nm0ca3qls642vx7yjsv35rkxqzj2vgds52sqgpverl"
|
||||
assert_log_contains fips-acl-container-a "context=inbound_handshake"
|
||||
assert_log_contains fips-acl-container-a "decision=denylist match"
|
||||
|
||||
log "ACL allowlist integration test passed"
|
||||
|
||||
@@ -6,8 +6,6 @@
|
||||
networks:
|
||||
bt-net:
|
||||
driver: bridge
|
||||
labels:
|
||||
- "com.corganlabs.fips-ci=1"
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.99.0.0/24
|
||||
@@ -27,7 +25,7 @@ x-boringtun-common: &boringtun-common
|
||||
services:
|
||||
alice:
|
||||
<<: *boringtun-common
|
||||
container_name: bt-alice${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: bt-alice
|
||||
hostname: alice
|
||||
environment:
|
||||
- ROLE=alice
|
||||
@@ -38,7 +36,7 @@ services:
|
||||
|
||||
bob:
|
||||
<<: *boringtun-common
|
||||
container_name: bt-bob${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: bt-bob
|
||||
hostname: bob
|
||||
environment:
|
||||
- ROLE=bob
|
||||
|
||||
@@ -10,14 +10,14 @@ PARALLEL="${PARALLEL:-1}"
|
||||
echo "=== boringtun iperf3 throughput (single TCP stream, ${DURATION}s) ==="
|
||||
|
||||
# Run iperf3 server on alice (background), client on bob.
|
||||
docker exec -d bt-alice${FIPS_CI_NAME_SUFFIX:-} iperf3 -s -1 -B 10.99.0.1 -p 5201
|
||||
docker exec -d bt-alice iperf3 -s -1 -B 10.99.0.1 -p 5201
|
||||
sleep 1
|
||||
|
||||
# wait for tun handshake to settle (boringtun + WG keepalive)
|
||||
sleep 2
|
||||
|
||||
# Client: bob → alice over WG (10.99.0.1)
|
||||
OUT=$(docker exec bt-bob${FIPS_CI_NAME_SUFFIX:-} iperf3 -c 10.99.0.1 -p 5201 -t "$DURATION" -P "$PARALLEL" -J)
|
||||
OUT=$(docker exec bt-bob iperf3 -c 10.99.0.1 -p 5201 -t "$DURATION" -P "$PARALLEL" -J)
|
||||
|
||||
# Pull SUM bps.
|
||||
MBPS=$(echo "$OUT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(f\"{d['end']['sum_received']['bits_per_second'] / 1_000_000:.2f}\")")
|
||||
|
||||
+46
-63
@@ -18,7 +18,7 @@ Ethernet). Logs are collected and analyzed automatically.
|
||||
|
||||
```bash
|
||||
./testing/chaos/scripts/build.sh
|
||||
./testing/chaos/scripts/chaos.sh churn-mixed
|
||||
./testing/chaos/scripts/chaos.sh smoke-10
|
||||
```
|
||||
|
||||
## Available Scenarios
|
||||
@@ -29,10 +29,12 @@ Random topologies with increasing stressor intensity.
|
||||
|
||||
| Scenario | Nodes | Topology | Duration | Netem | Link Flaps | Traffic | Node Churn | Bandwidth |
|
||||
| -------- | ----- | ---------------- | -------- | ----- | ---------- | ------- | ---------- | --------- |
|
||||
| smoke-10 | 10 | random_geometric | 60s | -- | -- | -- | -- | -- |
|
||||
| chaos-10 | 10 | random_geometric | 120s | yes | yes | yes | -- | -- |
|
||||
| churn-10 | 10 | random_geometric | 600s | yes | yes | yes | yes | -- |
|
||||
| churn-20 | 20 | erdos_renyi | 600s | yes | yes | yes | yes | yes |
|
||||
|
||||
- **smoke-10**: Baseline sanity check. No stressors, just verify tree convergence.
|
||||
- **chaos-10**: Network degradation (5-50ms delay, 0-2% loss), link flaps (max 2
|
||||
down, 10-30s), and iperf traffic (max 3 concurrent). Netem mutates 30% of
|
||||
links every 15-30s between normal and degraded policies.
|
||||
@@ -42,22 +44,41 @@ Random topologies with increasing stressor intensity.
|
||||
simultaneously, bandwidth tiers (1/10/100/1000 Mbps), `protect_connectivity`
|
||||
disabled (partitions allowed).
|
||||
|
||||
### Cost-based parent selection — retired, now sans-IO unit tests
|
||||
### Cost-based parent selection
|
||||
|
||||
The cost-selection scenarios (cost-avoidance, depth-vs-cost, bottleneck-parent,
|
||||
cost-reeval, cost-stability, mixed-technology) were retired on 2026-07-23.
|
||||
Their subject was the pure `TreeState::evaluate_parent` decision — which parent
|
||||
wins on `effective_depth = depth + link_cost`, when periodic re-evaluation
|
||||
switches, and when hysteresis suppresses a flap. A Docker mesh could not test
|
||||
that reliably: the root is whichever node holds the smallest `NodeAddr`, MMP
|
||||
costs take several measurement windows to settle, and hold-down plus hysteresis
|
||||
timing all confound the outcome (a deterministic `link_swap` attempt still
|
||||
produced zero periodic switches in a full run).
|
||||
Explicit topologies with heterogeneous link types (fiber, Bluetooth, WiFi) to
|
||||
test that the spanning tree selects optimal parents based on link cost.
|
||||
|
||||
That logic is now covered by deterministic sans-IO unit tests in
|
||||
`src/tree/tests.rs` (`test_evaluate_parent_cost_*`, `..._hysteresis_*`,
|
||||
`..._effective_depth_*`), which run in the cargo quartet on every commit and
|
||||
can each be shown to fail by breaking the cost or hysteresis logic.
|
||||
| Scenario | Nodes | Shape | Link types | Duration | What it tests |
|
||||
| ----------------- | ----- | --------------- | ------------------------ | -------- | ------------------------------------------------------------------- |
|
||||
| cost-avoidance | 4 | Diamond | Fiber + Bluetooth | 120s | n04 picks fiber parent (n03) over Bluetooth parent (n02) |
|
||||
| depth-vs-cost | 4 | Linear tree | Fiber + Bluetooth | 120s | Cost tradeoff: depth vs. Bluetooth link quality |
|
||||
| bottleneck-parent | 10 | Tree with BT | Fiber + Bluetooth | 120s | n06 avoids Bluetooth bottleneck via n02, picks fiber via n03 |
|
||||
| cost-mixed-7node | 7 | Multi-type tree | Fiber + Bluetooth + WiFi | 180s | n06 prefers fiber (n03) over WiFi (n04) |
|
||||
| cost-reeval | 4 | Diamond | Fiber (mutated) | 180s | Periodic re-evaluation triggers parent switch (reeval_interval=15s) |
|
||||
| cost-stability | 4 | Diamond | WiFi (all) | 180s | Hysteresis prevents flapping when costs vary within 20% band |
|
||||
|
||||
- **cost-avoidance**, **depth-vs-cost**: Minimal scenarios validating the core
|
||||
cost formula. Bluetooth (L2CAP) links use 15-40ms delay and 2-8% loss;
|
||||
fiber uses 1-5ms delay and 0-1% loss.
|
||||
- **bottleneck-parent**: Larger topology where some nodes have both fiber and
|
||||
Bluetooth paths to choose from, and one node (n09) is stuck with Bluetooth
|
||||
(no alternative).
|
||||
- **cost-mixed-7node**: Three link technologies in one mesh. Traffic enabled.
|
||||
- **cost-reeval**: Netem mutation (50% fraction, every 12-18s) degrades random
|
||||
links. FIPS override sets `reeval_interval_secs=15` so periodic re-evaluation
|
||||
catches cost asymmetry. Look for `trigger=periodic` in logs.
|
||||
- **cost-stability**: All links are WiFi. Mutation swings costs between
|
||||
`slightly_better` and `slightly_worse` — within the hysteresis band. Expect
|
||||
≤ 5 parent switches over 180s.
|
||||
|
||||
### Mixed-technology
|
||||
|
||||
Larger explicit topologies combining multiple link technologies.
|
||||
|
||||
| Scenario | Nodes | Link types | Duration | Netem mutation | What it tests |
|
||||
| ---------------- | ----- | ------------------------ | -------- | -------------- | ------------------------------------------------ |
|
||||
| mixed-technology | 10 | Fiber + Bluetooth + WiFi | 180s | 20%/30-60s | Tree convergence across heterogeneous link types |
|
||||
|
||||
### Transport-specific
|
||||
|
||||
@@ -67,12 +88,19 @@ Explicit topologies exercising non-UDP transports.
|
||||
| ------------- | ----- | -------------- | ----- | -------- | ----- | ---------- | ------------------------------------------ |
|
||||
| ethernet-only | 4 | Ethernet | Ring | 90s | yes | -- | AF_PACKET transport with beacon discovery |
|
||||
| ethernet-mesh | 6 | UDP + Ethernet | Mesh | 120s | yes | yes | Mixed UDP/Ethernet, netem mutation + flaps |
|
||||
| tcp-only | 4 | TCP | Ring | 90s | yes | -- | TCP transport with static peer config |
|
||||
| tcp-chain | 4 | TCP | Chain | 90s | yes | -- | TCP multi-hop routing through chain |
|
||||
| tcp-mesh | 6 | UDP + TCP | Mesh | 120s | yes | yes | Mixed UDP/TCP, netem mutation + flaps |
|
||||
|
||||
- **ethernet-only**: 4-node ring on raw Ethernet (AF_PACKET). Peers discovered
|
||||
via beacons, not static config. Minimal netem (1-5ms delay).
|
||||
- **ethernet-mesh**: Mirrors `tcp-mesh` topology but with Ethernet instead of
|
||||
TCP. UDP edges use static config; Ethernet edges use beacon discovery.
|
||||
- **tcp-only**: 4-node ring using TCP on port 8443. Tests connect-on-send,
|
||||
FMP framing over TCP, and reconnection. Netem enabled (1-10ms delay, 0-1%
|
||||
loss).
|
||||
- **tcp-chain**: 4-node linear chain, all TCP. Tests multi-hop routing over
|
||||
TCP-only mesh.
|
||||
- **tcp-mesh**: 6-node mesh with 4 UDP and 3 TCP edges. Both transports use
|
||||
static peer config. Netem mutation (30% fraction, every 20-40s) and link
|
||||
flaps (1 link max, 10-20s down).
|
||||
@@ -96,14 +124,8 @@ detection.
|
||||
- **ecn-ab-on / ecn-ab-off**: Paired scenarios with identical conditions
|
||||
(6-node tree, 10 Mbps egress, 1000 kbps ingress policing, 10ms link
|
||||
delay, 8 KB recv buffer) differing only in `ecn.enabled`.
|
||||
`ecn-ab-compare.sh` runs both and prints a side-by-side of throughput
|
||||
and congestion counters. It is a manual tool, not a test: it asserts
|
||||
nothing and no runner invokes it. The "+10.2% recv throughput with ECN
|
||||
enabled" figure once recorded here is not reproducible from anything on
|
||||
disk — the script read a fixed `sim-results/ecn-ab-on/` path while the
|
||||
runner has written timestamped directories since 2026-03-20, and no
|
||||
ecn-ab result directory survives. The path bug is fixed; the figure is
|
||||
left out until a run produces one.
|
||||
`ecn-ab-test.sh` runs both and compares throughput and congestion
|
||||
counters. Initial results: +10.2% recv throughput with ECN enabled.
|
||||
|
||||
### Ingress Traffic Control
|
||||
|
||||
@@ -235,51 +257,12 @@ since they use beacon discovery.
|
||||
Results written to `sim-results/` (configurable via
|
||||
`logging.output_dir`):
|
||||
|
||||
- `status.txt` -- How the run ended, plus the scenario, the seed and the
|
||||
container names it used; one `key=value` per line
|
||||
- `analysis.txt` -- Summary: panics, errors, sessions, metrics
|
||||
- `metadata.txt` -- Seed, node count, edges, adjacency list
|
||||
- `runner.log` -- Orchestration events (topology, netem, churn, traffic) with timestamps
|
||||
- `fips-node-nXX.log` -- Per-node log output
|
||||
|
||||
The `status` field reads:
|
||||
|
||||
- `completed` -- ran for its configured duration
|
||||
- `interrupted` -- a signal cut the run short, so the artifacts are real
|
||||
but describe less time than the scenario asked for
|
||||
- `aborted` -- the run raised part way through; same caveat, and
|
||||
`runner.log` carries the traceback
|
||||
- `setup-failed` -- the containers never started
|
||||
- `teardown-failed` -- the mesh ran but its logs or analysis could not be
|
||||
produced
|
||||
|
||||
A `setup-failed` directory holds `runner.log` and `status.txt` and nothing
|
||||
else. Nothing is harvested, because container names are global to the host
|
||||
and reading them after a failed setup describes whichever run holds them
|
||||
now. So `analysis.txt` in a result directory is proof that this scenario's
|
||||
own mesh existed. A directory with no `status.txt` was written before this
|
||||
was the case and says nothing either way.
|
||||
|
||||
Exit codes:
|
||||
|
||||
- `0` -- Ran to completion, no panics, every assertion passed
|
||||
- `1` -- The scenario file could not be loaded, or a second interrupt
|
||||
arrived while the first was being handled
|
||||
- `2` -- Panics found in the collected node logs. Also what the argument
|
||||
parser exits with when it rejects the command line, before any run starts
|
||||
- `3` -- A post-run assertion failed
|
||||
- `4` -- Setup, warmup, the simulation loop or teardown raised, so the run
|
||||
did not complete; `runner.log` carries the traceback
|
||||
|
||||
Codes 2 and 3 describe what a mesh that ran did. Code 4 says there is
|
||||
nothing to describe, and takes precedence over both. Code 2 is dual-use:
|
||||
a run that never started cannot have panicked, so read it together with
|
||||
whether `runner.log` exists.
|
||||
|
||||
A run stopped by a signal exits on this same ladder rather than one of its
|
||||
own: what it collected before stopping is still worth reporting, and
|
||||
`status.txt` says it was cut short. `chaos.sh` reports 130 for a Ctrl-C of
|
||||
its own accord.
|
||||
Exit code 0 on success, 2 if panics detected.
|
||||
|
||||
## Creating Custom Scenarios
|
||||
|
||||
|
||||
@@ -1,27 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# ECN A/B Throughput Comparison (a manual tool, NOT a test)
|
||||
# ECN A/B Throughput Test
|
||||
#
|
||||
# Runs two identical chaos scenarios — one with ECN enabled, one disabled —
|
||||
# and prints a side-by-side of iperf3 throughput and congestion counters.
|
||||
# and compares iperf3 throughput and congestion counter results.
|
||||
#
|
||||
# Renamed from ecn-ab-test.sh on 2026-07-23. The old name claimed a verdict
|
||||
# this has never produced: it asserts nothing, applies no threshold, and no
|
||||
# runner invokes it. Naming it a test made it look like coverage.
|
||||
#
|
||||
# It also could not have worked. It read sim-results/ecn-ab-on/... while the
|
||||
# runner has written sim-results/<timestamp>-<scenario>/ since 2026-03-20, so
|
||||
# it has found neither input for at least four months, and there is not one
|
||||
# archived ecn-ab result directory on disk. That path bug is fixed below.
|
||||
#
|
||||
# WHAT IS STILL MISSING, and why it was not added: turning this into a real
|
||||
# test needs a threshold — how much throughput ECN should buy, or how much
|
||||
# lower the congestion counters should run — and there is no corpus to derive
|
||||
# one from, precisely because the tool has never produced a kept result. A
|
||||
# number invented here would assert the author's guess. The prerequisite is a
|
||||
# calibration run set, and that is a protocol question about what ECN is
|
||||
# expected to deliver, not a harness one.
|
||||
#
|
||||
# Usage: ./ecn-ab-compare.sh [--seed N] [--duration N]
|
||||
# Usage: ./ecn-ab-test.sh [--seed N] [--duration N]
|
||||
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
@@ -41,12 +24,12 @@ echo ""
|
||||
|
||||
# --- Run A: ECN ON ---
|
||||
echo "--- Phase A: ECN ENABLED ---"
|
||||
sudo python3 -m sim scenarios/ecn-ab-on.yaml "${EXTRA_ARGS[@]}"
|
||||
sudo python3 -m sim scenarios/ecn-ab-on.yaml "${EXTRA_ARGS[@]}" || true
|
||||
echo ""
|
||||
|
||||
# --- Run B: ECN OFF ---
|
||||
echo "--- Phase B: ECN DISABLED ---"
|
||||
sudo python3 -m sim scenarios/ecn-ab-off.yaml "${EXTRA_ARGS[@]}"
|
||||
sudo python3 -m sim scenarios/ecn-ab-off.yaml "${EXTRA_ARGS[@]}" || true
|
||||
echo ""
|
||||
|
||||
# --- Compare results ---
|
||||
@@ -54,27 +37,7 @@ echo "=== Results ==="
|
||||
echo ""
|
||||
|
||||
python3 - <<'PYEOF'
|
||||
import glob
|
||||
import json
|
||||
|
||||
|
||||
def latest(scenario, filename):
|
||||
"""Newest run directory for a scenario, or None.
|
||||
|
||||
The runner writes sim-results/<timestamp>-<scenario>/, so a fixed path
|
||||
such as sim-results/ecn-ab-on/ has never matched anything. Sorting the
|
||||
glob works because the timestamp is the leading, fixed-width component.
|
||||
"""
|
||||
dirs = sorted(glob.glob(f"sim-results/*-{scenario}"))
|
||||
if not dirs:
|
||||
print(f" no run directory found for {scenario}")
|
||||
return None
|
||||
path = f"{dirs[-1]}/{filename}"
|
||||
if not glob.glob(path):
|
||||
print(f" {scenario}: {filename} missing from {dirs[-1]}")
|
||||
return None
|
||||
return path
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
@@ -129,10 +92,8 @@ def print_sessions(label, sessions):
|
||||
print(f" {'':>14} completed={n} incomplete={incomplete}")
|
||||
return total_recv / n
|
||||
|
||||
on_path = latest("ecn-ab-on", "iperf3-results.json")
|
||||
off_path = latest("ecn-ab-off", "iperf3-results.json")
|
||||
on_results = load_results(on_path) if on_path else []
|
||||
off_results = load_results(off_path) if off_path else []
|
||||
on_results = load_results("sim-results/ecn-ab-on/iperf3-results.json")
|
||||
off_results = load_results("sim-results/ecn-ab-off/iperf3-results.json")
|
||||
|
||||
on_sessions = extract_throughput(on_results)
|
||||
off_sessions = extract_throughput(off_results)
|
||||
@@ -150,10 +111,8 @@ if avg_on and avg_off:
|
||||
|
||||
# Congestion counters
|
||||
print("Congestion Counters (final snapshot):")
|
||||
for label, scenario in [("ECN ON", "ecn-ab-on"), ("ECN OFF", "ecn-ab-off")]:
|
||||
path = latest(scenario, "congestion-snapshot-final.json")
|
||||
if path is None:
|
||||
continue
|
||||
for label, path in [("ECN ON", "sim-results/ecn-ab-on/congestion-snapshot-final.json"),
|
||||
("ECN OFF", "sim-results/ecn-ab-off/congestion-snapshot-final.json")]:
|
||||
snap = load_congestion(path)
|
||||
if not snap:
|
||||
print(f" {label}: no snapshot")
|
||||
@@ -0,0 +1,92 @@
|
||||
# Bottleneck Parent: 10-node focused Bluetooth bottleneck test
|
||||
#
|
||||
# Explicit topology with two Bluetooth (L2CAP) links that create
|
||||
# bottleneck parent candidates. Tests that nodes avoid choosing
|
||||
# Bluetooth parents when fiber alternatives exist at the same or
|
||||
# slightly greater depth.
|
||||
#
|
||||
# Topology:
|
||||
#
|
||||
# n01 (root)
|
||||
# / | \ \
|
||||
# f f f f
|
||||
# / | \ \
|
||||
# n02 n03 n04 n05
|
||||
# | / | \ |
|
||||
# BT f f BT
|
||||
# | / | \ |
|
||||
# n06 n07 n08 n09
|
||||
# |
|
||||
# f
|
||||
# |
|
||||
# n10
|
||||
#
|
||||
# Edges and link types:
|
||||
# Fiber: n01-n02, n01-n03, n01-n04, n01-n05,
|
||||
# n03-n06, n03-n07, n04-n08, n08-n10
|
||||
# Bluetooth: n02-n06, n05-n09
|
||||
#
|
||||
# Cross-links: n03-n08 (fiber) — gives n08 a fiber alternative to n04
|
||||
#
|
||||
# Test subjects:
|
||||
# - n06 has Bluetooth (n02) and fiber (n03) at depth 1 — should pick n03
|
||||
# - n09 has only Bluetooth (n05) — no alternative, stuck with BT parent
|
||||
|
||||
scenario:
|
||||
name: "bottleneck-parent"
|
||||
seed: 42
|
||||
duration_secs: 60
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 10
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n01, n04]
|
||||
- [n01, n05]
|
||||
- [n02, n06]
|
||||
- [n03, n06]
|
||||
- [n03, n07]
|
||||
- [n03, n08]
|
||||
- [n04, n08]
|
||||
- [n05, n09]
|
||||
- [n08, n10]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# Bluetooth (L2CAP) links
|
||||
- edges: ["n02-n06", "n05-n09"]
|
||||
policy:
|
||||
delay_ms: [15, 40]
|
||||
jitter_ms: [5, 15]
|
||||
loss_pct: [2, 8]
|
||||
mutation:
|
||||
interval_secs: {min: 30, max: 60}
|
||||
fraction: 0.2
|
||||
policies:
|
||||
normal:
|
||||
delay_ms: [1, 5]
|
||||
loss_pct: [0, 0.5]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 2
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 10}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -67,66 +67,6 @@ bandwidth:
|
||||
enabled: true
|
||||
tiers_mbps: [1, 10, 100, 1000]
|
||||
|
||||
# Baseline: the mesh came up, agreed on a root, and took parents. This
|
||||
# asserts nothing about which nodes survive the churn; it exists so that
|
||||
# a run in which the mesh never formed cannot report success, which
|
||||
# until now it could, because this scenario carried no assertions at
|
||||
# all.
|
||||
#
|
||||
# This one is calibrated rather than derived, because a scenario that
|
||||
# stops and starts nodes on purpose does not hold a single spanning
|
||||
# tree.
|
||||
#
|
||||
# Calibrated 2026-07-26 against fourteen runs that recorded a baseline
|
||||
# verdict, read from their assertions.txt files. Every one used this
|
||||
# file's fixed seed 42 and therefore an identical chaos schedule, so the
|
||||
# spread below is container timing rather than differing scenarios:
|
||||
#
|
||||
# distinct roots 1 2 3 4 5 -> 2 3 5 3 1 runs
|
||||
# nodes parented 9 8 7 6 5 -> the exact complement, in all 14
|
||||
# sessions 12 to 20, minimum 12
|
||||
#
|
||||
# The previous ceiling of 4 roots sat at roughly the 93rd percentile of
|
||||
# that distribution: one run in fourteen exceeded it and four sat at or
|
||||
# above it, so it reddened a share of runs whatever the daemon did. It
|
||||
# was set from six runs whose observed maximum was 3, which is how a
|
||||
# threshold one step outside a small sample ends up inside the real one.
|
||||
#
|
||||
# The ceiling is now 6, one step beyond the observed maximum of 5, and
|
||||
# the parented floor is its complement at 4. That still fails a mesh
|
||||
# that collapsed — seven or more of ten nodes islanded — which is the
|
||||
# only thing this assertion was ever meant to catch. Do not read a pass
|
||||
# as convergence: four of the six gating scenarios assert a floor of
|
||||
# this kind, and it means the mesh formed and nobody errored.
|
||||
#
|
||||
# min_sessions stays at 10 against an observed minimum of 12. It has
|
||||
# never fired and there is no evidence it is mis-set, but the margin is
|
||||
# thin and a future failure there should be read as calibration before
|
||||
# it is read as a defect.
|
||||
#
|
||||
# Tighten any of these only against a larger sample, and against one
|
||||
# gathered at the invocation CI actually runs.
|
||||
#
|
||||
# READ THIS BEFORE RETUNING: the numbers above describe the invocation CI
|
||||
# gates on, which is not this file's own defaults. ci-local runs it as
|
||||
# "churn-mixed --nodes 10 --duration 120", and --nodes sed-patches
|
||||
# num_nodes in a copy of this file before the scenario is loaded, so the
|
||||
# gating run is a 10-node 120-second one while a bare `chaos.sh
|
||||
# churn-mixed` runs the 20-node 600-second scenario written here. This is
|
||||
# the only scenario CI overrides that way.
|
||||
#
|
||||
# The floors hold for both but are calibrated for the smaller. A bare
|
||||
# 20-node run clears them easily: 20 answering, 1 root, 19 parented, 69
|
||||
# sessions, measured 2026-07-23. Retuning against numbers like those would
|
||||
# put them past what the 10-node gating run can reach, and CI would go red
|
||||
# while a manual run stayed green.
|
||||
assertions:
|
||||
baseline:
|
||||
min_nodes_reporting: 10
|
||||
max_roots: 6
|
||||
min_nodes_parented: 4
|
||||
min_sessions: 10
|
||||
|
||||
logging:
|
||||
rust_log: "debug"
|
||||
output_dir: "./sim-results"
|
||||
|
||||
@@ -7,22 +7,17 @@
|
||||
# Congestion detection signals exercised:
|
||||
# 1. MMP loss detection: netem loss exceeds the 5% loss_threshold,
|
||||
# triggering detect_congestion() via MMP metrics on transit nodes.
|
||||
# 2. Ingress policing: tc policer on the receive side drops excess
|
||||
# 2. Kernel socket drops: small recv_buf_size (8KB) combined with
|
||||
# traffic saturation causes SO_RXQ_OVFL on the UDP socket,
|
||||
# triggering the transport drop detection path.
|
||||
# 3. Ingress policing: tc policer on the receive side drops excess
|
||||
# inbound packets, creating bursty arrival patterns.
|
||||
# 3. ECN CE marking under the shaped bottleneck queue.
|
||||
#
|
||||
# The kernel socket-drop signal (SO_RXQ_OVFL) is NOT exercised here. This
|
||||
# scenario's 1 Mbps cap and ingress policer, which its ECN/MMP signals
|
||||
# require, make socket overflow impossible. It also cannot be provoked
|
||||
# deterministically anywhere in Docker — a fresh daemon reader keeps up
|
||||
# with container-speed traffic — so the FIPS drop-detection logic is
|
||||
# unit-tested directly in src/node/tests/unit.rs instead. See the note at
|
||||
# the assertions block below.
|
||||
#
|
||||
# ECN is explicitly enabled via fips_overrides.
|
||||
#
|
||||
# Success criteria (verified via post-run congestion snapshot):
|
||||
# - congestion_detected > 0 on at least one forwarding node
|
||||
# - kernel_drop_events > 0 on at least one node
|
||||
# - ce_forwarded > 0 on transit nodes
|
||||
# - ce_received > 0 on destination nodes
|
||||
#
|
||||
@@ -96,56 +91,6 @@ traffic:
|
||||
node_churn:
|
||||
enabled: false
|
||||
|
||||
# Three of the four success criteria above, encoded. Until now all four
|
||||
# existed only as that comment and were checked by nothing, so the scenario
|
||||
# could not fail on any of them.
|
||||
#
|
||||
# The floors are 1 node each because that is what the criteria say ("on at
|
||||
# least one forwarding node", "on transit nodes", "on destination nodes").
|
||||
# They are deliberately not tightened to the observed counts: the criterion
|
||||
# is the spec, and a floor invented from six runs would assert something
|
||||
# nobody wrote down.
|
||||
#
|
||||
# What the floors are worth, from the archived corpus. The six runs that
|
||||
# carry a status.txt -- the only ones provably completed, all between
|
||||
# 2026-07-22 22:14 and 2026-07-23 02:02 -- meet all three with 5 to 9 nodes
|
||||
# reporting each signal, so the margin over a floor of 1 is comfortable.
|
||||
# The 176 older runs meet none of them. Those older runs did reach teardown
|
||||
# (each wrote an analysis.txt), and ECN landed 2026-03-05, before all but
|
||||
# one of them, so they are valid runs that observed no congestion rather
|
||||
# than runs that died early. What changed on 2026-07-22 is not established:
|
||||
# this file has not been touched since it was created, and neither have
|
||||
# netem.py, traffic.py or control.py. Worth knowing before trusting a green
|
||||
# result here, and worth its own investigation.
|
||||
assertions:
|
||||
congestion_signals:
|
||||
min_nodes_detected: 1
|
||||
min_nodes_ce_forwarded: 1
|
||||
min_nodes_ce_received: 1
|
||||
|
||||
# The kernel socket-drop criterion is NOT asserted here and no longer
|
||||
# belongs to this scenario. The FIPS drop-DETECTION logic is unit-tested in
|
||||
# src/node/tests/unit.rs (2026-07-23); the kernel dropping datagrams is a
|
||||
# kernel behaviour, not FIPS's to test, and could not be provoked in Docker.
|
||||
#
|
||||
# Why it could never be met here: across all 182 archived runs of this
|
||||
# scenario, including the six that meet the three signals above, no node
|
||||
# ever reported a non-zero kernel_drop_events. SO_RXQ_OVFL counts datagrams
|
||||
# arriving at a FULL receive queue, and the 1 Mbps cap sets the arrival rate
|
||||
# to 125 kB/s per link. Linux doubles a requested SO_RCVBUF, so the queue is
|
||||
# 8192 bytes and filling it would take ~65 ms of reader stall (~22 ms even at
|
||||
# the busiest node's 3 Mbps aggregate). Traffic volume cannot cause the
|
||||
# overflow because the volume is capped below the rate the buffer drains, and
|
||||
# the ingress policer discards excess in tc before the socket ever sees it.
|
||||
# An unshaped attempt (congestion-drops, 2026-07-23) recorded zero raw drops
|
||||
# on every node even with a 4 KB buffer and heavy iperf: a fresh daemon
|
||||
# reader keeps up, so the overflow cannot be provoked deterministically. That
|
||||
# is why the detection edge is tested as a sans-IO unit test.
|
||||
#
|
||||
# The recv_buf_size: 4096 override below is kept because the three asserted
|
||||
# signals were validated with it in place; it is inert for socket overflow
|
||||
# under this scenario's cap.
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# Cost-Based Parent Selection: Bottleneck Avoidance Test
|
||||
#
|
||||
# Topology (explicit 4-node diamond):
|
||||
#
|
||||
# n01 (root — smallest addr)
|
||||
# / \
|
||||
# fiber fiber
|
||||
# / \
|
||||
# n02 n03
|
||||
# \ /
|
||||
# BT fiber
|
||||
# \ /
|
||||
# n04 (test subject)
|
||||
#
|
||||
# n04 has two candidate parents at depth 1: n02 (via Bluetooth L2CAP)
|
||||
# and n03 (via fiber). Cost-based selection should pick n03 because:
|
||||
# effective_depth(n02) = 1 + ~1.4 (Bluetooth) = ~2.4
|
||||
# effective_depth(n03) = 1 + ~1.01 (fiber) = ~2.01
|
||||
#
|
||||
# Validation: tree snapshot shows n04's parent is n03.
|
||||
|
||||
scenario:
|
||||
name: "cost-avoidance"
|
||||
seed: 42
|
||||
duration_secs: 45
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n02, n04]
|
||||
- [n03, n04]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# Bluetooth (L2CAP) link from n02 to n04
|
||||
- edges: ["n02-n04"]
|
||||
policy:
|
||||
delay_ms: [15, 40]
|
||||
jitter_ms: [5, 15]
|
||||
loss_pct: [2, 8]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 2
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 10}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,86 @@
|
||||
# Periodic Cost Re-evaluation Test
|
||||
#
|
||||
# Topology (explicit 4-node diamond):
|
||||
#
|
||||
# n01 (root)
|
||||
# / \
|
||||
# fiber fiber
|
||||
# / \
|
||||
# n02 n03
|
||||
# \ /
|
||||
# fiber fiber
|
||||
# \ /
|
||||
# n04 (test subject)
|
||||
#
|
||||
# Initial state: All links are fiber. n04 has two candidate parents at
|
||||
# depth 1: n02 and n03. Both have identical costs (~1.01), so n04 picks
|
||||
# n02 (smaller NodeAddr, tiebreak rule).
|
||||
#
|
||||
# Mutation: Stochastic netem mutation with a single "degraded" policy
|
||||
# (Bluetooth-like: 15-40ms delay, 2-8% loss). With fraction=0.5, on
|
||||
# average 2 of 4 edges degrade each round. Over 12 mutation rounds
|
||||
# (180s / 15s interval), n02-n04 will be degraded in some rounds.
|
||||
#
|
||||
# Expected behavior: When n02-n04 is degraded and n03-n04 stays fiber
|
||||
# (or vice versa), periodic re-evaluation detects the cost asymmetry
|
||||
# and switches parents. Look for "trigger = periodic" in logs.
|
||||
#
|
||||
# FIPS overrides: reeval_interval_secs=15 (vs default 60) to increase
|
||||
# the chance of catching a cost asymmetry within the mutation window.
|
||||
#
|
||||
# Validation: grep n04 logs for "Parent switched via periodic cost
|
||||
# re-evaluation" (trigger=periodic). If mutation never creates enough
|
||||
# asymmetry in a particular seed, the test still validates that periodic
|
||||
# re-eval runs without interference.
|
||||
|
||||
scenario:
|
||||
name: "cost-reeval"
|
||||
seed: 42
|
||||
duration_secs: 180
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n02, n04]
|
||||
- [n03, n04]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like baseline
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
mutation:
|
||||
interval_secs: {min: 12, max: 18}
|
||||
fraction: 0.5
|
||||
policies:
|
||||
degraded:
|
||||
delay_ms: [15, 40]
|
||||
jitter_ms: [5, 15]
|
||||
loss_pct: [2, 8]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 1
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 10}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
|
||||
fips_overrides:
|
||||
node:
|
||||
tree:
|
||||
reeval_interval_secs: 15
|
||||
@@ -0,0 +1,72 @@
|
||||
# Cost-Based Parent Selection: Hysteresis Stability Test
|
||||
#
|
||||
# Topology (explicit 4-node diamond, symmetric):
|
||||
#
|
||||
# n01 (root)
|
||||
# / \
|
||||
# wifi wifi
|
||||
# / \
|
||||
# n02 n03
|
||||
# \ /
|
||||
# wifi wifi
|
||||
# \ /
|
||||
# n04 (test subject)
|
||||
#
|
||||
# All links are WiFi-like with similar characteristics. Aggressive
|
||||
# netem mutation shifts link qualities every 10-20s, but the changes
|
||||
# stay within the 20% hysteresis band. n04 should pick one parent
|
||||
# and mostly stick with it — flapping indicates insufficient hysteresis.
|
||||
#
|
||||
# Validation: count "Parent switched" in n04 logs, expect <= 5 switches
|
||||
# over the full 180s duration.
|
||||
|
||||
scenario:
|
||||
name: "cost-stability"
|
||||
seed: 42
|
||||
duration_secs: 180
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n02, n04]
|
||||
- [n03, n04]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# WiFi baseline
|
||||
delay_ms: [5, 20]
|
||||
jitter_ms: [2, 5]
|
||||
loss_pct: [1, 3]
|
||||
mutation:
|
||||
interval_secs: {min: 10, max: 20}
|
||||
fraction: 1.0
|
||||
policies:
|
||||
slightly_better:
|
||||
delay_ms: [3, 8]
|
||||
jitter_ms: [1, 3]
|
||||
loss_pct: [0, 1]
|
||||
slightly_worse:
|
||||
delay_ms: [15, 25]
|
||||
jitter_ms: [3, 8]
|
||||
loss_pct: [2, 4]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 2
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 15}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,71 @@
|
||||
# Cost-Based Parent Selection: Deeper Fiber Beats Shallow Bluetooth
|
||||
#
|
||||
# Topology (explicit 4-node):
|
||||
#
|
||||
# n01 (root)
|
||||
# / \
|
||||
# fiber BT
|
||||
# / \
|
||||
# n02 n04 (test subject)
|
||||
# | /
|
||||
# fiber fiber
|
||||
# | /
|
||||
# n03
|
||||
#
|
||||
# n04 has two candidate parents:
|
||||
# - n01 (root, depth 0) via Bluetooth L2CAP: effective_depth = 0 + ~1.4 = ~1.4
|
||||
# - n03 (depth 2) via fiber: effective_depth = 2 + ~1.01 = ~3.01
|
||||
#
|
||||
# Without cost-based selection, n04 would pick n01 (depth 0 < depth 2).
|
||||
# With cost-based selection and these Bluetooth impairments, n04 may
|
||||
# still pick n01 since the Bluetooth cost (~1.4) is modest. This tests
|
||||
# that the cost formula correctly weighs depth against link quality.
|
||||
#
|
||||
# Validation: tree snapshot shows n04's parent selection reflects the
|
||||
# actual cost tradeoff between depth and link quality.
|
||||
|
||||
scenario:
|
||||
name: "depth-vs-cost"
|
||||
seed: 42
|
||||
duration_secs: 45
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 4
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n02, n03]
|
||||
- [n03, n04]
|
||||
- [n01, n04]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# Bluetooth (L2CAP) link from n01 to n04
|
||||
- edges: ["n01-n04"]
|
||||
policy:
|
||||
delay_ms: [15, 40]
|
||||
jitter_ms: [5, 15]
|
||||
loss_pct: [2, 8]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 1
|
||||
interval_secs: {min: 15, max: 30}
|
||||
duration_secs: {min: 5, max: 10}
|
||||
parallel_streams: 2
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -62,21 +62,6 @@ link_flaps:
|
||||
traffic:
|
||||
enabled: false
|
||||
|
||||
# Baseline: the mesh came up, agreed on a root, and took parents. This
|
||||
# asserts nothing about Ethernet link behaviour under flaps; it exists
|
||||
# so that a run in which the mesh never formed cannot report success,
|
||||
# which until now it could, because this scenario carried no assertions
|
||||
# at all.
|
||||
#
|
||||
# Six nodes, one root, five parented in all six provably-completed
|
||||
# archived runs. The other assertion covering Ethernet transport in
|
||||
# CI; see ethernet-only for why that matters.
|
||||
assertions:
|
||||
baseline:
|
||||
min_nodes_reporting: 6
|
||||
max_roots: 1
|
||||
min_nodes_parented: 5
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
|
||||
@@ -41,26 +41,6 @@ link_flaps:
|
||||
traffic:
|
||||
enabled: false
|
||||
|
||||
# Baseline: the mesh came up, agreed on a root, and took parents. This
|
||||
# asserts nothing about what Ethernet transport does; it exists so that
|
||||
# a run in which the mesh never formed cannot report success, which
|
||||
# until now it could, because this scenario carried no assertions at
|
||||
# all.
|
||||
#
|
||||
# A 4-node mesh forms a spanning tree: one root and three nodes
|
||||
# with a parent. All six provably-completed archived runs show exactly
|
||||
# that, so these are the shape of a converged mesh rather than a
|
||||
# tolerance fitted to observations. This is the only assertion covering
|
||||
# Ethernet transport anywhere in CI, and it covers the control plane
|
||||
# only: traffic is disabled above, so no datagram crosses an Ethernet
|
||||
# link in any test. Framing, the length field that trims NIC minimum-
|
||||
# frame padding, and AEAD over Ethernet are all unexercised as a result.
|
||||
assertions:
|
||||
baseline:
|
||||
min_nodes_reporting: 4
|
||||
max_roots: 1
|
||||
min_nodes_parented: 3
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Mixed Technology: 10-node heterogeneous network
|
||||
#
|
||||
# Explicit topology with Bluetooth (L2CAP), WiFi, and fiber links.
|
||||
# Tests that cost-based parent selection produces a tree favoring
|
||||
# low-cost paths when multiple link technologies coexist.
|
||||
#
|
||||
# Topology:
|
||||
#
|
||||
# n01 (root)
|
||||
# / | \
|
||||
# f f f
|
||||
# / | \
|
||||
# n02 n03 n04
|
||||
# | \ | \ | \
|
||||
# f BT f f BT f
|
||||
# | \ | | \ |
|
||||
# n05 n06 n07 n08 n09
|
||||
# \ /
|
||||
# wifi---wifi
|
||||
# n10
|
||||
#
|
||||
# Edges and link types:
|
||||
# Fiber: n01-n02, n01-n03, n01-n04, n02-n05, n03-n07, n04-n09
|
||||
# Bluetooth: n02-n06, n04-n08
|
||||
# WiFi: n03-n06, n08-n10
|
||||
# Fiber: n03-n08, n06-n10
|
||||
#
|
||||
# Test subjects:
|
||||
# - n06 has fiber (n03) and Bluetooth (n02) parents — should pick n03
|
||||
# - n08 has fiber (n03) and Bluetooth (n04) parents — should pick n03
|
||||
#
|
||||
# Netem mutation shifts fiber-only links between normal and degraded.
|
||||
|
||||
scenario:
|
||||
name: "mixed-technology"
|
||||
seed: 42
|
||||
duration_secs: 90
|
||||
|
||||
topology:
|
||||
algorithm: explicit
|
||||
num_nodes: 10
|
||||
params:
|
||||
adjacency:
|
||||
- [n01, n02]
|
||||
- [n01, n03]
|
||||
- [n01, n04]
|
||||
- [n02, n05]
|
||||
- [n02, n06]
|
||||
- [n03, n06]
|
||||
- [n03, n07]
|
||||
- [n03, n08]
|
||||
- [n04, n08]
|
||||
- [n04, n09]
|
||||
- [n06, n10]
|
||||
- [n08, n10]
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
netem:
|
||||
enabled: true
|
||||
default_policy:
|
||||
# Fiber-like defaults
|
||||
delay_ms: [1, 5]
|
||||
jitter_ms: [0, 1]
|
||||
loss_pct: [0, 0.5]
|
||||
link_policies:
|
||||
# Bluetooth (L2CAP) links
|
||||
- edges: ["n02-n06", "n04-n08"]
|
||||
policy:
|
||||
delay_ms: [15, 40]
|
||||
jitter_ms: [5, 15]
|
||||
loss_pct: [2, 8]
|
||||
# WiFi links (moderate latency, low loss)
|
||||
- edges: ["n03-n06", "n08-n10"]
|
||||
policy:
|
||||
delay_ms: [5, 20]
|
||||
jitter_ms: [2, 5]
|
||||
loss_pct: [1, 3]
|
||||
mutation:
|
||||
interval_secs: {min: 30, max: 60}
|
||||
fraction: 0.2
|
||||
policies:
|
||||
normal:
|
||||
delay_ms: [1, 10]
|
||||
loss_pct: [0, 1]
|
||||
degraded:
|
||||
delay_ms: [50, 100]
|
||||
jitter_ms: [10, 30]
|
||||
loss_pct: [3, 8]
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: true
|
||||
max_concurrent: 3
|
||||
interval_secs: {min: 10, max: 30}
|
||||
duration_secs: {min: 5, max: 15}
|
||||
parallel_streams: 4
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -0,0 +1,27 @@
|
||||
scenario:
|
||||
name: "smoke-10"
|
||||
seed: 42
|
||||
duration_secs: 30
|
||||
|
||||
topology:
|
||||
num_nodes: 10
|
||||
algorithm: random_geometric
|
||||
params:
|
||||
radius: 0.5
|
||||
ensure_connected: true
|
||||
subnet: "172.20.0.0/24"
|
||||
ip_start: 10
|
||||
|
||||
# Phase 2+ features (disabled for MVP)
|
||||
netem:
|
||||
enabled: false
|
||||
|
||||
link_flaps:
|
||||
enabled: false
|
||||
|
||||
traffic:
|
||||
enabled: false
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
@@ -61,19 +61,6 @@ link_flaps:
|
||||
traffic:
|
||||
enabled: false
|
||||
|
||||
# Baseline: the mesh came up, agreed on a root, and took parents. This
|
||||
# asserts nothing about TCP transport specifically; it exists so that a
|
||||
# run in which the mesh never formed cannot report success, which until
|
||||
# now it could, because this scenario carried no assertions at all.
|
||||
#
|
||||
# Six nodes, one root, five parented in all six provably-completed
|
||||
# archived runs.
|
||||
assertions:
|
||||
baseline:
|
||||
min_nodes_reporting: 6
|
||||
max_roots: 1
|
||||
min_nodes_parented: 5
|
||||
|
||||
logging:
|
||||
rust_log: "info"
|
||||
output_dir: "./sim-results"
|
||||
|
||||
@@ -65,7 +65,6 @@ VERBOSE=""
|
||||
SEED=""
|
||||
DURATION=""
|
||||
NODES=""
|
||||
SUBNET=""
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
@@ -73,7 +72,6 @@ while [ $# -gt 0 ]; do
|
||||
--seed) SEED="$2"; shift 2 ;;
|
||||
--duration) DURATION="$2"; shift 2 ;;
|
||||
--nodes) NODES="$2"; shift 2 ;;
|
||||
--subnet) SUBNET="$2"; shift 2 ;;
|
||||
--list) list_scenarios ;;
|
||||
-*) echo "Error: Unknown option '$1'" >&2; usage ;;
|
||||
*)
|
||||
@@ -125,11 +123,7 @@ if ! docker info &> /dev/null; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A harness that scopes its build context per run passes it in
|
||||
# FIPS_BUILD_CONTEXT and stops writing to the shared directory, so checking the
|
||||
# shared one would either fail on a clean checkout or, worse, pass while
|
||||
# reading a stale binary that is not the one under test.
|
||||
DOCKER_DIR="${FIPS_BUILD_CONTEXT:-$CHAOS_DIR/../docker}"
|
||||
DOCKER_DIR="$CHAOS_DIR/../docker"
|
||||
if [ ! -f "$DOCKER_DIR/fips" ]; then
|
||||
echo "Error: FIPS binary not found at $DOCKER_DIR/fips" >&2
|
||||
echo "Run testing/scripts/build.sh first" >&2
|
||||
@@ -141,7 +135,6 @@ PYTHON_ARGS=("$SCENARIO_FILE")
|
||||
[ -n "$VERBOSE" ] && PYTHON_ARGS+=("$VERBOSE")
|
||||
[ -n "$SEED" ] && PYTHON_ARGS+=("--seed" "$SEED")
|
||||
[ -n "$DURATION" ] && PYTHON_ARGS+=("--duration" "$DURATION")
|
||||
[ -n "$SUBNET" ] && PYTHON_ARGS+=("--subnet" "$SUBNET")
|
||||
|
||||
echo "=== FIPS Stochastic Simulation ==="
|
||||
echo ""
|
||||
@@ -150,7 +143,6 @@ echo " File: $SCENARIO_FILE"
|
||||
[ -n "$SEED" ] && echo " Seed: $SEED (override)"
|
||||
[ -n "$DURATION" ] && echo " Duration: ${DURATION}s (override)"
|
||||
[ -n "$NODES" ] && echo " Nodes: $NODES (override)"
|
||||
[ -n "$SUBNET" ] && echo " Subnet: $SUBNET (override)"
|
||||
echo ""
|
||||
|
||||
# If --nodes is specified, create a patched copy of the scenario file
|
||||
|
||||
@@ -25,13 +25,6 @@ def main():
|
||||
"--duration", type=int, default=None,
|
||||
help="Override scenario duration in seconds",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--subnet", type=str, default=None,
|
||||
help="Pin the topology subnet CIDR (e.g. 10.30.0.0/24) instead of "
|
||||
"claiming a free one. The sim normally claims a range so "
|
||||
"concurrent runs cannot collide; pin only when you need a known "
|
||||
"range, and expect a hard failure if it is already taken.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
level = logging.DEBUG if args.verbose else logging.INFO
|
||||
@@ -55,20 +48,10 @@ def main():
|
||||
print("Error: --duration must be >= 1", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
scenario.duration_secs = args.duration
|
||||
if args.subnet is not None:
|
||||
# Pinning opts out of claiming. The runner still *creates* the network,
|
||||
# so a range already in use fails loudly here rather than silently
|
||||
# overlapping a concurrent run.
|
||||
scenario.topology.pinned_subnet = args.subnet
|
||||
|
||||
runner = SimRunner(scenario)
|
||||
result = runner.run()
|
||||
|
||||
# Liveness before content. A simulation that raised on its way up, or part
|
||||
# way through, has no panic count or assertion outcome worth reporting, and
|
||||
# saying so is more useful than whatever partial content it did produce.
|
||||
if runner.aborted:
|
||||
sys.exit(4)
|
||||
if result and result.panics:
|
||||
sys.exit(2)
|
||||
if runner.assertions_failed:
|
||||
|
||||
@@ -18,15 +18,7 @@ import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .control import snapshot_all_bloom
|
||||
from .scenario import (
|
||||
BaselineAssertion,
|
||||
BloomSendRateAssertion,
|
||||
CongestionSignalsAssertion,
|
||||
MaxErrorsAssertion,
|
||||
MaxParentSwitchesAssertion,
|
||||
MinParentSwitchesAssertion,
|
||||
TreeParentsAssertion,
|
||||
)
|
||||
from .scenario import BloomSendRateAssertion, MinParentSwitchesAssertion
|
||||
from .topology import SimTopology
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -139,278 +131,6 @@ class BloomSendRateMonitor:
|
||||
)
|
||||
|
||||
|
||||
def evaluate_max_parent_switches(
|
||||
cfg: MaxParentSwitchesAssertion,
|
||||
parent_switch_count: int,
|
||||
scope: str,
|
||||
) -> AssertionOutcome:
|
||||
"""Stability ceiling on parent switches over the run.
|
||||
|
||||
``scope`` describes what was counted and appears in the message, so a
|
||||
reader can tell a per-node result from a mesh-wide one. Resolving a
|
||||
per-node scope to a real node is the caller's job, and so is failing
|
||||
loudly when it cannot: an unresolvable node would count zero switches
|
||||
and sail under any ceiling without having observed anything.
|
||||
"""
|
||||
if parent_switch_count <= cfg.max_total:
|
||||
return AssertionOutcome(
|
||||
name="max_parent_switches",
|
||||
passed=True,
|
||||
detail=(
|
||||
f"PASS max_parent_switches: {parent_switch_count} switches "
|
||||
f"({scope}) <= ceiling {cfg.max_total}"
|
||||
),
|
||||
)
|
||||
return AssertionOutcome(
|
||||
name="max_parent_switches",
|
||||
passed=False,
|
||||
detail=(
|
||||
f"FAIL max_parent_switches: {parent_switch_count} switches "
|
||||
f"({scope}) > ceiling {cfg.max_total} — the tree is reparenting "
|
||||
f"more than the hysteresis band should allow. Check whether a "
|
||||
f"cost change smaller than the hysteresis margin is still "
|
||||
f"triggering a switch."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def evaluate_baseline(
|
||||
cfg: BaselineAssertion,
|
||||
snapshot: dict | None,
|
||||
sessions: int,
|
||||
) -> AssertionOutcome:
|
||||
"""Floor on the mesh having formed: nodes answered, agreed a root, took parents."""
|
||||
if not snapshot:
|
||||
return AssertionOutcome(
|
||||
name="baseline",
|
||||
passed=False,
|
||||
detail=(
|
||||
"FAIL baseline: no final tree snapshot was taken, so nothing "
|
||||
"about the mesh was observed. This is a harness failure."
|
||||
),
|
||||
)
|
||||
|
||||
reporting = len(snapshot)
|
||||
roots = {v.get("root") for v in snapshot.values() if v.get("root")}
|
||||
parented = sum(
|
||||
1 for v in snapshot.values()
|
||||
if v.get("parent") and v.get("parent") != v.get("my_node_addr")
|
||||
)
|
||||
|
||||
parts, failures = [], []
|
||||
|
||||
def note(ok, text):
|
||||
parts.append(text)
|
||||
if not ok:
|
||||
failures.append(text)
|
||||
|
||||
if cfg.min_nodes_reporting is not None:
|
||||
note(reporting >= cfg.min_nodes_reporting,
|
||||
f"{reporting} node(s) answered (need {cfg.min_nodes_reporting})")
|
||||
if cfg.max_roots is not None:
|
||||
note(len(roots) <= cfg.max_roots and len(roots) >= 1,
|
||||
f"{len(roots)} distinct root(s) (allowed {cfg.max_roots})")
|
||||
if cfg.min_nodes_parented is not None:
|
||||
note(parented >= cfg.min_nodes_parented,
|
||||
f"{parented} node(s) have a parent (need {cfg.min_nodes_parented})")
|
||||
if cfg.min_sessions is not None:
|
||||
note(sessions >= cfg.min_sessions,
|
||||
f"{sessions} session(s) established (need {cfg.min_sessions})")
|
||||
|
||||
summary = "; ".join(parts)
|
||||
if failures:
|
||||
return AssertionOutcome(
|
||||
name="baseline",
|
||||
passed=False,
|
||||
detail=(
|
||||
f"FAIL baseline: {'; '.join(failures)}. Full: {summary}"
|
||||
),
|
||||
)
|
||||
return AssertionOutcome(
|
||||
name="baseline", passed=True, detail=f"PASS baseline: {summary}"
|
||||
)
|
||||
|
||||
|
||||
def evaluate_tree_parents(
|
||||
cfg: TreeParentsAssertion,
|
||||
snapshot: dict | None,
|
||||
) -> AssertionOutcome:
|
||||
"""Check each node's parent in the final tree snapshot.
|
||||
|
||||
Parents are compared by node address, resolved from the snapshot's own
|
||||
``my_node_addr`` fields, so the check does not depend on the display
|
||||
name a node happened to publish.
|
||||
|
||||
Every way of not knowing the answer is a failure: no snapshot, the
|
||||
node absent from it, the expected parent absent from it, or the node
|
||||
still claiming to be its own root. Each of those produces the same
|
||||
"no match" that a genuinely wrong parent does, and only saying so
|
||||
separately keeps a harness problem from reading as a routing verdict.
|
||||
"""
|
||||
if not snapshot:
|
||||
return AssertionOutcome(
|
||||
name="tree_parents",
|
||||
passed=False,
|
||||
detail=(
|
||||
"FAIL tree_parents: no final tree snapshot was taken, so no "
|
||||
"node's parent was observed. This is a harness failure, not "
|
||||
"a statement about the tree."
|
||||
),
|
||||
)
|
||||
|
||||
addr_of = {
|
||||
nid: data.get("my_node_addr")
|
||||
for nid, data in snapshot.items()
|
||||
if data.get("my_node_addr")
|
||||
}
|
||||
id_of = {addr: nid for nid, addr in addr_of.items()}
|
||||
|
||||
good, bad = [], []
|
||||
for child, want_parent in sorted(cfg.expected.items()):
|
||||
entry = snapshot.get(child)
|
||||
if entry is None:
|
||||
bad.append(
|
||||
f"{child} is absent from the snapshot ({len(snapshot)} node(s) "
|
||||
f"present: {', '.join(sorted(snapshot))})"
|
||||
)
|
||||
continue
|
||||
want_addr = addr_of.get(want_parent)
|
||||
if want_addr is None:
|
||||
bad.append(
|
||||
f"{child}: expected parent {want_parent} is absent from the "
|
||||
f"snapshot, so its address cannot be resolved"
|
||||
)
|
||||
continue
|
||||
got_addr = entry.get("parent")
|
||||
if got_addr == entry.get("my_node_addr"):
|
||||
bad.append(
|
||||
f"{child} is its own parent — it still believes it is root, "
|
||||
f"so the tree never converged around it (wanted {want_parent})"
|
||||
)
|
||||
continue
|
||||
if got_addr == want_addr:
|
||||
good.append(f"{child}->{want_parent}")
|
||||
continue
|
||||
got_id = id_of.get(got_addr) or entry.get("parent_display_name") or got_addr
|
||||
bad.append(f"{child} chose {got_id}, wanted {want_parent}")
|
||||
|
||||
if bad:
|
||||
detail = f"FAIL tree_parents: {'; '.join(bad)}"
|
||||
if good:
|
||||
detail += f". Correct: {', '.join(good)}"
|
||||
return AssertionOutcome(name="tree_parents", passed=False, detail=detail)
|
||||
return AssertionOutcome(
|
||||
name="tree_parents",
|
||||
passed=True,
|
||||
detail=f"PASS tree_parents: {', '.join(good)}",
|
||||
)
|
||||
|
||||
|
||||
_CONGESTION_FLOORS = (
|
||||
("min_nodes_detected", "congestion_detected"),
|
||||
("min_nodes_ce_forwarded", "ce_forwarded"),
|
||||
("min_nodes_ce_received", "ce_received"),
|
||||
)
|
||||
|
||||
|
||||
def evaluate_congestion_signals(
|
||||
cfg: CongestionSignalsAssertion,
|
||||
snapshot: dict | None,
|
||||
) -> AssertionOutcome:
|
||||
"""Floors on how many nodes observed each congestion counter.
|
||||
|
||||
``snapshot`` is the final congestion snapshot keyed by node id. None
|
||||
means the snapshot never ran, which fails: a missing snapshot and a
|
||||
mesh that observed no congestion produce the same zero counts, and
|
||||
treating them alike is how an assertion comes to pass on an absence
|
||||
of evidence.
|
||||
"""
|
||||
if not snapshot:
|
||||
return AssertionOutcome(
|
||||
name="congestion_signals",
|
||||
passed=False,
|
||||
detail=(
|
||||
"FAIL congestion_signals: no final congestion snapshot was "
|
||||
"taken, so no node was observed at all. This is a harness "
|
||||
"failure, not a statement about congestion."
|
||||
),
|
||||
)
|
||||
|
||||
parts, failures = [], []
|
||||
for attr, counter in _CONGESTION_FLOORS:
|
||||
floor = getattr(cfg, attr)
|
||||
if floor is None:
|
||||
continue
|
||||
hits = sorted(
|
||||
nid for nid, data in snapshot.items()
|
||||
if (data.get("congestion") or {}).get(counter, 0) > 0
|
||||
)
|
||||
parts.append(f"{counter}: {len(hits)} node(s) >0 (floor {floor})")
|
||||
if len(hits) < floor:
|
||||
failures.append(
|
||||
f"{counter} non-zero on {len(hits)} node(s), need {floor}"
|
||||
)
|
||||
else:
|
||||
parts[-1] += f" [{', '.join(hits)}]"
|
||||
|
||||
summary = "; ".join(parts)
|
||||
if failures:
|
||||
return AssertionOutcome(
|
||||
name="congestion_signals",
|
||||
passed=False,
|
||||
detail=(
|
||||
f"FAIL congestion_signals: {'; '.join(failures)} — across "
|
||||
f"{len(snapshot)} node(s) sampled. Full counts: {summary}"
|
||||
),
|
||||
)
|
||||
return AssertionOutcome(
|
||||
name="congestion_signals",
|
||||
passed=True,
|
||||
detail=f"PASS congestion_signals: {summary}",
|
||||
)
|
||||
|
||||
|
||||
def evaluate_max_errors(
|
||||
cfg: MaxErrorsAssertion,
|
||||
errors: list[tuple[str, str]],
|
||||
) -> AssertionOutcome:
|
||||
"""Ceiling on ERROR-level log lines across the whole mesh.
|
||||
|
||||
``errors`` is the ``AnalysisResult.errors`` list of ``(source, line)``
|
||||
pairs rather than a bare count, so a failure can name the nodes and
|
||||
quote the lines. A ceiling breach that only reports a number sends the
|
||||
reader back to the logs it was supposed to save them reading.
|
||||
"""
|
||||
count = len(errors)
|
||||
if count <= cfg.max_total:
|
||||
return AssertionOutcome(
|
||||
name="max_errors",
|
||||
passed=True,
|
||||
detail=(
|
||||
f"PASS max_errors: {count} ERROR line(s) mesh-wide <= "
|
||||
f"ceiling {cfg.max_total}"
|
||||
),
|
||||
)
|
||||
|
||||
per_node: dict[str, int] = {}
|
||||
for source, _line in errors:
|
||||
per_node[source] = per_node.get(source, 0) + 1
|
||||
worst = sorted(per_node.items(), key=lambda kv: -kv[1])
|
||||
breakdown = ", ".join(f"{src}={n}" for src, n in worst)
|
||||
samples = "\n".join(
|
||||
f" [{src}] {line.strip()}" for src, line in errors[:5]
|
||||
)
|
||||
return AssertionOutcome(
|
||||
name="max_errors",
|
||||
passed=False,
|
||||
detail=(
|
||||
f"FAIL max_errors: {count} ERROR line(s) mesh-wide > ceiling "
|
||||
f"{cfg.max_total} — per node: {breakdown}. First "
|
||||
f"{min(5, count)}:\n{samples}"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def evaluate_min_parent_switches(
|
||||
cfg: MinParentSwitchesAssertion,
|
||||
parent_switch_count: int,
|
||||
@@ -427,7 +147,7 @@ def evaluate_min_parent_switches(
|
||||
passed=True,
|
||||
detail=(
|
||||
f"PASS min_parent_switches: {parent_switch_count} switches "
|
||||
f"(mesh-wide) >= floor {cfg.min_total}"
|
||||
f">= floor {cfg.min_total}"
|
||||
),
|
||||
)
|
||||
return AssertionOutcome(
|
||||
@@ -435,8 +155,7 @@ def evaluate_min_parent_switches(
|
||||
passed=False,
|
||||
detail=(
|
||||
f"FAIL min_parent_switches: {parent_switch_count} switches "
|
||||
f"(mesh-wide) < floor {cfg.min_total} — harness did not induce "
|
||||
f"sufficient "
|
||||
f"< floor {cfg.min_total} — harness did not induce sufficient "
|
||||
f"parent flapping; bloom-rate assertion would be trivially "
|
||||
f"true. Check tree-snapshot-warmup.json: did the expected "
|
||||
f"node win the root election?"
|
||||
|
||||
@@ -10,13 +10,8 @@ from .scenario import Scenario
|
||||
from .topology import SimTopology
|
||||
|
||||
# Image name for the pre-built FIPS test image.
|
||||
#
|
||||
# A harness that has already built an image passes it in FIPS_TEST_IMAGE, and
|
||||
# it is then the caller's image: the runner uses it and must not rebuild it.
|
||||
# Unset means a bare run, where the shared tag is the right name and the runner
|
||||
# still builds it. Read at import, which is safe because the simulation always
|
||||
# starts as a child process with the environment already set.
|
||||
FIPS_SIM_IMAGE = os.environ.get("FIPS_TEST_IMAGE", "fips-test:latest")
|
||||
# The runner builds this once before starting containers.
|
||||
FIPS_SIM_IMAGE = "fips-test:latest"
|
||||
|
||||
# Jinja2 template for the compose file.
|
||||
# Uses a pre-built image instead of per-service build to support large topologies.
|
||||
@@ -24,12 +19,10 @@ _COMPOSE_TEMPLATE = Template(
|
||||
"""\
|
||||
networks:
|
||||
fips-net:
|
||||
# External, and created by the sim rather than by compose. The range has to
|
||||
# be *claimed* -- attempt-create, advance on docker's own overlap error --
|
||||
# so that two concurrent runs cannot select the same one, and only the
|
||||
# process that creates the network can do that. See sim/netclaim.py.
|
||||
external: true
|
||||
name: {{ network_name }}
|
||||
driver: bridge
|
||||
ipam:
|
||||
config:
|
||||
- subnet: {{ subnet }}
|
||||
|
||||
x-fips-common: &fips-common
|
||||
image: {{ image }}
|
||||
@@ -52,7 +45,7 @@ services:
|
||||
{% for node in nodes %}
|
||||
{{ node.node_id }}:
|
||||
<<: *fips-common
|
||||
container_name: {{ topology.container_name(node.node_id) }}
|
||||
container_name: fips-node-{{ node.node_id }}
|
||||
hostname: {{ node.node_id }}
|
||||
volumes:
|
||||
- ./{{ node.node_id }}.yaml:/etc/fips/fips.yaml:ro
|
||||
@@ -69,14 +62,8 @@ def generate_compose(
|
||||
topology: SimTopology,
|
||||
scenario: Scenario,
|
||||
output_dir: str,
|
||||
network_name: str,
|
||||
) -> str:
|
||||
"""Render docker-compose.yml and write to output_dir. Returns the file path.
|
||||
|
||||
``network_name`` is the docker network the sim has already claimed; the
|
||||
compose file refers to it as external rather than declaring a subnet, so
|
||||
that the claim and the creation are the same operation.
|
||||
"""
|
||||
"""Render docker-compose.yml and write to output_dir. Returns the file path."""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
nodes = [topology.nodes[nid] for nid in sorted(topology.nodes)]
|
||||
@@ -88,12 +75,11 @@ def generate_compose(
|
||||
)
|
||||
|
||||
content = _COMPOSE_TEMPLATE.render(
|
||||
network_name=network_name,
|
||||
subnet=scenario.topology.subnet,
|
||||
rust_log=scenario.logging.rust_log,
|
||||
image=FIPS_SIM_IMAGE,
|
||||
nodes=nodes,
|
||||
resolv_conf=resolv_conf,
|
||||
topology=topology,
|
||||
)
|
||||
|
||||
path = os.path.join(output_dir, "docker-compose.yml")
|
||||
|
||||
@@ -7,34 +7,6 @@ import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# `docker compose` renders its progress UI on stderr, so a failed command can
|
||||
# have hundreds of lines of captured output behind it. Log the tail: whatever
|
||||
# the daemon refused is the last thing it wrote.
|
||||
OUTPUT_TAIL_CHARS = 2000
|
||||
|
||||
|
||||
def _tail(text: str) -> str:
|
||||
"""Trim captured output to its last few KB for logging."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return "(empty)"
|
||||
if len(text) <= OUTPUT_TAIL_CHARS:
|
||||
return text
|
||||
return "...\n" + text[-OUTPUT_TAIL_CHARS:]
|
||||
|
||||
|
||||
def _decode(output) -> str:
|
||||
"""Render captured output as text.
|
||||
|
||||
A timed-out command hands back what it had written as bytes, even when
|
||||
the call asked for text, so the timeout path cannot assume either.
|
||||
"""
|
||||
if output is None:
|
||||
return ""
|
||||
if isinstance(output, bytes):
|
||||
return output.decode(errors="replace")
|
||||
return output
|
||||
|
||||
|
||||
class DockerExecError(Exception):
|
||||
def __init__(self, container, cmd, returncode, stderr):
|
||||
@@ -75,50 +47,16 @@ def docker_compose(
|
||||
timeout: int = 300,
|
||||
check: bool = True,
|
||||
) -> subprocess.CompletedProcess:
|
||||
"""Run a docker compose command with the given compose file.
|
||||
|
||||
Output is captured, so a failure's stderr is logged here before anything
|
||||
else sees it. `CalledProcessError` reports only the argv and the exit
|
||||
status, and a `check=False` caller reads neither, so without this the one
|
||||
place the daemon says what it objected to -- a container name already in
|
||||
use, an unusable subnet, a missing image -- is captured and then thrown
|
||||
away. Nothing about the success path changes.
|
||||
"""
|
||||
"""Run a docker compose command with the given compose file."""
|
||||
cmd = ["docker", "compose", "-f", compose_file] + args
|
||||
log.info("Running: %s", " ".join(cmd))
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
# A timeout raises from inside communicate(), so the return-code
|
||||
# branch below never runs and this is the only chance to say what
|
||||
# the command had managed to emit. The partials arrive as bytes
|
||||
# even under text=True.
|
||||
log.error(
|
||||
"%s timed out after %ds\nstderr: %s\nstdout: %s",
|
||||
" ".join(cmd),
|
||||
timeout,
|
||||
_tail(_decode(e.stderr)),
|
||||
_tail(_decode(e.stdout)),
|
||||
)
|
||||
raise
|
||||
if result.returncode != 0:
|
||||
log.error(
|
||||
"%s exited %d\nstderr: %s\nstdout: %s",
|
||||
" ".join(cmd),
|
||||
result.returncode,
|
||||
_tail(result.stderr),
|
||||
_tail(result.stdout),
|
||||
)
|
||||
if check:
|
||||
raise subprocess.CalledProcessError(
|
||||
result.returncode, cmd, result.stdout, result.stderr
|
||||
)
|
||||
return result
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def is_container_running(container: str) -> bool:
|
||||
|
||||
@@ -8,4 +8,4 @@ _TESTING_DIR = os.path.join(os.path.dirname(__file__), "..", "..")
|
||||
if _TESTING_DIR not in sys.path:
|
||||
sys.path.insert(0, _TESTING_DIR)
|
||||
|
||||
from lib.derive_keys import derive, derive_full # noqa: E402,F401
|
||||
from lib.derive_keys import derive # noqa: E402
|
||||
|
||||
+11
-38
@@ -29,18 +29,9 @@ __all__ = ["AnalysisResult", "analyze_logs", "collect_logs", "write_sim_metadata
|
||||
|
||||
|
||||
def collect_logs(container_names: list[str], output_dir: str) -> dict[str, str]:
|
||||
"""Collect all output (stdout + stderr) from all containers.
|
||||
|
||||
Raises RuntimeError if any container's output could not be read, or if
|
||||
there was nothing to read. `docker logs` writes its own failures to
|
||||
stderr and exits non-zero, so without the returncode check the daemon's
|
||||
"No such container" reply was stored as though it were the node's log
|
||||
and analysed as a mesh with no panics, no errors and no sessions --
|
||||
which reads exactly like a clean run.
|
||||
"""
|
||||
"""Collect all output (stdout + stderr) from all containers."""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
logs = {}
|
||||
failed = []
|
||||
|
||||
for name in container_names:
|
||||
try:
|
||||
@@ -50,35 +41,17 @@ def collect_logs(container_names: list[str], output_dir: str) -> dict[str, str]:
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except Exception as e:
|
||||
raw = result.stdout + result.stderr
|
||||
log_text = strip_ansi(raw)
|
||||
logs[name] = log_text
|
||||
|
||||
path = os.path.join(output_dir, f"{name}.log")
|
||||
with open(path, "w") as f:
|
||||
f.write(log_text)
|
||||
|
||||
except (subprocess.TimeoutExpired, Exception) as e:
|
||||
log.warning("Failed to collect logs from %s: %s", name, e)
|
||||
failed.append(name)
|
||||
continue
|
||||
|
||||
if result.returncode != 0:
|
||||
log.warning(
|
||||
"docker logs %s exited %d: %s",
|
||||
name, result.returncode, result.stderr.strip(),
|
||||
)
|
||||
failed.append(name)
|
||||
continue
|
||||
|
||||
# A container's own stderr comes back on our stderr, so both
|
||||
# streams are log content on the success path.
|
||||
log_text = strip_ansi(result.stdout + result.stderr)
|
||||
logs[name] = log_text
|
||||
|
||||
path = os.path.join(output_dir, f"{name}.log")
|
||||
with open(path, "w") as f:
|
||||
f.write(log_text)
|
||||
|
||||
if failed:
|
||||
raise RuntimeError(
|
||||
f"could not collect logs from {len(failed)}/{len(container_names)} "
|
||||
f"containers: {', '.join(failed)}"
|
||||
)
|
||||
if not logs:
|
||||
raise RuntimeError("no container logs were collected")
|
||||
logs[name] = ""
|
||||
|
||||
return logs
|
||||
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
"""Scoping suffix for names that live in a global namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def name_suffix() -> str:
|
||||
"""Return the suffix appended to globally-scoped names.
|
||||
|
||||
Docker container names and the generated-config directory are shared
|
||||
across every simulation running on the host, so the harness exports
|
||||
``FIPS_CI_NAME_SUFFIX`` to keep concurrent runs and concurrent
|
||||
scenarios apart. The suffix is empty when the variable is unset, so a
|
||||
bare ``chaos.sh`` run renders exactly the same names as it always has.
|
||||
"""
|
||||
return os.environ.get("FIPS_CI_NAME_SUFFIX", "")
|
||||
|
||||
|
||||
def veth_token(suffix: str) -> str:
|
||||
"""Shorten a name suffix to four hex characters.
|
||||
|
||||
Host interface names have only 15 characters to work with, far fewer
|
||||
than the suffix needs, so concurrent scenarios are told apart by a
|
||||
hash of it instead. Empty for an empty suffix, so a bare run's
|
||||
interface names are unchanged.
|
||||
"""
|
||||
if not suffix:
|
||||
return ""
|
||||
return hashlib.sha1(suffix.encode()).hexdigest()[:4]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Print the token for each suffix given on the command line, one per
|
||||
# line. ci-cleanup.sh reaps host interfaces by token and calls this
|
||||
# rather than re-deriving the hash, so widening the token here cannot
|
||||
# leave the reaper matching the old width.
|
||||
for arg in sys.argv[1:]:
|
||||
print(veth_token(arg))
|
||||
@@ -1,108 +0,0 @@
|
||||
"""Claim a free /24 for a scenario's network, atomically.
|
||||
|
||||
Two concurrent chaos runs used to request identical ranges: `ci-local.sh`
|
||||
derived each child's subnet from its position in the suite list, so both runs
|
||||
walked `10.30.0` through `10.30.12` and collided on every one of them. A run
|
||||
index or a hashed offset only makes a collision unlikely, and a collision is
|
||||
precisely the failure being removed, so this claims instead.
|
||||
|
||||
Claim-and-advance: attempt to create the network on a candidate range and, on
|
||||
docker's own "Pool overlaps" error, move to the next. Docker's address pool is
|
||||
then the arbiter and an overlap between two concurrent runs is *impossible*
|
||||
rather than improbable. The pattern is taken from `testing/sidecar/scripts/
|
||||
test-sidecar.sh:75-98`, which has been carrying it since 2026-07-19.
|
||||
|
||||
The claim has to happen before the topology is generated, not before the
|
||||
compose file is written: node IPs derive from the subnet inside
|
||||
`generate_topology`, and traffic shaping keys its filters on those addresses.
|
||||
Claiming later yields a network on one range and `tc` filters on another, which
|
||||
does not fail at bring-up and instead leaves the shaping matching nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Stay inside 10.30.0.0/16, which ci-local.sh already documents as clear of
|
||||
# docker's default pool (172.17-31, 192.168) and of the fixed-subnet suites in
|
||||
# 172.x. Sidecar has claimed 10.40.0.0/16; do not overlap it.
|
||||
NET_BASE = "10.30"
|
||||
NET_CANDIDATES = 200
|
||||
|
||||
|
||||
class NetworkClaimError(RuntimeError):
|
||||
"""No range could be claimed, or docker refused for another reason."""
|
||||
|
||||
|
||||
def claim_network(
|
||||
name: str,
|
||||
labels: dict[str, str] | None = None,
|
||||
candidates: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Create `name` on the first free /24 and return its CIDR.
|
||||
|
||||
`candidates` pins the search to an explicit list, which is how `--subnet`
|
||||
opts out of claiming. A single pinned candidate that is already taken
|
||||
raises rather than advancing, so pinning fails loudly instead of silently
|
||||
overlapping a concurrent run.
|
||||
|
||||
Raises NetworkClaimError if every candidate is taken, or immediately if
|
||||
docker fails for any reason other than an address-pool conflict. Retrying
|
||||
a real error 200 times would bury the reason for it.
|
||||
"""
|
||||
ranges = candidates or [
|
||||
f"{NET_BASE}.{i}.0/24" for i in range(NET_CANDIDATES)
|
||||
]
|
||||
label_args: list[str] = []
|
||||
for key, value in (labels or {}).items():
|
||||
label_args += ["--label", f"{key}={value}"]
|
||||
|
||||
for subnet in ranges:
|
||||
proc = subprocess.run(
|
||||
["docker", "network", "create", "--subnet", subnet]
|
||||
+ label_args
|
||||
+ [name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode == 0:
|
||||
log.info("Claimed network %s on %s", name, subnet)
|
||||
return subnet
|
||||
err = (proc.stderr or "") + (proc.stdout or "")
|
||||
if "ool overlaps" in err:
|
||||
continue
|
||||
raise NetworkClaimError(
|
||||
f"docker network create {name} on {subnet} failed: {err.strip()}"
|
||||
)
|
||||
|
||||
if candidates:
|
||||
raise NetworkClaimError(
|
||||
f"pinned subnet(s) {', '.join(candidates)} already in use; "
|
||||
f"another run holds the range"
|
||||
)
|
||||
raise NetworkClaimError(
|
||||
f"no free /24 in {NET_BASE}.0.0/16 after {NET_CANDIDATES} attempts"
|
||||
)
|
||||
|
||||
|
||||
def remove_network(name: str) -> None:
|
||||
"""Remove a claimed network, tolerating its absence.
|
||||
|
||||
The network is `external:` to the generated compose file, so `compose down`
|
||||
leaves it behind and this is the only thing that reclaims the range.
|
||||
Failures are logged rather than raised: teardown runs on the failure path
|
||||
too, and losing a /24 is a leak, not a reason to mask the original error.
|
||||
"""
|
||||
proc = subprocess.run(
|
||||
["docker", "network", "rm", name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
err = ((proc.stderr or "") + (proc.stdout or "")).strip()
|
||||
if "not found" in err or "No such network" in err:
|
||||
return
|
||||
log.warning("Could not remove network %s: %s", name, err)
|
||||
@@ -144,27 +144,6 @@ class NetemManager:
|
||||
len(self._edge_overrides),
|
||||
)
|
||||
|
||||
# Edges the periodic mutation must never touch (canonical "nXX-nYY").
|
||||
# Unlike a link_policy typo, which merely fails to shape a link, a typo
|
||||
# here would silently leave an asserted link unprotected and reintroduce
|
||||
# the exact flakiness the exclusion exists to remove — so an unknown
|
||||
# edge is a hard error, not a warning.
|
||||
self._mutation_exclude: set[str] = set()
|
||||
for edge_str in config.mutation.exclude_edges:
|
||||
canonical = "-".join(sorted(edge_str.split("-")))
|
||||
if canonical not in topo_edge_strs:
|
||||
raise ValueError(
|
||||
f"netem.mutation.exclude_edges references {edge_str!r}, "
|
||||
"which is not an edge in the topology"
|
||||
)
|
||||
self._mutation_exclude.add(canonical)
|
||||
if self._mutation_exclude:
|
||||
log.info(
|
||||
"Mutation excludes %d edge(s): %s",
|
||||
len(self._mutation_exclude),
|
||||
", ".join(sorted(self._mutation_exclude)),
|
||||
)
|
||||
|
||||
def _htb_rate(self, node_id: str, peer_id: str) -> str:
|
||||
"""Return the HTB rate string for a link direction."""
|
||||
rate = self._edge_rates.get((node_id, peer_id), 0)
|
||||
@@ -391,12 +370,10 @@ class NetemManager:
|
||||
if not self.config.mutation.policies:
|
||||
return
|
||||
|
||||
# Only consider edges where both endpoints are up, and never the
|
||||
# explicitly excluded ones (links an assertion depends on).
|
||||
# Only consider edges where both endpoints are up
|
||||
live_edges = [
|
||||
(a, b) for a, b in self.topology.edges
|
||||
if a not in self.down_nodes and b not in self.down_nodes
|
||||
and "-".join(sorted([a, b])) not in self._mutation_exclude
|
||||
]
|
||||
if not live_edges:
|
||||
return
|
||||
|
||||
+15
-277
@@ -12,16 +12,7 @@ import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
from .assertions import (
|
||||
AssertionOutcome,
|
||||
BloomSendRateMonitor,
|
||||
evaluate_baseline,
|
||||
evaluate_congestion_signals,
|
||||
evaluate_max_errors,
|
||||
evaluate_max_parent_switches,
|
||||
evaluate_min_parent_switches,
|
||||
evaluate_tree_parents,
|
||||
)
|
||||
from .assertions import AssertionOutcome, BloomSendRateMonitor, evaluate_min_parent_switches
|
||||
from .compose import generate_compose
|
||||
from .config_gen import write_configs
|
||||
from .control import snapshot_all_congestion, snapshot_all_mmp, snapshot_all_trees
|
||||
@@ -29,8 +20,6 @@ from .docker_exec import docker_compose
|
||||
from .link_swap import LinkSwapManager
|
||||
from .links import LinkManager
|
||||
from .logs import AnalysisResult, analyze_logs, collect_logs, write_sim_metadata
|
||||
from .naming import name_suffix
|
||||
from .netclaim import claim_network, remove_network
|
||||
from .netem import NetemManager
|
||||
from .nodes import NodeManager
|
||||
from .peer_churn import PeerChurnManager
|
||||
@@ -48,24 +37,9 @@ class SimRunner:
|
||||
self.rng = random.Random(scenario.seed)
|
||||
self.topology: SimTopology | None = None
|
||||
self.compose_file: str | None = None
|
||||
# Claimed in _setup; the compose file refers to it as external, so
|
||||
# `compose down` does not remove it and teardown must.
|
||||
self.network_name: str | None = None
|
||||
self.output_dir: str = self._resolve_output_dir(scenario)
|
||||
self._interrupted = False
|
||||
|
||||
# Set when setup, warmup or the simulation loop raises. The exception
|
||||
# is logged rather than propagated, so nothing else on the return path
|
||||
# of run() carries the fact that the simulation did not complete.
|
||||
self.aborted = False
|
||||
|
||||
# Whether this run ever owned containers. Teardown runs from a
|
||||
# finally, so it runs after a failed setup too, and container names
|
||||
# are global: without this it would harvest whatever currently
|
||||
# answers to them. topology and compose_file are both set well
|
||||
# before the containers start and so cannot stand in for it.
|
||||
self._containers_started = False
|
||||
|
||||
# Shared set of currently-down node IDs (updated by NodeManager,
|
||||
# read by NetemManager, LinkManager, TrafficManager)
|
||||
self._down_nodes: set[str] = set()
|
||||
@@ -82,51 +56,6 @@ class SimRunner:
|
||||
# Post-run assertion monitors (sampled near end of run).
|
||||
self.bloom_rate_monitor: BloomSendRateMonitor | None = None
|
||||
self.assertion_outcomes: list[AssertionOutcome] = []
|
||||
# Set by the final snapshot. None means the snapshot never ran,
|
||||
# which the congestion assertion must treat as a failure rather
|
||||
# than as an absence of congestion.
|
||||
self.final_congestion: dict | None = None
|
||||
self.final_tree: dict | None = None
|
||||
|
||||
def _evaluate_max_parent_switches(
|
||||
self, cfg, parent_switches: list[tuple[str, str]]
|
||||
) -> AssertionOutcome:
|
||||
"""Count parent switches in the configured scope and apply the ceiling.
|
||||
|
||||
A per-node scope is resolved through the topology and fails
|
||||
explicitly if the node id is not in it. That is the whole reason
|
||||
this lives here rather than in assertions.py: filtering log lines
|
||||
by an unknown container name yields zero matches, and zero
|
||||
trivially satisfies a ceiling, so a typo in ``node:`` would turn
|
||||
the assertion into an unconditional pass.
|
||||
|
||||
Note the limit of that guard. It covers an unresolvable node id
|
||||
and nothing else. A node that is in the topology but whose log is
|
||||
empty, or whose log level suppressed the event, still counts zero
|
||||
and still passes. Only the misspelling is caught here; the log
|
||||
level is guarded at load time, and an empty log for a live node
|
||||
is not guarded at all.
|
||||
"""
|
||||
if cfg.node is None:
|
||||
return evaluate_max_parent_switches(
|
||||
cfg, len(parent_switches), "mesh-wide"
|
||||
)
|
||||
|
||||
if cfg.node not in self.topology.nodes:
|
||||
known = ", ".join(sorted(self.topology.nodes))
|
||||
return AssertionOutcome(
|
||||
name="max_parent_switches",
|
||||
passed=False,
|
||||
detail=(
|
||||
f"FAIL max_parent_switches: node '{cfg.node}' is not in "
|
||||
f"this topology (nodes: {known}). Nothing was counted, so "
|
||||
f"the ceiling was never actually tested."
|
||||
),
|
||||
)
|
||||
|
||||
source = self.topology.container_name(cfg.node)
|
||||
count = sum(1 for src, _ in parent_switches if src == source)
|
||||
return evaluate_max_parent_switches(cfg, count, f"node {cfg.node}")
|
||||
|
||||
@staticmethod
|
||||
def _resolve_output_dir(scenario: Scenario) -> str:
|
||||
@@ -153,12 +82,7 @@ class SimRunner:
|
||||
self._warmup()
|
||||
self._simulation_loop()
|
||||
except Exception:
|
||||
# Log rather than re-raise: the default excepthook writes to stderr
|
||||
# and would not reach the file handler installed in _setup(), so a
|
||||
# re-raise would lose the traceback from runner.log, which is the
|
||||
# artifact that survives into CI. The flag carries the abort out.
|
||||
log.exception("Simulation failed")
|
||||
self.aborted = True
|
||||
finally:
|
||||
result = self._teardown()
|
||||
|
||||
@@ -189,28 +113,6 @@ class SimRunner:
|
||||
logging.getLogger().addHandler(fh)
|
||||
log.info("Runner log: %s", runner_log_path)
|
||||
|
||||
# 0. Claim this run's network range, before anything derives from it.
|
||||
#
|
||||
# The ordering is not obvious and it matters: node IPs are computed
|
||||
# from the subnet inside generate_topology, and traffic shaping keys
|
||||
# its filters on those addresses. Claiming after the topology exists
|
||||
# would give a network on one range and `tc` filters on another, which
|
||||
# does not fail at bring-up — it silently leaves the shaping matching
|
||||
# nothing.
|
||||
self.network_name = f"fips-sim{name_suffix()}-net"
|
||||
s.topology.subnet = claim_network(
|
||||
self.network_name,
|
||||
labels={
|
||||
"com.corganlabs.fips-ci": "1",
|
||||
"com.corganlabs.fips-ci.run": os.environ.get(
|
||||
"FIPS_CI_RUN_ID", "manual"
|
||||
),
|
||||
},
|
||||
candidates=(
|
||||
[s.topology.pinned_subnet] if s.topology.pinned_subnet else None
|
||||
),
|
||||
)
|
||||
|
||||
# 1. Generate topology
|
||||
log.info(
|
||||
"Generating %d-node %s topology (seed=%d)...",
|
||||
@@ -231,17 +133,9 @@ class SimRunner:
|
||||
log.info(" %s: peers=%s", nid, ",".join(peers))
|
||||
|
||||
# 2. Generate configs
|
||||
#
|
||||
# The directory carries the same suffix as the container names, so
|
||||
# parallel scenarios cannot overwrite each other's compose file
|
||||
# between writing it and starting containers from it.
|
||||
docker_network_dir = os.path.join(os.path.dirname(__file__), "..")
|
||||
config_dir = os.path.normpath(
|
||||
os.path.join(
|
||||
docker_network_dir,
|
||||
"generated-configs",
|
||||
f"sim{self.topology.name_suffix}",
|
||||
)
|
||||
os.path.join(docker_network_dir, "generated-configs", "sim")
|
||||
)
|
||||
# Select ephemeral identity nodes (if peer churn enabled)
|
||||
self._ephemeral_nodes: set[str] = set()
|
||||
@@ -263,44 +157,21 @@ class SimRunner:
|
||||
log.info("Wrote node configs to %s", config_dir)
|
||||
|
||||
# 3. Generate docker-compose.yml
|
||||
self.compose_file = generate_compose(
|
||||
self.topology, self.scenario, config_dir, self.network_name
|
||||
)
|
||||
self.compose_file = generate_compose(self.topology, self.scenario, config_dir)
|
||||
log.info("Wrote %s", self.compose_file)
|
||||
|
||||
# 4. Obtain the test image (once, rather than per-service at scale).
|
||||
#
|
||||
# Building it is right for a bare run and wrong under a harness. When
|
||||
# FIPS_TEST_IMAGE is set the image belongs to the caller, and every
|
||||
# scenario of a parallel run would otherwise rebuild it into one shared
|
||||
# name from one shared context — so assert it exists and fail loudly if
|
||||
# it does not, rather than manufacture a substitute nobody asked for.
|
||||
# 4. Build the test image once (avoids per-service build at scale)
|
||||
log.info("Building Docker image...")
|
||||
from .compose import FIPS_SIM_IMAGE
|
||||
if os.environ.get("FIPS_TEST_IMAGE"):
|
||||
log.info("Using caller-supplied image %s", FIPS_SIM_IMAGE)
|
||||
probe = subprocess.run(
|
||||
["docker", "image", "inspect", FIPS_SIM_IMAGE],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
if probe.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"FIPS_TEST_IMAGE names {FIPS_SIM_IMAGE}, which is not present. "
|
||||
"The harness that set it is expected to have built it."
|
||||
)
|
||||
else:
|
||||
log.info("Building Docker image...")
|
||||
docker_dir = os.environ.get("FIPS_BUILD_CONTEXT") or os.path.join(
|
||||
os.path.dirname(__file__), "..", "..", "docker"
|
||||
)
|
||||
subprocess.run(
|
||||
["docker", "build", "-t", FIPS_SIM_IMAGE, docker_dir],
|
||||
check=True,
|
||||
)
|
||||
docker_dir = os.path.join(os.path.dirname(__file__), "..", "..", "docker")
|
||||
subprocess.run(
|
||||
["docker", "build", "-t", FIPS_SIM_IMAGE, docker_dir],
|
||||
check=True,
|
||||
)
|
||||
|
||||
# 5. Start containers
|
||||
log.info("Starting %d containers...", len(self.topology.nodes))
|
||||
docker_compose(self.compose_file, ["up", "-d"])
|
||||
self._containers_started = True
|
||||
|
||||
# 6. Set up veth pairs for Ethernet edges (before netem)
|
||||
#
|
||||
@@ -538,69 +409,11 @@ class SimRunner:
|
||||
def assertions_failed(self) -> bool:
|
||||
return any(not o.passed for o in self.assertion_outcomes)
|
||||
|
||||
def _run_status(self) -> str:
|
||||
"""Name for how the run itself ended, before teardown is considered."""
|
||||
if self.aborted:
|
||||
return "aborted"
|
||||
if self._interrupted:
|
||||
return "interrupted"
|
||||
return "completed"
|
||||
|
||||
def _write_status(self, status: str) -> None:
|
||||
"""Record how the run ended, for a reader who has only this directory.
|
||||
|
||||
One field per line so it can be grepped. The container names are
|
||||
included because they are the handle on everything else the run
|
||||
touched, and because a directory that names them cannot be mistaken
|
||||
for a directory assembled from some other scenario's mesh.
|
||||
|
||||
Never raises. Both callers run this immediately before stopping the
|
||||
containers, so letting a full disk or a vanished output directory
|
||||
out of here would leak the mesh it was about to tear down.
|
||||
"""
|
||||
try:
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
path = os.path.join(self.output_dir, "status.txt")
|
||||
with open(path, "w") as f:
|
||||
f.write(f"status={status}\n")
|
||||
f.write(f"scenario={self.scenario.name}\n")
|
||||
f.write(f"seed={self.scenario.seed}\n")
|
||||
f.write(f"containers=fips-node-nNN{name_suffix()}\n")
|
||||
except Exception:
|
||||
log.exception("Could not write status file")
|
||||
|
||||
def _release_network(self) -> None:
|
||||
"""Give this run's claimed /24 back.
|
||||
|
||||
The compose file declares the network `external:`, so `compose down`
|
||||
leaves it alone and nothing else reclaims the range. Called from both
|
||||
teardown paths, including the setup-failed one, because a run that
|
||||
fell over after claiming still holds a range.
|
||||
"""
|
||||
if self.network_name:
|
||||
remove_network(self.network_name)
|
||||
|
||||
def _teardown(self) -> AnalysisResult | None:
|
||||
"""Stop dynamic elements, collect logs, analyze, stop containers."""
|
||||
if not self._containers_started:
|
||||
# There is no mesh to harvest. Snapshots, logs, analysis and
|
||||
# assertions all address containers by a name that is global to
|
||||
# the host, so running them here would describe whoever holds
|
||||
# those names now and leave a plausible report of a run that
|
||||
# never happened. Leaving them out makes the presence of
|
||||
# analysis.txt proof that this scenario's own mesh existed.
|
||||
self._write_status("setup-failed")
|
||||
if self.compose_file:
|
||||
# `up -d` can fail part way through, so this run may own
|
||||
# containers or a network even with no mesh to speak of.
|
||||
log.info("Stopping containers...")
|
||||
docker_compose(self.compose_file, ["down"], check=False)
|
||||
self._release_network()
|
||||
return None
|
||||
|
||||
result = None
|
||||
status = self._run_status()
|
||||
try:
|
||||
|
||||
if self.topology and self.compose_file:
|
||||
# Evaluate post-run assertions before doing any teardown so
|
||||
# control sockets are still reachable.
|
||||
self._evaluate_assertions()
|
||||
@@ -647,10 +460,7 @@ class SimRunner:
|
||||
print(result.summary())
|
||||
|
||||
# Log-derived assertions (evaluated after analyze_logs so
|
||||
# parent_switches and similar are populated). See
|
||||
# _evaluate_max_parent_switches for why the per-node variant
|
||||
# resolves its node against the topology rather than filtering
|
||||
# optimistically.
|
||||
# parent_switches and similar are populated).
|
||||
mps_cfg = self.scenario.assertions.min_parent_switches
|
||||
if mps_cfg is not None:
|
||||
outcome = evaluate_min_parent_switches(
|
||||
@@ -662,60 +472,6 @@ class SimRunner:
|
||||
else:
|
||||
log.error("%s", outcome.detail)
|
||||
|
||||
xps_cfg = self.scenario.assertions.max_parent_switches
|
||||
if xps_cfg is not None:
|
||||
outcome = self._evaluate_max_parent_switches(
|
||||
xps_cfg, result.parent_switches
|
||||
)
|
||||
self.assertion_outcomes.append(outcome)
|
||||
if outcome.passed:
|
||||
log.info("%s", outcome.detail)
|
||||
else:
|
||||
log.error("%s", outcome.detail)
|
||||
|
||||
# Applied to every scenario by default, so this is the one
|
||||
# assertion that is present even when the YAML declares no
|
||||
# assertions block at all.
|
||||
err_cfg = self.scenario.assertions.max_errors
|
||||
if err_cfg is not None:
|
||||
outcome = evaluate_max_errors(err_cfg, result.errors)
|
||||
self.assertion_outcomes.append(outcome)
|
||||
if outcome.passed:
|
||||
log.info("%s", outcome.detail)
|
||||
else:
|
||||
log.error("%s", outcome.detail)
|
||||
|
||||
bl_cfg = self.scenario.assertions.baseline
|
||||
if bl_cfg is not None:
|
||||
outcome = evaluate_baseline(
|
||||
bl_cfg, self.final_tree, len(result.sessions_established)
|
||||
)
|
||||
self.assertion_outcomes.append(outcome)
|
||||
if outcome.passed:
|
||||
log.info("%s", outcome.detail)
|
||||
else:
|
||||
log.error("%s", outcome.detail)
|
||||
|
||||
tp_cfg = self.scenario.assertions.tree_parents
|
||||
if tp_cfg is not None:
|
||||
outcome = evaluate_tree_parents(tp_cfg, self.final_tree)
|
||||
self.assertion_outcomes.append(outcome)
|
||||
if outcome.passed:
|
||||
log.info("%s", outcome.detail)
|
||||
else:
|
||||
log.error("%s", outcome.detail)
|
||||
|
||||
cong_cfg = self.scenario.assertions.congestion_signals
|
||||
if cong_cfg is not None:
|
||||
outcome = evaluate_congestion_signals(
|
||||
cong_cfg, self.final_congestion
|
||||
)
|
||||
self.assertion_outcomes.append(outcome)
|
||||
if outcome.passed:
|
||||
log.info("%s", outcome.detail)
|
||||
else:
|
||||
log.error("%s", outcome.detail)
|
||||
|
||||
# Write assertion outcomes
|
||||
if self.assertion_outcomes:
|
||||
assertions_path = os.path.join(self.output_dir, "assertions.txt")
|
||||
@@ -742,26 +498,14 @@ class SimRunner:
|
||||
if self.veth_mgr:
|
||||
log.info("Cleaning up veth pairs...")
|
||||
self.veth_mgr.teardown_all()
|
||||
except Exception:
|
||||
# Same reasoning as run(): logging keeps the traceback in
|
||||
# runner.log, where re-raising would send it to stderr instead
|
||||
# and past the exit ladder, which would report a harvest that
|
||||
# fell over as a bad command line.
|
||||
log.exception("Teardown failed")
|
||||
self.aborted = True
|
||||
status = "teardown-failed"
|
||||
result = None
|
||||
finally:
|
||||
# Status first: it is the one artifact that must exist whatever
|
||||
# else happens, and stopping containers can still time out.
|
||||
self._write_status(status)
|
||||
|
||||
# Stop containers
|
||||
log.info("Stopping containers...")
|
||||
docker_compose(
|
||||
self.compose_file,
|
||||
["down"],
|
||||
check=False,
|
||||
)
|
||||
self._release_network()
|
||||
|
||||
return result
|
||||
|
||||
@@ -777,12 +521,6 @@ class SimRunner:
|
||||
tree_path = os.path.join(self.output_dir, f"tree-snapshot-{label}.json")
|
||||
mmp_path = os.path.join(self.output_dir, f"mmp-snapshot-{label}.json")
|
||||
congestion_path = os.path.join(self.output_dir, f"congestion-snapshot-{label}.json")
|
||||
# Retained for the congestion assertion, which needs the same
|
||||
# responses the file gets. Reading the file back would let a write
|
||||
# failure present as an assertion that saw nothing.
|
||||
if label == "final":
|
||||
self.final_congestion = congestion_snap
|
||||
self.final_tree = tree_snap
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
with open(tree_path, "w") as f:
|
||||
json.dump(tree_snap, f, indent=2)
|
||||
|
||||
@@ -3,16 +3,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import yaml
|
||||
|
||||
# Node ids are rendered nNN by the topology generator, zero-padded to two
|
||||
# digits. Matching the shape here keeps a typo such as "no4" or "n4x" from
|
||||
# reaching an assertion as a node that simply never appears.
|
||||
_NODE_ID_RE = re.compile(r"n\d+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Range:
|
||||
@@ -44,18 +38,6 @@ class TopologyConfig:
|
||||
# When set, each edge is randomly assigned a transport based on weights.
|
||||
# Only valid for non-explicit algorithms (explicit uses per-edge syntax).
|
||||
transport_mix: dict[str, float] | None = None
|
||||
# Assign derived identities in NodeAddr order so n01 holds the smallest and
|
||||
# is therefore the root every scenario diagram draws. Default on: without it
|
||||
# the root is whichever node happens to hold the smallest key, which is
|
||||
# effectively arbitrary and leaves any scenario that reasons about a
|
||||
# specific root (e.g. bloom-storm's diamond, or a baseline max_roots check)
|
||||
# describing a tree that does not form. Set false where election from an
|
||||
# arbitrary key distribution is itself the subject.
|
||||
pin_root: bool = True
|
||||
# Set by --subnet to opt out of claiming a free range. Not a scenario-file
|
||||
# key: a scenario that hardcoded its range would reintroduce the collision
|
||||
# claiming exists to remove.
|
||||
pinned_subnet: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -73,11 +55,6 @@ class NetemMutationConfig:
|
||||
interval_secs: Range = field(default_factory=lambda: Range(15, 30))
|
||||
fraction: float = 0.3
|
||||
policies: dict[str, NetemPolicy] = field(default_factory=dict)
|
||||
# Edges the periodic mutation must never touch, "nXX-nYY" strings. Use it
|
||||
# to pin the links a tree_parents assertion depends on so a random
|
||||
# degradation cannot flip the very comparison being asserted. Validated
|
||||
# against the topology in NetemManager — an unknown edge is a hard error.
|
||||
exclude_edges: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -226,114 +203,12 @@ class MinParentSwitchesAssertion:
|
||||
min_total: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class MaxParentSwitchesAssertion:
|
||||
"""Stability ceiling: fail if parent switches exceed ``max_total``.
|
||||
|
||||
``node`` scopes the count to a single node id (e.g. ``n04``) instead
|
||||
of the mesh-wide total. The distinction is not cosmetic: a criterion
|
||||
written about one node's log is a different number from the sum over
|
||||
every node, and checking the sum against a per-node threshold
|
||||
silently tests something other than what was specified.
|
||||
"""
|
||||
|
||||
max_total: int = 0
|
||||
node: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaselineAssertion:
|
||||
"""Floor on the mesh having formed at all.
|
||||
|
||||
Deliberately weak and deliberately universal. It does not describe any
|
||||
scenario's subject; it says the nodes came up, agreed on a root, and
|
||||
took parents. Its value is that a scenario with no other assertion
|
||||
still cannot pass while the mesh is dead, which is the state twelve of
|
||||
thirteen scenarios were previously unable to distinguish from success.
|
||||
|
||||
``max_roots`` is how many distinct root values the snapshot may carry.
|
||||
One means the whole mesh agreed; a churn scenario that partitions on
|
||||
purpose needs a higher number, and setting it above one is a statement
|
||||
that partition is expected rather than an oversight.
|
||||
"""
|
||||
|
||||
min_nodes_reporting: int | None = None
|
||||
max_roots: int | None = None
|
||||
min_nodes_parented: int | None = None
|
||||
min_sessions: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TreeParentsAssertion:
|
||||
"""Expected parent per node in the final tree snapshot.
|
||||
|
||||
``expected`` maps a node id to the node id its parent must be, e.g.
|
||||
``{"n04": "n03"}``. Both sides are node ids rather than node
|
||||
addresses: an address is a per-run key derived from the generated
|
||||
identity, so a scenario could not name one ahead of time.
|
||||
|
||||
A node absent from the snapshot fails rather than being skipped. That
|
||||
case is not hypothetical — more than half the archived runs of these
|
||||
scenarios have no entry for the node under test, and a skip would
|
||||
have reported those as satisfied.
|
||||
"""
|
||||
|
||||
expected: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CongestionSignalsAssertion:
|
||||
"""Floors on how many nodes observed each congestion signal.
|
||||
|
||||
Each floor is the number of nodes whose final congestion snapshot
|
||||
reports a non-zero counter, not the counter's own magnitude: the
|
||||
scenario's written criteria are about *whether* a signal reached a
|
||||
class of node, and a single node with a huge count would satisfy a
|
||||
magnitude test while proving the signal never propagated.
|
||||
|
||||
A floor left unset is not asserted. At least one must be set, since a
|
||||
block that asserts nothing is the failure this assertion exists to
|
||||
remove.
|
||||
"""
|
||||
|
||||
min_nodes_detected: int | None = None
|
||||
min_nodes_ce_forwarded: int | None = None
|
||||
min_nodes_ce_received: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class MaxErrorsAssertion:
|
||||
"""Ceiling on ERROR-level lines across every node's log.
|
||||
|
||||
Unlike the other assertions this one is applied to every scenario
|
||||
whether or not the YAML asks for it, because it is a floor on what a
|
||||
green run means rather than a property of one scenario: without it a
|
||||
run in which every node errors on every line still exits 0.
|
||||
|
||||
``max_total`` defaults to 0, which is what the archived corpus
|
||||
supports. Across 2416 archived run directories no node log contains a
|
||||
single ERROR-level line, while the sibling WARN counter extracted by
|
||||
the same code path ranges from 0 to 1135 — so 0 is an observed value
|
||||
and not an aspiration, and the counter is known to discriminate.
|
||||
|
||||
A scenario that legitimately induces errors raises the ceiling in its
|
||||
own YAML and must say at that site why the errors are expected.
|
||||
"""
|
||||
|
||||
max_total: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssertionsConfig:
|
||||
"""Optional post-run assertions evaluated against control-socket data."""
|
||||
|
||||
bloom_send_rate: BloomSendRateAssertion | None = None
|
||||
min_parent_switches: MinParentSwitchesAssertion | None = None
|
||||
max_parent_switches: MaxParentSwitchesAssertion | None = None
|
||||
max_errors: MaxErrorsAssertion | None = None
|
||||
congestion_signals: CongestionSignalsAssertion | None = None
|
||||
tree_parents: TreeParentsAssertion | None = None
|
||||
baseline: BaselineAssertion | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -364,105 +239,15 @@ class Scenario:
|
||||
fips_overrides: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
# Keys each fixed-schema section understands. A key absent from these sets is a
|
||||
# typo, and silently ignoring it leaves the block default-constructed while the
|
||||
# YAML still looks correct — a mistyped "assertion:" disarms a scenario's only
|
||||
# assertions, a mistyped "link_flap:" turns off the chaos it was meant to inject.
|
||||
#
|
||||
# Four mappings are deliberately NOT listed, because their keys are names the
|
||||
# scenario author chooses rather than a schema: netem.mutation.policies,
|
||||
# link_swap.policies, topology.transport_mix and assertions.tree_parents (whose
|
||||
# keys are node ids). Two sub-trees are passed through whole and are likewise
|
||||
# not checked: fips_overrides and topology.params. tree_parents is not thereby
|
||||
# unchecked -- both sides of every entry are validated as node ids that exist in
|
||||
# this scenario's topology, which is the check that matters for it.
|
||||
#
|
||||
# NOTE: adding a new assertion type means registering it in TWO places below —
|
||||
# _SECTION_KEYS["assertions"] (so the block accepts its name) and _ASSERTION_KEYS
|
||||
# (so its own members are checked) — or scenarios using it are rejected at load.
|
||||
_TOP_KEYS = {
|
||||
"scenario", "topology", "netem", "link_flaps", "traffic", "node_churn",
|
||||
"peer_churn", "bandwidth", "ingress", "link_swap", "assertions", "logging",
|
||||
"fips_overrides",
|
||||
}
|
||||
_SECTION_KEYS = {
|
||||
"scenario": {"name", "seed", "duration_secs"},
|
||||
"topology": {
|
||||
"num_nodes", "algorithm", "params", "ensure_connected", "subnet",
|
||||
"ip_start", "default_transport", "transport_mix", "pin_root",
|
||||
},
|
||||
"netem": {"enabled", "default_policy", "link_policies", "mutation"},
|
||||
"netem.link_policies[]": {"edges", "policy", "policy_name"},
|
||||
"netem.mutation": {"interval_secs", "fraction", "policies", "exclude_edges"},
|
||||
"link_flaps": {
|
||||
"enabled", "interval_secs", "max_down_links", "down_duration_secs",
|
||||
"protect_connectivity",
|
||||
},
|
||||
"traffic": {
|
||||
"enabled", "max_concurrent", "interval_secs", "duration_secs",
|
||||
"parallel_streams",
|
||||
},
|
||||
"node_churn": {
|
||||
"enabled", "interval_secs", "max_down_nodes", "down_duration_secs",
|
||||
"protect_connectivity",
|
||||
},
|
||||
"peer_churn": {"enabled", "interval_secs", "ephemeral_fraction"},
|
||||
"bandwidth": {"enabled", "tiers_mbps"},
|
||||
"ingress": {"enabled", "tiers_kbps", "burst_bytes"},
|
||||
"link_swap": {"enabled", "interval_secs", "policies", "edges"},
|
||||
"link_swap.edges[]": {"edge", "policy"},
|
||||
"assertions": {
|
||||
"bloom_send_rate", "min_parent_switches", "max_parent_switches",
|
||||
"max_errors", "congestion_signals", "tree_parents", "baseline",
|
||||
},
|
||||
"logging": {"rust_log", "output_dir"},
|
||||
}
|
||||
_ASSERTION_KEYS = {
|
||||
"bloom_send_rate": {"window_secs", "max_per_node"},
|
||||
"min_parent_switches": {"min_total"},
|
||||
"max_parent_switches": {"max_total", "node"},
|
||||
"max_errors": {"max_total"},
|
||||
"congestion_signals": {
|
||||
"min_nodes_detected", "min_nodes_ce_forwarded", "min_nodes_ce_received",
|
||||
},
|
||||
"baseline": {
|
||||
"min_nodes_reporting", "max_roots", "min_nodes_parented", "min_sessions",
|
||||
},
|
||||
}
|
||||
_NETEM_POLICY_KEYS = {
|
||||
"delay_ms", "jitter_ms", "loss_pct", "duplicate_pct", "reorder_pct",
|
||||
"corrupt_pct",
|
||||
}
|
||||
_RANGE_KEYS = {"min", "max"}
|
||||
|
||||
|
||||
def _reject_unknown(data, known, context: str):
|
||||
"""Raise unless every key in `data` is one the loader understands."""
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
f"{context}: section is present but empty; give it a body or remove it"
|
||||
)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{context}: expected a mapping, got {type(data).__name__}")
|
||||
unknown = sorted(str(k) for k in set(data) - set(known))
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"{context}: unknown key(s): {', '.join(unknown)} "
|
||||
f"(known: {', '.join(sorted(known))})"
|
||||
)
|
||||
|
||||
|
||||
def _parse_range(data, name: str) -> Range:
|
||||
"""Parse a {min, max} dict into a Range."""
|
||||
if isinstance(data, dict):
|
||||
_reject_unknown(data, _RANGE_KEYS, name)
|
||||
return Range(min=float(data["min"]), max=float(data["max"]))
|
||||
raise ValueError(f"{name}: expected {{min, max}} dict, got {type(data).__name__}")
|
||||
|
||||
|
||||
def _parse_netem_policy(data: dict, name: str = "netem policy") -> NetemPolicy:
|
||||
def _parse_netem_policy(data: dict) -> NetemPolicy:
|
||||
"""Parse a netem policy from a dict with [min, max] lists or {min, max} dicts."""
|
||||
_reject_unknown(data, _NETEM_POLICY_KEYS, name)
|
||||
policy = NetemPolicy()
|
||||
for attr in (
|
||||
"delay_ms",
|
||||
@@ -488,22 +273,16 @@ def load_scenario(path: str) -> Scenario:
|
||||
with open(path) as f:
|
||||
raw = yaml.safe_load(f)
|
||||
|
||||
if raw is None:
|
||||
raise ValueError(f"{path}: file is empty")
|
||||
_reject_unknown(raw, _TOP_KEYS, "(top level)")
|
||||
|
||||
s = Scenario()
|
||||
|
||||
# Scenario section
|
||||
sc = raw.get("scenario", {})
|
||||
_reject_unknown(sc, _SECTION_KEYS["scenario"], "scenario")
|
||||
s.name = sc.get("name", os.path.splitext(os.path.basename(path))[0])
|
||||
s.seed = int(sc.get("seed", 42))
|
||||
s.duration_secs = int(sc.get("duration_secs", 120))
|
||||
|
||||
# Topology section
|
||||
tc = raw.get("topology", {})
|
||||
_reject_unknown(tc, _SECTION_KEYS["topology"], "topology")
|
||||
s.topology.num_nodes = int(tc.get("num_nodes", 10))
|
||||
s.topology.algorithm = tc.get("algorithm", "random_geometric")
|
||||
s.topology.params = tc.get("params", {})
|
||||
@@ -511,13 +290,6 @@ def load_scenario(path: str) -> Scenario:
|
||||
s.topology.subnet = tc.get("subnet", "172.20.0.0/24")
|
||||
s.topology.ip_start = int(tc.get("ip_start", 10))
|
||||
s.topology.default_transport = tc.get("default_transport", "udp")
|
||||
if "pin_root" in tc:
|
||||
pin = tc["pin_root"]
|
||||
if not isinstance(pin, bool):
|
||||
raise ValueError(
|
||||
f"topology.pin_root must be a boolean, got {pin!r}"
|
||||
)
|
||||
s.topology.pin_root = pin
|
||||
if "transport_mix" in tc:
|
||||
mix = tc["transport_mix"]
|
||||
if not isinstance(mix, dict) or not mix:
|
||||
@@ -526,44 +298,33 @@ def load_scenario(path: str) -> Scenario:
|
||||
|
||||
# Netem section
|
||||
nc = raw.get("netem", {})
|
||||
_reject_unknown(nc, _SECTION_KEYS["netem"], "netem")
|
||||
s.netem.enabled = nc.get("enabled", False)
|
||||
if "default_policy" in nc:
|
||||
s.netem.default_policy = _parse_netem_policy(
|
||||
nc["default_policy"], "netem.default_policy"
|
||||
)
|
||||
s.netem.default_policy = _parse_netem_policy(nc["default_policy"])
|
||||
if "link_policies" in nc:
|
||||
for lp_data in nc["link_policies"]:
|
||||
_reject_unknown(
|
||||
lp_data, _SECTION_KEYS["netem.link_policies[]"], "netem.link_policies[]"
|
||||
)
|
||||
override = LinkPolicyOverride(
|
||||
edges=lp_data.get("edges", []),
|
||||
)
|
||||
if "policy" in lp_data:
|
||||
override.policy = _parse_netem_policy(
|
||||
lp_data["policy"], "netem.link_policies[].policy"
|
||||
)
|
||||
override.policy = _parse_netem_policy(lp_data["policy"])
|
||||
if "policy_name" in lp_data:
|
||||
override.policy_name = lp_data["policy_name"]
|
||||
s.netem.link_policies.append(override)
|
||||
if "mutation" in nc:
|
||||
mc = nc["mutation"]
|
||||
_reject_unknown(mc, _SECTION_KEYS["netem.mutation"], "netem.mutation")
|
||||
s.netem.mutation.interval_secs = _parse_range(
|
||||
mc.get("interval_secs", {"min": 15, "max": 30}), "netem.mutation.interval_secs"
|
||||
)
|
||||
s.netem.mutation.fraction = float(mc.get("fraction", 0.3))
|
||||
s.netem.mutation.exclude_edges = list(mc.get("exclude_edges", []))
|
||||
if "policies" in mc:
|
||||
s.netem.mutation.policies = {
|
||||
name: _parse_netem_policy(pdata, f"netem.mutation.policies.{name}")
|
||||
name: _parse_netem_policy(pdata)
|
||||
for name, pdata in mc["policies"].items()
|
||||
}
|
||||
|
||||
# Link flaps section
|
||||
lf = raw.get("link_flaps", {})
|
||||
_reject_unknown(lf, _SECTION_KEYS["link_flaps"], "link_flaps")
|
||||
s.link_flaps.enabled = lf.get("enabled", False)
|
||||
if "interval_secs" in lf:
|
||||
s.link_flaps.interval_secs = _parse_range(lf["interval_secs"], "link_flaps.interval_secs")
|
||||
@@ -576,7 +337,6 @@ def load_scenario(path: str) -> Scenario:
|
||||
|
||||
# Traffic section
|
||||
tf = raw.get("traffic", {})
|
||||
_reject_unknown(tf, _SECTION_KEYS["traffic"], "traffic")
|
||||
s.traffic.enabled = tf.get("enabled", False)
|
||||
s.traffic.max_concurrent = int(tf.get("max_concurrent", 3))
|
||||
if "interval_secs" in tf:
|
||||
@@ -587,7 +347,6 @@ def load_scenario(path: str) -> Scenario:
|
||||
|
||||
# Node churn section
|
||||
nc2 = raw.get("node_churn", {})
|
||||
_reject_unknown(nc2, _SECTION_KEYS["node_churn"], "node_churn")
|
||||
s.node_churn.enabled = nc2.get("enabled", False)
|
||||
if "interval_secs" in nc2:
|
||||
s.node_churn.interval_secs = _parse_range(nc2["interval_secs"], "node_churn.interval_secs")
|
||||
@@ -600,7 +359,6 @@ def load_scenario(path: str) -> Scenario:
|
||||
|
||||
# Peer churn section
|
||||
pc = raw.get("peer_churn", {})
|
||||
_reject_unknown(pc, _SECTION_KEYS["peer_churn"], "peer_churn")
|
||||
s.peer_churn.enabled = pc.get("enabled", False)
|
||||
if "interval_secs" in pc:
|
||||
s.peer_churn.interval_secs = _parse_range(pc["interval_secs"], "peer_churn.interval_secs")
|
||||
@@ -608,7 +366,6 @@ def load_scenario(path: str) -> Scenario:
|
||||
|
||||
# Bandwidth section
|
||||
bw = raw.get("bandwidth", {})
|
||||
_reject_unknown(bw, _SECTION_KEYS["bandwidth"], "bandwidth")
|
||||
s.bandwidth.enabled = bw.get("enabled", False)
|
||||
if "tiers_mbps" in bw:
|
||||
tiers = bw["tiers_mbps"]
|
||||
@@ -618,7 +375,6 @@ def load_scenario(path: str) -> Scenario:
|
||||
|
||||
# Ingress section
|
||||
ig = raw.get("ingress", {})
|
||||
_reject_unknown(ig, _SECTION_KEYS["ingress"], "ingress")
|
||||
s.ingress.enabled = ig.get("enabled", False)
|
||||
if "tiers_kbps" in ig:
|
||||
tiers = ig["tiers_kbps"]
|
||||
@@ -629,22 +385,18 @@ def load_scenario(path: str) -> Scenario:
|
||||
|
||||
# Link swap section (deterministic asymmetric link-cost flapping).
|
||||
ls = raw.get("link_swap", {})
|
||||
_reject_unknown(ls, _SECTION_KEYS["link_swap"], "link_swap")
|
||||
s.link_swap.enabled = ls.get("enabled", False)
|
||||
if "interval_secs" in ls:
|
||||
s.link_swap.interval_secs = float(ls["interval_secs"])
|
||||
if "policies" in ls:
|
||||
s.link_swap.policies = {
|
||||
name: _parse_netem_policy(pdata, f"link_swap.policies.{name}")
|
||||
name: _parse_netem_policy(pdata)
|
||||
for name, pdata in ls["policies"].items()
|
||||
}
|
||||
if "edges" in ls:
|
||||
for edata in ls["edges"]:
|
||||
if not isinstance(edata, dict):
|
||||
raise ValueError("link_swap.edges entries must be dicts")
|
||||
_reject_unknown(
|
||||
edata, _SECTION_KEYS["link_swap.edges[]"], "link_swap.edges[]"
|
||||
)
|
||||
edge = str(edata.get("edge", ""))
|
||||
policy = str(edata.get("policy", ""))
|
||||
if not edge or not policy:
|
||||
@@ -653,202 +405,20 @@ def load_scenario(path: str) -> Scenario:
|
||||
|
||||
# Assertions section (post-run control-socket-based checks).
|
||||
asrt = raw.get("assertions", {})
|
||||
_reject_unknown(asrt, _SECTION_KEYS["assertions"], "assertions")
|
||||
if "bloom_send_rate" in asrt:
|
||||
bsr = asrt["bloom_send_rate"]
|
||||
_reject_unknown(
|
||||
bsr, _ASSERTION_KEYS["bloom_send_rate"], "assertions.bloom_send_rate"
|
||||
)
|
||||
s.assertions.bloom_send_rate = BloomSendRateAssertion(
|
||||
window_secs=int(bsr.get("window_secs", 30)),
|
||||
max_per_node=int(bsr.get("max_per_node", 30)),
|
||||
)
|
||||
if "min_parent_switches" in asrt:
|
||||
mps = asrt["min_parent_switches"]
|
||||
_reject_unknown(
|
||||
mps, _ASSERTION_KEYS["min_parent_switches"],
|
||||
"assertions.min_parent_switches",
|
||||
)
|
||||
s.assertions.min_parent_switches = MinParentSwitchesAssertion(
|
||||
min_total=int(mps.get("min_total", 1)),
|
||||
)
|
||||
if "max_parent_switches" in asrt:
|
||||
xps = asrt["max_parent_switches"]
|
||||
_reject_unknown(
|
||||
xps, _ASSERTION_KEYS["max_parent_switches"],
|
||||
"assertions.max_parent_switches",
|
||||
)
|
||||
if "max_total" not in xps:
|
||||
raise ValueError(
|
||||
"assertions.max_parent_switches: max_total is required "
|
||||
"(a defaulted ceiling would assert an arbitrary number)"
|
||||
)
|
||||
max_total = xps["max_total"]
|
||||
# bool is a subclass of int, and `max_total: yes` would coerce to 1
|
||||
# -- a ceiling low enough to change the verdict, arrived at by typo.
|
||||
if isinstance(max_total, bool) or not isinstance(max_total, int):
|
||||
raise ValueError(
|
||||
f"assertions.max_parent_switches: max_total must be a "
|
||||
f"non-negative integer, got {max_total!r}"
|
||||
)
|
||||
if max_total < 0:
|
||||
raise ValueError(
|
||||
f"assertions.max_parent_switches: max_total must be "
|
||||
f"non-negative, got {max_total}"
|
||||
)
|
||||
# Distinguish "key absent" from "key present but empty". Only the
|
||||
# first means mesh-wide. YAML renders a bare `node:` as None, and
|
||||
# treating that as absent would silently swap a per-node ceiling
|
||||
# for a mesh-wide one -- the exact conflation this assertion was
|
||||
# added to stop, and a live flake rather than a theoretical one:
|
||||
# archived runs of this scenario reach 6 mesh-wide against an n04
|
||||
# maximum of 2.
|
||||
node = None
|
||||
if "node" in xps:
|
||||
node = xps["node"]
|
||||
if not isinstance(node, str) or not node.strip():
|
||||
raise ValueError(
|
||||
f"assertions.max_parent_switches: node must be a node id "
|
||||
f"string such as 'n04', got {node!r}. Omit the key "
|
||||
f"entirely for the mesh-wide total."
|
||||
)
|
||||
node = node.strip()
|
||||
s.assertions.max_parent_switches = MaxParentSwitchesAssertion(
|
||||
max_total=max_total,
|
||||
node=node,
|
||||
)
|
||||
if "max_errors" in asrt:
|
||||
mev = asrt["max_errors"]
|
||||
_reject_unknown(
|
||||
mev, _ASSERTION_KEYS["max_errors"], "assertions.max_errors",
|
||||
)
|
||||
if "max_total" not in mev:
|
||||
raise ValueError(
|
||||
"assertions.max_errors: max_total is required (the point of "
|
||||
"overriding the default ceiling is to name the new number)"
|
||||
)
|
||||
err_total = mev["max_total"]
|
||||
if isinstance(err_total, bool) or not isinstance(err_total, int):
|
||||
raise ValueError(
|
||||
f"assertions.max_errors: max_total must be a non-negative "
|
||||
f"integer, got {err_total!r}"
|
||||
)
|
||||
if err_total < 0:
|
||||
raise ValueError(
|
||||
f"assertions.max_errors: max_total must be non-negative, "
|
||||
f"got {err_total}"
|
||||
)
|
||||
s.assertions.max_errors = MaxErrorsAssertion(max_total=err_total)
|
||||
else:
|
||||
# Default-on. See MaxErrorsAssertion for why this one assertion is
|
||||
# applied without being asked for: it is the floor on what a green
|
||||
# run means, and a scenario that has to opt in is a scenario that
|
||||
# can forget to.
|
||||
s.assertions.max_errors = MaxErrorsAssertion()
|
||||
if "congestion_signals" in asrt:
|
||||
cs = asrt["congestion_signals"]
|
||||
_reject_unknown(
|
||||
cs, _ASSERTION_KEYS["congestion_signals"],
|
||||
"assertions.congestion_signals",
|
||||
)
|
||||
floors = {}
|
||||
for key in _ASSERTION_KEYS["congestion_signals"]:
|
||||
if key not in cs:
|
||||
continue
|
||||
val = cs[key]
|
||||
if isinstance(val, bool) or not isinstance(val, int):
|
||||
raise ValueError(
|
||||
f"assertions.congestion_signals.{key}: must be a positive "
|
||||
f"integer number of nodes, got {val!r}"
|
||||
)
|
||||
if val < 1:
|
||||
raise ValueError(
|
||||
f"assertions.congestion_signals.{key}: must be at least 1, "
|
||||
f"got {val}. A floor of 0 is satisfied by a mesh that "
|
||||
f"observed nothing, which is the case this assertion exists "
|
||||
f"to catch; omit the key instead."
|
||||
)
|
||||
floors[key] = val
|
||||
if not floors:
|
||||
raise ValueError(
|
||||
"assertions.congestion_signals: set at least one floor "
|
||||
"(min_nodes_detected, min_nodes_ce_forwarded, "
|
||||
"min_nodes_ce_received); a block with none asserts nothing"
|
||||
)
|
||||
s.assertions.congestion_signals = CongestionSignalsAssertion(**floors)
|
||||
if "tree_parents" in asrt:
|
||||
tp = asrt["tree_parents"]
|
||||
if not isinstance(tp, dict) or not tp:
|
||||
raise ValueError(
|
||||
"assertions.tree_parents: give it at least one "
|
||||
"'<node>: <expected parent>' entry; an empty block asserts "
|
||||
"nothing"
|
||||
)
|
||||
expected = {}
|
||||
for child, parent in tp.items():
|
||||
for role, val in (("node", child), ("parent", parent)):
|
||||
if not isinstance(val, str) or not _NODE_ID_RE.fullmatch(val):
|
||||
raise ValueError(
|
||||
f"assertions.tree_parents: {role} {val!r} is not a node "
|
||||
f"id of the form 'n04'"
|
||||
)
|
||||
idx = int(val[1:])
|
||||
if idx < 1 or idx > s.topology.num_nodes:
|
||||
raise ValueError(
|
||||
f"assertions.tree_parents: {role} '{val}' is outside "
|
||||
f"this scenario's {s.topology.num_nodes} nodes. An "
|
||||
f"assertion about a node that cannot exist would fail "
|
||||
f"for the wrong reason every run."
|
||||
)
|
||||
if child == parent:
|
||||
raise ValueError(
|
||||
f"assertions.tree_parents: '{child}' is given itself as "
|
||||
f"its parent. A node is its own parent only when it "
|
||||
f"believes it is root, which is what an unconverged tree "
|
||||
f"looks like; assert that some other way."
|
||||
)
|
||||
expected[child] = parent
|
||||
s.assertions.tree_parents = TreeParentsAssertion(expected=expected)
|
||||
if "baseline" in asrt:
|
||||
bl = asrt["baseline"]
|
||||
_reject_unknown(bl, _ASSERTION_KEYS["baseline"], "assertions.baseline")
|
||||
vals = {}
|
||||
for key in _ASSERTION_KEYS["baseline"]:
|
||||
if key not in bl:
|
||||
continue
|
||||
val = bl[key]
|
||||
if isinstance(val, bool) or not isinstance(val, int):
|
||||
raise ValueError(
|
||||
f"assertions.baseline.{key}: must be an integer, "
|
||||
f"got {val!r}"
|
||||
)
|
||||
floor = 1 if key == "max_roots" else 0
|
||||
if val < floor:
|
||||
raise ValueError(
|
||||
f"assertions.baseline.{key}: must be at least {floor}, "
|
||||
f"got {val}"
|
||||
)
|
||||
vals[key] = val
|
||||
if not vals:
|
||||
raise ValueError(
|
||||
"assertions.baseline: set at least one of "
|
||||
+ ", ".join(sorted(_ASSERTION_KEYS["baseline"]))
|
||||
+ "; a block with none asserts nothing"
|
||||
)
|
||||
if vals.get("min_nodes_parented", 0) > 0 or vals.get("max_roots"):
|
||||
n = s.topology.num_nodes
|
||||
if vals.get("min_nodes_parented", 0) > n - 1:
|
||||
raise ValueError(
|
||||
f"assertions.baseline.min_nodes_parented: "
|
||||
f"{vals['min_nodes_parented']} exceeds {n - 1}, the most a "
|
||||
f"{n}-node mesh can reach — the root is its own parent, so "
|
||||
f"this could never pass"
|
||||
)
|
||||
s.assertions.baseline = BaselineAssertion(**vals)
|
||||
|
||||
# Logging section
|
||||
lg = raw.get("logging", {})
|
||||
_reject_unknown(lg, _SECTION_KEYS["logging"], "logging")
|
||||
s.logging.rust_log = lg.get("rust_log", "info")
|
||||
s.logging.output_dir = lg.get("output_dir", "./sim-results")
|
||||
|
||||
@@ -861,65 +431,8 @@ def load_scenario(path: str) -> Scenario:
|
||||
return s
|
||||
|
||||
|
||||
_SUPPRESSING_LOG_LEVELS = ("off", "error", "warn")
|
||||
|
||||
|
||||
def _validate_parent_switch_observability(s: Scenario):
|
||||
"""Refuse a parent-switch assertion the log level cannot observe.
|
||||
|
||||
The events these assertions count are emitted at ``info``. A scenario
|
||||
that declares one while setting ``logging.rust_log`` to ``warn`` or
|
||||
below counts zero switches: the minimum assertion then fails for the
|
||||
wrong reason, and the maximum assertion passes without observing
|
||||
anything, which is the failure mode the whole assertion effort exists
|
||||
to remove. Catch it at load rather than after the run.
|
||||
|
||||
Deliberately conservative. It rejects only a default level that is
|
||||
demonstrably too coarse, and says nothing about per-target directives
|
||||
such as ``info,fips::node=debug``, which raise verbosity rather than
|
||||
lower it.
|
||||
"""
|
||||
if (
|
||||
s.assertions.min_parent_switches is None
|
||||
and s.assertions.max_parent_switches is None
|
||||
):
|
||||
return
|
||||
default_level = s.logging.rust_log.split(",")[0].strip().lower()
|
||||
if default_level in _SUPPRESSING_LOG_LEVELS:
|
||||
raise ValueError(
|
||||
f"logging.rust_log is '{s.logging.rust_log}', whose default level "
|
||||
f"'{default_level}' suppresses the info-level parent-switch events "
|
||||
f"that a parent-switch assertion counts. The assertion would see "
|
||||
f"zero switches regardless of what the tree did. Use 'info' or "
|
||||
f"more verbose, or drop the assertion."
|
||||
)
|
||||
|
||||
|
||||
def _validate_error_observability(s: Scenario):
|
||||
"""Refuse an error ceiling the log level cannot observe.
|
||||
|
||||
Narrower than the parent-switch guard on purpose: ERROR lines survive
|
||||
every level except ``off``, so ``off`` is the only setting that turns
|
||||
this assertion into one that counts zero whatever the mesh did. Since
|
||||
the ceiling is applied by default, a scenario silencing its logs would
|
||||
otherwise acquire an assertion that cannot fail.
|
||||
"""
|
||||
if s.assertions.max_errors is None:
|
||||
return
|
||||
default_level = s.logging.rust_log.split(",")[0].strip().lower()
|
||||
if default_level == "off":
|
||||
raise ValueError(
|
||||
"logging.rust_log is 'off', which suppresses the ERROR-level "
|
||||
"lines the max_errors assertion counts. The assertion would see "
|
||||
"zero errors regardless of what the mesh did. Raise the level, "
|
||||
"or state a deliberate override in assertions.max_errors."
|
||||
)
|
||||
|
||||
|
||||
def _validate(s: Scenario):
|
||||
"""Validate scenario constraints."""
|
||||
_validate_parent_switch_observability(s)
|
||||
_validate_error_observability(s)
|
||||
if s.topology.num_nodes < 2:
|
||||
raise ValueError("topology.num_nodes must be >= 2")
|
||||
if s.topology.num_nodes > 250:
|
||||
|
||||
@@ -7,8 +7,7 @@ import random
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .keys import derive_full
|
||||
from .naming import name_suffix, veth_token
|
||||
from .keys import derive
|
||||
from .scenario import TopologyConfig
|
||||
|
||||
|
||||
@@ -29,18 +28,6 @@ class SimTopology:
|
||||
edges: set[tuple[str, str]] = field(default_factory=set)
|
||||
# Per-edge transport type; edges not in this dict default to "udp"
|
||||
edge_transport: dict[tuple[str, str], str] = field(default_factory=dict)
|
||||
# Suffix scoping globally-visible names to this run and scenario; empty
|
||||
# outside the CI harness, which keeps a bare run's names unchanged.
|
||||
name_suffix: str = ""
|
||||
|
||||
@property
|
||||
def veth_token(self) -> str:
|
||||
"""Short stand-in for the suffix, for names bound by IFNAMSIZ.
|
||||
|
||||
Derived rather than stored so no caller can build a topology whose
|
||||
host names are scoped differently from its container names.
|
||||
"""
|
||||
return veth_token(self.name_suffix)
|
||||
|
||||
def transport_for_edge(self, a: str, b: str) -> str:
|
||||
"""Get the transport type for an edge (defaults to 'udp')."""
|
||||
@@ -120,27 +107,7 @@ class SimTopology:
|
||||
return not connected
|
||||
|
||||
def container_name(self, node_id: str) -> str:
|
||||
return f"fips-node-{node_id}{self.name_suffix}"
|
||||
|
||||
def veth_host_name(self, node_a: str, node_b: str, end: str) -> str:
|
||||
"""Generate the host-namespace veth name for one end of an edge.
|
||||
|
||||
Format: ``vh{token}{NN}{MM}{end}`` (max 15 chars for IFNAMSIZ).
|
||||
Host interfaces are global, so the token keeps a scenario from
|
||||
deleting a concurrent scenario's pair; it is empty outside the CI
|
||||
harness, yielding the same "vh0104a" this has always produced.
|
||||
|
||||
``node_a`` and ``node_b`` must be in canonical edge order. Unlike
|
||||
``veth_interface_name()`` this is not symmetric: the far end is
|
||||
``end="b"`` on the same ordering, so swapping the arguments names
|
||||
an interface that does not exist.
|
||||
"""
|
||||
nn_local = node_a.replace("n", "")
|
||||
nn_peer = node_b.replace("n", "")
|
||||
name = f"vh{self.veth_token}{nn_local}{nn_peer}{end}"
|
||||
if len(name) > 15:
|
||||
raise ValueError(f"veth host name too long: {name!r} ({len(name)} > 15)")
|
||||
return name
|
||||
return f"fips-node-{node_id}"
|
||||
|
||||
def directed_outbound(self) -> dict[str, list[str]]:
|
||||
"""Assign each static-config edge to exactly one node for outbound connection.
|
||||
@@ -203,30 +170,12 @@ def generate_topology(
|
||||
n = config.num_nodes
|
||||
subnet_base = config.subnet.rsplit(".", 1)[0] # "172.20.0"
|
||||
|
||||
# Create nodes with IPs and keys.
|
||||
#
|
||||
# The mesh roots itself at the numerically smallest NodeAddr
|
||||
# (`src/tree/state.rs:363-390`), which is a hash of the node's public key
|
||||
# and so bears no relation to the node numbering. Every scenario diagram in
|
||||
# this tree draws n01 at the top, and before this ordering was applied the
|
||||
# root landed on an arbitrary node in most scenarios — which is why the
|
||||
# cost-selection scenarios that reasoned about a specific root could never
|
||||
# be turned into reliable assertions and were moved to sans-IO unit tests.
|
||||
#
|
||||
# So derive the identities from the mesh name as before, then *assign* them
|
||||
# in NodeAddr order: n01 receives the smallest and is the root, n02 the next,
|
||||
# and so on. The keys are unchanged and still deterministic; only which node
|
||||
# id holds which one changes. Scenarios that want an arbitrary root set
|
||||
# `pin_root: false` and keep exercising election.
|
||||
node_ids_ordered = [f"n{i + 1:02d}" for i in range(n)]
|
||||
identities = [derive_full(mesh_name, nid) for nid in node_ids_ordered]
|
||||
if config.pin_root:
|
||||
identities.sort(key=lambda t: t[2])
|
||||
|
||||
# Create nodes with IPs and keys
|
||||
nodes: dict[str, SimNode] = {}
|
||||
for i, node_id in enumerate(node_ids_ordered):
|
||||
for i in range(n):
|
||||
node_id = f"n{i + 1:02d}"
|
||||
docker_ip = f"{subnet_base}.{config.ip_start + i}"
|
||||
nsec, npub, _ = identities[i]
|
||||
nsec, npub = derive(mesh_name, node_id)
|
||||
nodes[node_id] = SimNode(
|
||||
node_id=node_id,
|
||||
docker_ip=docker_ip,
|
||||
@@ -270,14 +219,7 @@ def generate_topology(
|
||||
nodes[a].peers.append(b)
|
||||
nodes[b].peers.append(a)
|
||||
|
||||
# Read the environment once, here, so every name a run produces comes
|
||||
# from the same value.
|
||||
topo = SimTopology(
|
||||
nodes=nodes,
|
||||
edges=edges,
|
||||
edge_transport=edge_transport,
|
||||
name_suffix=name_suffix(),
|
||||
)
|
||||
topo = SimTopology(nodes=nodes, edges=edges, edge_transport=edge_transport)
|
||||
|
||||
# Connectivity check with retry
|
||||
if config.ensure_connected:
|
||||
|
||||
+10
-13
@@ -4,8 +4,7 @@ Creates veth pairs between Docker containers for Ethernet-transport
|
||||
edges. Each Ethernet edge gets a veth pair with one end moved into
|
||||
each container's network namespace. Naming:
|
||||
|
||||
Host (temporary): vh{token}{NN}{MM}a / vh{token}{NN}{MM}b
|
||||
(via SimTopology.veth_host_name())
|
||||
Host (temporary): vh{NN}{MM}a / vh{NN}{MM}b
|
||||
Container: ve-{local}-{peer} (via veth_interface_name())
|
||||
|
||||
After creation, the container-side MAC addresses are queried and
|
||||
@@ -114,7 +113,9 @@ class VethManager:
|
||||
if a != node_id and b != node_id:
|
||||
continue
|
||||
# Remove existing pair if any (host-side might still exist)
|
||||
host_a = self.topology.veth_host_name(a, b, "a")
|
||||
nn_a = a.replace("n", "")
|
||||
nn_b = b.replace("n", "")
|
||||
host_a = f"vh{nn_a}{nn_b}a"
|
||||
_run_host(["ip", "link", "delete", host_a], image, check=False)
|
||||
# Re-create
|
||||
self._create_veth_pair(a, b, image)
|
||||
@@ -141,14 +142,14 @@ class VethManager:
|
||||
return
|
||||
|
||||
# Generate names
|
||||
host_a = self.topology.veth_host_name(node_a, node_b, "a")
|
||||
host_b = self.topology.veth_host_name(node_a, node_b, "b")
|
||||
nn_a = node_a.replace("n", "")
|
||||
nn_b = node_b.replace("n", "")
|
||||
host_a = f"vh{nn_a}{nn_b}a"
|
||||
host_b = f"vh{nn_a}{nn_b}b"
|
||||
final_a = veth_interface_name(node_a, node_b)
|
||||
final_b = veth_interface_name(node_b, node_a)
|
||||
|
||||
# Clean up a stale pair left by this scenario. The token makes the
|
||||
# name unique to this run, so a pair orphaned by an earlier run is
|
||||
# no longer reclaimed here — `ci-cleanup.sh` reaps those instead.
|
||||
# Clean up any stale pair
|
||||
_run_host(["ip", "link", "delete", host_a], image, check=False)
|
||||
|
||||
# Create veth pair on host
|
||||
@@ -252,11 +253,7 @@ def _run_host(cmd: list[str], image: str, check: bool = True) -> bool:
|
||||
timeout=30,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
# Warning, not debug: the runner logs at INFO unless asked for
|
||||
# -v, so at debug this never reached runner.log and the callers
|
||||
# below report only that a pair could not be created. The
|
||||
# check=False deletes are expected to fail and stay silent.
|
||||
log.warning(
|
||||
log.debug(
|
||||
"ip cmd failed: %s -> %s",
|
||||
" ".join(cmd),
|
||||
result.stderr.strip(),
|
||||
|
||||
+68
-201
@@ -11,30 +11,19 @@
|
||||
# unreliable on GitHub-hosted runners.
|
||||
# tor-directory — same; live Tor dependency.
|
||||
#
|
||||
# What is compared, and at what granularity:
|
||||
# chaos — per scenario, plus its flags. GitHub fans each scenario
|
||||
# into its own matrix leg carrying `scenario:` (and
|
||||
# optionally `chaos_flags:`); local lists the same scenarios
|
||||
# in CHAOS_SUITES as "display scenario flags". The `suite:`
|
||||
# names differ cosmetically between runners and are ignored
|
||||
# — `scenario:` is the identity.
|
||||
# deb-install — per distro. GitHub splits into per-distro legs carrying
|
||||
# `scenario:`; local runs the same distro set in one suite,
|
||||
# enumerated by ALL_SCENARIOS in deb-install/test.sh.
|
||||
# everything else — per suite name.
|
||||
# Granularity-only differences folded before comparison (same coverage,
|
||||
# different matrix shape — NOT a divergence):
|
||||
# deb-install — GitHub splits into per-distro legs (deb-install-debian12/
|
||||
# debian13/ubuntu22/ubuntu24/ubuntu26); local runs the same
|
||||
# distro set in one suite. Folded to "deb-install".
|
||||
# chaos-* — GitHub fans each chaos scenario into its own matrix leg
|
||||
# (type: chaos); local runs them all via the one CHAOS_SUITES
|
||||
# path. The individual scenario names also differ cosmetically
|
||||
# between runners (e.g. chaos-smoke-10 vs churn-mixed-10).
|
||||
# Folded to a single "chaos" token on both sides.
|
||||
# dns-resolver — single leg / single suite both sides; runs all scenarios.
|
||||
#
|
||||
# dns-resolver is the one leg still compared at leg granularity rather than
|
||||
# per scenario: it is a single leg and a single suite on both sides, and it
|
||||
# runs all of its scenarios internally. Its scenario list is NOT cross-checked.
|
||||
#
|
||||
# The local suite set is discovered by sweeping ci-local.sh for *_SUITES arrays
|
||||
# rather than from a hardcoded list of variable names, and every run_suite
|
||||
# dispatch arm is then checked to have a backing array — a suite dispatched
|
||||
# without one is invisible to a name-list sweep, which is how a real divergence
|
||||
# went unnoticed.
|
||||
#
|
||||
# Exit 0 = parity clean. Exit 1 = unexpected divergence. Exit 2 = the guard
|
||||
# could not run (missing file or missing dependency); never treated as a pass.
|
||||
# Exit 0 = parity clean. Exit 1 = unexpected divergence (suite names printed).
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
set -uo pipefail
|
||||
|
||||
@@ -43,235 +32,113 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
CI_LOCAL="$SCRIPT_DIR/ci-local.sh"
|
||||
CI_YML="$PROJECT_ROOT/.github/workflows/ci.yml"
|
||||
DEB_TEST="$SCRIPT_DIR/deb-install/test.sh"
|
||||
|
||||
# Deliberate local-only allowlist (suites intentionally absent from GitHub).
|
||||
ALLOWLIST="tor-socks5 tor-directory"
|
||||
|
||||
for f in "$CI_LOCAL" "$CI_YML" "$DEB_TEST"; do
|
||||
for f in "$CI_LOCAL" "$CI_YML"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "check-ci-parity: missing file: $f" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo "check-ci-parity: python3 not found; cannot verify CI parity" >&2
|
||||
exit 2
|
||||
fi
|
||||
if ! python3 -c "import yaml" >/dev/null 2>&1; then
|
||||
echo "check-ci-parity: python3 module 'yaml' not found; cannot verify CI parity" >&2
|
||||
echo "check-ci-parity: install it with 'pip3 install pyyaml'" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
python3 - "$CI_LOCAL" "$CI_YML" "$DEB_TEST" "$ALLOWLIST" <<'PY'
|
||||
# Extract and normalize both suite sets in Python (robust YAML parse of the
|
||||
# matrix; regex extraction of the bash suite arrays). Folding rules above are
|
||||
# applied identically to both sides so only genuine divergence surfaces.
|
||||
python3 - "$CI_LOCAL" "$CI_YML" "$ALLOWLIST" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
ci_local_path, ci_yml_path, deb_test_path, allowlist_raw = sys.argv[1:5]
|
||||
ci_local_path, ci_yml_path, allowlist_raw = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
allowlist = set(allowlist_raw.split())
|
||||
|
||||
|
||||
def fold(name):
|
||||
"""Collapse granularity-only matrix shape into canonical suite identity."""
|
||||
if name.startswith("chaos-") or name == "chaos":
|
||||
return "chaos"
|
||||
if name.startswith("deb-install"):
|
||||
return "deb-install"
|
||||
return name
|
||||
|
||||
|
||||
# ── Local: parse the suite arrays from ci-local.sh ───────────────────────────
|
||||
with open(ci_local_path, encoding="utf-8") as fh:
|
||||
local_src = fh.read()
|
||||
with open(deb_test_path, encoding="utf-8") as fh:
|
||||
deb_src = fh.read()
|
||||
|
||||
|
||||
def bash_array_entries(var):
|
||||
"""Full entries of a bash array, in order, quotes stripped."""
|
||||
def bash_array(var):
|
||||
m = re.search(rf"^{var}=\((.*?)\)", local_src, re.MULTILINE | re.DOTALL)
|
||||
if not m:
|
||||
return []
|
||||
body = m.group(1)
|
||||
# Quoted entries (chaos uses "display scenario flags"): first token is name.
|
||||
quoted = re.findall(r'"([^"]*)"', body)
|
||||
if quoted:
|
||||
return [e.strip() for e in quoted if e.strip()]
|
||||
return [entry.split()[0] for entry in quoted if entry.strip()]
|
||||
return [tok for tok in body.split() if tok.strip()]
|
||||
|
||||
|
||||
def discovered_arrays():
|
||||
"""Every *_SUITES array in ci-local.sh, by name.
|
||||
|
||||
Swept rather than hardcoded: a suite whose array is not on a fixed list
|
||||
would otherwise be invisible to this guard.
|
||||
"""
|
||||
return {
|
||||
name + "_SUITES": bash_array_entries(name + "_SUITES")
|
||||
for name in re.findall(r"^([A-Z_]+)_SUITES=\(", local_src, re.MULTILINE)
|
||||
}
|
||||
|
||||
|
||||
arrays = discovered_arrays()
|
||||
|
||||
# ── Local side ───────────────────────────────────────────────────────────────
|
||||
# Chaos: "display scenario flags" — compare the scenario and its flags.
|
||||
local_chaos = {}
|
||||
for entry in arrays.get("CHAOS_SUITES", []):
|
||||
parts = entry.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
local_chaos[parts[1]] = " ".join(parts[2:])
|
||||
|
||||
# deb-install: one local suite that runs the distro set enumerated in its script.
|
||||
m = re.search(r'^ALL_SCENARIOS="([^"]*)"', deb_src, re.MULTILINE)
|
||||
local_deb = set(m.group(1).split()) if m else set()
|
||||
|
||||
# Everything else: suite names, with NAT stored bare and prefixed at use.
|
||||
local = set()
|
||||
for name, entries in arrays.items():
|
||||
if name in ("CHAOS_SUITES", "DEB_INSTALL_SUITES"):
|
||||
continue
|
||||
names = [e.split()[0] for e in entries]
|
||||
if name == "NAT_SUITES":
|
||||
local |= {f"nat-{n}" for n in names}
|
||||
else:
|
||||
local |= set(names)
|
||||
# Static, rekey, gateway, sidecar, acl, firewall, nostr, stun, dns, deb.
|
||||
for var in ("STATIC_SUITES", "REKEY_SUITES", "ADMISSION_SUITES",
|
||||
"GATEWAY_SUITES", "SIDECAR_SUITES", "ACL_SUITES",
|
||||
"FIREWALL_SUITES", "NOSTR_RELAY_SUITES", "STUN_FAULTS_SUITES",
|
||||
"DNS_RESOLVER_SUITES", "DEB_INSTALL_SUITES"):
|
||||
local.update(bash_array(var))
|
||||
# Chaos display names → fold to "chaos".
|
||||
for _ in bash_array("CHAOS_SUITES"):
|
||||
local.add("chaos")
|
||||
# NAT scenarios are stored bare (cone/symmetric/lan) and prefixed nat- at use.
|
||||
for scen in bash_array("NAT_SUITES"):
|
||||
local.add(f"nat-{scen}")
|
||||
# TOR_SUITES is the deliberate local-only set — excluded from the default path.
|
||||
|
||||
local = {fold(n) for n in local}
|
||||
|
||||
# ── GitHub: parse the integration matrix suite: values from ci.yml ───────────
|
||||
import yaml # noqa: E402
|
||||
|
||||
# ── GitHub side ──────────────────────────────────────────────────────────────
|
||||
with open(ci_yml_path, encoding="utf-8") as fh:
|
||||
doc = yaml.safe_load(fh)
|
||||
|
||||
include = doc["jobs"]["integration"]["strategy"]["matrix"]["include"]
|
||||
github_chaos, github_deb, github = {}, set(), set()
|
||||
malformed = []
|
||||
github = set()
|
||||
for leg in include:
|
||||
if "suite" not in leg and "scenario" not in leg:
|
||||
if "suite" not in leg:
|
||||
continue
|
||||
kind = str(leg.get("type", ""))
|
||||
if kind in ("chaos", "deb-install"):
|
||||
# scenario: is the identity for these; suite: is cosmetic.
|
||||
if "scenario" not in leg:
|
||||
malformed.append(f"{leg.get('suite', '(unnamed leg)')} has type "
|
||||
f"{kind} but no scenario:")
|
||||
continue
|
||||
if kind == "chaos":
|
||||
github_chaos[str(leg["scenario"])] = str(leg.get("chaos_flags", ""))
|
||||
else:
|
||||
github_deb.add(str(leg["scenario"]))
|
||||
elif "suite" in leg:
|
||||
github.add(str(leg["suite"]))
|
||||
# Chaos legs carry inconsistent suite: names (chaos-smoke-10 vs
|
||||
# churn-mixed-10) but a uniform type: chaos — fold via type, not name.
|
||||
if str(leg.get("type", "")) == "chaos":
|
||||
github.add("chaos")
|
||||
else:
|
||||
malformed.append(f"leg with scenario {leg['scenario']} has no suite: "
|
||||
f"and no chaos/deb-install type")
|
||||
github.add(fold(str(leg["suite"])))
|
||||
|
||||
# ── Dispatch cross-check: every run_suite arm needs a backing array ──────────
|
||||
# A suite dispatched without an array is invisible to the sweep above, so the
|
||||
# guard would report it as GitHub-only forever without ever naming the cause.
|
||||
body = re.search(r"^run_suite\(\).*?^\}", local_src, re.MULTILINE | re.DOTALL)
|
||||
dispatch_uncovered = []
|
||||
if body is None:
|
||||
print("check-ci-parity: could not locate run_suite() in ci-local.sh", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# Arms sit at one fixed indentation inside the case block. Pin to it, taken from
|
||||
# the first arm rather than assumed, so a body line that happens to end in ')'
|
||||
# cannot be read as an arm.
|
||||
arm_re = re.compile(r"^([ \t]+)['\"]?([a-z0-9|*_.-]+)['\"]?\)", re.MULTILINE)
|
||||
first = arm_re.search(body.group(0))
|
||||
if first is None:
|
||||
print("check-ci-parity: no dispatch arms found in run_suite()", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
indent = first.group(1)
|
||||
# An arm-shaped line at a different indent is not skipped silently: it would
|
||||
# make this check quietly stop covering a suite, which is the failure mode the
|
||||
# check exists to prevent.
|
||||
odd_arms = [
|
||||
m.group(2) for m in arm_re.finditer(body.group(0)) if m.group(1) != indent
|
||||
]
|
||||
if odd_arms:
|
||||
print("check-ci-parity: run_suite has arm-shaped lines at an unexpected "
|
||||
f"indent, so the dispatch check cannot be trusted: {', '.join(odd_arms)}",
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
known = (set(local) | set(local_chaos) | local_deb
|
||||
| {e.split()[0] for e in arrays.get("DEB_INSTALL_SUITES", [])})
|
||||
for m in arm_re.finditer(body.group(0)):
|
||||
if m.group(1) != indent:
|
||||
continue
|
||||
for arm in m.group(2).split("|"):
|
||||
if arm == "*":
|
||||
continue # the unknown-suite error arm, not a suite
|
||||
if arm == "chaos-*":
|
||||
# Dispatches any chaos-<name> through a fallback, so its vocabulary
|
||||
# is unbounded; the chaos scenario comparison covers it instead.
|
||||
continue
|
||||
if arm not in known:
|
||||
dispatch_uncovered.append(arm)
|
||||
|
||||
# ── Diff ─────────────────────────────────────────────────────────────────────
|
||||
# ── Diff (subtract allowlist from local before comparison) ───────────────────
|
||||
local_cmp = {n for n in local if n not in allowlist}
|
||||
|
||||
local_only = sorted(local_cmp - github)
|
||||
github_only = sorted(github - local_cmp)
|
||||
chaos_local_only = sorted(set(local_chaos) - set(github_chaos))
|
||||
chaos_github_only = sorted(set(github_chaos) - set(local_chaos))
|
||||
chaos_flag_drift = sorted(
|
||||
(s, local_chaos[s], github_chaos[s])
|
||||
for s in set(local_chaos) & set(github_chaos)
|
||||
if local_chaos[s] != github_chaos[s]
|
||||
)
|
||||
deb_local_only = sorted(local_deb - github_deb)
|
||||
deb_github_only = sorted(github_deb - local_deb)
|
||||
|
||||
problems = (local_only or github_only or chaos_local_only or chaos_github_only
|
||||
or chaos_flag_drift or deb_local_only or deb_github_only
|
||||
or dispatch_uncovered or malformed)
|
||||
|
||||
if problems:
|
||||
print("CI parity FAILED: the two runners do not cover the same work.\n")
|
||||
if local_only or github_only:
|
||||
print("CI parity FAILED: integration suite sets diverge.\n")
|
||||
if local_only:
|
||||
print(" Suites local-only (in ci-local.sh, missing from ci.yml, "
|
||||
"not in the deliberate allowlist):")
|
||||
print(" Local-only (in ci-local.sh, missing from ci.yml, "
|
||||
"not in deliberate allowlist):")
|
||||
for n in local_only:
|
||||
print(f" - {n}")
|
||||
if github_only:
|
||||
print(" Suites GitHub-only (in ci.yml, missing from the local default path):")
|
||||
print(" GitHub-only (in ci.yml, missing from local default path):")
|
||||
for n in github_only:
|
||||
print(f" - {n}")
|
||||
if chaos_local_only:
|
||||
print(" Chaos scenarios local-only:")
|
||||
for n in chaos_local_only:
|
||||
print(f" - {n}")
|
||||
if chaos_github_only:
|
||||
print(" Chaos scenarios GitHub-only:")
|
||||
for n in chaos_github_only:
|
||||
print(f" - {n}")
|
||||
if chaos_flag_drift:
|
||||
print(" Chaos scenarios whose flags differ between runners:")
|
||||
for name, lflags, gflags in chaos_flag_drift:
|
||||
print(f" - {name}: local '{lflags}' vs GitHub '{gflags}'")
|
||||
if deb_local_only:
|
||||
print(" deb-install distros local-only:")
|
||||
for n in deb_local_only:
|
||||
print(f" - {n}")
|
||||
if deb_github_only:
|
||||
print(" deb-install distros GitHub-only:")
|
||||
for n in deb_github_only:
|
||||
print(f" - {n}")
|
||||
if malformed:
|
||||
print(" Matrix legs this guard cannot identify:")
|
||||
for n in malformed:
|
||||
print(f" - {n}")
|
||||
if dispatch_uncovered:
|
||||
print(" run_suite dispatches these with no backing *_SUITES array, so "
|
||||
"this guard\n cannot see them in the local set:")
|
||||
for n in dispatch_uncovered:
|
||||
print(f" - {n}")
|
||||
print("\n Resolve by adding the suite to the other runner, by giving a "
|
||||
"dispatchable\n suite a *_SUITES array, or by adding it to the "
|
||||
"deliberate local-only\n allowlist in check-ci-parity.sh with a "
|
||||
"stated reason.")
|
||||
print("\n Resolve by adding the suite to the other runner, or by adding "
|
||||
"it\n to the deliberate local-only allowlist in "
|
||||
"check-ci-parity.sh with a\n stated reason.")
|
||||
sys.exit(1)
|
||||
|
||||
total = len(github) + len(github_chaos) + len(github_deb)
|
||||
print("CI parity OK: both runners cover the same work "
|
||||
print("CI parity OK: integration suite sets match "
|
||||
"(allowlist: " + ", ".join(sorted(allowlist)) + ").")
|
||||
print(f" {len(github)} suites, {len(github_chaos)} chaos scenarios "
|
||||
f"(flags compared), {len(github_deb)} deb-install distros "
|
||||
f"— {total} legs on each side.")
|
||||
print(f" {len(github)} canonical suites compared on each side.")
|
||||
sys.exit(0)
|
||||
PY
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ── Test-image scoping guard ────────────────────────────────────────────────
|
||||
# A local CI run builds fips-test:<run-id> and hands it to every suite through
|
||||
# FIPS_TEST_IMAGE / FIPS_TEST_APP_IMAGE. It does NOT write fips-test:latest,
|
||||
# deliberately: while a bridge back to that shared mutable name existed, a
|
||||
# consumer that named it directly kept working while resolving whichever
|
||||
# concurrent run wrote the tag last, and the verdict was then recorded against
|
||||
# a commit whose binaries had not run. Nothing in the harness compares a
|
||||
# running container's binary against the commit under test, so that failure is
|
||||
# silent and leaves no artifact.
|
||||
#
|
||||
# The bridge is gone, so a consumer that names the shared tag now fails loudly
|
||||
# at run time. This guard is the static half: it stops one being reintroduced,
|
||||
# because the reintroduction is invisible on any host where a hand build has
|
||||
# left an fips-test:latest lying around.
|
||||
#
|
||||
# What counts as a violation: a reference to the shared tag that is neither a
|
||||
# comment, nor a documented default of the ${FIPS_TEST_IMAGE:-...} form, nor in
|
||||
# a file on the allowlist below.
|
||||
#
|
||||
# Exit 0 = clean. Exit 1 = an unexpected reference. Exit 2 = the guard could
|
||||
# not run; never treated as a pass.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
# Files permitted to name the shared tag outright, each for a stated reason.
|
||||
# Every one of these is a hand-run path: none is reachable from ci-local.sh
|
||||
# with FIPS_TEST_IMAGE set.
|
||||
#
|
||||
# scripts/build.sh the developer build; :latest IS its product
|
||||
# sidecar/scripts/test-sidecar.sh hand build, behind its --skip-build guard,
|
||||
# which ci-local always passes
|
||||
# static/scripts/iperf-compare-refs.sh hand-run A/B against two refs
|
||||
# ci-cleanup.sh last-resort name for the image ip(8) runs in,
|
||||
# after the caller's and the run's have failed
|
||||
# check-image-scoping.sh this guard, which has to name what it looks
|
||||
# for; it resolves no image
|
||||
ALLOWED=(
|
||||
"scripts/build.sh"
|
||||
"sidecar/scripts/test-sidecar.sh"
|
||||
"static/scripts/iperf-compare-refs.sh"
|
||||
"ci-cleanup.sh"
|
||||
"check-image-scoping.sh"
|
||||
)
|
||||
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "check-image-scoping: git not available, cannot sweep" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! -d "$SCRIPT_DIR" ]]; then
|
||||
echo "check-image-scoping: $SCRIPT_DIR missing" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Tracked files only, and no documentation: prose naming the tag is describing
|
||||
# it, not resolving it.
|
||||
mapfile -t files < <(git -C "$SCRIPT_DIR/.." ls-files -- testing/ | grep -vE '\.md$')
|
||||
if [[ ${#files[@]} -eq 0 ]]; then
|
||||
echo "check-image-scoping: no tracked files under testing/, refusing to pass" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
violations=0
|
||||
for f in "${files[@]}"; do
|
||||
rel="${f#testing/}"
|
||||
skip=0
|
||||
for a in "${ALLOWED[@]}"; do
|
||||
[[ "$rel" == "$a" ]] && skip=1 && break
|
||||
done
|
||||
[[ $skip -eq 1 ]] && continue
|
||||
[[ -f "$SCRIPT_DIR/../$f" ]] || continue
|
||||
|
||||
while IFS= read -r hit; do
|
||||
n="${hit%%:*}"
|
||||
text="${hit#*:}"
|
||||
# A comment line is describing the tag, not resolving it. Shell, python
|
||||
# and yaml all use #; nothing under testing/ uses // for comments.
|
||||
[[ "$text" =~ ^[[:space:]]*# ]] && continue
|
||||
# The documented indirection: the shared tag as a FALLBACK, which is
|
||||
# what a bare hand run is supposed to get.
|
||||
[[ "$text" == *'${FIPS_TEST_IMAGE:-fips-test:latest}'* ]] && continue
|
||||
[[ "$text" == *'${FIPS_TEST_APP_IMAGE:-fips-test-app:latest}'* ]] && continue
|
||||
[[ "$text" == *'os.environ.get("FIPS_TEST_IMAGE", "fips-test:latest")'* ]] && continue
|
||||
echo "FAIL $f:$n names the shared test image directly:"
|
||||
echo " $text"
|
||||
violations=$((violations + 1))
|
||||
done < <(grep -n 'fips-test:latest\|fips-test-app:latest' "$SCRIPT_DIR/../$f" 2>/dev/null)
|
||||
done
|
||||
|
||||
if [[ $violations -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "check-image-scoping: $violations reference(s) to the shared mutable test image."
|
||||
echo "Read FIPS_TEST_IMAGE (default \${FIPS_TEST_IMAGE:-fips-test:latest}) instead."
|
||||
echo "A run that resolves the shared tag can execute a concurrent run's binaries"
|
||||
echo "and record the verdict against this commit."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "check-image-scoping: no unscoped references to the shared test image"
|
||||
exit 0
|
||||
@@ -1,252 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify that every daemon log string a test matches on still exists in src/.
|
||||
|
||||
A test that greps the daemon's log for a message the daemon no longer emits
|
||||
does not fail — it quietly stops observing anything, and an assertion built on
|
||||
it (especially one expecting a count of zero) passes for the wrong reason.
|
||||
That class has produced several findings, so it is checked mechanically here
|
||||
rather than re-discovered by reading.
|
||||
|
||||
The check extracts the string literals that test code matches against daemon
|
||||
log lines, reduces each to its longest literal run (patterns carry regex
|
||||
syntax), and requires that run to appear somewhere under src/. Anything that
|
||||
legitimately does not originate in src/ — runtime panic text, tracing's own
|
||||
level tokens — must be named in ALLOWED with a reason, so the exceptions are
|
||||
reviewable instead of invisible.
|
||||
|
||||
Usage: testing/check-log-strings.py [--verbose]
|
||||
Exit: 0 all matched strings are live, 1 otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SRC = REPO / "src"
|
||||
TESTING = REPO / "testing"
|
||||
|
||||
# Strings matched against log text that do not come from src/, each with the
|
||||
# reason it is legitimately absent. Anything here is exempt from the src/
|
||||
# existence requirement — keep the list short and the reasons specific.
|
||||
ALLOWED = {
|
||||
"panicked": "emitted by the Rust runtime's panic hook, not by our code",
|
||||
"PANIC": "panic-adjacent marker matched defensively alongside 'panicked'",
|
||||
"ERROR": "tracing's own level token, produced by the subscriber's formatter",
|
||||
" ERROR ": "tracing's own level token, produced by the subscriber's formatter",
|
||||
" WARN ": "tracing's own level token, produced by the subscriber's formatter",
|
||||
"Bootstrapped 100%": (
|
||||
"read from the tor-daemon container's log, not the fips daemon's — "
|
||||
"Tor's own bootstrap progress line"
|
||||
),
|
||||
"panicked at": "the Rust runtime's panic hook writes this, not our code",
|
||||
"RUST_BACKTRACE": "the runtime's backtrace hint, printed alongside a panic",
|
||||
"fatal runtime error": "emitted by the Rust runtime on an abort",
|
||||
}
|
||||
|
||||
# Shell helpers whose first argument is a pattern matched against daemon logs.
|
||||
SHELL_HELPERS = ("count_log_pattern", "assert_zero_count")
|
||||
|
||||
# A literal run shorter than this is too weak to search for meaningfully.
|
||||
MIN_ANCHOR = 8
|
||||
|
||||
|
||||
META = set("[](){}?*+.^$")
|
||||
|
||||
|
||||
def literal_anchors(pattern: str) -> list[str]:
|
||||
"""Longest literal run of each alternation branch of a grep pattern.
|
||||
|
||||
Walks the pattern rather than substituting, because escaping has to be
|
||||
resolved in the same pass as the split: `\\.` is a literal dot and ends
|
||||
nothing, while a bare `.` is a wildcard and ends the run. Unescaping first
|
||||
and splitting after would conflate the two and treat `directory.mode` as
|
||||
though it were literal text.
|
||||
"""
|
||||
branches, current, i = [], [], 0
|
||||
while i < len(pattern):
|
||||
ch = pattern[i]
|
||||
if ch == "\\" and i + 1 < len(pattern):
|
||||
nxt = pattern[i + 1]
|
||||
if nxt == "|": # BRE alternation
|
||||
branches.append("".join(current))
|
||||
current = []
|
||||
elif nxt in "wsdbWSDB": # a character class, not a literal
|
||||
current.append("\0")
|
||||
else:
|
||||
current.append(nxt)
|
||||
i += 2
|
||||
continue
|
||||
if ch == "|": # ERE alternation
|
||||
branches.append("".join(current))
|
||||
current = []
|
||||
elif ch in META:
|
||||
current.append("\0")
|
||||
else:
|
||||
current.append(ch)
|
||||
i += 1
|
||||
branches.append("".join(current))
|
||||
|
||||
anchors = []
|
||||
for branch in branches:
|
||||
runs = [r.strip() for r in branch.split("\0")]
|
||||
runs = [r for r in runs if r]
|
||||
if runs:
|
||||
anchors.append(max(runs, key=len))
|
||||
return anchors
|
||||
|
||||
|
||||
def python_candidates() -> list[tuple[Path, int, str]]:
|
||||
"""`"literal" in line` tests in testing/ python."""
|
||||
found = []
|
||||
for path in sorted(TESTING.rglob("*.py")):
|
||||
for n, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
for m in re.finditer(r'"([^"]+)"\s+in\s+line\b', line):
|
||||
found.append((path, n, m.group(1)))
|
||||
return found
|
||||
|
||||
|
||||
def shell_candidates() -> list[tuple[Path, int, str]]:
|
||||
"""Literal first argument to a log-matching shell helper."""
|
||||
helpers = "|".join(SHELL_HELPERS)
|
||||
# Quoted literal only; a variable argument is resolved elsewhere and is
|
||||
# reported as unscannable rather than silently skipped.
|
||||
literal = re.compile(rf"\b(?:{helpers})\s+(\"[^\"$]+\"|'[^']+')")
|
||||
variable = re.compile(rf"\b(?:{helpers})\s+[\"']?\$")
|
||||
found = []
|
||||
for path in sorted(TESTING.rglob("*.sh")):
|
||||
for n, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
if line.lstrip().startswith("#"):
|
||||
continue
|
||||
for m in literal.finditer(line):
|
||||
found.append((path, n, m.group(1)[1:-1]))
|
||||
if variable.search(line):
|
||||
found.append((path, n, None))
|
||||
return found
|
||||
|
||||
|
||||
def pattern_table_candidates() -> list[tuple[Path, int, str]]:
|
||||
"""Bash associative-array keys used as log patterns.
|
||||
|
||||
A suite that iterates a table of patterns into a log-matching helper hides
|
||||
every string behind a variable, so the helper rule above sees only
|
||||
`count_log_pattern "$pat"` and reports it unscannable. The keys are the
|
||||
patterns; read them where they are written.
|
||||
"""
|
||||
key = re.compile(r'^\s*\[\s*"([^"$]+)"\s*\]=')
|
||||
found = []
|
||||
for path in sorted(TESTING.rglob("*.sh")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if not any(h in text for h in SHELL_HELPERS):
|
||||
continue
|
||||
for n, line in enumerate(text.splitlines(), 1):
|
||||
if line.lstrip().startswith("#"):
|
||||
continue
|
||||
m = key.match(line)
|
||||
if m:
|
||||
found.append((path, n, m.group(1)))
|
||||
return found
|
||||
|
||||
|
||||
def grep_candidates() -> list[tuple[Path, int, str]]:
|
||||
"""Literal grep patterns whose input is daemon log text.
|
||||
|
||||
Scoped by what the grep READS, not by what the file mentions. A suite
|
||||
greps several unrelated sources — its own analyzer output, fipsctl JSON,
|
||||
Tor's log, ping output — and only the daemon's log has to correspond to a
|
||||
string in src/. Two shapes qualify: a grep piped directly from
|
||||
`docker logs`, and a grep fed a variable that was assigned from it.
|
||||
"""
|
||||
grep_lit = r"\bgrep\b[^|;]*?\s(\"[^\"$]+\"|'[^'$]+')"
|
||||
assign = re.compile(r"(\w+)=\"?\$\(\s*docker logs\b")
|
||||
found = []
|
||||
for path in sorted(TESTING.rglob("*.sh")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if "docker logs" not in text:
|
||||
continue
|
||||
log_vars = set(assign.findall(text))
|
||||
# A grep reading one of those variables, by herestring or by pipe.
|
||||
var_alt = "|".join(re.escape(v) for v in log_vars) or r"\0"
|
||||
reads_var = re.compile(rf"[\"']?\$\{{?(?:{var_alt})\b")
|
||||
for n, line in enumerate(text.splitlines(), 1):
|
||||
if line.lstrip().startswith("#"):
|
||||
continue
|
||||
if "docker logs" not in line and not reads_var.search(line):
|
||||
continue
|
||||
for m in re.finditer(grep_lit, line):
|
||||
found.append((path, n, m.group(1)[1:-1]))
|
||||
return found
|
||||
|
||||
|
||||
def main() -> int:
|
||||
verbose = "--verbose" in sys.argv
|
||||
src_text = "\n".join(
|
||||
p.read_text(encoding="utf-8", errors="replace")
|
||||
for p in SRC.rglob("*.rs")
|
||||
)
|
||||
|
||||
candidates = (
|
||||
python_candidates()
|
||||
+ shell_candidates()
|
||||
+ pattern_table_candidates()
|
||||
+ grep_candidates()
|
||||
)
|
||||
dead, checked, exempt, unscannable = [], 0, 0, 0
|
||||
|
||||
for path, lineno, raw in candidates:
|
||||
rel = path.relative_to(REPO)
|
||||
if raw is None:
|
||||
unscannable += 1
|
||||
if verbose:
|
||||
print(f" skip {rel}:{lineno}: pattern comes from a variable")
|
||||
continue
|
||||
if raw in ALLOWED:
|
||||
exempt += 1
|
||||
if verbose:
|
||||
print(f" allow {rel}:{lineno}: {raw!r} ({ALLOWED[raw]})")
|
||||
continue
|
||||
anchors = literal_anchors(raw)
|
||||
# An alternation may mix daemon strings with runtime ones, so the
|
||||
# allowlist applies per branch and not only to the whole pattern.
|
||||
if any(a in ALLOWED for a in anchors):
|
||||
exempt += 1
|
||||
if verbose:
|
||||
print(f" allow {rel}:{lineno}: {raw!r} (branch in ALLOWED)")
|
||||
anchors = [a for a in anchors if a not in ALLOWED]
|
||||
usable = [a for a in anchors if len(a) >= MIN_ANCHOR]
|
||||
if not usable:
|
||||
unscannable += 1
|
||||
if verbose:
|
||||
print(f" skip {rel}:{lineno}: {raw!r} has no literal run >= {MIN_ANCHOR}")
|
||||
continue
|
||||
checked += 1
|
||||
# Every alternation branch must be live: one dead branch is a matcher
|
||||
# that has silently narrowed.
|
||||
for anchor in usable:
|
||||
if anchor not in src_text:
|
||||
dead.append((rel, lineno, raw, anchor))
|
||||
elif verbose:
|
||||
print(f" ok {rel}:{lineno}: {anchor!r}")
|
||||
|
||||
print(
|
||||
f"log-string check: {checked} matched, {exempt} allowed, "
|
||||
f"{unscannable} unscannable, {len(dead)} dead"
|
||||
)
|
||||
if dead:
|
||||
print("\nStrings matched against daemon logs that src/ never emits:\n")
|
||||
for rel, lineno, raw, anchor in dead:
|
||||
print(f" {rel}:{lineno}")
|
||||
print(f" pattern: {raw!r}")
|
||||
print(f" missing: {anchor!r}")
|
||||
print(
|
||||
"\nEither correct the string to what the daemon emits, or add it to "
|
||||
"ALLOWED\nin this script with the reason it does not come from src/."
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,208 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Find shell functions that end in a logging call and whose status a caller tests.
|
||||
|
||||
A bash function's exit status is the status of its last command. A function
|
||||
whose last statement is `echo`/`log`/`info`/... therefore returns 0 on every
|
||||
path that reaches the end, and any caller written as `if func`, `func || fail`
|
||||
or `func && ...` has a gate that cannot fire.
|
||||
|
||||
This is not hypothetical and not a style rule. Two instances were found in one
|
||||
day in this tree:
|
||||
|
||||
* `run_chaos` ended in `record`, whose own last statement is an `echo`, so
|
||||
every chaos row was unconditionally green from 2026-03-09.
|
||||
* `build_fips_for_e2e` ended in `log`, so a failed binary extraction left the
|
||||
previous run's cache in place and five end-to-end legs tested the previous
|
||||
commit's code and reported green — a green run certifying software that was
|
||||
never built.
|
||||
|
||||
Both were repaired individually. This exists so the next one is caught by a
|
||||
gate rather than by a reader.
|
||||
|
||||
WHAT IT ENFORCES, stated precisely, because it is a convention and not a bug
|
||||
hunt: a function whose exit status a caller tests must END WITH AN EXPLICIT
|
||||
`return`. It must not leave its success value to be whatever the last logging
|
||||
call happened to produce.
|
||||
|
||||
That is deliberately stricter than "has a bug". Every instance in the tree when
|
||||
this check was written was in fact benign -- each failure path already returned
|
||||
early, so the trailing `echo` reported a genuine success. The point is that
|
||||
reading the function is the only way to know that, and the next edit that adds
|
||||
an unguarded command before the final log turns a benign shape into the exact
|
||||
defect above with nothing to notice it. An explicit terminal `return` costs one
|
||||
line and makes the class unreachable.
|
||||
|
||||
WHAT IT DELIBERATELY DOES NOT FLAG: a function ending in a logging call whose
|
||||
status nobody consumes. That is idiomatic and harmless — a reporting helper is
|
||||
supposed to end by reporting. The defect needs both halves, and scoping on the
|
||||
call sites rather than on the definitions is what keeps the finding list short
|
||||
enough to stay read. Same reasoning as check-log-strings.py scoping on what a
|
||||
grep reads rather than on what its file mentions.
|
||||
|
||||
Exit codes:
|
||||
0 — no function has both the shape and a status-testing caller
|
||||
1 — at least one does
|
||||
2 — the checker could not run (bad tree, unreadable file)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
TESTING = Path(__file__).resolve().parent
|
||||
|
||||
# Commands whose exit status is always 0 in practice and which exist to print.
|
||||
# `record` and `skip` are project helpers that end in `echo` themselves, so a
|
||||
# function ending in one of those inherits the same problem transitively.
|
||||
LOG_COMMANDS = {
|
||||
"echo", "printf", "log", "info", "warn", "warning", "note", "stage",
|
||||
"pass", "ok", "record", "skip", "report", "summary", "header",
|
||||
}
|
||||
|
||||
# Functions allowed to end in a logging call even though a caller tests them,
|
||||
# each with the reason. Keep this list short: a long one means the check has
|
||||
# been miscalibrated rather than satisfied.
|
||||
ALLOWED: dict[str, str] = {}
|
||||
|
||||
FUNC_RE = re.compile(r"^\s*(?:function\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*\(\)\s*\{")
|
||||
|
||||
|
||||
def function_bodies(lines: list[str]) -> list[tuple[str, int, list[str]]]:
|
||||
"""Yield (name, start_line, body_lines) for each top-level function.
|
||||
|
||||
Brace counting rather than a real parser. Good enough because the tree's
|
||||
functions are conventionally formatted, and a miscount fails safe: it
|
||||
produces a body that ends somewhere odd, which at worst misreads the last
|
||||
statement of one function rather than silently skipping every function.
|
||||
"""
|
||||
out = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
m = FUNC_RE.match(lines[i])
|
||||
if not m:
|
||||
i += 1
|
||||
continue
|
||||
name = m.group(1)
|
||||
depth = lines[i].count("{") - lines[i].count("}")
|
||||
body_start = i
|
||||
j = i + 1
|
||||
body: list[str] = []
|
||||
while j < len(lines) and depth > 0:
|
||||
depth += lines[j].count("{") - lines[j].count("}")
|
||||
if depth > 0:
|
||||
body.append(lines[j])
|
||||
j += 1
|
||||
out.append((name, body_start + 1, body))
|
||||
i = j
|
||||
return out
|
||||
|
||||
|
||||
def last_statement(body: list[str]) -> str | None:
|
||||
"""The last executable line of a function body, or None if there is none."""
|
||||
for line in reversed(body):
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
return stripped
|
||||
return None
|
||||
|
||||
|
||||
def leading_command(statement: str) -> str | None:
|
||||
"""The command word a statement starts with, ignoring shell decoration."""
|
||||
s = statement.strip()
|
||||
# A trailing-log defect cannot hide behind these, and treating them as the
|
||||
# command word would miss the real one.
|
||||
for prefix in ("}", "fi", "done", "esac", "else", ";;"):
|
||||
if s == prefix or s.startswith(prefix + " "):
|
||||
return None
|
||||
s = s.lstrip("&|;( ")
|
||||
m = re.match(r"([A-Za-z_][A-Za-z0-9_\-]*)", s)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def status_tested(name: str, all_text: str, defining_file: Path) -> list[str]:
|
||||
"""Call sites that consume the function's exit status, as evidence strings."""
|
||||
patterns = [
|
||||
(rf"\bif\s+{re.escape(name)}\b", "if <fn>"),
|
||||
(rf"\bif\s+!\s+{re.escape(name)}\b", "if ! <fn>"),
|
||||
(rf"\bwhile\s+{re.escape(name)}\b", "while <fn>"),
|
||||
(rf"^\s*{re.escape(name)}\s+[^\n|&]*\|\|", "<fn> ||"),
|
||||
(rf"^\s*{re.escape(name)}\s*\|\|", "<fn> ||"),
|
||||
(rf"^\s*{re.escape(name)}\s+[^\n|&]*&&", "<fn> &&"),
|
||||
(rf"^\s*{re.escape(name)}\s*&&", "<fn> &&"),
|
||||
(rf"\bif\s+\S*\s*{re.escape(name)}\b.*;\s*then", "if ... <fn> ; then"),
|
||||
# Command substitution. Found 2026-07-23 while fixing a function this
|
||||
# check reported clean: `total=$(count_log_pattern "$p") || { ... }`
|
||||
# consumes the status, but the line begins with the variable, so none
|
||||
# of the patterns above match it. That is not an exotic form — it is
|
||||
# how a shell function returns a *value*, and it is the shape of the
|
||||
# `|| true`-swallowed-failure family this check exists to catch.
|
||||
(rf"=\$\(\s*{re.escape(name)}\b", "<var>=$(<fn>)"),
|
||||
(rf"\bif\s+!?\s*\S*=?\$\(\s*{re.escape(name)}\b", "if <var>=$(<fn>)"),
|
||||
(rf"\[\s+\"?\$\(\s*{re.escape(name)}\b", "[ $(<fn>) ]"),
|
||||
]
|
||||
hits = []
|
||||
for pat, label in patterns:
|
||||
if re.search(pat, all_text, re.M):
|
||||
hits.append(label)
|
||||
return sorted(set(hits))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
sh_files = sorted(
|
||||
p for p in TESTING.rglob("*.sh")
|
||||
if "sim-results" not in p.parts and ".cache" not in p.parts
|
||||
)
|
||||
if not sh_files:
|
||||
print("trailing-log check: found no shell scripts to scan", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
corpus = {}
|
||||
for p in sh_files:
|
||||
try:
|
||||
corpus[p] = p.read_text(errors="replace")
|
||||
except OSError as e:
|
||||
print(f"trailing-log check: cannot read {p}: {e}", file=sys.stderr)
|
||||
return 2
|
||||
all_text = "\n".join(corpus.values())
|
||||
|
||||
findings = []
|
||||
scanned = 0
|
||||
shaped = 0
|
||||
for path, text in corpus.items():
|
||||
lines = text.splitlines()
|
||||
for name, lineno, body in function_bodies(lines):
|
||||
scanned += 1
|
||||
stmt = last_statement(body)
|
||||
if stmt is None:
|
||||
continue
|
||||
cmd = leading_command(stmt)
|
||||
if cmd is None or cmd not in LOG_COMMANDS:
|
||||
continue
|
||||
shaped += 1
|
||||
if name in ALLOWED:
|
||||
continue
|
||||
callers = status_tested(name, all_text, path)
|
||||
if callers:
|
||||
rel = path.relative_to(TESTING.parent)
|
||||
findings.append((rel, lineno, name, cmd, callers, stmt))
|
||||
|
||||
for rel, lineno, name, cmd, callers, stmt in sorted(findings):
|
||||
print(f"{rel}:{lineno}: {name}() has a caller that tests its status, "
|
||||
f"but ends in `{cmd}` rather than an explicit return")
|
||||
print(f" last statement: {stmt[:100]}")
|
||||
print(f" status consumed by: {', '.join(callers)}")
|
||||
print(f" fix: add an explicit `return 0` as the last statement. Its "
|
||||
f"success value is currently the trailing command's, which is 0 "
|
||||
f"whatever the function did.")
|
||||
|
||||
print(f"trailing-log check: {scanned} function(s) scanned, "
|
||||
f"{shaped} end in a logging call, {len(ALLOWED)} allowed, "
|
||||
f"{len(findings)} with a status-testing caller")
|
||||
return 1 if findings else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,340 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Reap FIPS CI resources: containers, networks, volumes, images, veth pairs.
|
||||
#
|
||||
# Force-removes everything created by ci-local.sh that is still around —
|
||||
# whether a run finished cleanly, was preempted (SIGTERM/SIGKILL), OOM-killed,
|
||||
# or crashed. Two complementary selectors make this robust no matter how a
|
||||
# prior run died:
|
||||
#
|
||||
# 1. The CI label com.corganlabs.fips-ci=1 (attached to every direct
|
||||
# `docker run`/network/volume ci-local drives). Every run additionally
|
||||
# stamps com.corganlabs.fips-ci.run=<run-id> on the same resources.
|
||||
# 2. The compose project-name prefix fipsci_ (every compose project ci-local
|
||||
# starts is named fipsci_<run-id>_<suite>, so its containers/networks/
|
||||
# volumes all carry com.docker.compose.project=fipsci_... and are named
|
||||
# with that prefix).
|
||||
#
|
||||
# The generic CI label is shared by every run on the host, so an unscoped label
|
||||
# sweep would tear down a CONCURRENT run's resources. So would an unscoped
|
||||
# compose-project sweep, and reap_containers runs BOTH. Neither flag alone is
|
||||
# therefore enough to spare a concurrent run: --run-id narrows only the label
|
||||
# sweep, --project-prefix only the project sweep, and whichever is left broad
|
||||
# reaps everything by itself. This cost three concurrent runs on 2026-07-29,
|
||||
# via a caller that passed --run-id alone and reasonably believed that scoped
|
||||
# it. --run-id now implies the matching project prefix, and --project-prefix
|
||||
# without --run-id is refused, so the dangerous half-scoped states are no
|
||||
# longer reachable. Passing neither is still the broad "reap everything" form.
|
||||
#
|
||||
# Host-namespace veth interfaces are the one non-docker resource reaped here.
|
||||
# The chaos simulation creates each pair in the host namespace and then moves
|
||||
# the two ends into container namespaces, so a run killed in between leaves the
|
||||
# pair behind, and nothing else on the box removes it. Their names carry a
|
||||
# short token derived from the scenario's name suffix, so --veth-suffixes takes
|
||||
# the suffixes a run used and reaps only the tokens those could produce. Only a
|
||||
# reap with neither --run-id nor --veth-suffixes matches every simulation veth
|
||||
# name, in step with the broad label sweep. --project-prefix does not scope the
|
||||
# veth sweep at all: a host interface carries no compose project.
|
||||
#
|
||||
# These interfaces are also the one resource reaped WITHOUT regard to the CI
|
||||
# label — they cannot carry one — so an unscoped reap takes the host ends of a
|
||||
# bare `chaos.sh` simulation's veth pairs too, even though it leaves that
|
||||
# simulation's unlabelled containers running. Scope with --veth-suffixes (or
|
||||
# just don't reap) while a bare simulation is up.
|
||||
#
|
||||
# Usage:
|
||||
# ci-cleanup.sh Reap ALL fips-ci resources (any run)
|
||||
# ci-cleanup.sh --run-id ID Scope the reap to one run: narrows the
|
||||
# label sweep to ID and, unless
|
||||
# --project-prefix says otherwise,
|
||||
# narrows the project sweep to that run
|
||||
# too. Does NOT scope the host veth
|
||||
# sweep — see --veth-suffixes
|
||||
# ci-cleanup.sh --project-prefix P Restrict the compose-project sweep to
|
||||
# names starting with P. Requires
|
||||
# --run-id: on its own it leaves the
|
||||
# label sweep broad, which reaps every
|
||||
# concurrent run regardless of P
|
||||
# ci-cleanup.sh --label L Override the CI label (default above)
|
||||
# ci-cleanup.sh --images "a b,c" Also `docker rmi -f` these image tags
|
||||
# (space- or comma-separated)
|
||||
# ci-cleanup.sh --veth-suffixes "a b" Restrict the host veth sweep to the
|
||||
# name suffixes a single run used
|
||||
# (space- or comma-separated). Without
|
||||
# it the sweep reaches every simulation
|
||||
# veth name on the host, including a
|
||||
# bare `chaos.sh` run's live ones
|
||||
#
|
||||
# Safe to run when there is nothing to reap, and safe to run repeatedly.
|
||||
# Also reachable as `ci-local.sh --reap`.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
LABEL="com.corganlabs.fips-ci=1"
|
||||
RUN_LABEL_KEY="com.corganlabs.fips-ci.run"
|
||||
PROJECT_PREFIX="fipsci_" # broad default: every CI run
|
||||
RUN_ID="" # broad default: every CI run
|
||||
IMAGES=""
|
||||
VETH_SUFFIXES="" # empty AND no --run-id: every simulation veth name
|
||||
# ip(8) runs inside this image, the same way the simulation creates the
|
||||
# interfaces, so the reap works wherever the simulation does. Any fips test
|
||||
# image will do; it is wanted only for its iproute2.
|
||||
#
|
||||
# Empty here and resolved after the argument loop, because the resolution has
|
||||
# to consider --veth-image. The old default of fips-test:latest is no longer
|
||||
# safe on its own: ci-local.sh does not write that tag, so on a host that has
|
||||
# only ever run the harness it need not exist at all, and the reap this script
|
||||
# advertises as the remedy for orphaned interfaces would be a permanent no-op.
|
||||
VETH_IMAGE=""
|
||||
PROJECT_PREFIX_GIVEN=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--label) LABEL="$2"; shift 2 ;;
|
||||
--project-prefix) PROJECT_PREFIX="$2"; PROJECT_PREFIX_GIVEN=1; shift 2 ;;
|
||||
--run-id) RUN_ID="$2"; shift 2 ;;
|
||||
--images) IMAGES="$2"; shift 2 ;;
|
||||
--veth-suffixes) VETH_SUFFIXES="$2"; shift 2 ;;
|
||||
--veth-image) VETH_IMAGE="$2"; shift 2 ;;
|
||||
-h|--help) sed -n '2,/^set /{ /^set /d; s/^# \?//; p }' "$0"; exit 0 ;;
|
||||
*) echo "Unknown option: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# A reap scoped on one axis and broad on the other still destroys every
|
||||
# concurrent run, because both selectors run. Close both half-scoped states
|
||||
# here rather than trusting each caller to pass the pair.
|
||||
if [[ -n "$RUN_ID" && "$PROJECT_PREFIX_GIVEN" -eq 0 ]]; then
|
||||
# Every run's projects are named "<base prefix><run id>_<suite>", so the
|
||||
# run id is all that is needed. Derived from the base rather than a second
|
||||
# copy of the literal, so the two cannot drift.
|
||||
PROJECT_PREFIX="${PROJECT_PREFIX}${RUN_ID}"
|
||||
fi
|
||||
if [[ -z "$RUN_ID" && "$PROJECT_PREFIX_GIVEN" -eq 1 ]]; then
|
||||
echo "ci-cleanup.sh: --project-prefix requires --run-id." >&2
|
||||
echo " Without it the label sweep stays broad and reaps every concurrent" >&2
|
||||
echo " run regardless of the prefix. Pass both, or neither for a full reap." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Resolve the image to run ip(8) in: the caller's choice, then the run's own
|
||||
# image, then any surviving fips test image. That last fallback is what keeps
|
||||
# an unscoped `ci-local.sh --reap` working — it execs this script from inside
|
||||
# its own argument loop, before the run identity is exported, so it can pass
|
||||
# neither. The empty case is handled at the point of use, which already warns
|
||||
# and skips rather than failing the sweep.
|
||||
if [[ -z "$VETH_IMAGE" ]]; then
|
||||
VETH_IMAGE="${FIPS_TEST_IMAGE:-}"
|
||||
fi
|
||||
if [[ -z "$VETH_IMAGE" ]] && command -v docker >/dev/null 2>&1; then
|
||||
VETH_IMAGE="$(timeout 10 docker image ls --format '{{.Repository}}:{{.Tag}}' fips-test 2>/dev/null | head -n1)"
|
||||
fi
|
||||
[[ -z "$VETH_IMAGE" ]] && VETH_IMAGE="fips-test:latest"
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
# No docker, nothing to reap.
|
||||
exit 0
|
||||
fi
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
# Daemon unreachable; treat as nothing to reap rather than wedging a caller.
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Each docker mutation is wrapped in `timeout` so a stuck daemon/resource can
|
||||
# never wedge a caller (ci-local's signal trap relies on this being bounded).
|
||||
TMO=30
|
||||
|
||||
# Selector for the label sweep. With --run-id it matches only the named run;
|
||||
# without it, every CI run. Note this is only half of what scopes a reap — the
|
||||
# compose-project sweep below is the other half, and both must be narrow.
|
||||
if [[ -n "$RUN_ID" ]]; then
|
||||
SWEEP_LABEL="${RUN_LABEL_KEY}=${RUN_ID}"
|
||||
else
|
||||
SWEEP_LABEL="$LABEL"
|
||||
fi
|
||||
|
||||
# Distinct compose project names (read off container labels) that start with
|
||||
# the configured prefix.
|
||||
ci_projects() {
|
||||
docker ps -a --format '{{.Label "com.docker.compose.project"}}' 2>/dev/null \
|
||||
| grep -E "^${PROJECT_PREFIX}" | sort -u
|
||||
}
|
||||
|
||||
reap_containers() {
|
||||
# By CI label.
|
||||
docker ps -aq --filter "label=${SWEEP_LABEL}" 2>/dev/null \
|
||||
| xargs -r timeout "$TMO" docker rm -f >/dev/null 2>&1 || true
|
||||
# By compose project (carried even when container_name is explicit).
|
||||
local p
|
||||
for p in $(ci_projects); do
|
||||
docker ps -aq --filter "label=com.docker.compose.project=${p}" 2>/dev/null \
|
||||
| xargs -r timeout "$TMO" docker rm -f >/dev/null 2>&1 || true
|
||||
done
|
||||
}
|
||||
|
||||
reap_networks() {
|
||||
docker network ls -q --filter "label=${SWEEP_LABEL}" 2>/dev/null \
|
||||
| xargs -r timeout "$TMO" docker network rm >/dev/null 2>&1 || true
|
||||
# Compose networks are named <project>_<net> → match by name prefix so
|
||||
# orphaned networks (whose containers are already gone) are still caught.
|
||||
docker network ls --format '{{.Name}}' 2>/dev/null | grep -E "^${PROJECT_PREFIX}" \
|
||||
| xargs -r timeout "$TMO" docker network rm >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
reap_volumes() {
|
||||
docker volume ls -q --filter "label=${SWEEP_LABEL}" 2>/dev/null \
|
||||
| xargs -r timeout "$TMO" docker volume rm >/dev/null 2>&1 || true
|
||||
docker volume ls --format '{{.Name}}' 2>/dev/null | grep -E "^${PROJECT_PREFIX}" \
|
||||
| xargs -r timeout "$TMO" docker volume rm >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# Every failure below leaves interfaces behind rather than widening the sweep,
|
||||
# so each one is silent by construction. Say so on stderr instead, or a reap
|
||||
# that reclaimed nothing looks exactly like a reap that had nothing to reclaim.
|
||||
veth_warn() { echo "ci-cleanup: host veth sweep skipped: $*" >&2; }
|
||||
|
||||
# One node id, exactly as chaos/sim/topology.py renders it (f"n{i+1:02d}"):
|
||||
# zero-padded to two digits, and never zero-padded beyond that. Spelling it out
|
||||
# keeps shapes the simulation cannot emit (vh01020a) out of the sweep.
|
||||
VETH_NODE_ID='(0[0-9]|[1-9][0-9]+)'
|
||||
|
||||
# Regex matching the host veth names to remove. With --veth-suffixes it covers
|
||||
# only the tokens those suffixes hash to, so a concurrent run's interfaces —
|
||||
# which carry a different token — cannot match. The token derivation is read
|
||||
# from the simulation itself rather than repeated here, so widening it cannot
|
||||
# leave this matching the old width. Empty output means "reap nothing".
|
||||
#
|
||||
# Two producers, two shapes. The chaos simulation makes vh{token}{NN}{MM}{a,b};
|
||||
# the NAT lab (nat/scripts/setup-topology.sh) makes vn{a,b}{token}{0,1}, using
|
||||
# the RUN-wide suffix rather than any chaos scenario's. Widening this regex is
|
||||
# only half the fix: the token set is derived separately below, so a suffix
|
||||
# list carrying no NAT suffix leaves the NAT half matching nothing while
|
||||
# looking correct. ci-local.sh's ci_teardown therefore appends the run-wide
|
||||
# suffix to --veth-suffixes.
|
||||
veth_pattern() {
|
||||
# vh{token}{NN}{MM}{a,b}: the token is 4 hex or wholly absent — never a
|
||||
# part of one — and the two node ids follow. Anchored and shaped this
|
||||
# tightly so the unscoped sweep cannot reach an interface the simulation
|
||||
# never made. Broad only for an unscoped reap: once --run-id names a
|
||||
# single run, no missing or empty suffix list may widen this back out to
|
||||
# every run.
|
||||
if [[ -z "$RUN_ID" && -z "$VETH_SUFFIXES" ]]; then
|
||||
printf '^vh([0-9a-f]{4})?%s%s[ab]$|^vn[ab]([0-9a-f]{4})?[01]$' \
|
||||
"$VETH_NODE_ID" "$VETH_NODE_ID"
|
||||
return 0
|
||||
fi
|
||||
if [[ -z "$VETH_SUFFIXES" ]]; then
|
||||
veth_warn "--run-id given with no --veth-suffixes"
|
||||
return 0
|
||||
fi
|
||||
local sfx=() tok alt="" out rc
|
||||
read -ra sfx <<< "${VETH_SUFFIXES//,/ }"
|
||||
if [[ ${#sfx[@]} -eq 0 ]]; then
|
||||
veth_warn "--veth-suffixes is empty"
|
||||
return 0
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
veth_warn "python3 not found, cannot derive interface tokens"
|
||||
return 0
|
||||
fi
|
||||
# SCRIPT_DIR comes from $0, so invoking this script through a symlink or a
|
||||
# copy on $PATH points PYTHONPATH at a tree with no simulation in it. Say
|
||||
# which tree was tried when the derivation fails, rather than no-opping.
|
||||
out="$(PYTHONPATH="$SCRIPT_DIR/chaos" python3 -m sim.naming "${sfx[@]}" 2>&1)"
|
||||
rc=$?
|
||||
if [[ $rc -ne 0 ]]; then
|
||||
veth_warn "token derivation failed under $SCRIPT_DIR/chaos: $out"
|
||||
return 0
|
||||
fi
|
||||
for tok in $out; do
|
||||
[[ "$tok" =~ ^[0-9a-f]{4}$ ]] && alt="${alt:+$alt|}$tok"
|
||||
done
|
||||
if [[ -z "$alt" ]]; then
|
||||
veth_warn "no interface tokens derived from: ${sfx[*]}"
|
||||
return 0
|
||||
fi
|
||||
printf '^vh(%s)%s%s[ab]$|^vn[ab](%s)[01]$' \
|
||||
"$alt" "$VETH_NODE_ID" "$VETH_NODE_ID" "$alt"
|
||||
}
|
||||
|
||||
# ip(8) in a privileged --net=host container, matching how the simulation
|
||||
# creates these interfaces (see chaos/sim/veth.py).
|
||||
veth_ip() {
|
||||
timeout "$TMO" docker run --rm --privileged --net=host \
|
||||
--entrypoint ip "$VETH_IMAGE" "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
# The same, taking `ip -batch` commands on stdin so any number of deletes costs
|
||||
# one container. -force keeps ip going past an interface that vanished under us.
|
||||
veth_ip_batch() {
|
||||
timeout "$TMO" docker run --rm -i --privileged --net=host \
|
||||
--entrypoint ip "$VETH_IMAGE" -force -batch - >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
reap_veths() {
|
||||
local pattern
|
||||
pattern="$(veth_pattern)"
|
||||
[[ -z "$pattern" ]] && return 0
|
||||
# Without the image there is no way to run ip(8). Orphans can outlive it —
|
||||
# `docker image prune -a`, a build host that prunes between runs, or a run
|
||||
# whose per-run image was already reaped all remove it while interfaces are
|
||||
# still up — so this is a real skip, not "nothing was ever run here".
|
||||
if ! docker image inspect "$VETH_IMAGE" >/dev/null 2>&1; then
|
||||
veth_warn "image $VETH_IMAGE not present, cannot run ip(8)"
|
||||
return 0
|
||||
fi
|
||||
# Both ends of a pair match, but deleting either removes both, so collapse
|
||||
# each pair to one delete and issue the lot in a single container. The
|
||||
# whole script runs under a caller-imposed timeout, and a container spawn
|
||||
# per name would put a large orphan set at risk of exhausting it. `sort -u`
|
||||
# orders `...a` before `...b`, so the surviving end of a pair the
|
||||
# simulation was killed part-way through moving is still the one picked.
|
||||
local names
|
||||
names="$(veth_ip -o link show \
|
||||
| sed -E 's/^[0-9]+: ([^:@]+).*/\1/' \
|
||||
| grep -E "$pattern" | sort -u \
|
||||
| awk '{ k = substr($0, 1, length($0) - 1)
|
||||
if (!(k in seen)) { seen[k] = 1; print } }')"
|
||||
[[ -z "$names" ]] && return 0
|
||||
sed 's/^/link delete /' <<< "$names" | veth_ip_batch
|
||||
}
|
||||
|
||||
reap_images() {
|
||||
[[ -z "$IMAGES" ]] && return 0
|
||||
local imgs
|
||||
read -ra imgs <<< "${IMAGES//,/ }"
|
||||
[[ ${#imgs[@]} -eq 0 ]] && return 0
|
||||
timeout "$TMO" docker rmi -f "${imgs[@]}" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# Per-run build contexts left in the working tree. ci-local.sh removes its own
|
||||
# from the EXIT trap, but the CI worker sends SIGKILL after SIGTERM and a
|
||||
# SIGKILL runs no trap, so a preempted run can leave an 18 MB directory behind
|
||||
# with nothing else that would ever notice it.
|
||||
#
|
||||
# Scoped mode takes only the named run's. Broad mode cannot tell a live run's
|
||||
# context from an abandoned one by name, so it goes by age instead: a run lasts
|
||||
# well under an hour, and a day is far outside that.
|
||||
reap_build_contexts() {
|
||||
local dir
|
||||
if [[ -n "$RUN_ID" ]]; then
|
||||
dir="$SCRIPT_DIR/docker-$RUN_ID"
|
||||
[[ -d "$dir" ]] && rm -rf "$dir"
|
||||
return 0
|
||||
fi
|
||||
while IFS= read -r dir; do
|
||||
[[ -n "$dir" ]] && rm -rf "$dir"
|
||||
done < <(find "$SCRIPT_DIR" -maxdepth 1 -type d -name 'docker-*' -mtime +0 2>/dev/null)
|
||||
return 0
|
||||
}
|
||||
|
||||
# Order matters: containers reference networks/volumes, so drop them first, and
|
||||
# the veth sweep needs an image to run ip(8) in, so it precedes the image reap.
|
||||
reap_containers
|
||||
reap_networks
|
||||
reap_volumes
|
||||
reap_veths
|
||||
reap_images
|
||||
reap_build_contexts
|
||||
|
||||
exit 0
|
||||
+117
-662
File diff suppressed because it is too large
Load Diff
+12
-49
@@ -32,7 +32,7 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
CACHE_DIR="$SCRIPT_DIR/.cache"
|
||||
DEB_CACHE_DIR="$CACHE_DIR/deb"
|
||||
|
||||
# Timeouts. Each wait loop below exits as soon as its condition is met.
|
||||
# Timeouts
|
||||
BOOT_TIMEOUT=30
|
||||
SERVICE_TIMEOUT=20
|
||||
DAEMON_TIMEOUT=15
|
||||
@@ -65,7 +65,6 @@ start_systemd_container_with_tun() {
|
||||
local name="$1" image="$2"
|
||||
cleanup_container "$name"
|
||||
docker run -d --name "$name" \
|
||||
--label com.corganlabs.fips-ci=1 \
|
||||
--privileged \
|
||||
--cgroupns=host \
|
||||
--device /dev/net/tun \
|
||||
@@ -75,26 +74,15 @@ start_systemd_container_with_tun() {
|
||||
}
|
||||
|
||||
wait_for_systemd() {
|
||||
local name="$1" state
|
||||
local name="$1"
|
||||
for _i in $(seq 1 "$BOOT_TIMEOUT"); do
|
||||
# `is-system-running` exits non-zero for `degraded` (a unit failed to
|
||||
# start -- e.g. systemd-modules-load, which cannot load kernel modules
|
||||
# inside a container -- even though the system did finish booting). This
|
||||
# script runs `set -o pipefail`, so a piped `grep` would inherit that
|
||||
# non-zero exit and reject an acceptable state, which timed out the
|
||||
# newest distros (they reach `degraded`, older ones reach `running`).
|
||||
# Capture the state string and test it directly instead of the pipe.
|
||||
state=$(docker exec "$name" systemctl is-system-running --wait 2>/dev/null || true)
|
||||
case "$state" in
|
||||
running | degraded) return 0 ;;
|
||||
esac
|
||||
if docker exec "$name" systemctl is-system-running --wait 2>/dev/null | grep -qE 'running|degraded'; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# A boot that never reached `running` or `degraded` is not a warning: every
|
||||
# check after this point reads a system that may not have started its units,
|
||||
# and returning 0 here made the timeout indistinguishable from a clean boot.
|
||||
echo " ERROR: systemd did not reach running state in ${BOOT_TIMEOUT}s" >&2
|
||||
return 1
|
||||
echo " WARNING: systemd did not reach running state in ${BOOT_TIMEOUT}s (may still work)"
|
||||
return 0
|
||||
}
|
||||
|
||||
wait_for_service_active() {
|
||||
@@ -198,7 +186,7 @@ _run_deb_install_scenario() {
|
||||
local distro_label="$1"
|
||||
local base_image="$2"
|
||||
|
||||
local name="fips-deb-test-${distro_label}${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-deb-test-${distro_label}"
|
||||
local image="fips-deb-test:${distro_label}"
|
||||
log ".deb install: ${base_image}"
|
||||
|
||||
@@ -240,32 +228,18 @@ DOCKERFILE
|
||||
rm -f "$CACHE_DIR/deb-for-image"
|
||||
|
||||
start_systemd_container_with_tun "$name" "$image"
|
||||
wait_for_systemd "$name" || {
|
||||
fail "systemd did not boot in $name; the install checks below would read an unstarted system"
|
||||
cleanup_container "$name"
|
||||
return
|
||||
}
|
||||
wait_for_systemd "$name"
|
||||
|
||||
# Install the .deb. apt handles dependencies (libc6, systemd,
|
||||
# libdbus-1-3) and runs the maintainer scripts (postinst →
|
||||
# systemctl enable fips.service; fips-dns.service starts and
|
||||
# runs fips-dns-setup).
|
||||
log "Installing .deb (apt install /opt/fips-deb/${deb_basename})"
|
||||
# `|| true` here used to discard apt's exit status, and an empty capture (a
|
||||
# failed `docker exec`) matches neither error pattern below, so a install
|
||||
# that never ran reached `pass "apt install completed"`. Keep the capture on
|
||||
# failure so the diagnostics below can print it, but remember the status.
|
||||
local install_output install_rc=0
|
||||
local install_output
|
||||
install_output=$(docker exec "$name" bash -c "
|
||||
apt-get update >/dev/null 2>&1
|
||||
cd /opt/fips-deb && apt-get install -y --no-install-recommends ./${deb_basename} 2>&1
|
||||
") || install_rc=$?
|
||||
if [ "$install_rc" -ne 0 ]; then
|
||||
fail "apt install exited $install_rc"
|
||||
echo "$install_output" | tail -20
|
||||
cleanup_container "$name"
|
||||
return
|
||||
fi
|
||||
") || true
|
||||
if echo "$install_output" | grep -qE "^E:|errors? were encountered"; then
|
||||
fail "apt install reported errors"
|
||||
echo "$install_output" | tail -20
|
||||
@@ -515,15 +489,4 @@ echo "════════════════════════
|
||||
echo "Results: $PASS passed, $FAIL failed, $SKIP skipped"
|
||||
echo "═══════════════════════════════════════"
|
||||
|
||||
# A skip must not read as a pass. Nothing calls skip() today, so SKIP is
|
||||
# always 0 and this changes no current outcome -- which is exactly why it is
|
||||
# worth adding now: the helper and the counter already existed and were
|
||||
# reported, so a later skip path would have printed "N skipped" next to a
|
||||
# zero exit and looked like coverage. Gate on it before that happens.
|
||||
if [ "$SKIP" -ne 0 ]; then
|
||||
echo "FAIL: $SKIP check(s) skipped; a skipped check is not a passed one." >&2
|
||||
echo " Either make the check run here, or remove it and record the" >&2
|
||||
echo " gap deliberately rather than skipping it at runtime." >&2
|
||||
fi
|
||||
|
||||
[ "$FAIL" -eq 0 ] && [ "$SKIP" -eq 0 ]
|
||||
[ "$FAIL" -eq 0 ]
|
||||
|
||||
@@ -67,7 +67,6 @@ start_systemd_container() {
|
||||
local name="$1" image="$2"
|
||||
cleanup_container "$name"
|
||||
docker run -d --name "$name" \
|
||||
--label com.corganlabs.fips-ci=1 \
|
||||
--privileged \
|
||||
--cgroupns=host \
|
||||
-v /sys/fs/cgroup:/sys/fs/cgroup:rw \
|
||||
@@ -80,7 +79,6 @@ start_systemd_container_with_tun() {
|
||||
local name="$1" image="$2"
|
||||
cleanup_container "$name"
|
||||
docker run -d --name "$name" \
|
||||
--label com.corganlabs.fips-ci=1 \
|
||||
--privileged \
|
||||
--cgroupns=host \
|
||||
--device /dev/net/tun \
|
||||
@@ -270,49 +268,13 @@ DOCKERFILE
|
||||
fi
|
||||
|
||||
log "Extracting fips + fips-gateway binaries from builder image"
|
||||
# Drop the previous run's binaries before extracting. Without this, a failed
|
||||
# extraction below leaves them in place, they satisfy the caller's -x check,
|
||||
# and the e2e scenarios silently exercise the previous commit's code.
|
||||
rm -f "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"
|
||||
|
||||
# stderr goes to its own file rather than into $cid: docker prints
|
||||
# warnings (a platform mismatch, say) on success too, and folding them
|
||||
# into the id would leave every later reference pointing at nothing.
|
||||
local cid err errfile
|
||||
errfile=$(mktemp)
|
||||
if ! cid=$(docker create "$builder_tag" 2>"$errfile"); then
|
||||
echo " ERROR: docker create failed: $(cat "$errfile")"
|
||||
rm -f "$errfile"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$errfile"
|
||||
|
||||
local rc=0 spec bin dest
|
||||
for spec in "fips:$FIPS_BIN_CACHE" "fips-gateway:$FIPS_GATEWAY_BIN_CACHE"; do
|
||||
bin="${spec%%:*}"
|
||||
dest="${spec#*:}"
|
||||
if ! err=$(docker cp "$cid:/src/target/release/$bin" "$dest" 2>&1); then
|
||||
echo " ERROR: extracting $bin from the builder image failed: $err"
|
||||
rc=1
|
||||
fi
|
||||
done
|
||||
local cid
|
||||
cid=$(docker create "$builder_tag")
|
||||
docker cp "$cid:/src/target/release/fips" "$FIPS_BIN_CACHE" >/dev/null 2>&1
|
||||
docker cp "$cid:/src/target/release/fips-gateway" "$FIPS_GATEWAY_BIN_CACHE" >/dev/null 2>&1
|
||||
docker rm "$cid" >/dev/null
|
||||
[ "$rc" -eq 0 ] || return 1
|
||||
|
||||
if ! chmod +x "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"; then
|
||||
echo " ERROR: chmod +x failed on the extracted binaries"
|
||||
return 1
|
||||
fi
|
||||
|
||||
for dest in "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"; do
|
||||
if [ ! -s "$dest" ] || [ ! -x "$dest" ]; then
|
||||
echo " ERROR: extracted binary missing, empty or not executable: $dest"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
chmod +x "$FIPS_BIN_CACHE" "$FIPS_GATEWAY_BIN_CACHE"
|
||||
log "Cached fips ($(stat -c %s "$FIPS_BIN_CACHE") bytes) + fips-gateway ($(stat -c %s "$FIPS_GATEWAY_BIN_CACHE") bytes)"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
@@ -320,7 +282,7 @@ DOCKERFILE
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
test_debian12_resolved() {
|
||||
local name="fips-dns-test-deb12-resolved${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-deb12-resolved"
|
||||
local image="fips-dns-test:debian12-resolved"
|
||||
log "Debian 12 + systemd-resolved (expects global-drop-in)"
|
||||
|
||||
@@ -348,7 +310,7 @@ DOCKERFILE
|
||||
}
|
||||
|
||||
test_debian13_resolved() {
|
||||
local name="fips-dns-test-deb13-resolved${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-deb13-resolved"
|
||||
local image="fips-dns-test:debian13-resolved"
|
||||
log "Debian 13 (trixie) + systemd-resolved (expects global-drop-in)"
|
||||
|
||||
@@ -376,7 +338,7 @@ DOCKERFILE
|
||||
}
|
||||
|
||||
test_ubuntu22_resolved() {
|
||||
local name="fips-dns-test-u22-resolved${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-u22-resolved"
|
||||
local image="fips-dns-test:ubuntu22-resolved"
|
||||
log "Ubuntu 22.04 + systemd-resolved (expects global-drop-in)"
|
||||
|
||||
@@ -406,7 +368,7 @@ DOCKERFILE
|
||||
}
|
||||
|
||||
test_ubuntu24_resolved() {
|
||||
local name="fips-dns-test-u24-resolved${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-u24-resolved"
|
||||
local image="fips-dns-test:ubuntu24-resolved"
|
||||
log "Ubuntu 24.04 + systemd-resolved (expects global-drop-in)"
|
||||
|
||||
@@ -434,7 +396,7 @@ DOCKERFILE
|
||||
}
|
||||
|
||||
test_ubuntu26_resolved() {
|
||||
local name="fips-dns-test-u26-resolved${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-u26-resolved"
|
||||
local image="fips-dns-test:ubuntu26-resolved"
|
||||
log "Ubuntu 26.04 + systemd-resolved (expects global-drop-in)"
|
||||
|
||||
@@ -462,7 +424,7 @@ DOCKERFILE
|
||||
}
|
||||
|
||||
test_dnsmasq() {
|
||||
local name="fips-dns-test-dnsmasq${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-dnsmasq"
|
||||
local image="fips-dns-test:dnsmasq"
|
||||
log "Debian 12 + dnsmasq standalone"
|
||||
|
||||
@@ -528,7 +490,7 @@ DOCKERFILE
|
||||
}
|
||||
|
||||
test_nm_dnsmasq() {
|
||||
local name="fips-dns-test-nm-dnsmasq${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-nm-dnsmasq"
|
||||
local image="fips-dns-test:nm-dnsmasq"
|
||||
log "Fedora + NetworkManager + dnsmasq plugin"
|
||||
|
||||
@@ -591,7 +553,7 @@ DOCKERFILE
|
||||
}
|
||||
|
||||
test_no_resolver() {
|
||||
local name="fips-dns-test-none${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-none"
|
||||
local image="fips-dns-test:none"
|
||||
log "Debian 12 bare (no resolver)"
|
||||
|
||||
@@ -661,7 +623,7 @@ _run_e2e_scenario() {
|
||||
local base_image="$2"
|
||||
local apt_packages="$3"
|
||||
|
||||
local name="fips-dns-test-e2e-${distro_label}${FIPS_CI_NAME_SUFFIX:-}"
|
||||
local name="fips-dns-test-e2e-${distro_label}"
|
||||
local image="fips-dns-test:e2e-${distro_label}"
|
||||
log "End-to-end: ${base_image} + systemd-resolved + real fips + fips-gateway + dig"
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
generated-configs*
|
||||
generated-configs
|
||||
|
||||
@@ -24,18 +24,10 @@ actively DROP'd by the fips chain (not silently unrouted).
|
||||
|
||||
Two FIPS nodes peered over UDP on a Docker bridge network:
|
||||
|
||||
| Container | Hostname | Firewall |
|
||||
|-------------------------|----------|----------|
|
||||
| `fips-fw-container-a` | `host-a` | none (probe) |
|
||||
| `fips-fw-container-b` | `host-b` | `fips.nft` + drop-in |
|
||||
|
||||
The bridge network requests no subnet, so docker assigns one from its own
|
||||
address pool and two concurrent runs never contend for a fixed range. No
|
||||
node's IPv4 address is therefore known before startup, and the generated peer
|
||||
stanzas address each other by docker hostname, resolved through the
|
||||
container's dnsmasq to docker's embedded DNS. The firewall assertions
|
||||
themselves are unaffected: they run over the fips0 overlay, whose addresses
|
||||
are derived from the node npubs.
|
||||
| Container | Hostname | docker IPv4 | Firewall |
|
||||
|-------------------------|----------|---------------|----------|
|
||||
| `fips-fw-container-a` | `host-a` | 172.32.0.10 | none (probe) |
|
||||
| `fips-fw-container-b` | `host-b` | 172.32.0.11 | `fips.nft` + drop-in |
|
||||
|
||||
`node-b` mounts the production `packaging/common/fips.nft` read-only at
|
||||
`/etc/fips/fips.nft`, plus a drop-in at `/etc/fips/fips.d/services.nft`
|
||||
@@ -92,7 +84,7 @@ PASS: fips-fw-container-b: fips.nft baseline + drop-in loaded
|
||||
=== Case (c): ICMPv6 echo-request to firewalled node
|
||||
PASS: (c) ICMPv6 ping node-a → node-b accepted
|
||||
=== Case (a): unallowed inbound TCP/8000 from node-a → node-b
|
||||
PASS: (a) inbound TCP/8000 dropped (curl rc=28, timed out as expected)
|
||||
PASS: (a) inbound TCP/8000 blocked (curl rc=28)
|
||||
=== Case (b): node-b initiates outbound TCP, expects reply via conntrack
|
||||
PASS: (b) outbound from node-b got HTTP 200 via conntrack reply path
|
||||
=== Case (d): drop-in allowlisted TCP/22 from node-a → node-b
|
||||
@@ -116,6 +108,4 @@ docker compose -f testing/firewall/docker-compose.yml down
|
||||
|
||||
## Generated fixture location
|
||||
|
||||
`testing/firewall/generated-configs/` (gitignored), or
|
||||
`generated-configs<suffix>/` when `FIPS_CI_NAME_SUFFIX` is set, which is how
|
||||
concurrent runs keep their fixtures apart.
|
||||
`testing/firewall/generated-configs/` (gitignored).
|
||||
|
||||
@@ -1,30 +1,14 @@
|
||||
networks:
|
||||
# No subnet is requested: docker assigns one from the daemon's address pool.
|
||||
# A fixed request is honoured verbatim, so two runs asking for the same range
|
||||
# collide on "Pool overlaps" — which is why the generated peer addresses are
|
||||
# docker hostnames (host-a, host-b) rather than literal IPs. Docker's embedded
|
||||
# DNS is per-network, so the same hostname in two concurrent runs resolves
|
||||
# inside each run's own subnet.
|
||||
#
|
||||
# That removes one of the two obstacles to running this suite twice at once,
|
||||
# not both. The compose project name is still fixed, so two runs that do not
|
||||
# set COMPOSE_PROJECT_NAME share a project, and the second `up` recreates the
|
||||
# first's containers while either `down` removes both. The local CI runner
|
||||
# scopes it externally (run_firewall in ci-local.sh); a bare hand run does
|
||||
# not. Do not read the floating subnet as making concurrent bare runs safe.
|
||||
fw-net:
|
||||
driver: bridge
|
||||
labels:
|
||||
- "com.corganlabs.fips-ci=1"
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.32.0.0/24
|
||||
|
||||
x-fips-common: &fips-common
|
||||
build:
|
||||
# The harness scopes its build context per run and passes it here; the
|
||||
# shared directory is the hand-run default. Compose resolves a relative
|
||||
# value against THIS file's directory, so the harness must export an
|
||||
# absolute path.
|
||||
context: ${FIPS_BUILD_CONTEXT:-../docker}
|
||||
image: ${FIPS_TEST_IMAGE:-fips-test:latest}
|
||||
context: ../docker
|
||||
image: fips-test:latest
|
||||
entrypoint: ["/usr/local/bin/entrypoint.sh"]
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
@@ -41,26 +25,28 @@ x-fips-common: &fips-common
|
||||
services:
|
||||
service-a:
|
||||
<<: *fips-common
|
||||
container_name: fips-fw-container-a${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: fips-fw-container-a
|
||||
hostname: host-a
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-a/fips.key:/etc/fips/fips.key:ro
|
||||
- ./generated-configs/node-a/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs/node-a/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs/node-a/fips.key:/etc/fips/fips.key:ro
|
||||
networks:
|
||||
- fw-net
|
||||
fw-net:
|
||||
ipv4_address: 172.32.0.10
|
||||
|
||||
service-b:
|
||||
<<: *fips-common
|
||||
container_name: fips-fw-container-b${FIPS_CI_NAME_SUFFIX:-}
|
||||
container_name: fips-fw-container-b
|
||||
hostname: host-b
|
||||
volumes:
|
||||
- ../docker/resolv.conf:/etc/resolv.conf:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.key:/etc/fips/fips.key:ro
|
||||
- ./generated-configs/node-b/hosts:/etc/fips/hosts:ro
|
||||
- ./generated-configs/node-b/fips.yaml:/etc/fips/fips.yaml:ro
|
||||
- ./generated-configs/node-b/fips.key:/etc/fips/fips.key:ro
|
||||
- ../../packaging/common/fips.nft:/etc/fips/fips.nft:ro
|
||||
- ./generated-configs${FIPS_CI_NAME_SUFFIX:-}/node-b/fips.d:/etc/fips/fips.d:ro
|
||||
- ./generated-configs/node-b/fips.d:/etc/fips/fips.d:ro
|
||||
networks:
|
||||
- fw-net
|
||||
fw-net:
|
||||
ipv4_address: 172.32.0.11
|
||||
|
||||
@@ -9,12 +9,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
# Scoped by the per-run suffix because this directory is wiped and rewritten
|
||||
# below: two runs sharing one output directory would delete each other's
|
||||
# fixtures out from under running containers. Unset (a bare hand run, or the
|
||||
# GitHub-hosted path) it collapses to the historical "generated-configs".
|
||||
GENERATED_DIR="$SCRIPT_DIR/generated-configs${FIPS_CI_NAME_SUFFIX:-}"
|
||||
GENERATED_DIR="$SCRIPT_DIR/generated-configs"
|
||||
|
||||
# Deterministic test identities (mirrors the acl-allowlist style).
|
||||
NPUB_A="npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
|
||||
@@ -37,10 +32,6 @@ node-b $NPUB_B
|
||||
EOF
|
||||
}
|
||||
|
||||
# Peers are addressed by the docker hostname the compose file assigns (host-a,
|
||||
# host-b), not by IP. The network requests no subnet so that two concurrent
|
||||
# runs cannot collide on one address range, which means neither node's address
|
||||
# is knowable before `docker compose up`.
|
||||
echo "Generating firewall fixtures..."
|
||||
rm -rf "$GENERATED_DIR"
|
||||
|
||||
@@ -67,7 +58,7 @@ peers:
|
||||
alias: "node-b"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-b:2121"
|
||||
addr: "172.32.0.11:2121"
|
||||
connect_policy: auto_connect
|
||||
EOF
|
||||
|
||||
@@ -98,7 +89,7 @@ peers:
|
||||
alias: "node-a"
|
||||
addresses:
|
||||
- transport: udp
|
||||
addr: "host-a:2121"
|
||||
addr: "172.32.0.10:2121"
|
||||
connect_policy: auto_connect
|
||||
EOF
|
||||
|
||||
|
||||
+17
-99
@@ -22,8 +22,8 @@ TESTING_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml"
|
||||
GENERATE_CONFIGS="$SCRIPT_DIR/generate-configs.sh"
|
||||
|
||||
CONTAINER_A="fips-fw-container-a${FIPS_CI_NAME_SUFFIX:-}"
|
||||
CONTAINER_B="fips-fw-container-b${FIPS_CI_NAME_SUFFIX:-}"
|
||||
CONTAINER_A="fips-fw-container-a"
|
||||
CONTAINER_B="fips-fw-container-b"
|
||||
|
||||
NPUB_A="npub1sjlh2c3x9w7kjsqg2ay080n2lff2uvt325vpan33ke34rn8l5jcqawh57m"
|
||||
NPUB_B="npub1tdwa4vjrjl33pcjdpf2t4p027nl86xrx24g4d3avg4vwvayr3g8qhd84le"
|
||||
@@ -81,46 +81,21 @@ wait_for_fips0() {
|
||||
fail "$container fips0 did not come up within ${timeout}s"
|
||||
}
|
||||
|
||||
# Connected-peer count for a container, or the empty string if it did not
|
||||
# answer.
|
||||
#
|
||||
# Empty is deliberately distinct from a real 0. An `|| echo 0` fallback here
|
||||
# would make "the container is unreachable, or its daemon never came up" and
|
||||
# "the daemon answered, and the answer was zero" the same value, so any caller
|
||||
# expecting zero would be satisfied on the first iteration without the
|
||||
# property it is checking ever being observed. No caller in this file expects
|
||||
# zero today, which is exactly why the fallback has to go now rather than when
|
||||
# one is added. Same shape as the acl-allowlist suite's read_connected_peers.
|
||||
read_connected_peers() {
|
||||
local container="$1"
|
||||
docker exec "$container" fipsctl show peers 2>/dev/null \
|
||||
| python3 -c 'import json,sys; data=json.load(sys.stdin); print(sum(1 for p in data.get("peers", []) if p.get("connectivity") == "connected"))' 2>/dev/null \
|
||||
|| true
|
||||
}
|
||||
|
||||
# Wait for the peer count on a container to reach the expected value.
|
||||
wait_for_peers_exact() {
|
||||
local container="$1"
|
||||
local expected_count="$2"
|
||||
local timeout="${3:-30}"
|
||||
|
||||
local count="" answered=false
|
||||
for _ in $(seq 1 "$timeout"); do
|
||||
count=$(read_connected_peers "$container")
|
||||
if [ -n "$count" ]; then
|
||||
answered=true
|
||||
if [ "$count" -eq "$expected_count" ]; then
|
||||
return 0
|
||||
fi
|
||||
local count
|
||||
count=$(docker exec "$container" fipsctl show peers 2>/dev/null \
|
||||
| python3 -c 'import json,sys; data=json.load(sys.stdin); print(sum(1 for p in data.get("peers", []) if p.get("connectivity") == "connected"))' 2>/dev/null || echo 0)
|
||||
if [ "$count" -eq "$expected_count" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ "$answered" = false ]; then
|
||||
fail "$container never answered a peer query in ${timeout}s, so a count of $expected_count was never actually observed"
|
||||
fi
|
||||
docker exec "$container" fipsctl show peers >&2 || true
|
||||
fail "$container did not reach $expected_count connected peers in ${timeout}s (last answer: $count)"
|
||||
fail "$container did not reach $expected_count connected peers in ${timeout}s"
|
||||
}
|
||||
|
||||
# Resolve `<npub>.fips` inside a container and print the AAAA answer.
|
||||
@@ -149,20 +124,8 @@ assert_baseline_loaded() {
|
||||
listing="$(docker exec "$container" nft list table inet fips)"
|
||||
# Default-deny is achieved via the trailing `counter drop` (chain
|
||||
# policy is `accept` for return-on-non-fips0 to work safely).
|
||||
# Require the verdict, not just the counter: `counter packets` alone
|
||||
# matches a counter rule with any verdict, including one that accepts.
|
||||
# nft renders the rule as `counter packets N bytes M drop`.
|
||||
#
|
||||
# This checks that a counted drop rule exists somewhere in the table,
|
||||
# not that it is the baseline's trailing one. A drop-in under
|
||||
# /etc/fips/fips.d/ contributing its own counted drop would satisfy
|
||||
# this even with the baseline rule removed. Asserting the rule's
|
||||
# position is a larger change than this finding calls for; the
|
||||
# drop-counter check at the end of the run is the one that reads the
|
||||
# trailing rule specifically.
|
||||
if ! printf '%s' "$listing" \
|
||||
| grep -qE 'counter packets [0-9]+ bytes [0-9]+ drop'; then
|
||||
fail "$container: trailing 'counter drop' rule missing from inet fips"
|
||||
if ! printf '%s' "$listing" | grep -q 'counter packets'; then
|
||||
fail "$container: counter drop rule missing from inet fips"
|
||||
fi
|
||||
if ! printf '%s' "$listing" | grep -q 'iifname != "fips0" return'; then
|
||||
fail "$container: non-fips0 early return rule missing"
|
||||
@@ -191,15 +154,7 @@ log "Generating firewall fixtures"
|
||||
|
||||
log "Starting firewall harness"
|
||||
docker compose -f "$COMPOSE_FILE" down >/dev/null 2>&1 || true
|
||||
# --build only on the hand path. Under a harness, --skip-build means the caller
|
||||
# has already built the image this compose file names, and rebuilding it here
|
||||
# would overwrite that image from whatever the shared build context happens to
|
||||
# hold — which is how a suite ends up certifying binaries it was never given.
|
||||
if [ "$SKIP_BUILD" = false ]; then
|
||||
docker compose -f "$COMPOSE_FILE" up -d --build
|
||||
else
|
||||
docker compose -f "$COMPOSE_FILE" up -d
|
||||
fi
|
||||
docker compose -f "$COMPOSE_FILE" up -d --build
|
||||
|
||||
log "Waiting for fips0 on both nodes"
|
||||
wait_for_fips0 "$CONTAINER_A" 40
|
||||
@@ -236,15 +191,7 @@ fi
|
||||
# ── (a) Unallowed inbound is dropped ───────────────────────────────────
|
||||
log "Case (a): unallowed inbound TCP/${UNALLOWED_PORT} from node-a → node-b"
|
||||
# python3 http.server is already listening on :: per entrypoint default mode.
|
||||
#
|
||||
# The rule is a DROP, so the SYN is discarded with no RST and curl must
|
||||
# hit --max-time and exit 28. Assert exactly that rather than "any
|
||||
# non-zero rc": a REJECT, a closed port, an unroutable address or a
|
||||
# missing listener all fail too, with rc 7 or similar, and accepting
|
||||
# those would let the suite report a blocked connection when nothing was
|
||||
# ever blocked. Only 28 distinguishes "silently dropped" from "failed for
|
||||
# some other reason".
|
||||
CURL_TIMEOUT_RC=28
|
||||
# Use curl --max-time 5 — must time out (exit 28) or otherwise fail.
|
||||
set +e
|
||||
docker exec "$CONTAINER_A" curl -6 --silent --output /dev/null \
|
||||
--max-time 5 "http://[${ADDR_B}]:${UNALLOWED_PORT}/"
|
||||
@@ -253,30 +200,21 @@ set -e
|
||||
if [ "$RC" -eq 0 ]; then
|
||||
fail "(a) connection to ${UNALLOWED_PORT} succeeded but should have been DROP'd (rc=0)"
|
||||
fi
|
||||
if [ "$RC" -ne "$CURL_TIMEOUT_RC" ]; then
|
||||
fail "(a) connection to ${UNALLOWED_PORT} failed with curl rc=$RC, expected $CURL_TIMEOUT_RC (timeout). A DROP produces no RST, so anything else means the connection failed for a reason other than the firewall dropping it"
|
||||
fi
|
||||
pass "(a) inbound TCP/${UNALLOWED_PORT} dropped (curl rc=$RC, timed out as expected)"
|
||||
pass "(a) inbound TCP/${UNALLOWED_PORT} blocked (curl rc=$RC)"
|
||||
|
||||
# ── (b) Outbound-initiated flow + conntrack reply ──────────────────────
|
||||
log "Case (b): node-b initiates outbound TCP, expects reply via conntrack"
|
||||
# node-b → node-a:8000 on the fips overlay. node-a has http.server on
|
||||
# [::]:8000 and is NOT firewalled, so this is purely a test of node-b's
|
||||
# outbound + ct state established,related path on the way back.
|
||||
#
|
||||
# mktemp rather than a fixed /tmp name: two concurrent runs of this suite
|
||||
# would otherwise share one host file, and either one's `rm` between the
|
||||
# other's write and read leaves an empty read that fails the http_code check
|
||||
# for a reason that has nothing to do with the firewall.
|
||||
CURL_OUT="$(mktemp)"
|
||||
set +e
|
||||
docker exec "$CONTAINER_B" curl -6 --silent --max-time 5 \
|
||||
--output /dev/null --write-out '%{http_code}' \
|
||||
"http://[${ADDR_A}]:${OUTBOUND_TARGET_PORT}/" >"$CURL_OUT" 2>/dev/null
|
||||
"http://[${ADDR_A}]:${OUTBOUND_TARGET_PORT}/" >/tmp/fw_b_rc 2>/dev/null
|
||||
RC=$?
|
||||
set -e
|
||||
HTTP_CODE="$(cat "$CURL_OUT" 2>/dev/null || true)"
|
||||
rm -f "$CURL_OUT"
|
||||
HTTP_CODE="$(cat /tmp/fw_b_rc 2>/dev/null || true)"
|
||||
rm -f /tmp/fw_b_rc
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
fail "(b) outbound from node-b failed (curl rc=$RC, http=$HTTP_CODE) — conntrack reply path broken"
|
||||
fi
|
||||
@@ -299,28 +237,8 @@ fi
|
||||
|
||||
# ── Drop-counter sanity ────────────────────────────────────────────────
|
||||
log "Drop counter incremented (case a should have ticked it)"
|
||||
# Scope to a drop rule rather than any counter rule: `/counter packets/`
|
||||
# alone takes the first counter rule of whatever verdict.
|
||||
#
|
||||
# Extract the count by position within the matched text, not by field
|
||||
# number. `$3` assumes the line starts with `counter`, so on a rule like
|
||||
# `tcp dport 9 counter packets 42 bytes 3000 drop` it prints the port,
|
||||
# not the packet count -- a plausible shape, since fips.nft includes
|
||||
# /etc/fips/fips.d/*.nft ahead of the trailing drop and a drop-in may
|
||||
# legitimately add its own counted drop.
|
||||
#
|
||||
# Take the LAST match, not the first. Case (a) is an unallowed inbound
|
||||
# that matches no accept rule and falls through to the baseline
|
||||
# default-deny, which fips.nft emits as the final rule of the chain
|
||||
# (packaging/common/fips.nft, `counter drop` after the drop-in include).
|
||||
# A drop-in's own counted drop would otherwise be read instead, and it
|
||||
# has nothing to do with what case (a) exercised.
|
||||
DROP_PKTS="$(docker exec "$CONTAINER_B" nft list table inet fips \
|
||||
| awk 'match($0, /counter packets [0-9]+ bytes [0-9]+ drop/) {
|
||||
line = substr($0, RSTART); sub(/^counter packets /, "", line);
|
||||
split(line, f, " "); val = f[1]
|
||||
}
|
||||
END { if (val != "") print val }')"
|
||||
| awk '/counter packets/ && !seen { print $3; seen=1 }')"
|
||||
if [ -z "${DROP_PKTS:-}" ] || [ "$DROP_PKTS" -lt 1 ]; then
|
||||
fail "drop counter is $DROP_PKTS — case (a) should have produced drops"
|
||||
fi
|
||||
|
||||
@@ -162,13 +162,6 @@ REKEY_SETTLE=12 # FSP-cutover settle budget (Phase 6)
|
||||
# strict assertion sweep runs. A genuinely stuck pair still fails — the
|
||||
# poll times out and the recording sweep captures it.
|
||||
POST_REKEY_TIMEOUT=45
|
||||
# Progress-aware stall budget for the convergence detector
|
||||
# (wait_for_full_baseline → wait_until_connected). If no additional pair
|
||||
# becomes reachable for this long while more than the near-converged
|
||||
# slack of pairs is still down, the detector gives up early instead of
|
||||
# burning the whole convergence/post-rekey window; any progress resets
|
||||
# the clock, so a slow-but-converging mesh under netem keeps polling.
|
||||
RECONVERGE_STALL=15
|
||||
LOG_POLL_INTERVAL=2
|
||||
|
||||
# Data-plane continuity stream (control-differential). Streams run a
|
||||
@@ -441,27 +434,25 @@ ping_all_pairs() {
|
||||
done
|
||||
}
|
||||
|
||||
# Convergence detector probe: one full all-pairs ping sweep in the
|
||||
# "convergence" context (which ping_all_pairs deliberately does NOT
|
||||
# record as a failure), setting PASSED/FAILED for wait_until_connected.
|
||||
_baseline_probe() {
|
||||
ping_all_pairs quiet 1 "convergence"
|
||||
}
|
||||
|
||||
# Poll until every directed pair pings clean, or until timeout. This is a
|
||||
# convergence DETECTOR — used at establishment (Phase 1) and after each
|
||||
# rekey (Phases 3/5). It pings with the "convergence" context, which
|
||||
# ping_all_pairs deliberately does not record as a failure; the caller
|
||||
# runs a separate strict assertion sweep afterwards.
|
||||
#
|
||||
# Delegates to the shared progress-aware wait_until_connected so the
|
||||
# deadline extends while more pairs are still coming up and gives up fast
|
||||
# on a genuine stall, instead of the prior fixed-deadline poll that could
|
||||
# false-time-out under heavy CI contention even while still converging.
|
||||
# Returns 0 once every pair is reachable, 1 on stall/timeout.
|
||||
wait_for_full_baseline() {
|
||||
local timeout="$1"
|
||||
wait_until_connected _baseline_probe "$timeout" "$RECONVERGE_STALL"
|
||||
local start=$SECONDS
|
||||
local best_passed=0 best_failed="$NUM_DIRECTED"
|
||||
while (( SECONDS - start < timeout )); do
|
||||
ping_all_pairs quiet 1 "convergence"
|
||||
if [ "$PASSED" -gt "$best_passed" ]; then
|
||||
best_passed="$PASSED"; best_failed="$FAILED"
|
||||
fi
|
||||
[ "$FAILED" -eq 0 ] && return 0
|
||||
sleep 1
|
||||
done
|
||||
PASSED="$best_passed"; FAILED="$best_failed"
|
||||
return 1
|
||||
}
|
||||
|
||||
phase_result() {
|
||||
@@ -476,24 +467,13 @@ phase_result() {
|
||||
}
|
||||
|
||||
# Count a pattern across all node logs.
|
||||
#
|
||||
# A node whose logs cannot be read makes the whole count unusable rather than
|
||||
# contributing 0. The previous form ended each read `| grep -cE … || true`, so a
|
||||
# failed `docker logs` yielded 0 for that node and the eight expect-zero
|
||||
# assertions in the GLOBAL_PATTERNS loop read a clean result from a node that was
|
||||
# never consulted. Same defect and same fix as rekey-test.sh.
|
||||
count_log_pattern() {
|
||||
local pattern="$1" total=0 n logs count
|
||||
local pattern="$1" total=0 n count
|
||||
for n in "${NODES[@]}"; do
|
||||
if ! logs=$(docker logs "${CONTAINER[$n]}" 2>&1); then
|
||||
echo "unreadable:${CONTAINER[$n]}"
|
||||
return 1
|
||||
fi
|
||||
count=$(grep -cE "$pattern" <<<"$logs" || true)
|
||||
count=$(docker logs "${CONTAINER[$n]}" 2>&1 | grep -cE "$pattern" || true)
|
||||
total=$((total + count))
|
||||
done
|
||||
echo "$total"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Per-node count of a pattern.
|
||||
@@ -779,15 +759,8 @@ echo ""
|
||||
|
||||
# ── Phase 4: second rekey cycle ──────────────────────────────────────
|
||||
|
||||
echo "Phase 4: Second rekey cycle (waiting up to ${SECOND_REKEY_WAIT}s for the next cutover)"
|
||||
# Poll for the next FMP cutover beyond what Phases 2/3 already saw, using
|
||||
# the same pre/post cutover-count delta convention as the control window
|
||||
# (Phase 1b), instead of a blind sleep. Bounded by SECOND_REKEY_WAIT so a
|
||||
# stalled rekey falls through to the strict Phase 5/6 assertions.
|
||||
fmp_cutovers_before="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')"
|
||||
wait_for_log_pattern_count \
|
||||
"Rekey cutover complete \(initiator\), K-bit flipped" \
|
||||
"$((fmp_cutovers_before + 1))" "$SECOND_REKEY_WAIT" || true
|
||||
echo "Phase 4: Second rekey cycle (waiting ${SECOND_REKEY_WAIT}s)"
|
||||
sleep "$SECOND_REKEY_WAIT"
|
||||
echo ""
|
||||
|
||||
echo "Phase 5: Post-second-rekey connectivity (reconverge within ${POST_REKEY_TIMEOUT}s)"
|
||||
@@ -857,20 +830,15 @@ declare -A GLOBAL_PATTERNS=(
|
||||
["ERROR"]="error-level log lines"
|
||||
["unknown FMP version|Unknown FMP version"]="unknown-FMP-version drops"
|
||||
["MMP link teardown"]="MMP link teardowns"
|
||||
["Excessive decryption failures"]="excessive-decryption-failure removals"
|
||||
["Excessive decrypt failures"]="excessive-decrypt-failure removals"
|
||||
["Session AEAD decryption failed"]="FSP AEAD decrypt failures"
|
||||
["Rekey msg2 processing failed"]="rekey msg2 failures"
|
||||
["Handshake failed|handshake failed"]="handshake failures"
|
||||
["Handshake failed|handshake failed|Handshake error"]="handshake failures"
|
||||
)
|
||||
|
||||
for pat in "${!GLOBAL_PATTERNS[@]}"; do
|
||||
desc="${GLOBAL_PATTERNS[$pat]}"
|
||||
if ! total="$(count_log_pattern "$pat")"; then
|
||||
echo " FAIL $desc: node logs unreadable ($total), zero not established"
|
||||
FAILED=$((FAILED + 1))
|
||||
INTEROP_FAILURES+=("[log] $desc: node logs unreadable ($total)")
|
||||
continue
|
||||
fi
|
||||
total="$(count_log_pattern "$pat")"
|
||||
if [ "$total" -eq 0 ]; then
|
||||
echo " PASS $desc: 0"
|
||||
PASSED=$((PASSED + 1))
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user