diff --git a/src/node/handlers/forwarding.rs b/src/node/handlers/forwarding.rs index e96dfc0..96d3185 100644 --- a/src/node/handlers/forwarding.rs +++ b/src/node/handlers/forwarding.rs @@ -84,6 +84,12 @@ impl Node { self.stats_mut() .forwarding .record_drop_no_route(payload.len()); + debug!( + src = %self.peer_display_name(&datagram.src_addr), + dest = %self.peer_display_name(&datagram.dest_addr), + bytes = payload.len(), + "Dropping transit SessionDatagram: no route to destination" + ); self.send_routing_error(&datagram).await; return; } diff --git a/testing/README.md b/testing/README.md index 8e09664..fb5dce8 100644 --- a/testing/README.md +++ b/testing/README.md @@ -54,3 +54,21 @@ testing. Scenarios are defined in YAML and executed via a Python harness that manages the full lifecycle: topology generation, Docker orchestration, fault scheduling, log collection, and analysis. + +### [interop/](interop/) -- Mixed-Version Interop Harness + +On-demand harness that runs an N-node full mesh from a node-spec where +each node can run a different build of the FIPS daemon, then attributes +every FMP/FSP/rekey/connectivity failure to a specific version pair +(same-version vs MIXED). Used to catch interop regressions between +builds, not as a per-commit CI gate; not part of `ci-local.sh`. + +### [mesh-lab/](mesh-lab/) -- Mesh Reliability Lab + +On-demand harness that runs a chosen integration suite N times under a +configurable host-pressure profile (idle / light / github-runner- +equivalent / heavy via `stress-ng`), per-container netem impairment, +and optional trace-level RUST_LOG, capturing per-rep diagnostics and a +mechanism-match summary across the run. Used for statistical reliability +characterization of known flake classes under calibrated stress, not as +a per-commit gate; not part of `ci-local.sh`. diff --git a/testing/interop/.gitignore b/testing/interop/.gitignore new file mode 100644 index 0000000..5d2520e --- /dev/null +++ b/testing/interop/.gitignore @@ -0,0 +1,10 @@ +# Generated per-node configs, generated compose, manifests +# (generated-configs/ is ignored wholesale, so docker-compose.generated.yml +# and nodes.env are covered). +generated-configs/ + +# Per-ref worktree + build-context scratch space (build-images.sh) +.build/ + +# Netem stress-loop artifacts (interop-stress.sh) +.stress-runs/ diff --git a/testing/interop/README.md b/testing/interop/README.md new file mode 100644 index 0000000..e07a8fa --- /dev/null +++ b/testing/interop/README.md @@ -0,0 +1,292 @@ +# Mixed-Version Interop Test Harness + +A CI lab harness for **mixed-version interoperability** testing. It runs an +N-node full mesh where nodes run **different builds** of the FIPS daemon, and +checks that every pair of versions interoperates — FMP link, FSP session, +connectivity, and rekey survival — without a failure a same-version pair +would not have. + +## What it tests + +The static/rekey suites bake one binary set for all nodes, so they only ever +test a version against itself. This harness breaks that assumption: each node +runs an image built from its own git ref. The harness then looks for +**interop regressions** — places where two *different* versions fail to +interoperate: + +- FMP handshake failures across versions +- `unknown FMP version` drops +- FSP / FMP AEAD decrypt failures +- replay storms / excessive-decrypt-failure removals +- link or session teardowns +- asymmetric connectivity drops +- rekey (FMP link + FSP session) that completes within a version but stalls + or breaks across versions + +Every failure is **attributed to a specific version pair**, classified +same-version vs MIXED. The summary states whether failures are +mixed-version-only (a genuine interop regression), both mixed and same +(general instability), or same-version-only (a build is unstable even +against itself). + +## The node-spec + +The harness is parameterized by a **node-spec**: a multiset of image slots, +of size >= 2. Each slot is one of `a`, `b`, `c`, and the same slot may appear +more than once. The slots resolve to the three images built by +`build-images.sh`: + +| Slot | Role | Intended ref | +| ---- | ------------------- | -------------------------------- | +| `a` | version under test | the branch tip / commit to vet | +| `b` | parent / comparison | parent commit on the same branch | +| `c` | release baseline | latest release tag (`v0.3.0`) | + +`build-images.sh` is **unchanged** — it always builds exactly three images +from three refs. A node-spec like `a a b c` resolves to the *same* three +images; only the mesh topology grows. + +### Node identity vs image slot + +Node identity and image slot are **separate**: + +- A spec entry is a slot letter. +- Node id = ``, ordinal counting occurrences of that slot, + 1-based. +- Container name = `fips-interop-`. +- IPv4 = `172.30.0.1`, index = 0-based position in the spec + (`.10`, `.11`, `.12`, ...). +- Each node maps to its image slot: `a1`, `a2` → `fips-interop:a`. +- A pair is **same-version** iff the two nodes' slots resolve to the same + built SHA (read from `.build/refs.env`); else **MIXED**. + +Two same-slot nodes get **distinct identities** — `derive_keys.py` is keyed +by node id, so `a1` and `a2` are different npubs even though they run the +same binary. + +### Example specs + +| Node-spec | Node ids | Pairs | +| --------- | --------------- | ------------------------------------------------ | +| `a b c` | `a1 b1 c1` | 3 pairs, all MIXED — today's triangle (default). | +| `a a b c` | `a1 a2 b1 c1` | 6 pairs: `a1↔a2` **same** + 5 MIXED. | +| `a a a` | `a1 a2 a3` | 3 pairs, all **same** — a same-version flake rig.| + +### Why the `a a b c` control topology matters + +The triangle `a b c` has *no same-version pair* — every pair is mixed, so +under packet loss you cannot tell an interop regression from generic loss +noise. The `a a b c` spec adds a **control arm**: the `a1↔a2` pair runs +identical binaries. Under a netem stress loop: + +- a failure on a mixed pair the control pair does **not** share → an interop + regression; +- a failure both the mixed pairs and the control pair share → loss-induced + instability, not version-specific. + +That control pair is what makes a stress run *interpretable*. It is the +default node-spec for `interop-stress.sh`. + +The `a a a` spec is the degenerate case: all-same-version, used purely as a +**flake rig** — exercising a single build against itself under loss to find +loss-induced instability with no version variable at all. + +## Files + +| File | Purpose | +| ----------------------- | -------------------------------------------------------------- | +| `build-images.sh` | Build `fips-interop:a/b/c`, one Docker image per git ref. | +| `generate-configs.sh` | Generate per-node configs, the generated compose, manifests. | +| `interop-test.sh` | Test driver: bring up, converge, rekey, analyze, attribute. | +| `interop-stress.sh` | Netem stress loop: N reps, pass rate, mixed-vs-same attribution.| +| `README.md` | This document. | + +Generated at runtime: `generated-configs/` (per-node configs + +`docker-compose.generated.yml` + `nodes.env` + `npubs.env`), `.build/`, +`.stress-runs/`. The root for these three is selected by the +`FIPS_INTEROP_RUNS_DIR` environment variable — see +[Scratch directory location](#scratch-directory-location) below. + +The static `docker-compose.yml` is gone — the compose file is now generated +per node-spec into `generated-configs/docker-compose.generated.yml`. + +## How per-node images work + +`build-images.sh ` (unchanged — always three refs): + +1. For each ref, `git worktree add --detach` a temp checkout of the repo. +2. `cargo build --release` the four binaries (`fips`, `fipsctl`, `fipstop`, + `fips-gateway`) in that worktree. +3. Copy the binaries into a build context alongside the shared + `testing/docker/Dockerfile` (and `entrypoint.sh`, `resolv.conf`). +4. `docker build` it, tagging `fips-interop:` and labelling the image + with its ref + short SHA. +5. Remove the temp worktree (done per-ref so peak disk stays at one worktree). + +It also writes `.build/refs.env`, recording each slot's ref and SHA. The +driver reads it to know which pairs are mixed-version. (If absent, it falls +back to the image labels.) + +## How to run it + +### Build the images (once per ref set) + +```bash +cd /dpool/src/clabs/nostr/fips +bash testing/interop/build-images.sh +``` + +Example: `A` = tip of `fix/fsp-rekey-overlapping-epoch`, `B` = `maint`, +`C` = release tag `v0.3.0`: + +```bash +bash testing/interop/build-images.sh fix/fsp-rekey-overlapping-epoch maint v0.3.0 +``` + +### Run a single mesh + +```bash +bash testing/interop/interop-test.sh [node-spec...] +``` + +`node-spec` defaults to `a b c` (the original triangle). Examples: + +```bash +bash testing/interop/interop-test.sh # a b c — 3-node triangle +bash testing/interop/interop-test.sh a a b c # 4-node, one control pair +bash testing/interop/interop-test.sh a a a # 3-node same-version flake rig +``` + +The driver regenerates configs automatically whenever the requested +node-spec differs from the one on disk, so changing the spec just works. +A run takes a few minutes (driven by `REKEY_AFTER_SECS`, default 35, times +two rekey cycles). + +### Run the netem stress loop + +```bash +FIPS_INTEROP_NETEM="delay 10ms 5ms 25% loss 2%" \ + bash testing/interop/interop-stress.sh [--reps N] [node-spec...] +``` + +- `--reps N` — repetitions (default 10). +- `node-spec` — default `a a b c` (the control topology). +- Reps run **serially** — the harness uses fixed container names and a fixed + Docker network, so two reps must never overlap. +- If `FIPS_INTEROP_NETEM` is unset the script warns (a stress run normally + wants netem) but still runs a clean baseline loop. + +Each rep invokes `interop-test.sh` with netem set and captures its full +output and exit code; a rep passes iff `interop-test.sh` exits 0. Artifacts +land in `testing/interop/.stress-runs//`: + +- `rep-NN/driver.log` — full driver output for every rep. +- `rep-NN/docker-.log` — per-container `docker logs` for **failed** + reps only. +- `summary.txt` — the aggregate report. + +The aggregate report gives reps run, passed/failed counts and an integer +pass rate, then tallies — across all failed reps — connectivity failures by +pair kind (mixed vs same), and a verdict: + +- **mixed pairs only, never the control pair** → interop-regression signal; +- **both mixed and same pairs** → loss-induced general instability; +- **same-version control pair only** → the build is unstable against itself. + +`interop-stress.sh` exits **non-zero only** for the interop-regression +signal. A sub-100% pass rate under loss is expected and is not by itself a +failure, so every other outcome exits 0. + +### Options + +| Variable | Effect | +| ---------------------------- | ------------------------------------------------- | +| `FIPS_INTEROP_NETEM` | tc-netem string applied to each container's eth0, e.g. `"delay 10ms 5ms 25% loss 1%"`. Passed through `interop-stress.sh` to `interop-test.sh`. | +| `REKEY_AFTER_SECS` | Rekey interval for generated configs (default 35).| +| `FIPS_INTEROP_KEEP_UP` | `1` = leave containers running after the test. | +| `FIPS_INTEROP_KEEP_WORKTREES`| `1` = keep `build-images.sh` worktrees (debug). | +| `FIPS_INTEROP_RUNS_DIR` | Root for the three scratch dirs — see [Scratch directory location](#scratch-directory-location). | + +The netem hook reuses the flake-lab mechanism (`docker exec ... tc qdisc` on +each container's `eth0`) — host-side bridge qdisc does not shape +inter-container port-to-port traffic, so the impairment must live inside the +containers. + +### Scratch directory location + +The three scratch dirs the harness writes — `.build/` (per-ref build +contexts and `refs.env`), `generated-configs/` (per-node configs + +generated compose + manifests), and `.stress-runs/` (stress-loop +artefacts) — are rooted under `FIPS_INTEROP_RUNS_DIR` when that +environment variable is set. With + +```bash +export FIPS_INTEROP_RUNS_DIR=/var/lib/fips-interop +``` + +all three land under `/var/lib/fips-interop/`, and the source tree +stays clean. + +When `FIPS_INTEROP_RUNS_DIR` is unset, each harness script falls back +to writing under `testing/interop/` itself and prints a one-line +stderr warning naming the variable. The in-tree paths are +`.gitignore`d, so accidentally running without the variable does not +dirty the checkout — but pointing the variable outside the source +tree is recommended, so lab runs do not interleave with the source +working copy at all. + +## How to read the output + +The driver runs six phases: + +| Phase | Check | +| ----- | ---------------------------------------------------------------- | +| 0 | Bring up the mesh (+ optional netem). | +| 1 | All nodes reach N-1 authenticated peers; all directed pairs ping over `fips0` (the definitive FSP-session check). | +| 2 | First FMP rekey cutover completes within the timeout. | +| 3 | All pairs still ping after the first rekey. | +| 4 | Wait out a second rekey cycle. | +| 5 | All pairs still ping after the second rekey. | +| 6 | Per-node / per-pair interop log analysis. | + +Phase 6 is the interop-specific part. It reports: + +- **Global health** — panics, `ERROR` lines, `unknown FMP version` drops, + link teardowns, decrypt failures, handshake failures, rekey-msg2 failures. + Any non-zero count is broken down per node, attributed to a specific build. +- **Rekey machinery exercised** — both FMP and FSP rekey cutovers fired. +- **Per-pair interop summary** — each unordered pair, classified + same-version vs MIXED, with whether it stayed healthy through the run. + +The final verdict lists every failure attributed to a specific +`x[ref@sha] <-> y[ref@sha]` pair or build, then states the attribution: + +- **mixed-version only** → a genuine interop regression. +- **both mixed and same** → general instability, not version-specific. +- **same-version only** → a build is unstable even against itself. + +Exit code is `0` only if every check passed and no per-pair failure was +recorded; non-zero otherwise, with a diagnostic dump (peer/link snapshots and +interop-relevant log tails for all nodes). + +## CI integration + +The harness is self-contained and slots into `.github/workflows/ci.yml` +alongside the existing `integration` matrix suites. A future matrix entry +would, per push to a PR branch: + +1. Resolve the three refs — e.g. `A = github.sha`, + `B = git rev-parse github.sha^`, `C = $(git describe --tags --abbrev=0)` or + a pinned release tag. +2. `bash testing/interop/build-images.sh "$A" "$B" "$C"`. +3. `bash testing/interop/interop-test.sh a a b c` for the control topology, + or `interop-stress.sh` for a loss sweep. +4. On failure, upload the diagnostic dump (or `.stress-runs/`) as an artifact. + +The three `cargo build --release` passes are the cost driver; on a CI runner +this suite is heavier than the single-image suites. Reasonable options are to +run it only on release branches / tags, gate it behind a label, or cache the +`fips-interop:c` (release) image since the release tag rarely moves. + +Until then the harness runs on demand locally — the same way the flake-lab is +used today. diff --git a/testing/interop/build-images.sh b/testing/interop/build-images.sh new file mode 100755 index 0000000..9107f4c --- /dev/null +++ b/testing/interop/build-images.sh @@ -0,0 +1,232 @@ +#!/bin/bash +# Mixed-version interop harness: build three Docker images, one per git ref. +# +# Given three git refs, this script builds the FIPS daemon from each ref and +# produces three distinct Docker images: +# +# fips-interop:a <- ref A ("version under test") +# fips-interop:b <- ref B ("parent") +# fips-interop:c <- ref C ("release") +# +# It deliberately does NOT reuse the fips-test:latest tag — that tag is owned +# by the flake-lab and the static suite, which bake a single binary set for +# all nodes. This harness needs a different binary set per node, so each ref +# gets its own tag. +# +# Mechanism: +# 1. For each ref, `git worktree add --detach` a temp checkout of this repo. +# 2. `cargo build --release` the four binaries in that worktree. +# 3. Copy fips/fipsctl/fipstop/fips-gateway into a per-ref build context. +# 4. `docker build` that context with the testing/docker Dockerfile, tagging +# fips-interop:. +# 5. Remove the temp worktree. +# +# The harness reuses the testing/docker/Dockerfile (and its entrypoint.sh, +# resolv.conf), so the runtime environment is identical to the static and +# rekey suites — only the daemon binaries differ between the three images. +# +# Idempotent and safe to re-run: each invocation rebuilds all three images +# from scratch and cleans up its worktrees, including stale worktrees from a +# previous interrupted run. +# +# Usage: +# ./build-images.sh +# +# Example (the motivating run): +# ./build-images.sh fix/fsp-rekey-overlapping-epoch 79975d72 v0.3.0 +# +# Environment: +# FIPS_INTEROP_KEEP_WORKTREES=1 Keep temp worktrees after build (debug). +# CARGO_BUILD_JOBS=N Passed through to cargo if set. +# FIPS_INTEROP_RUNS_DIR=DIR Root for harness scratch dirs (.build/, +# .stress-runs/, generated-configs/). When +# unset, falls back to in-tree paths under +# testing/interop/ and prints a warning to +# stderr; set it to a path outside the +# source tree to keep generated artefacts +# out of the checkout. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DOCKER_CTX_SRC="$REPO_ROOT/testing/docker" + +# ── Scratch-dir root ───────────────────────────────────────────────── +# +# FIPS_INTEROP_RUNS_DIR controls where the harness writes its scratch +# directories (.build/, .stress-runs/, generated-configs/). When unset +# we fall back to in-tree paths under testing/interop/ and warn the +# operator, so the warning fires exactly once per invocation. When a +# parent script has already warned it exports _FIPS_INTEROP_WARNED=1 +# to suppress duplicate warnings in child scripts. +if [[ -n "${FIPS_INTEROP_RUNS_DIR:-}" ]]; then + RUNS_BASE="$FIPS_INTEROP_RUNS_DIR" + mkdir -p "$RUNS_BASE" +else + RUNS_BASE="$SCRIPT_DIR" + if [[ -z "${_FIPS_INTEROP_WARNED:-}" ]]; then + echo >&2 "WARNING: FIPS_INTEROP_RUNS_DIR not set; harness output will be written under the source tree at $RUNS_BASE. Set FIPS_INTEROP_RUNS_DIR to a path outside the source tree to avoid this." + export _FIPS_INTEROP_WARNED=1 + fi +fi + +# Per-ref tag slots. Order matters: slot[i] <- ref[i]. +SLOTS=(a b c) + +# ── Args ───────────────────────────────────────────────────────────── + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " >&2 + echo "" >&2 + echo " ref-a version under test -> fips-interop:a" >&2 + echo " ref-b parent / comparison -> fips-interop:b" >&2 + echo " ref-c release baseline -> fips-interop:c" >&2 + exit 1 +fi + +REFS=("$1" "$2" "$3") + +# ── Preflight ──────────────────────────────────────────────────────── + +if ! docker info >/dev/null 2>&1; then + echo "ERROR: Docker daemon is not reachable" >&2 + exit 2 +fi + +if ! command -v cargo >/dev/null 2>&1; then + echo "ERROR: cargo not found on PATH" >&2 + exit 2 +fi + +for ref in "${REFS[@]}"; do + if ! git -C "$REPO_ROOT" rev-parse --verify --quiet "${ref}^{commit}" >/dev/null; then + echo "ERROR: git ref '$ref' does not resolve to a commit" >&2 + exit 2 + fi +done + +# ── Worktree + build-context scratch space ─────────────────────────── + +WORK_BASE="$RUNS_BASE/.build" +mkdir -p "$WORK_BASE" + +# Track worktree paths for cleanup. +CREATED_WORKTREES=() + +cleanup() { + if [ "${FIPS_INTEROP_KEEP_WORKTREES:-}" = "1" ]; then + echo "" + echo "FIPS_INTEROP_KEEP_WORKTREES=1 — leaving worktrees in place:" + for wt in "${CREATED_WORKTREES[@]:-}"; do + [ -n "$wt" ] && echo " $wt" + done + return + fi + for wt in "${CREATED_WORKTREES[@]:-}"; do + [ -n "$wt" ] || continue + if [ -d "$wt" ]; then + git -C "$REPO_ROOT" worktree remove --force "$wt" 2>/dev/null \ + || rm -rf "$wt" + fi + done + git -C "$REPO_ROOT" worktree prune 2>/dev/null || true +} +trap cleanup EXIT + +# Prune any stale interop worktrees from a prior interrupted run before we +# start, so re-running the script is clean. +git -C "$REPO_ROOT" worktree prune 2>/dev/null || true + +# ── Per-ref build ──────────────────────────────────────────────────── + +build_one() { + local slot="$1" + local ref="$2" + local sha + sha="$(git -C "$REPO_ROOT" rev-parse --short "$ref")" + + echo "" + echo "=== Building slot '$slot' ref='$ref' sha=$sha ===" + + local wt="$WORK_BASE/worktree-$slot" + local ctx="$WORK_BASE/ctx-$slot" + + # Fresh worktree per build (remove a stale one first). + if [ -d "$wt" ]; then + git -C "$REPO_ROOT" worktree remove --force "$wt" 2>/dev/null \ + || rm -rf "$wt" + fi + git -C "$REPO_ROOT" worktree add --detach "$wt" "$ref" + CREATED_WORKTREES+=("$wt") + + # Build the four binaries from this ref. + local cargo_jobs_arg=() + if [ -n "${CARGO_BUILD_JOBS:-}" ]; then + cargo_jobs_arg=(--jobs "$CARGO_BUILD_JOBS") + fi + ( + cd "$wt" + cargo build --release "${cargo_jobs_arg[@]}" \ + --bin fips --bin fipsctl --bin fipstop --bin fips-gateway + ) + + # Assemble a build context: the testing/docker Dockerfile + support + # files from the MAIN checkout, with the per-ref binaries layered in. + rm -rf "$ctx" + mkdir -p "$ctx" + cp "$DOCKER_CTX_SRC/Dockerfile" "$ctx/Dockerfile" + cp "$DOCKER_CTX_SRC/entrypoint.sh" "$ctx/entrypoint.sh" + cp "$DOCKER_CTX_SRC/resolv.conf" "$ctx/resolv.conf" + for bin in fips fipsctl fipstop fips-gateway; do + cp "$wt/target/release/$bin" "$ctx/$bin" + chmod +x "$ctx/$bin" + done + + docker build \ + --label "fips.interop.slot=$slot" \ + --label "fips.interop.ref=$ref" \ + --label "fips.interop.sha=$sha" \ + -t "fips-interop:$slot" \ + "$ctx" + + # The worktree is large (target/ dir); remove it now rather than at + # exit so peak disk use stays at one worktree, not three. + if [ "${FIPS_INTEROP_KEEP_WORKTREES:-}" != "1" ]; then + git -C "$REPO_ROOT" worktree remove --force "$wt" 2>/dev/null \ + || rm -rf "$wt" + rm -rf "$ctx" + fi + + echo "=== Built fips-interop:$slot ($ref @ $sha) ===" +} + +# Record the ref->sha mapping so the driver and README can report what +# actually ran. Written before the builds so a partial failure still +# leaves a breadcrumb. +MANIFEST="$WORK_BASE/refs.env" +{ + echo "# Generated by build-images.sh on $(date -u +%Y-%m-%dT%H:%M:%SZ)" + for i in 0 1 2; do + slot="${SLOTS[$i]}" + ref="${REFS[$i]}" + sha="$(git -C "$REPO_ROOT" rev-parse --short "$ref")" + upper="$(echo "$slot" | tr '[:lower:]' '[:upper:]')" + echo "INTEROP_REF_${upper}=$ref" + echo "INTEROP_SHA_${upper}=$sha" + done +} > "$MANIFEST" + +for i in 0 1 2; do + build_one "${SLOTS[$i]}" "${REFS[$i]}" +done + +echo "" +echo "=== All three interop images built ===" +docker image ls --filter 'reference=fips-interop' \ + --format 'table {{.Repository}}:{{.Tag}}\t{{.ID}}\t{{.CreatedSince}}' +echo "" +echo "Ref manifest: $MANIFEST" +cat "$MANIFEST" +echo "" +echo "Next: bash testing/interop/interop-test.sh" diff --git a/testing/interop/generate-configs.sh b/testing/interop/generate-configs.sh new file mode 100755 index 0000000..dd32dd9 --- /dev/null +++ b/testing/interop/generate-configs.sh @@ -0,0 +1,303 @@ +#!/bin/bash +# Mixed-version interop harness: generate the per-node configs for a +# node-spec. +# +# A NODE-SPEC is a multiset of image slots (`a`, `b`, `c`), of size >= 2. +# Each spec entry is one node; the same slot may appear more than once. +# The slots resolve to the three Docker images built by build-images.sh: +# +# a -> fips-interop:a ("version under test") +# b -> fips-interop:b ("parent / comparison") +# c -> fips-interop:c ("release baseline") +# +# Node identity vs image slot are SEPARATE: +# - A spec entry is a slot letter. +# - Node id = , ordinal counting occurrences of that +# slot, 1-based. Spec `a a b c` -> node ids `a1 a2 b1 c1`. +# - Container name = fips-interop-. +# - IPv4 = 172.30.0.1, index = 0-based position in the spec +# (10, 11, 12, 13, ...). +# +# Topology is a FULL MESH: every node auto-connects to every other node. +# Node identities are derived deterministically via the shared +# derive_keys.py helper, keyed by a unique per-node string so that two +# nodes of the same slot (a1, a2) still get DISTINCT identities. +# +# Output goes to /generated-configs/ (see FIPS_INTEROP_RUNS_DIR +# under Environment, below): +# .yaml per-node daemon config (one per node) +# docker-compose.generated.yml generated compose, one service per node +# nodes.env manifest for the driver to source +# npubs.env NPUB_ map (per-node npubs) +# +# Generation is deterministic and idempotent: same node-spec in -> same +# files out. +# +# Usage: +# ./generate-configs.sh [node-spec...] +# +# node-spec space-separated slot letters, default `a b c`. +# +# Environment: +# REKEY_AFTER_SECS FSP/FMP rekey interval (default 35, matching +# the static rekey suite). +# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/, +# .stress-runs/, generated-configs/). When +# unset, falls back to in-tree paths under +# testing/interop/ and prints a warning to +# stderr; set it to a path outside the source +# tree to keep generated artefacts out of the +# checkout. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DERIVE_KEYS="$SCRIPT_DIR/../lib/derive_keys.py" + +# ── Scratch-dir root ───────────────────────────────────────────────── +# +# FIPS_INTEROP_RUNS_DIR controls where the harness writes its scratch +# directories (.build/, .stress-runs/, generated-configs/). When unset +# we fall back to in-tree paths under testing/interop/ and warn the +# operator, so the warning fires exactly once per invocation. When a +# parent script has already warned it exports _FIPS_INTEROP_WARNED=1 +# to suppress duplicate warnings in child scripts. +if [[ -n "${FIPS_INTEROP_RUNS_DIR:-}" ]]; then + RUNS_BASE="$FIPS_INTEROP_RUNS_DIR" + mkdir -p "$RUNS_BASE" +else + RUNS_BASE="$SCRIPT_DIR" + if [[ -z "${_FIPS_INTEROP_WARNED:-}" ]]; then + echo >&2 "WARNING: FIPS_INTEROP_RUNS_DIR not set; harness output will be written under the source tree at $RUNS_BASE. Set FIPS_INTEROP_RUNS_DIR to a path outside the source tree to avoid this." + export _FIPS_INTEROP_WARNED=1 + fi +fi + +OUT_DIR="$RUNS_BASE/generated-configs" + +REKEY_AFTER_SECS="${REKEY_AFTER_SECS:-35}" +MESH_NAME="fips-interop-mesh" +UDP_PORT=2121 +SUBNET_PREFIX="172.30.0.1" # node N gets ${SUBNET_PREFIX} + +# ── Args: the node-spec ────────────────────────────────────────────── + +SPEC=("$@") +if [ "${#SPEC[@]}" -eq 0 ]; then + SPEC=(a b c) +fi + +if [ "${#SPEC[@]}" -lt 2 ]; then + echo "ERROR: node-spec must have at least 2 nodes (got ${#SPEC[@]})" >&2 + echo "Usage: $0 [node-spec...] e.g. $0 a a b c" >&2 + exit 1 +fi + +for slot in "${SPEC[@]}"; do + case "$slot" in + a|b|c) ;; + *) + echo "ERROR: invalid slot '$slot' — must be one of a, b, c" >&2 + exit 1 + ;; + esac +done + +# ── Derive node ids, slots, containers, IPs from the spec ──────────── +# +# NODE_IDS preserves spec order. NODE_SLOT/NODE_IP/NODE_CTR are keyed by +# node id. + +NODE_IDS=() +declare -A NODE_SLOT NODE_IP NODE_CTR +declare -A SLOT_COUNT=([a]=0 [b]=0 [c]=0) + +index=0 +for slot in "${SPEC[@]}"; do + SLOT_COUNT[$slot]=$(( SLOT_COUNT[$slot] + 1 )) + nid="${slot}${SLOT_COUNT[$slot]}" + NODE_IDS+=("$nid") + NODE_SLOT[$nid]="$slot" + NODE_CTR[$nid]="fips-interop-$nid" + NODE_IP[$nid]="${SUBNET_PREFIX}${index}" + index=$(( index + 1 )) +done + +mkdir -p "$OUT_DIR" + +# Clear stale per-node configs from a previous (differently-shaped) spec +# so generation is idempotent — a smaller spec must not leave orphans. +rm -f "$OUT_DIR"/*.yaml + +# ── Phase 1: derive identities ─────────────────────────────────────── +# +# Key the derivation by node id (a1, a2, ...) so two same-slot nodes get +# distinct identities. + +declare -A NSEC NPUB +for nid in "${NODE_IDS[@]}"; do + keys="$(python3 "$DERIVE_KEYS" "$MESH_NAME" "$nid")" + NSEC[$nid]="$(echo "$keys" | sed -n 's/^nsec=//p')" + NPUB[$nid]="$(echo "$keys" | sed -n 's/^npub=//p')" +done + +# ── Phase 2: emit per-node configs ─────────────────────────────────── + +emit_peer_block() { + # Args: peer node-id + local pid="$1" + cat < "$cfg" + echo " generated $cfg" +done + +# ── Phase 3: emit the generated docker-compose file ────────────────── +# +# One service per node, image pinned to the node's slot. This supersedes +# the static docker-compose.yml. + +COMPOSE_FILE="$OUT_DIR/docker-compose.generated.yml" +{ + echo "# Mixed-version interop harness: generated compose, one service per node." + echo "#" + echo "# Generated by testing/interop/generate-configs.sh — DO NOT EDIT." + echo "# Node-spec: ${SPEC[*]}" + echo "#" + echo "# Each service is pinned to its node's image slot, so two nodes of the" + echo "# same slot (e.g. a1, a2) run the SAME daemon build. Netem is applied" + echo "# by the driver via 'docker exec ... tc qdisc' on each container eth0." + echo "" + echo "networks:" + echo " fips-interop-net:" + echo " driver: bridge" + echo " ipam:" + echo " config:" + echo " - subnet: 172.30.0.0/24" + echo "" + echo "x-fips-interop-common: &fips-interop-common" + echo " cap_add:" + echo " - NET_ADMIN" + echo " devices:" + echo " - /dev/net/tun:/dev/net/tun" + echo " sysctls:" + echo " - net.ipv6.conf.all.disable_ipv6=0" + echo " restart: \"no\"" + echo " environment:" + echo " - RUST_LOG=info,fips::node::handlers::rekey=debug,fips::node::handlers::handshake=debug" + echo "" + echo "services:" + for nid in "${NODE_IDS[@]}"; do + slot="${NODE_SLOT[$nid]}" + echo " $nid:" + echo " <<: *fips-interop-common" + echo " image: fips-interop:$slot" + echo " container_name: ${NODE_CTR[$nid]}" + echo " hostname: $nid" + echo " volumes:" + echo " - $REPO_ROOT/testing/docker/resolv.conf:/etc/resolv.conf:ro" + echo " - ./$nid.yaml:/etc/fips/fips.yaml:ro" + echo " networks:" + echo " fips-interop-net:" + echo " ipv4_address: ${NODE_IP[$nid]}" + done +} > "$COMPOSE_FILE" +echo " generated $COMPOSE_FILE" + +# ── Phase 4: nodes.env manifest ────────────────────────────────────── +# +# The driver sources this. NODE_IDS is the canonical ordered node list; +# the per-node maps are emitted as space-separated key:value pairs so the +# driver can rebuild associative arrays without re-deriving anything. + +MANIFEST="$OUT_DIR/nodes.env" +{ + echo "# Generated by testing/interop/generate-configs.sh" + echo "# Mesh name: $MESH_NAME" + echo "# Node-spec: ${SPEC[*]}" + echo "INTEROP_SPEC=\"${SPEC[*]}\"" + echo "INTEROP_NODE_IDS=\"${NODE_IDS[*]}\"" + # Per-node maps, one var each, space-separated 'nodeid:value' tokens. + { + printf 'INTEROP_NODE_SLOTS="' + for nid in "${NODE_IDS[@]}"; do printf '%s:%s ' "$nid" "${NODE_SLOT[$nid]}"; done + printf '"\n' + } + { + printf 'INTEROP_NODE_CONTAINERS="' + for nid in "${NODE_IDS[@]}"; do printf '%s:%s ' "$nid" "${NODE_CTR[$nid]}"; done + printf '"\n' + } + { + printf 'INTEROP_NODE_IPS="' + for nid in "${NODE_IDS[@]}"; do printf '%s:%s ' "$nid" "${NODE_IP[$nid]}"; done + printf '"\n' + } + { + printf 'INTEROP_NODE_NPUBS="' + for nid in "${NODE_IDS[@]}"; do printf '%s:%s ' "$nid" "${NPUB[$nid]}"; done + printf '"\n' + } +} > "$MANIFEST" +echo " generated $MANIFEST" + +# ── Phase 5: npubs.env — per-node npub map ─────────────────────────── + +ENV_FILE="$OUT_DIR/npubs.env" +{ + echo "# Generated by testing/interop/generate-configs.sh" + echo "# Mesh name: $MESH_NAME" + for nid in "${NODE_IDS[@]}"; do + upper="$(echo "$nid" | tr '[:lower:]' '[:upper:]')" + echo "NPUB_${upper}=${NPUB[$nid]}" + done +} > "$ENV_FILE" +echo " generated $ENV_FILE" + +echo "" +echo "Mesh configs ready (spec='${SPEC[*]}', rekey.after_secs=$REKEY_AFTER_SECS):" +for nid in "${NODE_IDS[@]}"; do + echo " $nid slot ${NODE_SLOT[$nid]} ${NODE_IP[$nid]} ${NPUB[$nid]}" +done diff --git a/testing/interop/interop-stress.sh b/testing/interop/interop-stress.sh new file mode 100755 index 0000000..b3554a3 --- /dev/null +++ b/testing/interop/interop-stress.sh @@ -0,0 +1,299 @@ +#!/bin/bash +# Mixed-version interop NETEM STRESS LOOP. +# +# Runs interop-test.sh for N repetitions under tc-netem packet loss / +# delay, then reports a pass rate plus a mixed-vs-same-pair failure +# attribution. +# +# Why repetitions: a single run under loss is a coin flip — convergence, +# rekey, and ping retries all interact with packet loss. Only across many +# reps does a SIGNAL separate from noise. The key is the CONTROL ARM: a +# node-spec with a same-version pair (the default `a a b c` gives a1<->a2) +# lets the loop distinguish: +# +# - failures on MIXED pairs only, never the same-version pair +# -> an interop regression — versions diverge under loss. +# - failures on BOTH mixed and same pairs +# -> loss-induced general instability, not version-specific. +# - failures on the same-version pair only +# -> the version under test is unstable even against itself. +# +# A sub-100% pass rate under loss is EXPECTED and is not, by itself, a +# failure. This script exits non-zero ONLY for the interop-regression +# signal (mixed-only failures). +# +# Reps run SERIALLY — interop-test.sh uses fixed container names and a +# fixed Docker network, so two reps must never overlap. +# +# Usage: +# ./interop-stress.sh [--reps N] [node-spec...] +# +# --reps N repetitions (default 10). +# node-spec slot letters (a/b/c), default `a a b c` (the control +# topology — one same-version pair + five mixed pairs). +# +# Environment: +# FIPS_INTEROP_NETEM tc-netem string passed through to interop-test.sh, +# which applies it per-container. A stress run +# normally wants this set; if unset the script +# warns but still runs (a clean baseline loop). +# REKEY_AFTER_SECS forwarded to interop-test.sh (default 35). +# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/, +# .stress-runs/, generated-configs/). When +# unset, falls back to in-tree paths under +# testing/interop/ and prints a warning to +# stderr; set it to a path outside the source +# tree to keep generated artefacts out of the +# checkout. +# +# Artifacts (per invocation): /.stress-runs// +# rep-NN/driver.log full interop-test.sh output for that rep. +# rep-NN/docker-.log per-container `docker logs` (FAILED reps only). +# summary.txt the final aggregate report. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DRIVER="$SCRIPT_DIR/interop-test.sh" + +# ── Scratch-dir root ───────────────────────────────────────────────── +# +# FIPS_INTEROP_RUNS_DIR controls where the harness writes its scratch +# directories (.build/, .stress-runs/, generated-configs/). When unset +# we fall back to in-tree paths under testing/interop/ and warn the +# operator, so the warning fires exactly once per invocation. When a +# parent script has already warned it exports _FIPS_INTEROP_WARNED=1 +# to suppress duplicate warnings in child scripts. +if [[ -n "${FIPS_INTEROP_RUNS_DIR:-}" ]]; then + INTEROP_RUNS_BASE="$FIPS_INTEROP_RUNS_DIR" + mkdir -p "$INTEROP_RUNS_BASE" +else + INTEROP_RUNS_BASE="$SCRIPT_DIR" + if [[ -z "${_FIPS_INTEROP_WARNED:-}" ]]; then + echo >&2 "WARNING: FIPS_INTEROP_RUNS_DIR not set; harness output will be written under the source tree at $INTEROP_RUNS_BASE. Set FIPS_INTEROP_RUNS_DIR to a path outside the source tree to avoid this." + export _FIPS_INTEROP_WARNED=1 + fi +fi + +RUNS_BASE="$INTEROP_RUNS_BASE/.stress-runs" + +# ── Args ───────────────────────────────────────────────────────────── + +REPS=10 +SPEC=() +while [ "$#" -gt 0 ]; do + case "$1" in + --reps) + REPS="${2:-}" + if ! [[ "$REPS" =~ ^[0-9]+$ ]] || [ "$REPS" -lt 1 ]; then + echo "ERROR: --reps needs a positive integer" >&2 + exit 1 + fi + shift 2 + ;; + --reps=*) + REPS="${1#--reps=}" + if ! [[ "$REPS" =~ ^[0-9]+$ ]] || [ "$REPS" -lt 1 ]; then + echo "ERROR: --reps needs a positive integer" >&2 + exit 1 + fi + shift + ;; + -h|--help) + sed -n '2,40p' "$0" + exit 0 + ;; + --) + shift + SPEC+=("$@") + break + ;; + -*) + echo "ERROR: unknown option '$1'" >&2 + exit 1 + ;; + *) + SPEC+=("$1") + shift + ;; + esac +done + +if [ "${#SPEC[@]}" -eq 0 ]; then + SPEC=(a a b c) +fi +SPEC_STR="${SPEC[*]}" + +# ── Preflight ──────────────────────────────────────────────────────── + +if [ ! -x "$DRIVER" ] && [ ! -f "$DRIVER" ]; then + echo "ERROR: driver not found: $DRIVER" >&2 + exit 2 +fi + +if [ -z "${FIPS_INTEROP_NETEM:-}" ]; then + echo "WARN: FIPS_INTEROP_NETEM is unset — a stress run normally wants" + echo " netem packet loss/delay. Running a clean (no-impairment)" + echo " loop anyway. Example:" + echo " FIPS_INTEROP_NETEM=\"delay 10ms 5ms 25% loss 2%\" $0" + echo "" +fi + +# ── Run directory ──────────────────────────────────────────────────── + +RUN_TS="$(date -u +%Y-%m-%dT%H-%M-%SZ)" +RUN_DIR="$RUNS_BASE/$RUN_TS" +mkdir -p "$RUN_DIR" + +echo "==============================================================" +echo " FIPS Interop Netem Stress Loop" +echo "==============================================================" +echo "" +echo "Node-spec : $SPEC_STR" +echo "Reps : $REPS" +echo "Netem : ${FIPS_INTEROP_NETEM:-}" +echo "Artifacts : $RUN_DIR" +echo "" + +# ── Serial rep loop ────────────────────────────────────────────────── + +PASS_COUNT=0 +FAIL_COUNT=0 +FAILED_REPS=() +# Per-kind connectivity-failure tallies, summed across all failed reps. +MIXED_FAILS=0 +SAME_FAILS=0 + +for ((rep = 1; rep <= REPS; rep++)); do + rep_id="$(printf 'rep-%02d' "$rep")" + rep_dir="$RUN_DIR/$rep_id" + mkdir -p "$rep_dir" + driver_log="$rep_dir/driver.log" + + echo "── $rep_id / $REPS ──────────────────────────────────────────" + + # Run the driver, capturing full output and exit code. Netem is + # passed through the environment; interop-test.sh applies it. + FIPS_INTEROP_NETEM="${FIPS_INTEROP_NETEM:-}" \ + bash "$DRIVER" "${SPEC[@]}" >"$driver_log" 2>&1 + rc=$? + + if [ "$rc" -eq 0 ]; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo " PASS (exit 0)" + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + FAILED_REPS+=("$rep_id") + echo " FAIL (exit $rc)" + + # Preserve each failed container's full docker logs. The driver + # uses fixed container names fips-interop-; harvest every + # container matching that prefix that still exists. + while read -r ctr; do + [ -n "$ctr" ] || continue + docker logs "$ctr" >"$rep_dir/docker-${ctr}.log" 2>&1 || true + done < <(docker ps -a --filter 'name=fips-interop-' \ + --format '{{.Names}}' 2>/dev/null) + + # Tally connectivity failures by pair kind, reusing the + # pair-attributed lines interop-test.sh prints. Each line is + # like: " - [baseline] MIXED pair a1[...] <-> c1[...]: ping ..." + m="$(grep -cE '^\s*-\s*\[[^]]+\]\s+MIXED pair' "$driver_log" || true)" + s="$(grep -cE '^\s*-\s*\[[^]]+\]\s+same pair' "$driver_log" || true)" + MIXED_FAILS=$((MIXED_FAILS + m)) + SAME_FAILS=$((SAME_FAILS + s)) + echo " connectivity-failure lines: mixed=$m same=$s" + fi +done + +echo "" + +# ── Aggregate report ───────────────────────────────────────────────── + +# Pass rate as an integer percent. +if [ "$REPS" -gt 0 ]; then + PASS_RATE=$(( PASS_COUNT * 100 / REPS )) +else + PASS_RATE=0 +fi + +# ── Verdict ────────────────────────────────────────────────────────── +# +# Count mixed vs same-version pairs from a completed rep's banner so the +# verdict can normalise for the pair-count asymmetry: a spec like +# `a a b c` has 5 mixed pairs but only 1 same-version control, so an +# isolated loss blip lands on a mixed pair ~5x more often by pure chance. +# A single mixed-only failure is therefore NOT a regression signal — a +# genuine interop regression shows a concentrated, repeated mixed-only +# pattern. Require at least REGRESSION_MIN_MIXED mixed failures before +# calling it; below that, a mixed-only result is reported as loss noise. + +REGRESSION_MIN_MIXED=3 + +MIXED_PAIRS=0 +SAME_PAIRS=0 +_banner="$RUN_DIR/rep-01/driver.log" +if [ -f "$_banner" ]; then + MIXED_PAIRS=$(grep -cE '^ MIXED ' "$_banner" || true) + SAME_PAIRS=$(grep -cE '^ same ' "$_banner" || true) +fi + +EXIT_CODE=0 +if [ "$MIXED_FAILS" -eq 0 ] && [ "$SAME_FAILS" -eq 0 ] && [ "$FAIL_COUNT" -eq 0 ]; then + VERDICT="ALL $REPS reps passed cleanly: every pair stayed healthy across" + VERDICT2="every rep under the applied netem profile." +elif [ "$MIXED_FAILS" -ge "$REGRESSION_MIN_MIXED" ] && [ "$SAME_FAILS" -eq 0 ]; then + VERDICT="INTEROP REGRESSION: $MIXED_FAILS connectivity failures on MIXED-version" + VERDICT2="pairs, none on the same-version control — a concentrated mixed-only pattern above the noise threshold. Versions diverge under loss." + EXIT_CODE=1 +elif [ "$MIXED_FAILS" -gt 0 ] && [ "$SAME_FAILS" -gt 0 ]; then + VERDICT="LOSS-INDUCED INSTABILITY: failures on both mixed ($MIXED_FAILS) and" + VERDICT2="same-version ($SAME_FAILS) pairs — general instability under loss, not version-specific." +elif [ "$SAME_FAILS" -gt 0 ]; then + VERDICT="SAME-VERSION INSTABILITY: $SAME_FAILS failure(s) on the same-version" + VERDICT2="control pair — a build is unstable even against itself, not an interop issue." +elif [ "$MIXED_FAILS" -gt 0 ]; then + VERDICT="NO INTEROP-REGRESSION SIGNAL: $MIXED_FAILS isolated mixed-pair failure(s)" + VERDICT2="across $FAIL_COUNT rep(s), none on the same-version control, below the regression threshold ($REGRESSION_MIN_MIXED). With $MIXED_PAIRS mixed pairs vs $SAME_PAIRS same-version, isolated loss blips land on mixed pairs more often — consistent with loss noise, not a regression." +else + VERDICT="NO connectivity-pair failures recorded, but $FAIL_COUNT rep(s) still" + VERDICT2="failed — on non-connectivity signatures (global-health log patterns or a missing rekey). Check the per-rep driver logs." +fi + +{ + echo "==============================================================" + echo " Interop Netem Stress — Aggregate Report" + echo "==============================================================" + echo "" + echo "Run : $RUN_TS" + echo "Node-spec : $SPEC_STR" + echo "Netem : ${FIPS_INTEROP_NETEM:-}" + echo "" + echo "Reps run : $REPS" + echo "Passed : $PASS_COUNT" + echo "Failed : $FAIL_COUNT" + echo "Pass rate : ${PASS_RATE}%" + if [ "${#FAILED_REPS[@]}" -gt 0 ]; then + echo "Failed reps: ${FAILED_REPS[*]}" + fi + echo "" + echo "-- Connectivity-failure attribution (summed over failed reps) --" + echo " mixed-version : $MIXED_FAILS failure(s) over $MIXED_PAIRS mixed pairs x $REPS reps" + echo " same-version : $SAME_FAILS failure(s) over $SAME_PAIRS same pair(s) x $REPS reps" + echo " regression threshold: >= $REGRESSION_MIN_MIXED mixed-only failures" + echo "" + echo "Verdict:" + echo " $VERDICT" + echo " $VERDICT2" + echo "" + if [ "$EXIT_CODE" -eq 0 ]; then + echo "Exit 0: no interop-regression signal (a sub-100% rate under loss" + echo " is expected and is not by itself a failure)." + else + echo "Exit 1: interop-regression signal present." + fi + echo "" + echo "Artifacts: $RUN_DIR" +} | tee "$RUN_DIR/summary.txt" + +exit "$EXIT_CODE" diff --git a/testing/interop/interop-test.sh b/testing/interop/interop-test.sh new file mode 100755 index 0000000..ecad4f6 --- /dev/null +++ b/testing/interop/interop-test.sh @@ -0,0 +1,700 @@ +#!/bin/bash +# Mixed-version interop test driver. +# +# Brings up an N-node full mesh from a NODE-SPEC (a multiset of image +# slots), verifies every pair interoperates — FMP link, FSP session, +# connectivity, rekey survival — and runs a per-node, per-pair log +# analysis tuned to surface INTEROP problems: handshake failures, +# FSP/FMP decrypt failures, `unknown FMP version`, replay storms, +# link/session teardowns. +# +# The harness's job is NOT to test a single version. It is to find any +# place where two DIFFERENT versions fail to interoperate in a way a +# same-version pair would not. Every failure is attributed to a specific +# (version-X <-> version-Y) pair, classified same-version vs MIXED. +# +# A node-spec like `a a b c` produces a same-version pair (a1<->a2) — the +# CONTROL ARM — alongside the mixed pairs. The control pair is what makes +# a netem run interpretable: a failure on a mixed pair that the control +# pair does not share is an interop regression; a failure both share is +# loss noise, not version-specific. +# +# Prerequisites: +# 1. bash build-images.sh (builds fips-interop:a/b/c) +# 2. configs are (re)generated automatically below for the given spec. +# +# Usage: +# ./interop-test.sh [node-spec...] +# +# node-spec space-separated slot letters (a/b/c), default `a b c`. +# e.g. `a a b c` (4-node, one same-version control pair), +# `a a a` (3-node same-version flake rig). +# +# Environment: +# FIPS_INTEROP_NETEM tc-netem arg string applied to every container's +# eth0, e.g. "delay 10ms 5ms 25% loss 1%". Unset = +# no impairment. +# FIPS_INTEROP_KEEP_UP 1 = leave containers running after the test (debug). +# REKEY_AFTER_SECS rekey interval to generate configs with (default 35). +# FIPS_INTEROP_RUNS_DIR Root for harness scratch dirs (.build/, +# .stress-runs/, generated-configs/). When +# unset, falls back to in-tree paths under +# testing/interop/ and prints a warning to +# stderr; set it to a path outside the source +# tree to keep generated artefacts out of the +# checkout. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +source "$SCRIPT_DIR/../lib/wait-converge.sh" + +# ── Scratch-dir root ───────────────────────────────────────────────── +# +# FIPS_INTEROP_RUNS_DIR controls where the harness writes its scratch +# directories (.build/, .stress-runs/, generated-configs/). When unset +# we fall back to in-tree paths under testing/interop/ and warn the +# operator, so the warning fires exactly once per invocation. When a +# parent script has already warned it exports _FIPS_INTEROP_WARNED=1 +# to suppress duplicate warnings in child scripts. +if [[ -n "${FIPS_INTEROP_RUNS_DIR:-}" ]]; then + RUNS_BASE="$FIPS_INTEROP_RUNS_DIR" + mkdir -p "$RUNS_BASE" +else + RUNS_BASE="$SCRIPT_DIR" + if [[ -z "${_FIPS_INTEROP_WARNED:-}" ]]; then + echo >&2 "WARNING: FIPS_INTEROP_RUNS_DIR not set; harness output will be written under the source tree at $RUNS_BASE. Set FIPS_INTEROP_RUNS_DIR to a path outside the source tree to avoid this." + export _FIPS_INTEROP_WARNED=1 + fi +fi + +GEN_DIR="$RUNS_BASE/generated-configs" +COMPOSE_FILE="$GEN_DIR/docker-compose.generated.yml" +NODES_ENV="$GEN_DIR/nodes.env" +REFS_ENV="$RUNS_BASE/.build/refs.env" + +REKEY_AFTER_SECS="${REKEY_AFTER_SECS:-35}" + +# ── Node-spec ──────────────────────────────────────────────────────── + +SPEC=("$@") +if [ "${#SPEC[@]}" -eq 0 ]; then + SPEC=(a b c) +fi +SPEC_STR="${SPEC[*]}" + +# ── Timing ─────────────────────────────────────────────────────────── + +CONVERGENCE_TIMEOUT=60 # wait for full mesh + clean ping baseline +PING_TIMEOUT=5 +MAX_PING_ATTEMPTS=4 # strict-ping retry (mirrors rekey-test.sh) +PING_RETRY_DELAY=1 +# First rekey should follow shortly after the interval once converged. +FIRST_REKEY_TIMEOUT=$((REKEY_AFTER_SECS + 20)) +SECOND_REKEY_WAIT=$((REKEY_AFTER_SECS + 5)) +REKEY_SETTLE=12 # FSP-cutover settle budget (Phase 6) +# Post-rekey reconvergence is polled, not fixed-slept: the mesh is given +# up to this long to restore full connectivity after a rekey before the +# strict assertion sweep runs. A genuinely stuck pair still fails — the +# poll times out and the recording sweep captures it. +POST_REKEY_TIMEOUT=45 +LOG_POLL_INTERVAL=2 + +# ── Counters ───────────────────────────────────────────────────────── + +PASSED=0 +FAILED=0 +TOTAL_PASSED=0 +TOTAL_FAILED=0 +# INTEROP_FAILURES collects human-readable, pair-attributed failure lines. +INTEROP_FAILURES=() + +# ── Preflight: docker + images ─────────────────────────────────────── + +if ! docker info >/dev/null 2>&1; then + echo "ERROR: Docker daemon is not reachable" >&2 + exit 2 +fi + +# A node-spec only ever references the three image slots, regardless of +# how many nodes it has. Check the slots actually present in the spec. +for slot in $(printf '%s\n' "${SPEC[@]}" | sort -u); do + case "$slot" in + a|b|c) ;; + *) echo "ERROR: invalid slot '$slot' in node-spec" >&2; exit 2 ;; + esac + if ! docker image inspect "fips-interop:$slot" >/dev/null 2>&1; then + echo "ERROR: image fips-interop:$slot not present." >&2 + echo "Build the three images first:" >&2 + echo " bash $SCRIPT_DIR/build-images.sh " >&2 + exit 2 + fi +done + +# ── (Re)generate configs for this node-spec ────────────────────────── +# +# Always regenerate when the spec on disk does not match the requested +# spec (or no manifest exists). Generation is deterministic and cheap. + +need_regen=1 +if [ -f "$NODES_ENV" ]; then + existing_spec="$(sed -n 's/^INTEROP_SPEC="\(.*\)"$/\1/p' "$NODES_ENV")" + if [ "$existing_spec" = "$SPEC_STR" ]; then + need_regen=0 + fi +fi +if [ "$need_regen" -eq 1 ]; then + echo "Generating mesh configs for spec '$SPEC_STR' (rekey.after_secs=$REKEY_AFTER_SECS)..." + REKEY_AFTER_SECS="$REKEY_AFTER_SECS" bash "$SCRIPT_DIR/generate-configs.sh" \ + "${SPEC[@]}" + echo "" +fi + +# ── Load the manifest ──────────────────────────────────────────────── +# +# nodes.env gives the ordered node list and the per-node slot/container/ +# ip/npub maps. npubs.env gives NPUB_ vars used by ping_one. + +# shellcheck disable=SC1090 +source "$NODES_ENV" +# shellcheck disable=SC1091 +source "$GEN_DIR/npubs.env" + +read -r -a NODES <<< "$INTEROP_NODE_IDS" + +declare -A SLOT_OF CONTAINER NODE_IP_OF NPUB_OF +parse_map() { + # parse_map + local -n _dst="$1" + local tok nid val + for tok in $2; do + nid="${tok%%:*}" + val="${tok#*:}" + _dst["$nid"]="$val" + done +} +parse_map SLOT_OF "$INTEROP_NODE_SLOTS" +parse_map CONTAINER "$INTEROP_NODE_CONTAINERS" +parse_map NODE_IP_OF "$INTEROP_NODE_IPS" +parse_map NPUB_OF "$INTEROP_NODE_NPUBS" + +# Unordered pairs of the mesh. +PAIRS=() +for ((i = 0; i < ${#NODES[@]}; i++)); do + for ((j = i + 1; j < ${#NODES[@]}; j++)); do + PAIRS+=("${NODES[$i]} ${NODES[$j]}") + done +done + +NUM_NODES="${#NODES[@]}" +NUM_PAIRS="${#PAIRS[@]}" +NUM_DIRECTED=$((NUM_PAIRS * 2)) +PEERS_EXPECTED=$((NUM_NODES - 1)) + +# ── Ref / version metadata ─────────────────────────────────────────── +# +# refs.env (written by build-images.sh) records what each SLOT was built +# from. If absent, fall back to the image labels, then to "unknown". +# These maps are keyed by SLOT (a/b/c), not by node id. + +declare -A SLOT_REF SLOT_SHA + +load_slot_metadata() { + local slot upper + for slot in a b c; do + SLOT_REF[$slot]="unknown" + SLOT_SHA[$slot]="unknown" + done + if [ -f "$REFS_ENV" ]; then + # shellcheck disable=SC1090 + source "$REFS_ENV" + fi + for slot in a b c; do + upper="$(echo "$slot" | tr '[:lower:]' '[:upper:]')" + local ref_var="INTEROP_REF_${upper}" + local sha_var="INTEROP_SHA_${upper}" + if [ -n "${!ref_var:-}" ]; then + SLOT_REF[$slot]="${!ref_var}" + SLOT_SHA[$slot]="${!sha_var:-unknown}" + continue + fi + local lbl_ref lbl_sha + lbl_ref="$(docker image inspect "fips-interop:$slot" \ + --format '{{ index .Config.Labels "fips.interop.ref" }}' 2>/dev/null || true)" + lbl_sha="$(docker image inspect "fips-interop:$slot" \ + --format '{{ index .Config.Labels "fips.interop.sha" }}' 2>/dev/null || true)" + [ -n "$lbl_ref" ] && SLOT_REF[$slot]="$lbl_ref" + [ -n "$lbl_sha" ] && SLOT_SHA[$slot]="$lbl_sha" + done +} + +# A pair is "mixed-version" iff the two nodes' image slots resolve to +# different built SHAs. SHA is the precise discriminator; ref names can +# differ yet point at the same commit. +pair_is_mixed() { + local n1="$1" n2="$2" + local s1="${SLOT_OF[$n1]}" s2="${SLOT_OF[$n2]}" + if [ "${SLOT_SHA[$s1]}" = "${SLOT_SHA[$s2]}" ] \ + && [ "${SLOT_SHA[$s1]}" != "unknown" ]; then + return 1 # same version + fi + return 0 # mixed (or unknown — treat as mixed for scrutiny) +} + +pair_kind() { + if pair_is_mixed "$1" "$2"; then + echo "MIXED" + else + echo "same" + fi +} + +pair_label() { + # e.g. "a1[A fix/...@3045212] <-> c1[C v0.3.0@b11b639]" + local n1="$1" n2="$2" + local s1="${SLOT_OF[$n1]}" s2="${SLOT_OF[$n2]}" + local u1 u2 + u1="$(echo "$s1" | tr '[:lower:]' '[:upper:]')" + u2="$(echo "$s2" | tr '[:lower:]' '[:upper:]')" + echo "${n1}[$u1 ${SLOT_REF[$s1]}@${SLOT_SHA[$s1]}] <-> ${n2}[$u2 ${SLOT_REF[$s2]}@${SLOT_SHA[$s2]}]" +} + +# ── Helpers ────────────────────────────────────────────────────────── + +# ping6 from one container to another node's mesh address (npub.fips). +ping_one() { + local from_node="$1" to_node="$2" quiet="${3:-}" + local max_attempts="${4:-1}" + local from_ctr="${CONTAINER[$from_node]}" + local to_npub="${NPUB_OF[$to_node]}" + local label="$from_node -> $to_node" + + local attempt=1 output rtt + while (( attempt <= max_attempts )); do + (( attempt > 1 )) && sleep "$PING_RETRY_DELAY" + if output=$(docker exec "$from_ctr" ping6 -c 1 -W "$PING_TIMEOUT" \ + "${to_npub}.fips" 2>&1); then + rtt=$(echo "$output" | grep -oE 'time=[0-9.]+' | cut -d= -f2) + if [ -z "$quiet" ]; then + echo " $label ... OK (${rtt:-?}ms${attempt:+, attempt $attempt})" + fi + PASSED=$((PASSED + 1)) + return 0 + fi + attempt=$((attempt + 1)) + done + if [ -z "$quiet" ]; then + echo " $label ... FAIL (after $max_attempts attempt(s))" + fi + FAILED=$((FAILED + 1)) + return 1 +} + +# All directed pairs of the mesh. Records pair-attributed failures into +# INTEROP_FAILURES, with the mixed/same classification. +ping_all_pairs() { + local quiet="${1:-}" max_attempts="${2:-1}" context="${3:-connectivity}" + PASSED=0 + FAILED=0 + local i j + for i in "${NODES[@]}"; do + for j in "${NODES[@]}"; do + [ "$i" = "$j" ] && continue + if ! ping_one "$i" "$j" "$quiet" "$max_attempts"; then + # The `convergence` context is the wait_for_full_baseline + # detector poll, NOT an assertion: a miss there only means + # "not converged yet, keep polling". Recording detector + # misses as interop failures is wrong — it false-fails + # every rep that takes a moment to converge. Only the + # strict assertion sweeps (baseline / post-rekey-*) record. + if [ "$context" != "convergence" ]; then + local kind + kind="$(pair_kind "$i" "$j")" + INTEROP_FAILURES+=("[$context] $kind pair $(pair_label "$i" "$j"): ping $i->$j FAILED") + fi + fi + done + done +} + +# Poll until every directed pair pings clean, or until timeout. This is a +# convergence DETECTOR — used at establishment (Phase 1) and after each +# rekey (Phases 3/5). It pings with the "convergence" context, which +# ping_all_pairs deliberately does not record as a failure; the caller +# runs a separate strict assertion sweep afterwards. +wait_for_full_baseline() { + local timeout="$1" + local start=$SECONDS + local best_passed=0 best_failed="$NUM_DIRECTED" + while (( SECONDS - start < timeout )); do + ping_all_pairs quiet 1 "convergence" + if [ "$PASSED" -gt "$best_passed" ]; then + best_passed="$PASSED"; best_failed="$FAILED" + fi + [ "$FAILED" -eq 0 ] && return 0 + sleep 1 + done + PASSED="$best_passed"; FAILED="$best_failed" + return 1 +} + +phase_result() { + local phase="$1" + TOTAL_PASSED=$((TOTAL_PASSED + PASSED)) + TOTAL_FAILED=$((TOTAL_FAILED + FAILED)) + if [ "$FAILED" -eq 0 ]; then + echo " PASS $phase: $PASSED/$((PASSED + FAILED))" + else + echo " FAIL $phase: $PASSED passed, $FAILED FAILED" + fi +} + +# Count a pattern across all node logs. +count_log_pattern() { + local pattern="$1" total=0 n count + for n in "${NODES[@]}"; do + count=$(docker logs "${CONTAINER[$n]}" 2>&1 | grep -cE "$pattern" || true) + total=$((total + count)) + done + echo "$total" +} + +# Per-node count of a pattern. +count_node_pattern() { + local node="$1" pattern="$2" + docker logs "${CONTAINER[$node]}" 2>&1 | grep -cE "$pattern" || true +} + +wait_for_log_pattern_count() { + local pattern="$1" min_count="$2" timeout="$3" + local start=$SECONDS + while (( SECONDS - start < timeout )); do + [ "$(count_log_pattern "$pattern")" -ge "$min_count" ] && return 0 + sleep "$LOG_POLL_INTERVAL" + done + [ "$(count_log_pattern "$pattern")" -ge "$min_count" ] +} + +dump_diagnostics() { + echo "" + echo "=== Peer / link snapshot ===" + local n s + for n in "${NODES[@]}"; do + s="${SLOT_OF[$n]}" + echo "--- $n (slot $s, ${SLOT_REF[$s]}@${SLOT_SHA[$s]}) ---" + docker exec "${CONTAINER[$n]}" fipsctl show peers 2>/dev/null || true + docker exec "${CONTAINER[$n]}" fipsctl show links 2>/dev/null || true + echo "" + done + echo "=== Recent node logs (interop-relevant lines) ===" + for n in "${NODES[@]}"; do + echo "--- $n ---" + docker logs "${CONTAINER[$n]}" 2>&1 \ + | grep -E "(ERROR|PANIC|panicked|FMP version|handshake|Handshake|decrypt|Decrypt|teardown|rekey|Rekey|K-bit|established|Established)" \ + | tail -40 + echo "" + done +} + +compose() { + docker compose -f "$COMPOSE_FILE" "$@" +} + +cleanup() { + if [ "${FIPS_INTEROP_KEEP_UP:-}" = "1" ]; then + echo "" + echo "FIPS_INTEROP_KEEP_UP=1 — leaving mesh running. Tear down with:" + echo " docker compose -f $COMPOSE_FILE down --volumes --remove-orphans" + return + fi + compose down --volumes --remove-orphans >/dev/null 2>&1 || true +} +trap cleanup EXIT +trap 'echo ""; echo "Interrupted"; exit 130' INT + +load_slot_metadata + +# ── Banner ─────────────────────────────────────────────────────────── + +echo "==============================================================" +echo " FIPS Mixed-Version Interop Test" +echo "==============================================================" +echo "" +echo "Node-spec: $SPEC_STR ($NUM_NODES nodes, $NUM_PAIRS pairs, $NUM_DIRECTED directed)" +echo "" +echo "Mesh nodes:" +for n in "${NODES[@]}"; do + s="${SLOT_OF[$n]}" + u="$(echo "$s" | tr '[:lower:]' '[:upper:]')" + echo " $n slot $u ${SLOT_REF[$s]} @ ${SLOT_SHA[$s]} (${NODE_IP_OF[$n]})" +done +echo "" +echo "Mesh pairs:" +for p in "${PAIRS[@]}"; do + read -r n1 n2 <<< "$p" + echo " $(pair_kind "$n1" "$n2") $(pair_label "$n1" "$n2")" +done +echo "" +if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then + echo "Netem impairment: $FIPS_INTEROP_NETEM" + echo "" +fi +echo "rekey.after_secs=$REKEY_AFTER_SECS" +echo "" + +# ── Phase 0: bring up the mesh ─────────────────────────────────────── + +echo "Phase 0: Starting mesh" +compose down --volumes --remove-orphans >/dev/null 2>&1 || true +if ! compose up -d; then + echo " FAIL compose up failed" + exit 1 +fi + +# Optional netem: applied via `docker exec ... tc qdisc` on each +# container's eth0 — host bridge qdisc does NOT shape inter-container +# port-to-port traffic, so the impairment must live inside each +# container. Same mechanism as testing/flake-lab/run-loop.sh. +if [ -n "${FIPS_INTEROP_NETEM:-}" ]; then + echo " Applying netem to each container eth0: $FIPS_INTEROP_NETEM" + for n in "${NODES[@]}"; do + if docker exec "${CONTAINER[$n]}" tc qdisc add dev eth0 root \ + netem ${FIPS_INTEROP_NETEM} >/dev/null 2>&1; then + echo " $n: netem applied" + else + echo " $n: WARN netem apply failed" + fi + done +fi +echo "" + +# ── Phase 1: FMP link + FSP session establishment ──────────────────── + +echo "Phase 1: Link/session establishment + connectivity baseline" +PASSED=0; FAILED=0 + +# Every node must reach N-1 authenticated peers. A peer in `show peers` +# is an authenticated FMP-link peer. +for n in "${NODES[@]}"; do + if ! wait_for_peers "${CONTAINER[$n]}" "$PEERS_EXPECTED" "$CONVERGENCE_TIMEOUT"; then + echo " $n did not reach $PEERS_EXPECTED authenticated peers" + fi +done + +# The all-pairs ping over fips0 is the definitive FSP-session check: in +# a full mesh every pair is a direct neighbor, so a successful ping +# proves that pair's FSP session carries traffic in that direction. +if wait_for_full_baseline "$CONVERGENCE_TIMEOUT"; then + ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline" + phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)" +else + echo " Best baseline before timeout: $PASSED/$((PASSED + FAILED))" + ping_all_pairs "" "$MAX_PING_ATTEMPTS" "baseline" + phase_result "Establishment baseline (all $NUM_DIRECTED directed pairs)" + dump_diagnostics + echo "" + echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ===" + echo "FAIL: mesh did not converge — see pair attribution above." + exit 1 +fi +echo "" + +# ── Phase 2: first rekey cycle ─────────────────────────────────────── + +echo "Phase 2: First rekey cycle (waiting up to ${FIRST_REKEY_TIMEOUT}s)" +PASSED=0; FAILED=0 +wait_for_log_pattern_count \ + "Rekey cutover complete \(initiator\), K-bit flipped" 1 \ + "$FIRST_REKEY_TIMEOUT" || true +fmp_cutovers="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')" +if [ "$fmp_cutovers" -ge 1 ]; then + echo " PASS FMP rekey initiator cutovers: $fmp_cutovers" + PASSED=$((PASSED + 1)) +else + echo " FAIL no FMP rekey cutover observed within ${FIRST_REKEY_TIMEOUT}s" + FAILED=$((FAILED + 1)) + INTEROP_FAILURES+=("[rekey] no FMP rekey cutover completed across the mesh") +fi +phase_result "First rekey cycle" +echo "" + +# ── Phase 3: post-first-rekey connectivity ─────────────────────────── + +echo "Phase 3: Post-first-rekey connectivity (reconverge within ${POST_REKEY_TIMEOUT}s)" +PASSED=0; FAILED=0 +# Poll until the mesh has reconverged after the rekey, then assert once. +# The detector poll records nothing (see ping_all_pairs); only the strict +# sweep below records, so a brief rekey-induced disruption is not a fail. +wait_for_full_baseline "$POST_REKEY_TIMEOUT" || true +ping_all_pairs "" "$MAX_PING_ATTEMPTS" "post-rekey-1" +phase_result "Post-first-rekey (all $NUM_DIRECTED directed pairs)" +echo "" + +# ── Phase 4: second rekey cycle ────────────────────────────────────── + +echo "Phase 4: Second rekey cycle (waiting ${SECOND_REKEY_WAIT}s)" +sleep "$SECOND_REKEY_WAIT" +echo "" + +echo "Phase 5: Post-second-rekey connectivity (reconverge within ${POST_REKEY_TIMEOUT}s)" +PASSED=0; FAILED=0 +wait_for_full_baseline "$POST_REKEY_TIMEOUT" || true +ping_all_pairs "" "$MAX_PING_ATTEMPTS" "post-rekey-2" +phase_result "Post-second-rekey (all $NUM_DIRECTED directed pairs)" +echo "" + +# ── Phase 6: per-node, per-pair interop log analysis ───────────────── +# +# This is the interop-specific analysis. For each unordered pair it +# reports same vs mixed-version, then scans BOTH endpoints' logs for +# interop failure signatures. A signature firing on a mixed pair that +# does not fire on a same-version pair is the harness's primary signal. + +echo "Phase 6: Interop log analysis" +PASSED=0; FAILED=0 + +# Give FSP rekey (which trails FMP rekey) a bounded chance to complete +# at least one cutover before the final assertions. +wait_for_log_pattern_count "FSP rekey cutover complete" 1 "$REKEY_SETTLE" || true + +# Global negative checks — these must be zero on EVERY node regardless +# of version pairing. A non-zero count is attributed to the node and, +# where the count is asymmetric across versions, flagged as interop. +echo "" +echo " -- Global health (all $NUM_NODES nodes) --" + +declare -A GLOBAL_PATTERNS=( + ["PANIC|panicked"]="panics" + ["ERROR"]="error-level log lines" + ["unknown FMP version|Unknown FMP version"]="unknown-FMP-version drops" + ["MMP link teardown"]="MMP link teardowns" + ["Excessive decrypt failures"]="excessive-decrypt-failure removals" + ["Session AEAD decryption failed"]="FSP AEAD decrypt failures" + ["Rekey msg2 processing failed"]="rekey msg2 failures" + ["Handshake failed|handshake failed|Handshake error"]="handshake failures" +) + +for pat in "${!GLOBAL_PATTERNS[@]}"; do + desc="${GLOBAL_PATTERNS[$pat]}" + total="$(count_log_pattern "$pat")" + if [ "$total" -eq 0 ]; then + echo " PASS $desc: 0" + PASSED=$((PASSED + 1)) + else + echo " FAIL $desc: $total (expected 0)" + FAILED=$((FAILED + 1)) + # Per-node breakdown so the count can be attributed to a build. + for n in "${NODES[@]}"; do + c="$(count_node_pattern "$n" "$pat")" + if [ "$c" -gt 0 ]; then + s="${SLOT_OF[$n]}" + u="$(echo "$s" | tr '[:lower:]' '[:upper:]')" + echo " $n [$u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}]: $c" + INTEROP_FAILURES+=("[log] node $n ($u ${SLOT_REF[$s]}@${SLOT_SHA[$s]}): $c x '$desc'") + fi + done + fi +done + +# Positive checks — the rekey machinery actually exercised both layers. +echo "" +echo " -- Rekey machinery exercised --" +fmp_total="$(count_log_pattern 'Rekey cutover complete \(initiator\), K-bit flipped')" +fsp_total="$(count_log_pattern 'FSP rekey cutover complete')" +if [ "$fmp_total" -ge 1 ]; then + echo " PASS FMP rekey cutovers across mesh: $fmp_total" + PASSED=$((PASSED + 1)) +else + echo " FAIL FMP rekey cutovers: $fmp_total (expected >= 1)" + FAILED=$((FAILED + 1)) +fi +if [ "$fsp_total" -ge 1 ]; then + echo " PASS FSP rekey cutovers across mesh: $fsp_total" + PASSED=$((PASSED + 1)) +else + # FSP rekey not completing is a soft signal: report it but only + # hard-fail if a pair also lost connectivity. The connectivity + # phases already cover the user-visible impact. + echo " WARN FSP rekey cutovers: $fsp_total (expected >= 1)" +fi + +# Per-pair summary: classify each unordered pair and report whether it +# stayed healthy through the run. "Healthy" = no connectivity failure +# recorded for either direction of the pair. +echo "" +echo " -- Per-pair interop summary --" +for p in "${PAIRS[@]}"; do + read -r n1 n2 <<< "$p" + kind="$(pair_kind "$n1" "$n2")" + label="$(pair_label "$n1" "$n2")" + pair_failed=0 + if [ "${#INTEROP_FAILURES[@]}" -gt 0 ]; then + for f in "${INTEROP_FAILURES[@]}"; do + case "$f" in + *"ping $n1->$n2 FAILED"*|*"ping $n2->$n1 FAILED"*) + pair_failed=1 ;; + esac + done + fi + if [ "$pair_failed" -eq 0 ]; then + echo " PASS $kind pair $label: stayed healthy" + PASSED=$((PASSED + 1)) + else + echo " FAIL $kind pair $label: connectivity failed during the run" + FAILED=$((FAILED + 1)) + fi +done + +phase_result "Interop log analysis" +echo "" + +# ── Summary ────────────────────────────────────────────────────────── + +echo "==============================================================" +echo " Results: $TOTAL_PASSED checks passed, $TOTAL_FAILED failed" +echo "==============================================================" + +# `${#arr[@]}` cannot be combined with `:-`; count into a plain var. +INTEROP_FAILURE_COUNT="${#INTEROP_FAILURES[@]}" + +if [ "$TOTAL_FAILED" -eq 0 ] && [ "$INTEROP_FAILURE_COUNT" -eq 0 ]; then + echo "" + echo "PASS: all versions interoperate cleanly across the mesh (spec '$SPEC_STR')." + exit 0 +fi + +echo "" +echo "INTEROP FAILURES (attributed to specific version pairs / builds):" +if [ "$INTEROP_FAILURE_COUNT" -gt 0 ]; then + printf '%s\n' "${INTEROP_FAILURES[@]}" | sort -u | sed 's/^/ - /' +else + echo " (no per-pair failure recorded; see phase output above)" +fi +echo "" + +# Highlight whether failures concentrate on mixed-version pairs — the +# defining interop signal. +mixed_hits=0 +same_hits=0 +if [ "$INTEROP_FAILURE_COUNT" -gt 0 ]; then + for f in "${INTEROP_FAILURES[@]}"; do + case "$f" in + *"MIXED pair"*) mixed_hits=$((mixed_hits + 1)) ;; + *"same pair"*) same_hits=$((same_hits + 1)) ;; + esac + done +fi +echo "Connectivity-failure attribution: mixed-version=$mixed_hits same-version=$same_hits" +if [ "$mixed_hits" -gt 0 ] && [ "$same_hits" -eq 0 ]; then + echo "=> Failures are MIXED-VERSION ONLY: a genuine interop regression." +elif [ "$mixed_hits" -gt 0 ] && [ "$same_hits" -gt 0 ]; then + echo "=> Failures hit both mixed and same-version pairs: likely a general" + echo " instability, not version-specific — investigate the common cause." +elif [ "$same_hits" -gt 0 ]; then + echo "=> Failures are SAME-VERSION ONLY: not an interop issue per se;" + echo " a build is unstable even against itself." +fi + +dump_diagnostics +exit 1 diff --git a/testing/mesh-lab/.gitignore b/testing/mesh-lab/.gitignore new file mode 100644 index 0000000..6dddbf7 --- /dev/null +++ b/testing/mesh-lab/.gitignore @@ -0,0 +1,5 @@ +# Raw per-rep diagnostic outputs are gigabyte-scale, ephemeral, and +# per-developer. The lab harness writes them here; we never commit them. +# Curated artifacts that prove a mechanism match get copied (by hand) +# into ../../workspace ../issues/YYYY-NNNN-diagnostics/ when relevant. +runs/ diff --git a/testing/mesh-lab/README.md b/testing/mesh-lab/README.md new file mode 100644 index 0000000..690efe2 --- /dev/null +++ b/testing/mesh-lab/README.md @@ -0,0 +1,141 @@ +# FIPS mesh-reliability lab + + + +Local reproduction infrastructure for chronic CI integration-test +flakiness. The goal: turn "happens occasionally on GitHub Actions" into +"reproduces deterministically under controlled local pressure," then fix +on bedrock instead of bumping timeouts. + +## Quick start + +Prerequisites — same as `testing/ci-local.sh`: + +- Docker daemon reachable. +- `fips-test:latest` and `fips-test-app:latest` Docker images built. The + easiest way to (re)build them is to run `testing/ci-local.sh + --build-only` once after a fresh checkout or after touching the + daemon source — the lab itself does not rebuild between reps. +- Python 3 with `pyyaml` and `jinja2` installed for the chaos suites + (`pip3 install --user pyyaml jinja2`). +- `stress-ng` on the host for pressure profiles other than `idle` + (`sudo apt-get install stress-ng`). + +Simplest invocation — single rekey rep on the idle profile (no CPU +pressure), output under a timestamped subdir of `runs/`: + +```bash +bash testing/mesh-lab/run-loop.sh rekey +``` + +Twenty reps under the github-runner-equivalent pressure profile (the +canonical Phase 1 acceptance-gate shape for the rekey Phase 5 +flake class): + +```bash +bash testing/mesh-lab/run-loop.sh rekey --reps 20 --profile github-runner-equivalent +``` + +The harness writes per-rep diagnostics to +`/runs//rep-NN/` (raw logs, container state, +exit codes) and a per-rep `summary.json` plus an aggregated +`/runs//summary.json` at the end. Where +`` lands is controlled by the `FIPS_MESH_LAB_RUNS_DIR` +environment variable (see below); by default it is the in-tree +`testing/mesh-lab/` directory. Raw artifacts are gitignored — they're +big and per-developer. The `summary.json` shape is compact so triage +doesn't require holding the raw log stream. + +## Suites supported + +Initial target set: + +- `rekey`, `rekey-accept-off`, `rekey-outbound-only` — rekey-suite + Phase 5 post-second-rekey connectivity flake class. +- `nat-lan` — two-node NAT-traversal handshake-completion flake class. +- `bloom-storm` — chaos scenario; logged for future opportunistic + checks, not in the initial active scope. + +Adding more is straightforward — see the `dispatch_suite` function in +[run-loop.sh](run-loop.sh). + +## Pressure profiles + +Defined in [pressure-profiles.sh](pressure-profiles.sh): + +- `idle` — no pressure. Baseline; should produce zero failures on a + healthy mesh. +- `light` — placeholder. Will be calibrated. +- `github-runner-equivalent` — placeholder. Will be calibrated to + approximate the headroom an `ubuntu-latest` GitHub runner has while + also juggling four parallel package-build workflows (estimate: + 2-core / ~7 GiB total, so 1 stress-ng worker + memory ballast). The + initial calibration target: this profile must reproduce the rekey + Phase 5 flake class at ≥20% rate over 20 reps with mechanism-match. +- `heavy` — placeholder. Worst-case pressure for stall-finding work. + +## Environment-variable knobs + +The harness reads three optional environment variables that shape what +each rep does, set them in the invoking shell: + +- **`FIPS_MESH_LAB_NETEM`** — netem argument string (e.g. + `"delay 10ms 5ms 25% loss 1%"`). When set, the harness runs + `tc qdisc add dev eth0 root netem ` inside each fips-node + container after `compose up`. Bridge-level qdisc on the docker + network does *not* shape inter-container traffic (Linux bridges + forward port-to-port without packets traversing the bridge + interface's egress qdisc), so per-container egress is the correct + injection point. + +- **`FIPS_MESH_LAB_TRACE`** — when set to any non-empty value, the + harness layers `compose-trace.yml` on top of the base + resource- + limits compose stack. That override bumps `RUST_LOG` to trace + level on the modules relevant to the rekey-class flake: `rekey`, + `handshake`, `forwarding`, `session`, `encrypted`, `mmp`. Use + only when capturing primary failure-moment evidence for mechanism + investigation — log volume increases substantially. Without this + knob, daemon logs only capture state transitions (rekey cutover, + K-bit flip), not per-datagram forwarding decisions, which makes + evidence collection for routing-state stalls effectively + impossible. + +- **`FIPS_MESH_LAB_RUNS_DIR`** — root directory for harness output + (the `runs//` tree). When unset, the harness falls back + to an in-tree path under `testing/mesh-lab/` and prints a warning + to stderr naming the variable and the fallback location. Set this + to a path outside the source tree (e.g. `/var/tmp/fips-mesh-lab` + or a path on a separate disk) to keep gigabyte-scale per-rep + artefacts out of the checkout. + +Example: + +```bash +FIPS_MESH_LAB_TRACE=1 \ +FIPS_MESH_LAB_RUNS_DIR=/var/tmp/fips-mesh-lab \ + bash testing/mesh-lab/run-loop.sh rekey-accept-off \ + --reps 20 --profile github-runner-equivalent +``` + +## Recipes + +`recipes/.yaml` files are commit-pinned reproduction recipes +the harness consumes. Each recipe declares the source SHA, the suite, +the pressure profile, the rep count, and the expected mechanism-match +rate, so a future operator can confirm "yes the lab still reproduces +this flake at the documented rate" with one command. None exist yet — +they're authored as concrete reproductions surface. + +## How this differs from `testing/ci-local.sh` + +`ci-local.sh` runs each suite exactly once in sequence (chaos +scenarios in parallel up to a job slot count), produces a pass/fail +matrix, and is the canonical "did anything regress" gate. The +mesh-lab runs the *same* per-suite test scripts (it does not +reimplement them) but in a loop, with deliberate host pressure +applied, and with rich per-rep diagnostic capture. They share the +Docker images and the test scripts. + +When in doubt, debug a single suite via `ci-local.sh --only ` +first to confirm the suite is healthy on idle, then graduate to the +mesh-lab when you want to chase a flake. diff --git a/testing/mesh-lab/compose-resource-limits.yml b/testing/mesh-lab/compose-resource-limits.yml new file mode 100644 index 0000000..b047708 --- /dev/null +++ b/testing/mesh-lab/compose-resource-limits.yml @@ -0,0 +1,58 @@ +# Compose override applied by the mesh-lab harness to constrain the +# rekey-family daemon containers to roughly the per-container slice +# they'd get on a GitHub Actions ubuntu-latest runner (2 cores / 7 GiB +# total, shared among 5 daemons + the test harness + OS overhead). +# +# Used by passing `-f testing/static/docker-compose.yml -f this-file` +# in the lab's `docker compose up` invocation. The base compose file +# is unmodified so ci-local.sh on a developer host stays unconstrained +# unless the developer deliberately opts into these limits. +# +# Sizing math: 5 daemons × 0.3 cpus = 1.5 cores; 5 daemons × 1 GiB = +# 5 GiB. Leaves ~0.5 cores and ~2 GiB for the test harness + OS on a +# 2-core / 7-GiB GHA runner, which matches reality on a fully-loaded +# CI box. +# +# Service names follow the per-profile naming in the base compose: +# the `rekey` profile uses rekey-a..rekey-e, `rekey-accept-off` uses +# rekey-accept-off-a..e, and `rekey-outbound-only` uses +# rekey-outbound-only-a..e. The override repeats the limits anchor +# across all three so any rekey-family profile picks them up. +# +# 2026-05-16 first cut: 0.3 cpus + 1 GiB per daemon, measured rate TBD. + +x-fips-limits: &fips-limits + cpus: 0.3 + mem_limit: 1g + +services: + rekey-a: + <<: *fips-limits + rekey-b: + <<: *fips-limits + rekey-c: + <<: *fips-limits + rekey-d: + <<: *fips-limits + rekey-e: + <<: *fips-limits + rekey-accept-off-a: + <<: *fips-limits + rekey-accept-off-b: + <<: *fips-limits + rekey-accept-off-c: + <<: *fips-limits + rekey-accept-off-d: + <<: *fips-limits + rekey-accept-off-e: + <<: *fips-limits + rekey-outbound-only-a: + <<: *fips-limits + rekey-outbound-only-b: + <<: *fips-limits + rekey-outbound-only-c: + <<: *fips-limits + rekey-outbound-only-d: + <<: *fips-limits + rekey-outbound-only-e: + <<: *fips-limits diff --git a/testing/mesh-lab/compose-trace.yml b/testing/mesh-lab/compose-trace.yml new file mode 100644 index 0000000..e9bc450 --- /dev/null +++ b/testing/mesh-lab/compose-trace.yml @@ -0,0 +1,73 @@ +# Compose override that bumps RUST_LOG to trace level on the modules +# relevant to rekey-class flake evidence collection: +# +# - fips::node::handlers::rekey — FMP/FSP rekey state machine +# - fips::node::handlers::handshake — Noise handshake (cross-init, +# dual-init, msg1/2/3 dispatch) +# - fips::node::handlers::forwarding — transit datagram forwarding, +# route lookup, drops +# - fips::node::handlers::session — FSP K-bit cutover, drain +# - fips::node::handlers::encrypted — FMP K-bit flip detection +# - fips::node::handlers::mmp — link liveness, SRTT, ETX +# +# Other modules stay at info to keep log volume manageable. The base +# docker-compose.yml's per-service RUST_LOG values (currently +# `info,fips::node::handlers::rekey=debug` for the rekey profile, plus +# handshake=debug on the variant profiles) are fully replaced by the +# values below via the YAML map form: docker compose's override +# semantics replace the map entry rather than merging individual +# environment keys. +# +# Service-name layout follows the base compose: per-profile services +# named `-` (rekey-a..e for the rekey profile, +# rekey-accept-off-a..e for the accept-off variant, +# rekey-outbound-only-a..e for the outbound-only variant). All 15 +# rekey-family services get the same trace-RUST_LOG value. +# +# Include this override alongside the base compose via: +# docker compose -f testing/static/docker-compose.yml \ +# -f testing/mesh-lab/compose-resource-limits.yml \ +# -f testing/mesh-lab/compose-trace.yml up -d +# +# The run-loop.sh harness includes it automatically when the mesh-lab +# environment variable FIPS_MESH_LAB_TRACE=1 is set. + +x-trace-rust-log: &trace-rust-log + RUST_LOG: "info,fips::node::handlers::rekey=trace,fips::node::handlers::handshake=trace,fips::node::handlers::forwarding=trace,fips::node::handlers::session=trace,fips::node::handlers::encrypted=trace,fips::node::handlers::mmp=trace" + +services: + # rekey profile + rekey-a: + environment: *trace-rust-log + rekey-b: + environment: *trace-rust-log + rekey-c: + environment: *trace-rust-log + rekey-d: + environment: *trace-rust-log + rekey-e: + environment: *trace-rust-log + + # rekey-accept-off profile + rekey-accept-off-a: + environment: *trace-rust-log + rekey-accept-off-b: + environment: *trace-rust-log + rekey-accept-off-c: + environment: *trace-rust-log + rekey-accept-off-d: + environment: *trace-rust-log + rekey-accept-off-e: + environment: *trace-rust-log + + # rekey-outbound-only profile + rekey-outbound-only-a: + environment: *trace-rust-log + rekey-outbound-only-b: + environment: *trace-rust-log + rekey-outbound-only-c: + environment: *trace-rust-log + rekey-outbound-only-d: + environment: *trace-rust-log + rekey-outbound-only-e: + environment: *trace-rust-log diff --git a/testing/mesh-lab/pressure-profiles.sh b/testing/mesh-lab/pressure-profiles.sh new file mode 100755 index 0000000..5cf14bd --- /dev/null +++ b/testing/mesh-lab/pressure-profiles.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# Pressure profiles for the mesh-lab. Sourced by run-loop.sh. +# +# Each profile is a function `pressure_start_` that spawns +# stress-ng (or whatever) in the background and writes its PID to the +# variable PRESSURE_PIDS (a bash array). The harness uses +# `pressure_stop` (defined at the bottom) to kill everything cleanly. +# +# Profiles are intentionally simple stubs until calibrated against +# actual reproduction of a known flake class. + +set -u + +PRESSURE_PIDS=() + +# Track which profile is active for the summary.json. +PRESSURE_PROFILE_ACTIVE="idle" + +# ── idle ───────────────────────────────────────────────────────────── +# No pressure. Baseline. A healthy mesh should run all reps green on +# this profile; failures here are a different bug class than what the +# lab is built to chase. + +pressure_start_idle() { + PRESSURE_PROFILE_ACTIVE="idle" + # nothing to do +} + +# ── light ──────────────────────────────────────────────────────────── +# Single stress-ng cpu worker at ~50% load. Placeholder; calibration +# will determine the right intensity for surfacing tail behavior +# without saturating the host. + +pressure_start_light() { + PRESSURE_PROFILE_ACTIVE="light" + require_stress_ng || return 1 + stress-ng --cpu 1 --cpu-load 50 --quiet & + PRESSURE_PIDS+=("$!") +} + +# ── github-runner-equivalent ──────────────────────────────────────── +# Calibrated to approximate the headroom a 2-core GitHub Actions +# ubuntu-latest runner has when also juggling four concurrent package- +# build workflows. Target effective per-host load: ~75% of cores +# consumed, ~30% of RAM allocated, so the test+daemon containers +# compete for ~2 effective cores and ~7-8 GiB headroom — close to a +# runner under quad-package-build contention. +# +# Calibration target: ≥20% mechanism-match rate over 5-rep pilots on +# the `rekey` suite. Calibration history (host = 8c/62GiB): +# +# 2026-05-16 first cut --cpu 1 --vm 1 --vm-bytes 512m +# 25 reps, 1 mechanism-match (4%) +# (stochastic pilot fire; not contention-driven) +# 2026-05-16 second cut --cpu 6 --vm 2 --vm-bytes 10g +# 5 reps, 0 mechanism-match +# (host load=8.25/8 cores, 20GiB resident; the +# mechanism did not surface even at saturation) +# +# Layered with the mesh-lab/compose-resource-limits.yml override +# (per-daemon cpus=0.3 + mem_limit=1g) and in-container tc qdisc netem +# (10ms ± 5ms delay + 1% loss), the rekey Phase 5 mechanism still did +# not reproduce above the stub's stochastic baseline on this host — +# implying the GHA fires are more likely code-side (tokio task stall, +# channel backpressure, mutex contention) than environment-side. + +pressure_start_github_runner_equivalent() { + PRESSURE_PROFILE_ACTIVE="github-runner-equivalent" + require_stress_ng || return 1 + # CPU pressure: six full-load cpu workers (75% of an 8-core host). + # On smaller hosts the per-worker share rises; on larger hosts the + # absolute headroom rises proportionally — the profile is anchored + # to "what's left after the pressure," not to a target host size. + stress-ng --cpu 6 --quiet & + PRESSURE_PIDS+=("$!") + # Memory pressure: two vm workers, 20 GiB total resident. Sized to + # eat ~30% of a 62 GiB host. --vm-keep holds pages resident rather + # than churning the allocator, which mirrors a peer workflow that + # has allocated and is using its working set, not one trashing it. + stress-ng --vm 2 --vm-bytes 10g --vm-keep --quiet & + PRESSURE_PIDS+=("$!") +} + +# ── heavy ──────────────────────────────────────────────────────────── +# Two cpu workers + memory pressure. Worst-case profile for stall- +# finding work once the milder profiles are characterized. + +pressure_start_heavy() { + PRESSURE_PROFILE_ACTIVE="heavy" + require_stress_ng || return 1 + stress-ng --cpu 2 --quiet & + PRESSURE_PIDS+=("$!") + stress-ng --vm 1 --vm-bytes 1g --vm-keep --quiet & + PRESSURE_PIDS+=("$!") +} + +# ── helpers ────────────────────────────────────────────────────────── + +require_stress_ng() { + if ! command -v stress-ng >/dev/null 2>&1; then + echo " ERROR: stress-ng not installed; cannot run pressure profile '$PRESSURE_PROFILE_ACTIVE'" >&2 + echo " install on Debian/Ubuntu: sudo apt-get install stress-ng" >&2 + return 1 + fi + return 0 +} + +pressure_start() { + local profile="$1" + PRESSURE_PIDS=() + case "$profile" in + idle) + pressure_start_idle ;; + light) + pressure_start_light ;; + github-runner-equivalent | github | gha) + pressure_start_github_runner_equivalent ;; + heavy) + pressure_start_heavy ;; + *) + echo " ERROR: unknown pressure profile '$profile'" >&2 + echo " available: idle, light, github-runner-equivalent, heavy" >&2 + return 1 ;; + esac +} + +pressure_stop() { + if [ "${#PRESSURE_PIDS[@]}" -eq 0 ]; then + return 0 + fi + for pid in "${PRESSURE_PIDS[@]}"; do + # kill the process group; stress-ng spawns children + kill -TERM "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true + done + # Brief grace, then SIGKILL stragglers + sleep 1 + for pid in "${PRESSURE_PIDS[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -KILL "$pid" 2>/dev/null || true + fi + done + PRESSURE_PIDS=() +} diff --git a/testing/mesh-lab/run-loop.sh b/testing/mesh-lab/run-loop.sh new file mode 100755 index 0000000..cb21fe3 --- /dev/null +++ b/testing/mesh-lab/run-loop.sh @@ -0,0 +1,496 @@ +#!/bin/bash +# FIPS mesh-reliability lab: run an integration suite N times under a +# configurable host-pressure profile, capture per-rep diagnostics, and +# produce a per-rep + aggregate summary.json compact enough for triage +# without holding gigabytes of raw log. +# +# See ./README.md for the full developer-facing description. +# +# Usage: +# run-loop.sh [--reps N] [--profile NAME] [--out DIR] +# +# suite One of: rekey, rekey-accept-off, rekey-outbound-only, +# nat-lan, bloom-storm. +# --reps N Number of repetitions (default 1). +# --profile Pressure profile name from pressure-profiles.sh (default +# idle). See pressure-profiles.sh for the full list. +# --out DIR Output directory (default /runs/; see +# FIPS_MESH_LAB_RUNS_DIR below for how the runs-base is +# chosen). +# +# Environment: +# FIPS_MESH_LAB_NETEM netem argument string applied via tc qdisc +# inside each fips-node container's eth0. +# FIPS_MESH_LAB_TRACE when set, layers compose-trace.yml on top +# of the base + resource-limits stack to +# bump RUST_LOG to trace on rekey/handshake/ +# forwarding/session/encrypted/mmp modules. +# FIPS_MESH_LAB_RUNS_DIR Root for harness output (runs/ and any +# other scratch). When unset, falls back to +# an in-tree path under testing/mesh-lab/ +# and prints a warning to stderr; set it to +# a path outside the source tree to keep +# generated artefacts out of the checkout. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/pressure-profiles.sh" + +# ── Scratch-dir root ───────────────────────────────────────────────── +# +# FIPS_MESH_LAB_RUNS_DIR controls where the harness writes its run +# output (runs//...). When unset we fall back to an +# in-tree path under testing/mesh-lab/ and warn the operator, so the +# warning fires exactly once per invocation. When a parent script +# has already warned it exports _FIPS_MESH_LAB_WARNED=1 to suppress +# duplicate warnings in child scripts. +if [[ -n "${FIPS_MESH_LAB_RUNS_DIR:-}" ]]; then + RUNS_BASE="$FIPS_MESH_LAB_RUNS_DIR" + mkdir -p "$RUNS_BASE" +else + RUNS_BASE="$SCRIPT_DIR" + if [[ -z "${_FIPS_MESH_LAB_WARNED:-}" ]]; then + echo >&2 "WARNING: FIPS_MESH_LAB_RUNS_DIR not set; harness output will be written under the source tree at $RUNS_BASE/runs/. Set FIPS_MESH_LAB_RUNS_DIR to a path outside the source tree to avoid this." + export _FIPS_MESH_LAB_WARNED=1 + fi +fi + +# ── Args ───────────────────────────────────────────────────────────── + +SUITE="" +REPS=1 +PROFILE="idle" +OUT_DIR="" + +usage() { + sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [ $# -gt 0 ]; do + case "$1" in + --reps) REPS="$2"; shift 2 ;; + --profile) PROFILE="$2"; shift 2 ;; + --out) OUT_DIR="$2"; shift 2 ;; + -h|--help) usage 0 ;; + --) shift; break ;; + -*) echo "unknown flag: $1" >&2; usage 1 ;; + *) + if [ -z "$SUITE" ]; then + SUITE="$1" + else + echo "unexpected positional arg: $1" >&2 + usage 1 + fi + shift ;; + esac +done + +if [ -z "$SUITE" ]; then + echo "missing required argument" >&2 + usage 1 +fi + +if [ -z "$OUT_DIR" ]; then + ts="$(date -u +%Y%m%dT%H%M%SZ)" + OUT_DIR="$RUNS_BASE/runs/${ts}-${SUITE}-${PROFILE}" +fi + +mkdir -p "$OUT_DIR" + +# Mirror the harness's own stdout/stderr to a per-run log. The +# per-rep setup/test-output/teardown captures only the in-container +# test side; this captures the wrapper-level signal (pressure-profile +# start/stop, OOM-killed child notifications from bash job control, +# aggregate summary, any preflight error). Without this, host-side +# diagnostics are lost the moment the host reboots and the kernel +# ring buffer rolls. +exec > >(tee -a "$OUT_DIR/run-loop.log") 2>&1 + +# ── Preflight ──────────────────────────────────────────────────────── + +require_docker() { + if ! docker info >/dev/null 2>&1; then + echo "ERROR: Docker daemon is not reachable" >&2 + exit 2 + fi +} + +require_test_image() { + if ! docker image inspect fips-test:latest >/dev/null 2>&1; then + echo "ERROR: fips-test:latest not present" >&2 + echo "Build it once with: bash testing/ci-local.sh --build-only" >&2 + exit 2 + fi +} + +require_docker +require_test_image + +# ── Suite-specific drivers and signature parsers ───────────────────── +# +# Each driver runs ONE rep of the suite. It must: +# - return 0 on suite-pass, non-zero on suite-fail +# - write the test stdout/stderr to ${REP_DIR}/test-output.log +# - capture container logs to ${REP_DIR}/docker-logs/ if relevant +# +# Each parser reads ${REP_DIR}/test-output.log and writes a JSON +# fragment (no enclosing braces) of suite-specific signature features +# into ${REP_DIR}/signature.json. Used by the aggregate summary to +# decide whether the failure matches the documented mechanism for the +# associated open issue. + +run_rekey_family() { + local variant="$1" # rekey, rekey-accept-off, rekey-outbound-only + local REP_DIR="$2" + local compose_profile="$variant" + local env_args=() + + case "$variant" in + rekey-accept-off) + env_args=(REKEY_TOPOLOGY=rekey-accept-off REKEY_ACCEPT_OFF_NODES=b) ;; + rekey-outbound-only) + env_args=(REKEY_TOPOLOGY=rekey-outbound-only REKEY_OUTBOUND_ONLY_NODES=b) ;; + esac + + # Lab compose stack: base compose + mesh-lab resource-limits override. + # The override pins each rekey-family daemon to roughly its GHA-runner + # share (0.3 cpus / 1 GiB), mimicking the constraint a 2-core / 7-GiB + # ubuntu-latest runner imposes. Base compose is unmodified so + # ci-local.sh stays unconstrained for day-to-day developer runs. + # + # Trace-logging override: set FIPS_MESH_LAB_TRACE=1 in the environment + # to bump RUST_LOG to trace level on the modules relevant to the + # rekey-class flake (rekey, handshake, forwarding, session, encrypted, + # mmp). Increases log volume substantially; use only when capturing + # primary failure-moment evidence for mechanism investigation. + local compose_args=( + -f testing/static/docker-compose.yml + -f testing/mesh-lab/compose-resource-limits.yml + --profile "$compose_profile" + ) + if [ -n "${FIPS_MESH_LAB_TRACE:-}" ]; then + compose_args=( + -f testing/static/docker-compose.yml + -f testing/mesh-lab/compose-resource-limits.yml + -f testing/mesh-lab/compose-trace.yml + --profile "$compose_profile" + ) + fi + + ( + cd "$REPO_ROOT" || exit 1 + env "${env_args[@]}" bash testing/static/scripts/generate-configs.sh "$variant" \ + >>"$REP_DIR/setup.log" 2>&1 + env "${env_args[@]}" bash testing/static/scripts/rekey-test.sh inject-config \ + >>"$REP_DIR/setup.log" 2>&1 + docker compose "${compose_args[@]}" up -d \ + >>"$REP_DIR/setup.log" 2>&1 + ) + + # Optional: apply tc qdisc netem inside each fips-node container's + # eth0. Set FIPS_MESH_LAB_NETEM to a netem argument string (e.g. + # "delay 10ms 5ms 25% loss 1%") to enable. Applied via `docker + # exec` because qdisc on the host-side docker bridge does NOT shape + # port-to-port inter-container traffic on the bridge — only traffic + # to/from the host's own IP. Egress qdisc on the container's own + # eth0 reliably shapes that container's outbound packets to peer + # containers. Containers already have NET_ADMIN cap (rekey needs + # it for TUN); tc is in the fips-test image. + local netem_applied=() + if [ -n "${FIPS_MESH_LAB_NETEM:-}" ]; then + for node in a b c d e; do + if docker exec "fips-node-$node" tc qdisc add dev eth0 root \ + netem ${FIPS_MESH_LAB_NETEM} >>"$REP_DIR/setup.log" 2>&1; then + netem_applied+=("fips-node-$node") + else + echo "WARN: netem apply failed on fips-node-$node" \ + >>"$REP_DIR/setup.log" + fi + done + if [ "${#netem_applied[@]}" -gt 0 ]; then + echo "applied netem on ${#netem_applied[@]}/5 nodes: $FIPS_MESH_LAB_NETEM" \ + >>"$REP_DIR/setup.log" + fi + fi + + local rc=0 + ( + cd "$REPO_ROOT" || exit 1 + env "${env_args[@]}" bash testing/static/scripts/rekey-test.sh + ) >"$REP_DIR/test-output.log" 2>&1 || rc=$? + + # Capture container logs before teardown + mkdir -p "$REP_DIR/docker-logs" + for node in a b c d e; do + docker logs "fips-node-$node" >"$REP_DIR/docker-logs/node-$node.log" 2>&1 || true + done + + # In-container netem disappears with the container itself on + # compose down, so no explicit teardown needed. + + ( + cd "$REPO_ROOT" || exit 1 + docker compose "${compose_args[@]}" \ + down --volumes --remove-orphans \ + >>"$REP_DIR/teardown.log" 2>&1 + ) + + return "$rc" +} + +parse_rekey() { + local REP_DIR="$1" + local log="$REP_DIR/test-output.log" + + # Phase 5 per-pair failures (e.g., "B → D ... FAIL" or + # "B → D ... FAIL (after 4 attempts)" when retries are enabled). + local phase5_failures + phase5_failures=$(awk ' + /^Phase 5:/ { in5=1; next } + /^Phase 6:/ { in5=0 } + in5 && /\.\.\. FAIL([[:space:]]|$)/ { print } + ' "$log" | sed 's/^ *//' | tr '\n' ',' | sed 's/,$//') + + # Phase 6 log analysis result + local phase6_status="unknown" + if grep -q '"Log analysis: .* passed"\|✓ Log analysis: ' "$log"; then + phase6_status="all-green" + fi + if grep -q 'ERROR\|PANIC\|panicked' "$log"; then + phase6_status="errors-observed" + fi + + # Late FSP K-bit cutover detection — scan node logs for cutover- + # complete events occurring within or after the Phase 5 settle + # window. The settle window is the 12 s before Phase 5's first + # ping_all. Without precise timing parsing in this first pass, we + # report the timestamps of all FSP K-bit-related events for the + # rep so a reviewer can match against the documented mechanism. + local late_fsp_events="" + for nodelog in "$REP_DIR"/docker-logs/node-*.log; do + [ -f "$nodelog" ] || continue + # Strip ANSI escape codes that docker logs preserves from the + # daemon's TTY-aware tracing-subscriber output; raw ESC bytes + # are invalid JSON string contents. + late_fsp_events+=$(grep -oE '[0-9-]+T[0-9:.]+Z.*(K-bit flip|FSP rekey cutover complete)' "$nodelog" \ + | sed -E 's/\x1b\[[0-9;]*[mK]//g' \ + | tail -5 \ + | tr '\n' ';') + late_fsp_events+="|" + done + + # Compose JSON via jq -n so embedded specials in any field are + # safely escaped rather than splatted as raw bytes. + jq -n \ + --arg pairs "$phase5_failures" \ + --arg phase6 "$phase6_status" \ + --arg events "$late_fsp_events" \ + '{phase5_failing_pairs: $pairs, phase6_log_analysis: $phase6, late_fsp_events_per_node_tail: $events}' \ + > "$REP_DIR/signature.json" +} + +# Heuristic mechanism-match check for the rekey Phase 5 flake class. +# True iff: +# - At least one Phase 5 ping fails +# - Phase 6 log analysis is all-green (no ERROR/PANIC noise) +mechanism_match_rekey() { + local REP_DIR="$1" + local sig="$REP_DIR/signature.json" + if [ ! -f "$sig" ]; then + echo " WARN: mechanism_match: $sig missing" >&2 + echo "false" + return + fi + # Surface invalid JSON loudly — silent jq failure with `|| echo ""` + # previously masked real mechanism matches when ANSI escapes leaked + # into the events field. + if ! jq -e . "$sig" >/dev/null 2>&1; then + echo " WARN: mechanism_match: $sig is invalid JSON" >&2 + echo "false" + return + fi + local pairs phase6 + pairs=$(jq -r '.phase5_failing_pairs' "$sig") + phase6=$(jq -r '.phase6_log_analysis' "$sig") + if [ -n "$pairs" ] && [ "$phase6" = "all-green" ]; then + echo "true" + else + echo "false" + fi +} + +run_nat_lan() { + local REP_DIR="$1" + local rc=0 + ( + cd "$REPO_ROOT" || exit 1 + bash testing/nat/scripts/nat-test.sh lan + ) >"$REP_DIR/test-output.log" 2>&1 || rc=$? + + mkdir -p "$REP_DIR/docker-logs" + for c in fips-nat-lan-a fips-nat-lan-b; do + docker logs "$c" >"$REP_DIR/docker-logs/$c.log" 2>&1 || true + done + + return "$rc" +} + +parse_nat_lan() { + local REP_DIR="$1" + local log="$REP_DIR/test-output.log" + + local peer_adoption_timeout="false" + if grep -q "TIMEOUT waiting for" "$log"; then + peer_adoption_timeout="true" + fi + + local cross_init_observed="false" + if grep -E "Connection initiated.*node-(a|b)" "$REP_DIR"/docker-logs/*.log 2>/dev/null \ + | awk '{print $1}' | sort -u | head -2 | wc -l | grep -q '2'; then + cross_init_observed="true" + fi + + cat <"$REP_DIR/signature.json" +{ + "peer_adoption_timeout": $peer_adoption_timeout, + "cross_init_observed": $cross_init_observed +} +EOF +} + +mechanism_match_nat_lan() { + local REP_DIR="$1" + local sig="$REP_DIR/signature.json" + [ -f "$sig" ] || return 1 + local timeout_seen + timeout_seen=$(jq -r '.peer_adoption_timeout' "$sig" 2>/dev/null || echo "false") + if [ "$timeout_seen" = "true" ]; then + echo "true" + else + echo "false" + fi +} + +# Dispatch — returns rc of suite, side-effects signature.json +dispatch_suite() { + local REP_DIR="$1" + case "$SUITE" in + rekey|rekey-accept-off|rekey-outbound-only) + local rc=0 + run_rekey_family "$SUITE" "$REP_DIR" || rc=$? + parse_rekey "$REP_DIR" + return "$rc" ;; + nat-lan) + local rc=0 + run_nat_lan "$REP_DIR" || rc=$? + parse_nat_lan "$REP_DIR" + return "$rc" ;; + *) + echo "ERROR: unsupported suite '$SUITE' in this lab harness (initial scaffolding)" >&2 + echo "Supported: rekey, rekey-accept-off, rekey-outbound-only, nat-lan" >&2 + return 99 ;; + esac +} + +# Mechanism-match heuristic per suite +dispatch_mechanism_match() { + local REP_DIR="$1" + case "$SUITE" in + rekey|rekey-accept-off|rekey-outbound-only) + mechanism_match_rekey "$REP_DIR" ;; + nat-lan) + mechanism_match_nat_lan "$REP_DIR" ;; + *) + echo "unknown" ;; + esac +} + +# ── Main loop ──────────────────────────────────────────────────────── + +echo "=== mesh-lab: suite=$SUITE reps=$REPS profile=$PROFILE ===" +echo " out: $OUT_DIR" +echo "" + +PASS_COUNT=0 +FAIL_COUNT=0 +MECH_MATCH_COUNT=0 + +# Trap to ensure pressure is always cleaned up even on Ctrl-C +trap 'pressure_stop; exit 130' INT TERM + +for rep in $(seq 1 "$REPS"); do + rep_padded=$(printf "%03d" "$rep") + REP_DIR="$OUT_DIR/rep-$rep_padded" + mkdir -p "$REP_DIR" + + started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + echo "--- rep $rep/$REPS (started $started_at) ---" + + pressure_start "$PROFILE" || { + echo " ERROR: pressure_start failed for profile '$PROFILE'" >&2 + exit 2 + } + + rc=0 + dispatch_suite "$REP_DIR" || rc=$? + + pressure_stop + + ended_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + mechanism_match=$(dispatch_mechanism_match "$REP_DIR") + + if [ "$rc" -eq 0 ]; then + result="pass" + PASS_COUNT=$((PASS_COUNT + 1)) + echo " rep $rep: PASS" + else + result="fail" + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo " rep $rep: FAIL (exit $rc, mechanism_match=$mechanism_match)" + fi + + if [ "$mechanism_match" = "true" ]; then + MECH_MATCH_COUNT=$((MECH_MATCH_COUNT + 1)) + fi + + cat <"$REP_DIR/summary.json" +{ + "rep": $rep, + "suite": "$SUITE", + "profile": "$PROFILE", + "started_at": "$started_at", + "ended_at": "$ended_at", + "exit_code": $rc, + "result": "$result", + "mechanism_match": $mechanism_match, + "signature_file": "signature.json" +} +EOF +done + +# ── Aggregate summary ──────────────────────────────────────────────── + +cat <"$OUT_DIR/summary.json" +{ + "suite": "$SUITE", + "profile": "$PROFILE", + "reps": $REPS, + "pass_count": $PASS_COUNT, + "fail_count": $FAIL_COUNT, + "mechanism_match_count": $MECH_MATCH_COUNT, + "pass_rate": $(awk -v p="$PASS_COUNT" -v r="$REPS" 'BEGIN{ printf "%.3f", p/r }'), + "fail_rate": $(awk -v f="$FAIL_COUNT" -v r="$REPS" 'BEGIN{ printf "%.3f", f/r }'), + "mechanism_match_rate": $(awk -v m="$MECH_MATCH_COUNT" -v r="$REPS" 'BEGIN{ printf "%.3f", m/r }') +} +EOF + +echo "" +echo "=== summary ===" +cat "$OUT_DIR/summary.json" +echo "" +echo "raw artifacts: $OUT_DIR/" diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index 0e26a99..46b2d5e 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -497,14 +497,30 @@ echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ===" if [ "$TOTAL_FAILED" -eq 0 ]; then exit 0 else - # Dump logs on failure for diagnostics + # Dump logs on failure for diagnostics. + # + # Wider pattern + larger line cap than the original head -30 (which + # truncated before the actual ping-failure timestamp on Phase 5 + # fires, making the post-cutover convergence window invisible to + # offline triage). Two passes per node: + # 1) rekey-related events across the whole run (cap 200 lines). + # 2) Last 80 lines unfiltered, to surface forwarding decisions, + # route lookups, congestion warnings, decrypt errors, and any + # other surrounding context near the failure point. echo "" - echo "=== Node logs (rekey-related) ===" + echo "=== Node logs (rekey-related, head -200) ===" for node in $NODES; do echo "--- node-$node ---" docker logs "fips-node-$node" 2>&1 | \ - grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit)" | \ - head -30 + grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit|no route|next hop|TTL exhausted|MTU exceeded|Congestion|decrypt|Decrypt|AEAD|Notify|drain|Drain|promot)" | \ + head -200 + echo "" + done + + echo "=== Node logs (last 80 lines, unfiltered) ===" + for node in $NODES; do + echo "--- node-$node ---" + docker logs "fips-node-$node" 2>&1 | tail -80 echo "" done exit 1