From 9c0dcd0f593f960ebd6ba01d1690698a18ba9628 Mon Sep 17 00:00:00 2001 From: Johnathan Corgan Date: Sat, 23 May 2026 14:34:11 +0000 Subject: [PATCH] testing: add mesh reliability lab harness with rekey-test diagnostic improvements --- testing/README.md | 10 + testing/mesh-lab/.gitignore | 5 + testing/mesh-lab/README.md | 141 ++++++ testing/mesh-lab/compose-resource-limits.yml | 58 +++ testing/mesh-lab/compose-trace.yml | 73 +++ testing/mesh-lab/pressure-profiles.sh | 143 ++++++ testing/mesh-lab/run-loop.sh | 496 +++++++++++++++++++ testing/static/scripts/rekey-test.sh | 24 +- 8 files changed, 946 insertions(+), 4 deletions(-) create mode 100644 testing/mesh-lab/.gitignore create mode 100644 testing/mesh-lab/README.md create mode 100644 testing/mesh-lab/compose-resource-limits.yml create mode 100644 testing/mesh-lab/compose-trace.yml create mode 100755 testing/mesh-lab/pressure-profiles.sh create mode 100755 testing/mesh-lab/run-loop.sh diff --git a/testing/README.md b/testing/README.md index 625cde1..fb5dce8 100644 --- a/testing/README.md +++ b/testing/README.md @@ -62,3 +62,13 @@ each node can run a different build of the FIPS daemon, then attributes every FMP/FSP/rekey/connectivity failure to a specific version pair (same-version vs MIXED). Used to catch interop regressions between builds, not as a per-commit CI gate; not part of `ci-local.sh`. + +### [mesh-lab/](mesh-lab/) -- Mesh Reliability Lab + +On-demand harness that runs a chosen integration suite N times under a +configurable host-pressure profile (idle / light / github-runner- +equivalent / heavy via `stress-ng`), per-container netem impairment, +and optional trace-level RUST_LOG, capturing per-rep diagnostics and a +mechanism-match summary across the run. Used for statistical reliability +characterization of known flake classes under calibrated stress, not as +a per-commit gate; not part of `ci-local.sh`. diff --git a/testing/mesh-lab/.gitignore b/testing/mesh-lab/.gitignore new file mode 100644 index 0000000..6dddbf7 --- /dev/null +++ b/testing/mesh-lab/.gitignore @@ -0,0 +1,5 @@ +# Raw per-rep diagnostic outputs are gigabyte-scale, ephemeral, and +# per-developer. The lab harness writes them here; we never commit them. +# Curated artifacts that prove a mechanism match get copied (by hand) +# into ../../workspace ../issues/YYYY-NNNN-diagnostics/ when relevant. +runs/ diff --git a/testing/mesh-lab/README.md b/testing/mesh-lab/README.md new file mode 100644 index 0000000..690efe2 --- /dev/null +++ b/testing/mesh-lab/README.md @@ -0,0 +1,141 @@ +# FIPS mesh-reliability lab + + + +Local reproduction infrastructure for chronic CI integration-test +flakiness. The goal: turn "happens occasionally on GitHub Actions" into +"reproduces deterministically under controlled local pressure," then fix +on bedrock instead of bumping timeouts. + +## Quick start + +Prerequisites — same as `testing/ci-local.sh`: + +- Docker daemon reachable. +- `fips-test:latest` and `fips-test-app:latest` Docker images built. The + easiest way to (re)build them is to run `testing/ci-local.sh + --build-only` once after a fresh checkout or after touching the + daemon source — the lab itself does not rebuild between reps. +- Python 3 with `pyyaml` and `jinja2` installed for the chaos suites + (`pip3 install --user pyyaml jinja2`). +- `stress-ng` on the host for pressure profiles other than `idle` + (`sudo apt-get install stress-ng`). + +Simplest invocation — single rekey rep on the idle profile (no CPU +pressure), output under a timestamped subdir of `runs/`: + +```bash +bash testing/mesh-lab/run-loop.sh rekey +``` + +Twenty reps under the github-runner-equivalent pressure profile (the +canonical Phase 1 acceptance-gate shape for the rekey Phase 5 +flake class): + +```bash +bash testing/mesh-lab/run-loop.sh rekey --reps 20 --profile github-runner-equivalent +``` + +The harness writes per-rep diagnostics to +`/runs//rep-NN/` (raw logs, container state, +exit codes) and a per-rep `summary.json` plus an aggregated +`/runs//summary.json` at the end. Where +`` lands is controlled by the `FIPS_MESH_LAB_RUNS_DIR` +environment variable (see below); by default it is the in-tree +`testing/mesh-lab/` directory. Raw artifacts are gitignored — they're +big and per-developer. The `summary.json` shape is compact so triage +doesn't require holding the raw log stream. + +## Suites supported + +Initial target set: + +- `rekey`, `rekey-accept-off`, `rekey-outbound-only` — rekey-suite + Phase 5 post-second-rekey connectivity flake class. +- `nat-lan` — two-node NAT-traversal handshake-completion flake class. +- `bloom-storm` — chaos scenario; logged for future opportunistic + checks, not in the initial active scope. + +Adding more is straightforward — see the `dispatch_suite` function in +[run-loop.sh](run-loop.sh). + +## Pressure profiles + +Defined in [pressure-profiles.sh](pressure-profiles.sh): + +- `idle` — no pressure. Baseline; should produce zero failures on a + healthy mesh. +- `light` — placeholder. Will be calibrated. +- `github-runner-equivalent` — placeholder. Will be calibrated to + approximate the headroom an `ubuntu-latest` GitHub runner has while + also juggling four parallel package-build workflows (estimate: + 2-core / ~7 GiB total, so 1 stress-ng worker + memory ballast). The + initial calibration target: this profile must reproduce the rekey + Phase 5 flake class at ≥20% rate over 20 reps with mechanism-match. +- `heavy` — placeholder. Worst-case pressure for stall-finding work. + +## Environment-variable knobs + +The harness reads three optional environment variables that shape what +each rep does, set them in the invoking shell: + +- **`FIPS_MESH_LAB_NETEM`** — netem argument string (e.g. + `"delay 10ms 5ms 25% loss 1%"`). When set, the harness runs + `tc qdisc add dev eth0 root netem ` inside each fips-node + container after `compose up`. Bridge-level qdisc on the docker + network does *not* shape inter-container traffic (Linux bridges + forward port-to-port without packets traversing the bridge + interface's egress qdisc), so per-container egress is the correct + injection point. + +- **`FIPS_MESH_LAB_TRACE`** — when set to any non-empty value, the + harness layers `compose-trace.yml` on top of the base + resource- + limits compose stack. That override bumps `RUST_LOG` to trace + level on the modules relevant to the rekey-class flake: `rekey`, + `handshake`, `forwarding`, `session`, `encrypted`, `mmp`. Use + only when capturing primary failure-moment evidence for mechanism + investigation — log volume increases substantially. Without this + knob, daemon logs only capture state transitions (rekey cutover, + K-bit flip), not per-datagram forwarding decisions, which makes + evidence collection for routing-state stalls effectively + impossible. + +- **`FIPS_MESH_LAB_RUNS_DIR`** — root directory for harness output + (the `runs//` tree). When unset, the harness falls back + to an in-tree path under `testing/mesh-lab/` and prints a warning + to stderr naming the variable and the fallback location. Set this + to a path outside the source tree (e.g. `/var/tmp/fips-mesh-lab` + or a path on a separate disk) to keep gigabyte-scale per-rep + artefacts out of the checkout. + +Example: + +```bash +FIPS_MESH_LAB_TRACE=1 \ +FIPS_MESH_LAB_RUNS_DIR=/var/tmp/fips-mesh-lab \ + bash testing/mesh-lab/run-loop.sh rekey-accept-off \ + --reps 20 --profile github-runner-equivalent +``` + +## Recipes + +`recipes/.yaml` files are commit-pinned reproduction recipes +the harness consumes. Each recipe declares the source SHA, the suite, +the pressure profile, the rep count, and the expected mechanism-match +rate, so a future operator can confirm "yes the lab still reproduces +this flake at the documented rate" with one command. None exist yet — +they're authored as concrete reproductions surface. + +## How this differs from `testing/ci-local.sh` + +`ci-local.sh` runs each suite exactly once in sequence (chaos +scenarios in parallel up to a job slot count), produces a pass/fail +matrix, and is the canonical "did anything regress" gate. The +mesh-lab runs the *same* per-suite test scripts (it does not +reimplement them) but in a loop, with deliberate host pressure +applied, and with rich per-rep diagnostic capture. They share the +Docker images and the test scripts. + +When in doubt, debug a single suite via `ci-local.sh --only ` +first to confirm the suite is healthy on idle, then graduate to the +mesh-lab when you want to chase a flake. diff --git a/testing/mesh-lab/compose-resource-limits.yml b/testing/mesh-lab/compose-resource-limits.yml new file mode 100644 index 0000000..b047708 --- /dev/null +++ b/testing/mesh-lab/compose-resource-limits.yml @@ -0,0 +1,58 @@ +# Compose override applied by the mesh-lab harness to constrain the +# rekey-family daemon containers to roughly the per-container slice +# they'd get on a GitHub Actions ubuntu-latest runner (2 cores / 7 GiB +# total, shared among 5 daemons + the test harness + OS overhead). +# +# Used by passing `-f testing/static/docker-compose.yml -f this-file` +# in the lab's `docker compose up` invocation. The base compose file +# is unmodified so ci-local.sh on a developer host stays unconstrained +# unless the developer deliberately opts into these limits. +# +# Sizing math: 5 daemons × 0.3 cpus = 1.5 cores; 5 daemons × 1 GiB = +# 5 GiB. Leaves ~0.5 cores and ~2 GiB for the test harness + OS on a +# 2-core / 7-GiB GHA runner, which matches reality on a fully-loaded +# CI box. +# +# Service names follow the per-profile naming in the base compose: +# the `rekey` profile uses rekey-a..rekey-e, `rekey-accept-off` uses +# rekey-accept-off-a..e, and `rekey-outbound-only` uses +# rekey-outbound-only-a..e. The override repeats the limits anchor +# across all three so any rekey-family profile picks them up. +# +# 2026-05-16 first cut: 0.3 cpus + 1 GiB per daemon, measured rate TBD. + +x-fips-limits: &fips-limits + cpus: 0.3 + mem_limit: 1g + +services: + rekey-a: + <<: *fips-limits + rekey-b: + <<: *fips-limits + rekey-c: + <<: *fips-limits + rekey-d: + <<: *fips-limits + rekey-e: + <<: *fips-limits + rekey-accept-off-a: + <<: *fips-limits + rekey-accept-off-b: + <<: *fips-limits + rekey-accept-off-c: + <<: *fips-limits + rekey-accept-off-d: + <<: *fips-limits + rekey-accept-off-e: + <<: *fips-limits + rekey-outbound-only-a: + <<: *fips-limits + rekey-outbound-only-b: + <<: *fips-limits + rekey-outbound-only-c: + <<: *fips-limits + rekey-outbound-only-d: + <<: *fips-limits + rekey-outbound-only-e: + <<: *fips-limits diff --git a/testing/mesh-lab/compose-trace.yml b/testing/mesh-lab/compose-trace.yml new file mode 100644 index 0000000..e9bc450 --- /dev/null +++ b/testing/mesh-lab/compose-trace.yml @@ -0,0 +1,73 @@ +# Compose override that bumps RUST_LOG to trace level on the modules +# relevant to rekey-class flake evidence collection: +# +# - fips::node::handlers::rekey — FMP/FSP rekey state machine +# - fips::node::handlers::handshake — Noise handshake (cross-init, +# dual-init, msg1/2/3 dispatch) +# - fips::node::handlers::forwarding — transit datagram forwarding, +# route lookup, drops +# - fips::node::handlers::session — FSP K-bit cutover, drain +# - fips::node::handlers::encrypted — FMP K-bit flip detection +# - fips::node::handlers::mmp — link liveness, SRTT, ETX +# +# Other modules stay at info to keep log volume manageable. The base +# docker-compose.yml's per-service RUST_LOG values (currently +# `info,fips::node::handlers::rekey=debug` for the rekey profile, plus +# handshake=debug on the variant profiles) are fully replaced by the +# values below via the YAML map form: docker compose's override +# semantics replace the map entry rather than merging individual +# environment keys. +# +# Service-name layout follows the base compose: per-profile services +# named `-` (rekey-a..e for the rekey profile, +# rekey-accept-off-a..e for the accept-off variant, +# rekey-outbound-only-a..e for the outbound-only variant). All 15 +# rekey-family services get the same trace-RUST_LOG value. +# +# Include this override alongside the base compose via: +# docker compose -f testing/static/docker-compose.yml \ +# -f testing/mesh-lab/compose-resource-limits.yml \ +# -f testing/mesh-lab/compose-trace.yml up -d +# +# The run-loop.sh harness includes it automatically when the mesh-lab +# environment variable FIPS_MESH_LAB_TRACE=1 is set. + +x-trace-rust-log: &trace-rust-log + RUST_LOG: "info,fips::node::handlers::rekey=trace,fips::node::handlers::handshake=trace,fips::node::handlers::forwarding=trace,fips::node::handlers::session=trace,fips::node::handlers::encrypted=trace,fips::node::handlers::mmp=trace" + +services: + # rekey profile + rekey-a: + environment: *trace-rust-log + rekey-b: + environment: *trace-rust-log + rekey-c: + environment: *trace-rust-log + rekey-d: + environment: *trace-rust-log + rekey-e: + environment: *trace-rust-log + + # rekey-accept-off profile + rekey-accept-off-a: + environment: *trace-rust-log + rekey-accept-off-b: + environment: *trace-rust-log + rekey-accept-off-c: + environment: *trace-rust-log + rekey-accept-off-d: + environment: *trace-rust-log + rekey-accept-off-e: + environment: *trace-rust-log + + # rekey-outbound-only profile + rekey-outbound-only-a: + environment: *trace-rust-log + rekey-outbound-only-b: + environment: *trace-rust-log + rekey-outbound-only-c: + environment: *trace-rust-log + rekey-outbound-only-d: + environment: *trace-rust-log + rekey-outbound-only-e: + environment: *trace-rust-log diff --git a/testing/mesh-lab/pressure-profiles.sh b/testing/mesh-lab/pressure-profiles.sh new file mode 100755 index 0000000..5cf14bd --- /dev/null +++ b/testing/mesh-lab/pressure-profiles.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# Pressure profiles for the mesh-lab. Sourced by run-loop.sh. +# +# Each profile is a function `pressure_start_` that spawns +# stress-ng (or whatever) in the background and writes its PID to the +# variable PRESSURE_PIDS (a bash array). The harness uses +# `pressure_stop` (defined at the bottom) to kill everything cleanly. +# +# Profiles are intentionally simple stubs until calibrated against +# actual reproduction of a known flake class. + +set -u + +PRESSURE_PIDS=() + +# Track which profile is active for the summary.json. +PRESSURE_PROFILE_ACTIVE="idle" + +# ── idle ───────────────────────────────────────────────────────────── +# No pressure. Baseline. A healthy mesh should run all reps green on +# this profile; failures here are a different bug class than what the +# lab is built to chase. + +pressure_start_idle() { + PRESSURE_PROFILE_ACTIVE="idle" + # nothing to do +} + +# ── light ──────────────────────────────────────────────────────────── +# Single stress-ng cpu worker at ~50% load. Placeholder; calibration +# will determine the right intensity for surfacing tail behavior +# without saturating the host. + +pressure_start_light() { + PRESSURE_PROFILE_ACTIVE="light" + require_stress_ng || return 1 + stress-ng --cpu 1 --cpu-load 50 --quiet & + PRESSURE_PIDS+=("$!") +} + +# ── github-runner-equivalent ──────────────────────────────────────── +# Calibrated to approximate the headroom a 2-core GitHub Actions +# ubuntu-latest runner has when also juggling four concurrent package- +# build workflows. Target effective per-host load: ~75% of cores +# consumed, ~30% of RAM allocated, so the test+daemon containers +# compete for ~2 effective cores and ~7-8 GiB headroom — close to a +# runner under quad-package-build contention. +# +# Calibration target: ≥20% mechanism-match rate over 5-rep pilots on +# the `rekey` suite. Calibration history (host = 8c/62GiB): +# +# 2026-05-16 first cut --cpu 1 --vm 1 --vm-bytes 512m +# 25 reps, 1 mechanism-match (4%) +# (stochastic pilot fire; not contention-driven) +# 2026-05-16 second cut --cpu 6 --vm 2 --vm-bytes 10g +# 5 reps, 0 mechanism-match +# (host load=8.25/8 cores, 20GiB resident; the +# mechanism did not surface even at saturation) +# +# Layered with the mesh-lab/compose-resource-limits.yml override +# (per-daemon cpus=0.3 + mem_limit=1g) and in-container tc qdisc netem +# (10ms ± 5ms delay + 1% loss), the rekey Phase 5 mechanism still did +# not reproduce above the stub's stochastic baseline on this host — +# implying the GHA fires are more likely code-side (tokio task stall, +# channel backpressure, mutex contention) than environment-side. + +pressure_start_github_runner_equivalent() { + PRESSURE_PROFILE_ACTIVE="github-runner-equivalent" + require_stress_ng || return 1 + # CPU pressure: six full-load cpu workers (75% of an 8-core host). + # On smaller hosts the per-worker share rises; on larger hosts the + # absolute headroom rises proportionally — the profile is anchored + # to "what's left after the pressure," not to a target host size. + stress-ng --cpu 6 --quiet & + PRESSURE_PIDS+=("$!") + # Memory pressure: two vm workers, 20 GiB total resident. Sized to + # eat ~30% of a 62 GiB host. --vm-keep holds pages resident rather + # than churning the allocator, which mirrors a peer workflow that + # has allocated and is using its working set, not one trashing it. + stress-ng --vm 2 --vm-bytes 10g --vm-keep --quiet & + PRESSURE_PIDS+=("$!") +} + +# ── heavy ──────────────────────────────────────────────────────────── +# Two cpu workers + memory pressure. Worst-case profile for stall- +# finding work once the milder profiles are characterized. + +pressure_start_heavy() { + PRESSURE_PROFILE_ACTIVE="heavy" + require_stress_ng || return 1 + stress-ng --cpu 2 --quiet & + PRESSURE_PIDS+=("$!") + stress-ng --vm 1 --vm-bytes 1g --vm-keep --quiet & + PRESSURE_PIDS+=("$!") +} + +# ── helpers ────────────────────────────────────────────────────────── + +require_stress_ng() { + if ! command -v stress-ng >/dev/null 2>&1; then + echo " ERROR: stress-ng not installed; cannot run pressure profile '$PRESSURE_PROFILE_ACTIVE'" >&2 + echo " install on Debian/Ubuntu: sudo apt-get install stress-ng" >&2 + return 1 + fi + return 0 +} + +pressure_start() { + local profile="$1" + PRESSURE_PIDS=() + case "$profile" in + idle) + pressure_start_idle ;; + light) + pressure_start_light ;; + github-runner-equivalent | github | gha) + pressure_start_github_runner_equivalent ;; + heavy) + pressure_start_heavy ;; + *) + echo " ERROR: unknown pressure profile '$profile'" >&2 + echo " available: idle, light, github-runner-equivalent, heavy" >&2 + return 1 ;; + esac +} + +pressure_stop() { + if [ "${#PRESSURE_PIDS[@]}" -eq 0 ]; then + return 0 + fi + for pid in "${PRESSURE_PIDS[@]}"; do + # kill the process group; stress-ng spawns children + kill -TERM "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true + done + # Brief grace, then SIGKILL stragglers + sleep 1 + for pid in "${PRESSURE_PIDS[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + kill -KILL "$pid" 2>/dev/null || true + fi + done + PRESSURE_PIDS=() +} diff --git a/testing/mesh-lab/run-loop.sh b/testing/mesh-lab/run-loop.sh new file mode 100755 index 0000000..cb21fe3 --- /dev/null +++ b/testing/mesh-lab/run-loop.sh @@ -0,0 +1,496 @@ +#!/bin/bash +# FIPS mesh-reliability lab: run an integration suite N times under a +# configurable host-pressure profile, capture per-rep diagnostics, and +# produce a per-rep + aggregate summary.json compact enough for triage +# without holding gigabytes of raw log. +# +# See ./README.md for the full developer-facing description. +# +# Usage: +# run-loop.sh [--reps N] [--profile NAME] [--out DIR] +# +# suite One of: rekey, rekey-accept-off, rekey-outbound-only, +# nat-lan, bloom-storm. +# --reps N Number of repetitions (default 1). +# --profile Pressure profile name from pressure-profiles.sh (default +# idle). See pressure-profiles.sh for the full list. +# --out DIR Output directory (default /runs/; see +# FIPS_MESH_LAB_RUNS_DIR below for how the runs-base is +# chosen). +# +# Environment: +# FIPS_MESH_LAB_NETEM netem argument string applied via tc qdisc +# inside each fips-node container's eth0. +# FIPS_MESH_LAB_TRACE when set, layers compose-trace.yml on top +# of the base + resource-limits stack to +# bump RUST_LOG to trace on rekey/handshake/ +# forwarding/session/encrypted/mmp modules. +# FIPS_MESH_LAB_RUNS_DIR Root for harness output (runs/ and any +# other scratch). When unset, falls back to +# an in-tree path under testing/mesh-lab/ +# and prints a warning to stderr; set it to +# a path outside the source tree to keep +# generated artefacts out of the checkout. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/pressure-profiles.sh" + +# ── Scratch-dir root ───────────────────────────────────────────────── +# +# FIPS_MESH_LAB_RUNS_DIR controls where the harness writes its run +# output (runs//...). When unset we fall back to an +# in-tree path under testing/mesh-lab/ and warn the operator, so the +# warning fires exactly once per invocation. When a parent script +# has already warned it exports _FIPS_MESH_LAB_WARNED=1 to suppress +# duplicate warnings in child scripts. +if [[ -n "${FIPS_MESH_LAB_RUNS_DIR:-}" ]]; then + RUNS_BASE="$FIPS_MESH_LAB_RUNS_DIR" + mkdir -p "$RUNS_BASE" +else + RUNS_BASE="$SCRIPT_DIR" + if [[ -z "${_FIPS_MESH_LAB_WARNED:-}" ]]; then + echo >&2 "WARNING: FIPS_MESH_LAB_RUNS_DIR not set; harness output will be written under the source tree at $RUNS_BASE/runs/. Set FIPS_MESH_LAB_RUNS_DIR to a path outside the source tree to avoid this." + export _FIPS_MESH_LAB_WARNED=1 + fi +fi + +# ── Args ───────────────────────────────────────────────────────────── + +SUITE="" +REPS=1 +PROFILE="idle" +OUT_DIR="" + +usage() { + sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [ $# -gt 0 ]; do + case "$1" in + --reps) REPS="$2"; shift 2 ;; + --profile) PROFILE="$2"; shift 2 ;; + --out) OUT_DIR="$2"; shift 2 ;; + -h|--help) usage 0 ;; + --) shift; break ;; + -*) echo "unknown flag: $1" >&2; usage 1 ;; + *) + if [ -z "$SUITE" ]; then + SUITE="$1" + else + echo "unexpected positional arg: $1" >&2 + usage 1 + fi + shift ;; + esac +done + +if [ -z "$SUITE" ]; then + echo "missing required argument" >&2 + usage 1 +fi + +if [ -z "$OUT_DIR" ]; then + ts="$(date -u +%Y%m%dT%H%M%SZ)" + OUT_DIR="$RUNS_BASE/runs/${ts}-${SUITE}-${PROFILE}" +fi + +mkdir -p "$OUT_DIR" + +# Mirror the harness's own stdout/stderr to a per-run log. The +# per-rep setup/test-output/teardown captures only the in-container +# test side; this captures the wrapper-level signal (pressure-profile +# start/stop, OOM-killed child notifications from bash job control, +# aggregate summary, any preflight error). Without this, host-side +# diagnostics are lost the moment the host reboots and the kernel +# ring buffer rolls. +exec > >(tee -a "$OUT_DIR/run-loop.log") 2>&1 + +# ── Preflight ──────────────────────────────────────────────────────── + +require_docker() { + if ! docker info >/dev/null 2>&1; then + echo "ERROR: Docker daemon is not reachable" >&2 + exit 2 + fi +} + +require_test_image() { + if ! docker image inspect fips-test:latest >/dev/null 2>&1; then + echo "ERROR: fips-test:latest not present" >&2 + echo "Build it once with: bash testing/ci-local.sh --build-only" >&2 + exit 2 + fi +} + +require_docker +require_test_image + +# ── Suite-specific drivers and signature parsers ───────────────────── +# +# Each driver runs ONE rep of the suite. It must: +# - return 0 on suite-pass, non-zero on suite-fail +# - write the test stdout/stderr to ${REP_DIR}/test-output.log +# - capture container logs to ${REP_DIR}/docker-logs/ if relevant +# +# Each parser reads ${REP_DIR}/test-output.log and writes a JSON +# fragment (no enclosing braces) of suite-specific signature features +# into ${REP_DIR}/signature.json. Used by the aggregate summary to +# decide whether the failure matches the documented mechanism for the +# associated open issue. + +run_rekey_family() { + local variant="$1" # rekey, rekey-accept-off, rekey-outbound-only + local REP_DIR="$2" + local compose_profile="$variant" + local env_args=() + + case "$variant" in + rekey-accept-off) + env_args=(REKEY_TOPOLOGY=rekey-accept-off REKEY_ACCEPT_OFF_NODES=b) ;; + rekey-outbound-only) + env_args=(REKEY_TOPOLOGY=rekey-outbound-only REKEY_OUTBOUND_ONLY_NODES=b) ;; + esac + + # Lab compose stack: base compose + mesh-lab resource-limits override. + # The override pins each rekey-family daemon to roughly its GHA-runner + # share (0.3 cpus / 1 GiB), mimicking the constraint a 2-core / 7-GiB + # ubuntu-latest runner imposes. Base compose is unmodified so + # ci-local.sh stays unconstrained for day-to-day developer runs. + # + # Trace-logging override: set FIPS_MESH_LAB_TRACE=1 in the environment + # to bump RUST_LOG to trace level on the modules relevant to the + # rekey-class flake (rekey, handshake, forwarding, session, encrypted, + # mmp). Increases log volume substantially; use only when capturing + # primary failure-moment evidence for mechanism investigation. + local compose_args=( + -f testing/static/docker-compose.yml + -f testing/mesh-lab/compose-resource-limits.yml + --profile "$compose_profile" + ) + if [ -n "${FIPS_MESH_LAB_TRACE:-}" ]; then + compose_args=( + -f testing/static/docker-compose.yml + -f testing/mesh-lab/compose-resource-limits.yml + -f testing/mesh-lab/compose-trace.yml + --profile "$compose_profile" + ) + fi + + ( + cd "$REPO_ROOT" || exit 1 + env "${env_args[@]}" bash testing/static/scripts/generate-configs.sh "$variant" \ + >>"$REP_DIR/setup.log" 2>&1 + env "${env_args[@]}" bash testing/static/scripts/rekey-test.sh inject-config \ + >>"$REP_DIR/setup.log" 2>&1 + docker compose "${compose_args[@]}" up -d \ + >>"$REP_DIR/setup.log" 2>&1 + ) + + # Optional: apply tc qdisc netem inside each fips-node container's + # eth0. Set FIPS_MESH_LAB_NETEM to a netem argument string (e.g. + # "delay 10ms 5ms 25% loss 1%") to enable. Applied via `docker + # exec` because qdisc on the host-side docker bridge does NOT shape + # port-to-port inter-container traffic on the bridge — only traffic + # to/from the host's own IP. Egress qdisc on the container's own + # eth0 reliably shapes that container's outbound packets to peer + # containers. Containers already have NET_ADMIN cap (rekey needs + # it for TUN); tc is in the fips-test image. + local netem_applied=() + if [ -n "${FIPS_MESH_LAB_NETEM:-}" ]; then + for node in a b c d e; do + if docker exec "fips-node-$node" tc qdisc add dev eth0 root \ + netem ${FIPS_MESH_LAB_NETEM} >>"$REP_DIR/setup.log" 2>&1; then + netem_applied+=("fips-node-$node") + else + echo "WARN: netem apply failed on fips-node-$node" \ + >>"$REP_DIR/setup.log" + fi + done + if [ "${#netem_applied[@]}" -gt 0 ]; then + echo "applied netem on ${#netem_applied[@]}/5 nodes: $FIPS_MESH_LAB_NETEM" \ + >>"$REP_DIR/setup.log" + fi + fi + + local rc=0 + ( + cd "$REPO_ROOT" || exit 1 + env "${env_args[@]}" bash testing/static/scripts/rekey-test.sh + ) >"$REP_DIR/test-output.log" 2>&1 || rc=$? + + # Capture container logs before teardown + mkdir -p "$REP_DIR/docker-logs" + for node in a b c d e; do + docker logs "fips-node-$node" >"$REP_DIR/docker-logs/node-$node.log" 2>&1 || true + done + + # In-container netem disappears with the container itself on + # compose down, so no explicit teardown needed. + + ( + cd "$REPO_ROOT" || exit 1 + docker compose "${compose_args[@]}" \ + down --volumes --remove-orphans \ + >>"$REP_DIR/teardown.log" 2>&1 + ) + + return "$rc" +} + +parse_rekey() { + local REP_DIR="$1" + local log="$REP_DIR/test-output.log" + + # Phase 5 per-pair failures (e.g., "B → D ... FAIL" or + # "B → D ... FAIL (after 4 attempts)" when retries are enabled). + local phase5_failures + phase5_failures=$(awk ' + /^Phase 5:/ { in5=1; next } + /^Phase 6:/ { in5=0 } + in5 && /\.\.\. FAIL([[:space:]]|$)/ { print } + ' "$log" | sed 's/^ *//' | tr '\n' ',' | sed 's/,$//') + + # Phase 6 log analysis result + local phase6_status="unknown" + if grep -q '"Log analysis: .* passed"\|✓ Log analysis: ' "$log"; then + phase6_status="all-green" + fi + if grep -q 'ERROR\|PANIC\|panicked' "$log"; then + phase6_status="errors-observed" + fi + + # Late FSP K-bit cutover detection — scan node logs for cutover- + # complete events occurring within or after the Phase 5 settle + # window. The settle window is the 12 s before Phase 5's first + # ping_all. Without precise timing parsing in this first pass, we + # report the timestamps of all FSP K-bit-related events for the + # rep so a reviewer can match against the documented mechanism. + local late_fsp_events="" + for nodelog in "$REP_DIR"/docker-logs/node-*.log; do + [ -f "$nodelog" ] || continue + # Strip ANSI escape codes that docker logs preserves from the + # daemon's TTY-aware tracing-subscriber output; raw ESC bytes + # are invalid JSON string contents. + late_fsp_events+=$(grep -oE '[0-9-]+T[0-9:.]+Z.*(K-bit flip|FSP rekey cutover complete)' "$nodelog" \ + | sed -E 's/\x1b\[[0-9;]*[mK]//g' \ + | tail -5 \ + | tr '\n' ';') + late_fsp_events+="|" + done + + # Compose JSON via jq -n so embedded specials in any field are + # safely escaped rather than splatted as raw bytes. + jq -n \ + --arg pairs "$phase5_failures" \ + --arg phase6 "$phase6_status" \ + --arg events "$late_fsp_events" \ + '{phase5_failing_pairs: $pairs, phase6_log_analysis: $phase6, late_fsp_events_per_node_tail: $events}' \ + > "$REP_DIR/signature.json" +} + +# Heuristic mechanism-match check for the rekey Phase 5 flake class. +# True iff: +# - At least one Phase 5 ping fails +# - Phase 6 log analysis is all-green (no ERROR/PANIC noise) +mechanism_match_rekey() { + local REP_DIR="$1" + local sig="$REP_DIR/signature.json" + if [ ! -f "$sig" ]; then + echo " WARN: mechanism_match: $sig missing" >&2 + echo "false" + return + fi + # Surface invalid JSON loudly — silent jq failure with `|| echo ""` + # previously masked real mechanism matches when ANSI escapes leaked + # into the events field. + if ! jq -e . "$sig" >/dev/null 2>&1; then + echo " WARN: mechanism_match: $sig is invalid JSON" >&2 + echo "false" + return + fi + local pairs phase6 + pairs=$(jq -r '.phase5_failing_pairs' "$sig") + phase6=$(jq -r '.phase6_log_analysis' "$sig") + if [ -n "$pairs" ] && [ "$phase6" = "all-green" ]; then + echo "true" + else + echo "false" + fi +} + +run_nat_lan() { + local REP_DIR="$1" + local rc=0 + ( + cd "$REPO_ROOT" || exit 1 + bash testing/nat/scripts/nat-test.sh lan + ) >"$REP_DIR/test-output.log" 2>&1 || rc=$? + + mkdir -p "$REP_DIR/docker-logs" + for c in fips-nat-lan-a fips-nat-lan-b; do + docker logs "$c" >"$REP_DIR/docker-logs/$c.log" 2>&1 || true + done + + return "$rc" +} + +parse_nat_lan() { + local REP_DIR="$1" + local log="$REP_DIR/test-output.log" + + local peer_adoption_timeout="false" + if grep -q "TIMEOUT waiting for" "$log"; then + peer_adoption_timeout="true" + fi + + local cross_init_observed="false" + if grep -E "Connection initiated.*node-(a|b)" "$REP_DIR"/docker-logs/*.log 2>/dev/null \ + | awk '{print $1}' | sort -u | head -2 | wc -l | grep -q '2'; then + cross_init_observed="true" + fi + + cat <"$REP_DIR/signature.json" +{ + "peer_adoption_timeout": $peer_adoption_timeout, + "cross_init_observed": $cross_init_observed +} +EOF +} + +mechanism_match_nat_lan() { + local REP_DIR="$1" + local sig="$REP_DIR/signature.json" + [ -f "$sig" ] || return 1 + local timeout_seen + timeout_seen=$(jq -r '.peer_adoption_timeout' "$sig" 2>/dev/null || echo "false") + if [ "$timeout_seen" = "true" ]; then + echo "true" + else + echo "false" + fi +} + +# Dispatch — returns rc of suite, side-effects signature.json +dispatch_suite() { + local REP_DIR="$1" + case "$SUITE" in + rekey|rekey-accept-off|rekey-outbound-only) + local rc=0 + run_rekey_family "$SUITE" "$REP_DIR" || rc=$? + parse_rekey "$REP_DIR" + return "$rc" ;; + nat-lan) + local rc=0 + run_nat_lan "$REP_DIR" || rc=$? + parse_nat_lan "$REP_DIR" + return "$rc" ;; + *) + echo "ERROR: unsupported suite '$SUITE' in this lab harness (initial scaffolding)" >&2 + echo "Supported: rekey, rekey-accept-off, rekey-outbound-only, nat-lan" >&2 + return 99 ;; + esac +} + +# Mechanism-match heuristic per suite +dispatch_mechanism_match() { + local REP_DIR="$1" + case "$SUITE" in + rekey|rekey-accept-off|rekey-outbound-only) + mechanism_match_rekey "$REP_DIR" ;; + nat-lan) + mechanism_match_nat_lan "$REP_DIR" ;; + *) + echo "unknown" ;; + esac +} + +# ── Main loop ──────────────────────────────────────────────────────── + +echo "=== mesh-lab: suite=$SUITE reps=$REPS profile=$PROFILE ===" +echo " out: $OUT_DIR" +echo "" + +PASS_COUNT=0 +FAIL_COUNT=0 +MECH_MATCH_COUNT=0 + +# Trap to ensure pressure is always cleaned up even on Ctrl-C +trap 'pressure_stop; exit 130' INT TERM + +for rep in $(seq 1 "$REPS"); do + rep_padded=$(printf "%03d" "$rep") + REP_DIR="$OUT_DIR/rep-$rep_padded" + mkdir -p "$REP_DIR" + + started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + echo "--- rep $rep/$REPS (started $started_at) ---" + + pressure_start "$PROFILE" || { + echo " ERROR: pressure_start failed for profile '$PROFILE'" >&2 + exit 2 + } + + rc=0 + dispatch_suite "$REP_DIR" || rc=$? + + pressure_stop + + ended_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + mechanism_match=$(dispatch_mechanism_match "$REP_DIR") + + if [ "$rc" -eq 0 ]; then + result="pass" + PASS_COUNT=$((PASS_COUNT + 1)) + echo " rep $rep: PASS" + else + result="fail" + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo " rep $rep: FAIL (exit $rc, mechanism_match=$mechanism_match)" + fi + + if [ "$mechanism_match" = "true" ]; then + MECH_MATCH_COUNT=$((MECH_MATCH_COUNT + 1)) + fi + + cat <"$REP_DIR/summary.json" +{ + "rep": $rep, + "suite": "$SUITE", + "profile": "$PROFILE", + "started_at": "$started_at", + "ended_at": "$ended_at", + "exit_code": $rc, + "result": "$result", + "mechanism_match": $mechanism_match, + "signature_file": "signature.json" +} +EOF +done + +# ── Aggregate summary ──────────────────────────────────────────────── + +cat <"$OUT_DIR/summary.json" +{ + "suite": "$SUITE", + "profile": "$PROFILE", + "reps": $REPS, + "pass_count": $PASS_COUNT, + "fail_count": $FAIL_COUNT, + "mechanism_match_count": $MECH_MATCH_COUNT, + "pass_rate": $(awk -v p="$PASS_COUNT" -v r="$REPS" 'BEGIN{ printf "%.3f", p/r }'), + "fail_rate": $(awk -v f="$FAIL_COUNT" -v r="$REPS" 'BEGIN{ printf "%.3f", f/r }'), + "mechanism_match_rate": $(awk -v m="$MECH_MATCH_COUNT" -v r="$REPS" 'BEGIN{ printf "%.3f", m/r }') +} +EOF + +echo "" +echo "=== summary ===" +cat "$OUT_DIR/summary.json" +echo "" +echo "raw artifacts: $OUT_DIR/" diff --git a/testing/static/scripts/rekey-test.sh b/testing/static/scripts/rekey-test.sh index 0e26a99..46b2d5e 100755 --- a/testing/static/scripts/rekey-test.sh +++ b/testing/static/scripts/rekey-test.sh @@ -497,14 +497,30 @@ echo "=== Results: $TOTAL_PASSED passed, $TOTAL_FAILED failed ===" if [ "$TOTAL_FAILED" -eq 0 ]; then exit 0 else - # Dump logs on failure for diagnostics + # Dump logs on failure for diagnostics. + # + # Wider pattern + larger line cap than the original head -30 (which + # truncated before the actual ping-failure timestamp on Phase 5 + # fires, making the post-cutover convergence window invisible to + # offline triage). Two passes per node: + # 1) rekey-related events across the whole run (cap 200 lines). + # 2) Last 80 lines unfiltered, to surface forwarding decisions, + # route lookups, congestion warnings, decrypt errors, and any + # other surrounding context near the failure point. echo "" - echo "=== Node logs (rekey-related) ===" + echo "=== Node logs (rekey-related, head -200) ===" for node in $NODES; do echo "--- node-$node ---" docker logs "fips-node-$node" 2>&1 | \ - grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit)" | \ - head -30 + grep -E "(rekey|Rekey|cross|Cross|teardown|ERROR|PANIC|K-bit|no route|next hop|TTL exhausted|MTU exceeded|Congestion|decrypt|Decrypt|AEAD|Notify|drain|Drain|promot)" | \ + head -200 + echo "" + done + + echo "=== Node logs (last 80 lines, unfiltered) ===" + for node in $NODES; do + echo "--- node-$node ---" + docker logs "fips-node-$node" 2>&1 | tail -80 echo "" done exit 1