testing: add boringtun throughput benchmark and iperf ref-compare harness

New testing/boringtun/ harness runs two Cloudflare BoringTun userspace
WireGuard containers with iperf3 between them, giving a single-hop
userspace tunnel baseline for comparison against FIPS throughput
numbers. Local WG key generation runs through the harness image so the
host needs no wireguard-tools.

New testing/static/scripts/iperf-compare-refs.sh builds two git refs
into separate fips-test:* images via git worktree and runs the same
static iperf topology against both, with RUNS-based repetition and
aggregate avg/min/max reporting.

testing/static/scripts/iperf-test.sh gains DURATION, PARALLEL,
SETTLE_SECONDS, IPERF_TIMEOUT env knobs and a per-path iperf timeout.
testing/static/docker-compose.yml selects the image under test via
FIPS_TEST_IMAGE; testing/scripts/build.sh respects CARGO_TARGET_DIR.

Author benchmark on aarch64 Docker Desktop:
  boringtun bob -> alice : 1000.13 Mbits/sec

testing/boringtun/.gitignore

@@ -0,0 +1 @@
+generated/

testing/boringtun/Dockerfile

@@ -0,0 +1,28 @@
+# Minimal boringtun-cli image for throughput benchmarking against FIPS.
+# Userspace WireGuard (boringtun) + wireguard-tools + iperf3 in one
+# image. Run two containers on a docker bridge, configure a WG peer
+# between them, then iperf3 client→server across the tunnel.
+
+FROM rust:1-bookworm AS build
+# 0.6.0 (crates.io HEAD) is from 2022 and turns in ~1.7 Mbps in
+# this harness — clearly broken. Pull the boringtun-cli binary from
+# the cloudflare HEAD instead; it gets the post-0.6 perf fixes and
+# the multi-threaded recv loop.
+RUN git clone --depth 1 https://github.com/cloudflare/boringtun /usr/src/boringtun \
+    && cargo install --path /usr/src/boringtun/boringtun-cli
+
+FROM debian:bookworm-slim
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        iperf3 \
+        iproute2 \
+        wireguard-tools \
+        iputils-ping \
+        netcat-openbsd \
+        ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=build /usr/local/cargo/bin/boringtun-cli /usr/local/bin/boringtun-cli
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

testing/boringtun/README.md

@@ -0,0 +1,29 @@
+# BoringTun Throughput Baseline
+
+This harness runs two userspace WireGuard peers with Cloudflare BoringTun and
+measures single-stream TCP throughput with `iperf3`. It is intended as a simple
+baseline for comparing FIPS tunnel throughput against another userspace tunnel.
+
+```bash
+docker build -t boringtun-test:latest testing/boringtun
+testing/boringtun/scripts/generate-keys.sh
+docker compose -f testing/boringtun/docker-compose.yml up -d
+testing/boringtun/scripts/bench.sh
+docker compose -f testing/boringtun/docker-compose.yml down
+```
+
+The generated WireGuard key material is written under
+`testing/boringtun/generated/` and is ignored by git.
+
+For FIPS-to-FIPS revision comparisons, use the static topology comparison
+script:
+
+```bash
+testing/static/scripts/iperf-compare-refs.sh origin/master HEAD mesh
+```
+
+That script builds each ref into a separate `fips-test:*` image, runs the
+same static `iperf3` topology against both images, and prints a bandwidth
+summary for each path. Override `DURATION`, `PARALLEL`, `SETTLE_SECONDS`, or
+`IPERF_TIMEOUT` in the environment when needed. Set `RUNS=3` or similar to
+repeat each ref and print aggregate results.

testing/boringtun/docker-compose.yml

@@ -0,0 +1,46 @@
+# Minimal 2-node boringtun setup for iperf3 throughput comparison.
+# Both containers join the same docker bridge; each runs a boringtun
+# userspace WireGuard tunnel and they peer with each other over UDP.
+# Inner WG IPs: 10.99.0.1/24 (alice) <-> 10.99.0.2/24 (bob).
+
+networks:
+  bt-net:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.99.0.0/24
+
+x-boringtun-common: &boringtun-common
+  image: boringtun-test:latest
+  cap_add:
+    - NET_ADMIN
+  devices:
+    - /dev/net/tun:/dev/net/tun
+  sysctls:
+    - net.ipv4.conf.all.forwarding=1
+  restart: "no"
+  env_file:
+    - ./generated/peers.env
+
+services:
+  alice:
+    <<: *boringtun-common
+    container_name: bt-alice
+    hostname: alice
+    environment:
+      - ROLE=alice
+      - PEER_HOST=bob
+    networks:
+      bt-net:
+        ipv4_address: 172.99.0.10
+
+  bob:
+    <<: *boringtun-common
+    container_name: bt-bob
+    hostname: bob
+    environment:
+      - ROLE=bob
+      - PEER_HOST=alice
+    networks:
+      bt-net:
+        ipv4_address: 172.99.0.11

testing/boringtun/entrypoint.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Bring up a boringtun-userspace WireGuard tunnel and then sleep
+# indefinitely so the benchmark scripts can drive it via docker exec.
+#
+# Required env:
+#   ROLE      "alice" or "bob"
+#   ALICE_WG_IP  WG inner IP for alice (e.g. 10.99.0.1/24)
+#   BOB_WG_IP    WG inner IP for bob (e.g. 10.99.0.2/24)
+#   ALICE_PUB   alice's WG public key
+#   BOB_PUB     bob's WG public key
+#   ALICE_PRIV  alice's WG private key
+#   BOB_PRIV    bob's WG private key
+#   PEER_HOST   the other container's hostname on the docker bridge
+#   PEER_PORT   the other container's WG UDP port (default 51820)
+#
+# boringtun-cli runs in foreground (--foreground) so wg-quick-style
+# configuration is done manually via `ip`, `wg`, and `wg set`.
+
+set -e
+
+PORT="${PEER_PORT:-51820}"
+case "$ROLE" in
+    alice)
+        OUR_IP="$ALICE_WG_IP"
+        OUR_PRIV="$ALICE_PRIV"
+        PEER_PUB="$BOB_PUB"
+        ;;
+    bob)
+        OUR_IP="$BOB_WG_IP"
+        OUR_PRIV="$BOB_PRIV"
+        PEER_PUB="$ALICE_PUB"
+        ;;
+    *)
+        echo "ROLE must be alice or bob, got '$ROLE'" >&2
+        exit 1
+        ;;
+esac
+
+echo "[$ROLE] starting boringtun-cli foreground on wg0"
+# `--foreground` keeps the userspace driver in the container
+# foreground. boringtun-cli sets WG_TUN_NAME_FILE to /tmp/wg0.name
+# when --foreground; we just hardcode the device name.
+boringtun-cli --foreground --disable-drop-privileges wg0 &
+BORINGTUN_PID=$!
+
+# wait for the tun device to appear
+for i in $(seq 1 50); do
+    if ip link show wg0 >/dev/null 2>&1; then
+        break
+    fi
+    sleep 0.1
+done
+
+echo "[$ROLE] configuring wg0 with $OUR_IP listening on $PORT"
+PRIV_FILE=$(mktemp)
+chmod 600 "$PRIV_FILE"
+printf '%s' "$OUR_PRIV" >"$PRIV_FILE"
+wg set wg0 private-key "$PRIV_FILE" listen-port "$PORT"
+rm -f "$PRIV_FILE"
+
+ip address add "$OUR_IP" dev wg0
+ip link set up dev wg0
+
+echo "[$ROLE] adding peer pubkey, endpoint $PEER_HOST:$PORT"
+wg set wg0 peer "$PEER_PUB" allowed-ips 10.99.0.0/24 endpoint "$PEER_HOST:$PORT" persistent-keepalive 25
+
+echo "[$ROLE] ready"
+wg show wg0
+
+# Park here so the container stays up; we'll run iperf3 etc via
+# docker exec.
+wait $BORINGTUN_PID

testing/boringtun/scripts/bench.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# End-to-end iperf3 bandwidth test between two boringtun containers.
+# Output mirrors testing/static/scripts/iperf-test.sh so the FIPS
+# numbers are directly comparable.
+set -euo pipefail
+
+DURATION="${DURATION:-10}"
+PARALLEL="${PARALLEL:-1}"
+
+echo "=== boringtun iperf3 throughput (single TCP stream, ${DURATION}s) ==="
+
+# Run iperf3 server on alice (background), client on bob.
+docker exec -d bt-alice iperf3 -s -1 -B 10.99.0.1 -p 5201
+sleep 1
+
+# wait for tun handshake to settle (boringtun + WG keepalive)
+sleep 2
+
+# Client: bob → alice over WG (10.99.0.1)
+OUT=$(docker exec bt-bob iperf3 -c 10.99.0.1 -p 5201 -t "$DURATION" -P "$PARALLEL" -J)
+
+# Pull SUM bps.
+MBPS=$(echo "$OUT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(f\"{d['end']['sum_received']['bits_per_second'] / 1_000_000:.2f}\")")
+
+echo "boringtun bob -> alice : ${MBPS} Mbits/sec"

testing/boringtun/scripts/generate-keys.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Generate WG keypairs for alice and bob and write the
+# generated/peers.env file the docker-compose.yml reads.
+# Idempotent: skips if file already exists.
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUT_DIR="$SCRIPT_DIR/../generated"
+ENV_FILE="$OUT_DIR/peers.env"
+
+if [ -f "$ENV_FILE" ]; then
+    echo "$ENV_FILE already exists; reusing keys"
+    exit 0
+fi
+
+mkdir -p "$OUT_DIR"
+
+# Use the built boringtun-test image to generate keys — host may
+# not have wireguard-tools installed. The `pubkey` step needs `-i`
+# so stdin is piped through; `genkey` doesn't need it but matching
+# flags keeps the two calls symmetric.
+GEN_CMD="docker run --rm --entrypoint wg boringtun-test:latest"
+PUB_CMD="docker run --rm -i --entrypoint wg boringtun-test:latest"
+
+ALICE_PRIV=$($GEN_CMD genkey)
+ALICE_PUB=$(printf '%s' "$ALICE_PRIV" | $PUB_CMD pubkey)
+BOB_PRIV=$($GEN_CMD genkey)
+BOB_PUB=$(printf '%s' "$BOB_PRIV" | $PUB_CMD pubkey)
+
+cat >"$ENV_FILE" <<EOF
+ALICE_PRIV=$ALICE_PRIV
+ALICE_PUB=$ALICE_PUB
+BOB_PRIV=$BOB_PRIV
+BOB_PUB=$BOB_PUB
+ALICE_WG_IP=10.99.0.1/24
+BOB_WG_IP=10.99.0.2/24
+PEER_PORT=51820
+EOF
+chmod 600 "$ENV_FILE"
+echo "wrote $ENV_FILE"

testing/scripts/build.sh

@@ -40,6 +40,8 @@ done
 UNAME_S=$(uname -s)
 CARGO_TARGET="x86_64-unknown-linux-musl"
 
+TARGET_ROOT="${CARGO_TARGET_DIR:-$PROJECT_ROOT/target}"
+
 if [ "$UNAME_S" = "Darwin" ]; then
     echo "Detected macOS host — using cross-compilation for Linux..."
 
@@ -57,12 +59,12 @@ if [ "$UNAME_S" = "Darwin" ]; then
     echo "Building FIPS for Linux (release) using cargo-zigbuild..."
     cargo zigbuild --release --target "$CARGO_TARGET" --manifest-path="$PROJECT_ROOT/Cargo.toml" "${CARGO_BUILD_ARGS[@]}"
 
-    TARGET_DIR="$PROJECT_ROOT/target/$CARGO_TARGET/release"
+    TARGET_DIR="$TARGET_ROOT/$CARGO_TARGET/release"
 else
     echo "Building FIPS (release)..."
     cargo build --release --manifest-path="$PROJECT_ROOT/Cargo.toml" "${CARGO_BUILD_ARGS[@]}"
 
-    TARGET_DIR="$PROJECT_ROOT/target/release"
+    TARGET_DIR="$TARGET_ROOT/release"
 fi
 
 echo "Copying binaries to $DOCKER_DIR/"

testing/static/README.md

@@ -270,6 +270,19 @@ Runs iperf3 with:
 - Parallel streams: 8 (`-P 8`)
 - Protocol: TCP over IPv6
 
+For before/after measurements across commits or branches:
+
+```bash
+./testing/static/scripts/iperf-compare-refs.sh origin/master HEAD mesh
+```
+
+The comparison script builds each ref into a separate Docker image, runs the
+same topology and `iperf3` settings for both images, and prints a bandwidth
+summary. Override `DURATION`, `PARALLEL`, `SETTLE_SECONDS`, `IPERF_TIMEOUT`,
+or `RUNS` in the environment when needed. `RUNS` is the total number of
+measurements per ref; for example, `RUNS=3` runs each ref three times and
+prints both per-run and aggregate tables.
+
 ## Network Impairment
 
 The `netem.sh` script simulates adverse network conditions using `tc`/`netem`

testing/static/docker-compose.yml

@@ -13,7 +13,7 @@ networks:
         - subnet: fd02::/64
 
 x-fips-common: &fips-common
-  image: fips-test:latest
+  image: ${FIPS_TEST_IMAGE:-fips-test:latest}
   cap_add:
     - NET_ADMIN
   devices:

testing/static/scripts/iperf-compare-refs.sh

@@ -0,0 +1,202 @@
+#!/bin/bash
+# Build two git refs into separate FIPS test images and run the same static
+# iperf3 topology against each one. This keeps performance PR evidence
+# repeatable: compare origin/master with a candidate branch using identical
+# configs, topology, duration, and parallelism.
+#
+# Usage:
+#   ./testing/static/scripts/iperf-compare-refs.sh <base-ref> <candidate-ref> [mesh|chain]
+#
+# Environment:
+#   DURATION=10        iperf3 duration passed through to iperf-test.sh
+#   PARALLEL=8         iperf3 parallel streams passed through to iperf-test.sh
+#   SETTLE_SECONDS=3   topology startup delay passed through to iperf-test.sh
+#   IPERF_TIMEOUT      per-path timeout, defaults to DURATION + 30
+#   RUNS=1             total measurement runs per ref
+set -euo pipefail
+
+if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
+    echo "Usage: $0 <base-ref> <candidate-ref> [mesh|chain]" >&2
+    exit 2
+fi
+
+BASE_REF="$1"
+CANDIDATE_REF="$2"
+PROFILE="${3:-mesh}"
+RUNS="${RUNS:-1}"
+
+if ! [[ "$RUNS" =~ ^[1-9][0-9]*$ ]]; then
+    echo "RUNS must be a positive integer" >&2
+    exit 2
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+COMPOSE_FILE="$PROJECT_ROOT/testing/static/docker-compose.yml"
+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/fips-iperf-compare.XXXXXX")"
+WORKTREES=()
+FAILED_RUNS=0
+
+cleanup() {
+    docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" down >/dev/null 2>&1 || true
+    for wt in "${WORKTREES[@]}"; do
+        git -C "$PROJECT_ROOT" worktree remove --force "$wt" >/dev/null 2>&1 || true
+    done
+    rm -rf "$TMP_DIR"
+}
+trap cleanup EXIT
+
+slug_ref() {
+    echo "$1" \
+        | tr '[:upper:]' '[:lower:]' \
+        | sed 's/[^a-z0-9_.-]/-/g; s/^-*//; s/-*$//' \
+        | cut -c1-48
+}
+
+image_tag() {
+    local label="$1"
+    local ref="$2"
+    local slug
+    slug="$(slug_ref "$ref")"
+    [ -n "$slug" ] || slug="ref"
+    printf 'fips-test:compare-%s-%s\n' "$label" "$slug"
+}
+
+build_ref_image() {
+    local label="$1"
+    local ref="$2"
+    local tag="$3"
+
+    local wt="$TMP_DIR/$label"
+    local target_dir="$PROJECT_ROOT/target/iperf-compare-$label-$(slug_ref "$ref")"
+
+    echo ""
+    echo "=== Building $label: $ref -> $tag ==="
+    git -C "$PROJECT_ROOT" worktree add --detach "$wt" "$ref"
+    WORKTREES+=("$wt")
+
+    mkdir -p "$target_dir"
+    ln -s "$target_dir" "$wt/target"
+    (
+        cd "$wt"
+        CARGO_TARGET_DIR="$wt/target" ./testing/scripts/build.sh
+    )
+    docker tag fips-test:latest "$tag"
+}
+
+run_profile() {
+    local label="$1"
+    local image="$2"
+    local run="$3"
+    local log="$TMP_DIR/$label-run-$run.log"
+    local duration="${DURATION:-10}"
+    local parallel="${PARALLEL:-8}"
+    local settle_seconds="${SETTLE_SECONDS:-3}"
+    local iperf_timeout="${IPERF_TIMEOUT:-$((duration + 30))}"
+
+    echo ""
+    echo "=== Running $label run $run/$RUNS with $image ==="
+    FIPS_TEST_IMAGE="$image" docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" up -d --force-recreate
+    if DURATION="$duration" PARALLEL="$parallel" \
+        SETTLE_SECONDS="$settle_seconds" IPERF_TIMEOUT="$iperf_timeout" \
+        "$SCRIPT_DIR/iperf-test.sh" "$PROFILE" | tee "$log"; then
+        :
+    else
+        local status="$?"
+        echo "=== $label exited with status $status ===" | tee -a "$log"
+        FAILED_RUNS=1
+    fi
+    docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" down
+}
+
+print_summary() {
+    local label="$1"
+    local run="$2"
+    local log="$TMP_DIR/$label-run-$run.log"
+
+    [ -f "$log" ] || return 0
+
+    awk -v label="$label" -v run="$run" '
+        /^=== .* ===$/ && $0 !~ /FIPS iperf3 Bandwidth Test/ && $0 !~ /Results:/ {
+            test=$0
+            sub(/^=== /, "", test)
+            sub(/ ===$/, "", test)
+        }
+        /^Bandwidth:/ {
+            print label "\t" run "\t" test "\t" $2 " " $3
+        }
+    ' "$log"
+}
+
+print_aggregate() {
+    local summary="$1"
+
+    awk -F '\t' '
+        NR == 1 { next }
+
+        function to_mbps(value, unit) {
+            if (unit ~ /^Gbits/) return value * 1000
+            if (unit ~ /^Mbits/) return value
+            if (unit ~ /^Kbits/) return value / 1000
+            return value
+        }
+
+        {
+            split($4, parts, " ")
+            mbps = to_mbps(parts[1] + 0, parts[2])
+            key = $1 SUBSEP $3
+            if (!(key in seen)) {
+                seen[key] = 1
+                order[++order_len] = key
+                refs[key] = $1
+                tests[key] = $3
+                min[key] = mbps
+                max[key] = mbps
+            }
+            count[key]++
+            sum[key] += mbps
+            if (mbps < min[key]) min[key] = mbps
+            if (mbps > max[key]) max[key] = mbps
+        }
+
+        END {
+            print "ref\ttest\truns\tavg_mbps\tmin_mbps\tmax_mbps"
+            for (i = 1; i <= order_len; i++) {
+                key = order[i]
+                printf "%s\t%s\t%d\t%.0f\t%.0f\t%.0f\n",
+                    refs[key], tests[key], count[key],
+                    sum[key] / count[key], min[key], max[key]
+            }
+        }
+    ' "$summary"
+}
+
+"$SCRIPT_DIR/generate-configs.sh" "$PROFILE"
+
+BASE_IMAGE="$(image_tag base "$BASE_REF")"
+CANDIDATE_IMAGE="$(image_tag candidate "$CANDIDATE_REF")"
+
+build_ref_image base "$BASE_REF" "$BASE_IMAGE"
+build_ref_image candidate "$CANDIDATE_REF" "$CANDIDATE_IMAGE"
+
+for run in $(seq 1 "$RUNS"); do
+    run_profile base "$BASE_IMAGE" "$run"
+    run_profile candidate "$CANDIDATE_IMAGE" "$run"
+done
+
+echo ""
+echo "=== Summary ==="
+SUMMARY_FILE="$TMP_DIR/summary.tsv"
+{
+    printf 'ref\trun\ttest\tbandwidth\n'
+    for run in $(seq 1 "$RUNS"); do
+        print_summary base "$run"
+        print_summary candidate "$run"
+    done
+} | tee "$SUMMARY_FILE"
+
+echo ""
+echo "=== Aggregate ==="
+print_aggregate "$SUMMARY_FILE"
+
+exit "$FAILED_RUNS"

testing/static/scripts/iperf-test.sh

@@ -18,8 +18,10 @@ if [ "$2" = "--live" ] || [ "$1" = "--live" ]; then
     [ "$1" = "--live" ] && PROFILE="mesh"
 fi
 
-DURATION=10
+DURATION="${DURATION:-10}"
-PARALLEL=8
+PARALLEL="${PARALLEL:-8}"
+SETTLE_SECONDS="${SETTLE_SECONDS:-3}"
+IPERF_TIMEOUT="${IPERF_TIMEOUT:-$((DURATION + 30))}"
 PASSED=0
 FAILED=0
 
@@ -47,7 +49,7 @@ iperf_test() {
     if [ "$LIVE_OUTPUT" = true ]; then
         # Show live output
         echo "Running iperf3 test (live output):"
-        if docker exec "fips-$client_node" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL"; then
+        if docker exec "fips-$client_node" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL"; then
             PASSED=$((PASSED + 1))
         else
             echo "FAIL"
@@ -57,7 +59,7 @@ iperf_test() {
         # Capture and summarize output
         echo -n "Running iperf3 test... "
         local output
-        if output=$(docker exec "fips-$client_node" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL" 2>&1); then
+        if output=$(docker exec "fips-$client_node" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL" 2>&1); then
             # Check if we got valid results
             if echo "$output" | grep -q "sender"; then
                 # Extract and display results (get SUM line for aggregate bandwidth)
@@ -83,8 +85,8 @@ echo "=== FIPS iperf3 Bandwidth Test ($PROFILE topology) ==="
 echo ""
 
 # Wait for nodes to converge
-echo "Waiting 3s for mesh convergence..."
+echo "Waiting ${SETTLE_SECONDS}s for mesh convergence..."
-sleep 3
+sleep "$SETTLE_SECONDS"
 
 if [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
     # Test key paths in mesh topology
@@ -120,4 +122,4 @@ fi
 
 echo ""
 echo "=== Results: $PASSED passed, $FAILED failed ==="
-[ "$FAILED" -eq 0 ] && exit 0 || exit 1
+[ "$FAILED" -eq 0 ] && exit 0 || exit 1