From b05c80e5f5f8690ef6167cf577ebe150193971d2 Mon Sep 17 00:00:00 2001 From: Martti Malmi Date: Fri, 15 May 2026 18:03:42 +0000 Subject: [PATCH] testing: add boringtun throughput benchmark and iperf ref-compare harness New testing/boringtun/ harness runs two Cloudflare BoringTun userspace WireGuard containers with iperf3 between them, giving a single-hop userspace tunnel baseline for comparison against FIPS throughput numbers. Local WG key generation runs through the harness image so the host needs no wireguard-tools. New testing/static/scripts/iperf-compare-refs.sh builds two git refs into separate fips-test:* images via git worktree and runs the same static iperf topology against both, with RUNS-based repetition and aggregate avg/min/max reporting. testing/static/scripts/iperf-test.sh gains DURATION, PARALLEL, SETTLE_SECONDS, IPERF_TIMEOUT env knobs and a per-path iperf timeout. testing/static/docker-compose.yml selects the image under test via FIPS_TEST_IMAGE; testing/scripts/build.sh respects CARGO_TARGET_DIR. Author benchmark on aarch64 Docker Desktop: boringtun bob -> alice : 1000.13 Mbits/sec --- testing/boringtun/.gitignore | 1 + testing/boringtun/Dockerfile | 28 +++ testing/boringtun/README.md | 29 +++ testing/boringtun/docker-compose.yml | 46 +++++ testing/boringtun/entrypoint.sh | 72 +++++++ testing/boringtun/scripts/bench.sh | 25 +++ testing/boringtun/scripts/generate-keys.sh | 40 ++++ testing/scripts/build.sh | 6 +- testing/static/README.md | 13 ++ testing/static/docker-compose.yml | 2 +- testing/static/scripts/iperf-compare-refs.sh | 202 +++++++++++++++++++ testing/static/scripts/iperf-test.sh | 16 +- 12 files changed, 470 insertions(+), 10 deletions(-) create mode 100644 testing/boringtun/.gitignore create mode 100644 testing/boringtun/Dockerfile create mode 100644 testing/boringtun/README.md create mode 100644 testing/boringtun/docker-compose.yml create mode 100755 testing/boringtun/entrypoint.sh create mode 100755 testing/boringtun/scripts/bench.sh create mode 100755 testing/boringtun/scripts/generate-keys.sh create mode 100755 testing/static/scripts/iperf-compare-refs.sh diff --git a/testing/boringtun/.gitignore b/testing/boringtun/.gitignore new file mode 100644 index 0000000..9ab870d --- /dev/null +++ b/testing/boringtun/.gitignore @@ -0,0 +1 @@ +generated/ diff --git a/testing/boringtun/Dockerfile b/testing/boringtun/Dockerfile new file mode 100644 index 0000000..0b44c21 --- /dev/null +++ b/testing/boringtun/Dockerfile @@ -0,0 +1,28 @@ +# Minimal boringtun-cli image for throughput benchmarking against FIPS. +# Userspace WireGuard (boringtun) + wireguard-tools + iperf3 in one +# image. Run two containers on a docker bridge, configure a WG peer +# between them, then iperf3 client→server across the tunnel. + +FROM rust:1-bookworm AS build +# 0.6.0 (crates.io HEAD) is from 2022 and turns in ~1.7 Mbps in +# this harness — clearly broken. Pull the boringtun-cli binary from +# the cloudflare HEAD instead; it gets the post-0.6 perf fixes and +# the multi-threaded recv loop. +RUN git clone --depth 1 https://github.com/cloudflare/boringtun /usr/src/boringtun \ + && cargo install --path /usr/src/boringtun/boringtun-cli + +FROM debian:bookworm-slim +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + iperf3 \ + iproute2 \ + wireguard-tools \ + iputils-ping \ + netcat-openbsd \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /usr/local/cargo/bin/boringtun-cli /usr/local/bin/boringtun-cli +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/testing/boringtun/README.md b/testing/boringtun/README.md new file mode 100644 index 0000000..aeb1be6 --- /dev/null +++ b/testing/boringtun/README.md @@ -0,0 +1,29 @@ +# BoringTun Throughput Baseline + +This harness runs two userspace WireGuard peers with Cloudflare BoringTun and +measures single-stream TCP throughput with `iperf3`. It is intended as a simple +baseline for comparing FIPS tunnel throughput against another userspace tunnel. + +```bash +docker build -t boringtun-test:latest testing/boringtun +testing/boringtun/scripts/generate-keys.sh +docker compose -f testing/boringtun/docker-compose.yml up -d +testing/boringtun/scripts/bench.sh +docker compose -f testing/boringtun/docker-compose.yml down +``` + +The generated WireGuard key material is written under +`testing/boringtun/generated/` and is ignored by git. + +For FIPS-to-FIPS revision comparisons, use the static topology comparison +script: + +```bash +testing/static/scripts/iperf-compare-refs.sh origin/master HEAD mesh +``` + +That script builds each ref into a separate `fips-test:*` image, runs the +same static `iperf3` topology against both images, and prints a bandwidth +summary for each path. Override `DURATION`, `PARALLEL`, `SETTLE_SECONDS`, or +`IPERF_TIMEOUT` in the environment when needed. Set `RUNS=3` or similar to +repeat each ref and print aggregate results. diff --git a/testing/boringtun/docker-compose.yml b/testing/boringtun/docker-compose.yml new file mode 100644 index 0000000..81c1220 --- /dev/null +++ b/testing/boringtun/docker-compose.yml @@ -0,0 +1,46 @@ +# Minimal 2-node boringtun setup for iperf3 throughput comparison. +# Both containers join the same docker bridge; each runs a boringtun +# userspace WireGuard tunnel and they peer with each other over UDP. +# Inner WG IPs: 10.99.0.1/24 (alice) <-> 10.99.0.2/24 (bob). + +networks: + bt-net: + driver: bridge + ipam: + config: + - subnet: 172.99.0.0/24 + +x-boringtun-common: &boringtun-common + image: boringtun-test:latest + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + sysctls: + - net.ipv4.conf.all.forwarding=1 + restart: "no" + env_file: + - ./generated/peers.env + +services: + alice: + <<: *boringtun-common + container_name: bt-alice + hostname: alice + environment: + - ROLE=alice + - PEER_HOST=bob + networks: + bt-net: + ipv4_address: 172.99.0.10 + + bob: + <<: *boringtun-common + container_name: bt-bob + hostname: bob + environment: + - ROLE=bob + - PEER_HOST=alice + networks: + bt-net: + ipv4_address: 172.99.0.11 diff --git a/testing/boringtun/entrypoint.sh b/testing/boringtun/entrypoint.sh new file mode 100755 index 0000000..a440125 --- /dev/null +++ b/testing/boringtun/entrypoint.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Bring up a boringtun-userspace WireGuard tunnel and then sleep +# indefinitely so the benchmark scripts can drive it via docker exec. +# +# Required env: +# ROLE "alice" or "bob" +# ALICE_WG_IP WG inner IP for alice (e.g. 10.99.0.1/24) +# BOB_WG_IP WG inner IP for bob (e.g. 10.99.0.2/24) +# ALICE_PUB alice's WG public key +# BOB_PUB bob's WG public key +# ALICE_PRIV alice's WG private key +# BOB_PRIV bob's WG private key +# PEER_HOST the other container's hostname on the docker bridge +# PEER_PORT the other container's WG UDP port (default 51820) +# +# boringtun-cli runs in foreground (--foreground) so wg-quick-style +# configuration is done manually via `ip`, `wg`, and `wg set`. + +set -e + +PORT="${PEER_PORT:-51820}" +case "$ROLE" in + alice) + OUR_IP="$ALICE_WG_IP" + OUR_PRIV="$ALICE_PRIV" + PEER_PUB="$BOB_PUB" + ;; + bob) + OUR_IP="$BOB_WG_IP" + OUR_PRIV="$BOB_PRIV" + PEER_PUB="$ALICE_PUB" + ;; + *) + echo "ROLE must be alice or bob, got '$ROLE'" >&2 + exit 1 + ;; +esac + +echo "[$ROLE] starting boringtun-cli foreground on wg0" +# `--foreground` keeps the userspace driver in the container +# foreground. boringtun-cli sets WG_TUN_NAME_FILE to /tmp/wg0.name +# when --foreground; we just hardcode the device name. +boringtun-cli --foreground --disable-drop-privileges wg0 & +BORINGTUN_PID=$! + +# wait for the tun device to appear +for i in $(seq 1 50); do + if ip link show wg0 >/dev/null 2>&1; then + break + fi + sleep 0.1 +done + +echo "[$ROLE] configuring wg0 with $OUR_IP listening on $PORT" +PRIV_FILE=$(mktemp) +chmod 600 "$PRIV_FILE" +printf '%s' "$OUR_PRIV" >"$PRIV_FILE" +wg set wg0 private-key "$PRIV_FILE" listen-port "$PORT" +rm -f "$PRIV_FILE" + +ip address add "$OUR_IP" dev wg0 +ip link set up dev wg0 + +echo "[$ROLE] adding peer pubkey, endpoint $PEER_HOST:$PORT" +wg set wg0 peer "$PEER_PUB" allowed-ips 10.99.0.0/24 endpoint "$PEER_HOST:$PORT" persistent-keepalive 25 + +echo "[$ROLE] ready" +wg show wg0 + +# Park here so the container stays up; we'll run iperf3 etc via +# docker exec. +wait $BORINGTUN_PID diff --git a/testing/boringtun/scripts/bench.sh b/testing/boringtun/scripts/bench.sh new file mode 100755 index 0000000..f8acbb9 --- /dev/null +++ b/testing/boringtun/scripts/bench.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# End-to-end iperf3 bandwidth test between two boringtun containers. +# Output mirrors testing/static/scripts/iperf-test.sh so the FIPS +# numbers are directly comparable. +set -euo pipefail + +DURATION="${DURATION:-10}" +PARALLEL="${PARALLEL:-1}" + +echo "=== boringtun iperf3 throughput (single TCP stream, ${DURATION}s) ===" + +# Run iperf3 server on alice (background), client on bob. +docker exec -d bt-alice iperf3 -s -1 -B 10.99.0.1 -p 5201 +sleep 1 + +# wait for tun handshake to settle (boringtun + WG keepalive) +sleep 2 + +# Client: bob → alice over WG (10.99.0.1) +OUT=$(docker exec bt-bob iperf3 -c 10.99.0.1 -p 5201 -t "$DURATION" -P "$PARALLEL" -J) + +# Pull SUM bps. +MBPS=$(echo "$OUT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(f\"{d['end']['sum_received']['bits_per_second'] / 1_000_000:.2f}\")") + +echo "boringtun bob -> alice : ${MBPS} Mbits/sec" diff --git a/testing/boringtun/scripts/generate-keys.sh b/testing/boringtun/scripts/generate-keys.sh new file mode 100755 index 0000000..0c0c7f5 --- /dev/null +++ b/testing/boringtun/scripts/generate-keys.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Generate WG keypairs for alice and bob and write the +# generated/peers.env file the docker-compose.yml reads. +# Idempotent: skips if file already exists. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="$SCRIPT_DIR/../generated" +ENV_FILE="$OUT_DIR/peers.env" + +if [ -f "$ENV_FILE" ]; then + echo "$ENV_FILE already exists; reusing keys" + exit 0 +fi + +mkdir -p "$OUT_DIR" + +# Use the built boringtun-test image to generate keys — host may +# not have wireguard-tools installed. The `pubkey` step needs `-i` +# so stdin is piped through; `genkey` doesn't need it but matching +# flags keeps the two calls symmetric. +GEN_CMD="docker run --rm --entrypoint wg boringtun-test:latest" +PUB_CMD="docker run --rm -i --entrypoint wg boringtun-test:latest" + +ALICE_PRIV=$($GEN_CMD genkey) +ALICE_PUB=$(printf '%s' "$ALICE_PRIV" | $PUB_CMD pubkey) +BOB_PRIV=$($GEN_CMD genkey) +BOB_PUB=$(printf '%s' "$BOB_PRIV" | $PUB_CMD pubkey) + +cat >"$ENV_FILE" < [mesh|chain] +# +# Environment: +# DURATION=10 iperf3 duration passed through to iperf-test.sh +# PARALLEL=8 iperf3 parallel streams passed through to iperf-test.sh +# SETTLE_SECONDS=3 topology startup delay passed through to iperf-test.sh +# IPERF_TIMEOUT per-path timeout, defaults to DURATION + 30 +# RUNS=1 total measurement runs per ref +set -euo pipefail + +if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then + echo "Usage: $0 [mesh|chain]" >&2 + exit 2 +fi + +BASE_REF="$1" +CANDIDATE_REF="$2" +PROFILE="${3:-mesh}" +RUNS="${RUNS:-1}" + +if ! [[ "$RUNS" =~ ^[1-9][0-9]*$ ]]; then + echo "RUNS must be a positive integer" >&2 + exit 2 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +COMPOSE_FILE="$PROJECT_ROOT/testing/static/docker-compose.yml" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/fips-iperf-compare.XXXXXX")" +WORKTREES=() +FAILED_RUNS=0 + +cleanup() { + docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" down >/dev/null 2>&1 || true + for wt in "${WORKTREES[@]}"; do + git -C "$PROJECT_ROOT" worktree remove --force "$wt" >/dev/null 2>&1 || true + done + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +slug_ref() { + echo "$1" \ + | tr '[:upper:]' '[:lower:]' \ + | sed 's/[^a-z0-9_.-]/-/g; s/^-*//; s/-*$//' \ + | cut -c1-48 +} + +image_tag() { + local label="$1" + local ref="$2" + local slug + slug="$(slug_ref "$ref")" + [ -n "$slug" ] || slug="ref" + printf 'fips-test:compare-%s-%s\n' "$label" "$slug" +} + +build_ref_image() { + local label="$1" + local ref="$2" + local tag="$3" + + local wt="$TMP_DIR/$label" + local target_dir="$PROJECT_ROOT/target/iperf-compare-$label-$(slug_ref "$ref")" + + echo "" + echo "=== Building $label: $ref -> $tag ===" + git -C "$PROJECT_ROOT" worktree add --detach "$wt" "$ref" + WORKTREES+=("$wt") + + mkdir -p "$target_dir" + ln -s "$target_dir" "$wt/target" + ( + cd "$wt" + CARGO_TARGET_DIR="$wt/target" ./testing/scripts/build.sh + ) + docker tag fips-test:latest "$tag" +} + +run_profile() { + local label="$1" + local image="$2" + local run="$3" + local log="$TMP_DIR/$label-run-$run.log" + local duration="${DURATION:-10}" + local parallel="${PARALLEL:-8}" + local settle_seconds="${SETTLE_SECONDS:-3}" + local iperf_timeout="${IPERF_TIMEOUT:-$((duration + 30))}" + + echo "" + echo "=== Running $label run $run/$RUNS with $image ===" + FIPS_TEST_IMAGE="$image" docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" up -d --force-recreate + if DURATION="$duration" PARALLEL="$parallel" \ + SETTLE_SECONDS="$settle_seconds" IPERF_TIMEOUT="$iperf_timeout" \ + "$SCRIPT_DIR/iperf-test.sh" "$PROFILE" | tee "$log"; then + : + else + local status="$?" + echo "=== $label exited with status $status ===" | tee -a "$log" + FAILED_RUNS=1 + fi + docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" down +} + +print_summary() { + local label="$1" + local run="$2" + local log="$TMP_DIR/$label-run-$run.log" + + [ -f "$log" ] || return 0 + + awk -v label="$label" -v run="$run" ' + /^=== .* ===$/ && $0 !~ /FIPS iperf3 Bandwidth Test/ && $0 !~ /Results:/ { + test=$0 + sub(/^=== /, "", test) + sub(/ ===$/, "", test) + } + /^Bandwidth:/ { + print label "\t" run "\t" test "\t" $2 " " $3 + } + ' "$log" +} + +print_aggregate() { + local summary="$1" + + awk -F '\t' ' + NR == 1 { next } + + function to_mbps(value, unit) { + if (unit ~ /^Gbits/) return value * 1000 + if (unit ~ /^Mbits/) return value + if (unit ~ /^Kbits/) return value / 1000 + return value + } + + { + split($4, parts, " ") + mbps = to_mbps(parts[1] + 0, parts[2]) + key = $1 SUBSEP $3 + if (!(key in seen)) { + seen[key] = 1 + order[++order_len] = key + refs[key] = $1 + tests[key] = $3 + min[key] = mbps + max[key] = mbps + } + count[key]++ + sum[key] += mbps + if (mbps < min[key]) min[key] = mbps + if (mbps > max[key]) max[key] = mbps + } + + END { + print "ref\ttest\truns\tavg_mbps\tmin_mbps\tmax_mbps" + for (i = 1; i <= order_len; i++) { + key = order[i] + printf "%s\t%s\t%d\t%.0f\t%.0f\t%.0f\n", + refs[key], tests[key], count[key], + sum[key] / count[key], min[key], max[key] + } + } + ' "$summary" +} + +"$SCRIPT_DIR/generate-configs.sh" "$PROFILE" + +BASE_IMAGE="$(image_tag base "$BASE_REF")" +CANDIDATE_IMAGE="$(image_tag candidate "$CANDIDATE_REF")" + +build_ref_image base "$BASE_REF" "$BASE_IMAGE" +build_ref_image candidate "$CANDIDATE_REF" "$CANDIDATE_IMAGE" + +for run in $(seq 1 "$RUNS"); do + run_profile base "$BASE_IMAGE" "$run" + run_profile candidate "$CANDIDATE_IMAGE" "$run" +done + +echo "" +echo "=== Summary ===" +SUMMARY_FILE="$TMP_DIR/summary.tsv" +{ + printf 'ref\trun\ttest\tbandwidth\n' + for run in $(seq 1 "$RUNS"); do + print_summary base "$run" + print_summary candidate "$run" + done +} | tee "$SUMMARY_FILE" + +echo "" +echo "=== Aggregate ===" +print_aggregate "$SUMMARY_FILE" + +exit "$FAILED_RUNS" diff --git a/testing/static/scripts/iperf-test.sh b/testing/static/scripts/iperf-test.sh index 74d1767..450db5e 100755 --- a/testing/static/scripts/iperf-test.sh +++ b/testing/static/scripts/iperf-test.sh @@ -18,8 +18,10 @@ if [ "$2" = "--live" ] || [ "$1" = "--live" ]; then [ "$1" = "--live" ] && PROFILE="mesh" fi -DURATION=10 -PARALLEL=8 +DURATION="${DURATION:-10}" +PARALLEL="${PARALLEL:-8}" +SETTLE_SECONDS="${SETTLE_SECONDS:-3}" +IPERF_TIMEOUT="${IPERF_TIMEOUT:-$((DURATION + 30))}" PASSED=0 FAILED=0 @@ -47,7 +49,7 @@ iperf_test() { if [ "$LIVE_OUTPUT" = true ]; then # Show live output echo "Running iperf3 test (live output):" - if docker exec "fips-$client_node" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL"; then + if docker exec "fips-$client_node" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL"; then PASSED=$((PASSED + 1)) else echo "FAIL" @@ -57,7 +59,7 @@ iperf_test() { # Capture and summarize output echo -n "Running iperf3 test... " local output - if output=$(docker exec "fips-$client_node" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL" 2>&1); then + if output=$(docker exec "fips-$client_node" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL" 2>&1); then # Check if we got valid results if echo "$output" | grep -q "sender"; then # Extract and display results (get SUM line for aggregate bandwidth) @@ -83,8 +85,8 @@ echo "=== FIPS iperf3 Bandwidth Test ($PROFILE topology) ===" echo "" # Wait for nodes to converge -echo "Waiting 3s for mesh convergence..." -sleep 3 +echo "Waiting ${SETTLE_SECONDS}s for mesh convergence..." +sleep "$SETTLE_SECONDS" if [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then # Test key paths in mesh topology @@ -120,4 +122,4 @@ fi echo "" echo "=== Results: $PASSED passed, $FAILED failed ===" -[ "$FAILED" -eq 0 ] && exit 0 || exit 1 \ No newline at end of file +[ "$FAILED" -eq 0 ] && exit 0 || exit 1