mirror of
https://github.com/jmcorgan/fips.git
synced 2026-08-09 08:14:42 +00:00
testing: add boringtun throughput benchmark and iperf ref-compare harness
New testing/boringtun/ harness runs two Cloudflare BoringTun userspace WireGuard containers with iperf3 between them, giving a single-hop userspace tunnel baseline for comparison against FIPS throughput numbers. Local WG key generation runs through the harness image so the host needs no wireguard-tools. New testing/static/scripts/iperf-compare-refs.sh builds two git refs into separate fips-test:* images via git worktree and runs the same static iperf topology against both, with RUNS-based repetition and aggregate avg/min/max reporting. testing/static/scripts/iperf-test.sh gains DURATION, PARALLEL, SETTLE_SECONDS, IPERF_TIMEOUT env knobs and a per-path iperf timeout. testing/static/docker-compose.yml selects the image under test via FIPS_TEST_IMAGE; testing/scripts/build.sh respects CARGO_TARGET_DIR. Author benchmark on aarch64 Docker Desktop: boringtun bob -> alice : 1000.13 Mbits/sec
This commit is contained in:
Martti Malmi
committed by
Johnathan Corgan
Johnathan Corgan
parent
09eb5ad6bf
commit
b05c80e5f5
Executable
+202
@@ -0,0 +1,202 @@
|
||||
#!/bin/bash
|
||||
# Build two git refs into separate FIPS test images and run the same static
|
||||
# iperf3 topology against each one. This keeps performance PR evidence
|
||||
# repeatable: compare origin/master with a candidate branch using identical
|
||||
# configs, topology, duration, and parallelism.
|
||||
#
|
||||
# Usage:
|
||||
# ./testing/static/scripts/iperf-compare-refs.sh <base-ref> <candidate-ref> [mesh|chain]
|
||||
#
|
||||
# Environment:
|
||||
# DURATION=10 iperf3 duration passed through to iperf-test.sh
|
||||
# PARALLEL=8 iperf3 parallel streams passed through to iperf-test.sh
|
||||
# SETTLE_SECONDS=3 topology startup delay passed through to iperf-test.sh
|
||||
# IPERF_TIMEOUT per-path timeout, defaults to DURATION + 30
|
||||
# RUNS=1 total measurement runs per ref
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
|
||||
echo "Usage: $0 <base-ref> <candidate-ref> [mesh|chain]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
BASE_REF="$1"
|
||||
CANDIDATE_REF="$2"
|
||||
PROFILE="${3:-mesh}"
|
||||
RUNS="${RUNS:-1}"
|
||||
|
||||
if ! [[ "$RUNS" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "RUNS must be a positive integer" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
COMPOSE_FILE="$PROJECT_ROOT/testing/static/docker-compose.yml"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/fips-iperf-compare.XXXXXX")"
|
||||
WORKTREES=()
|
||||
FAILED_RUNS=0
|
||||
|
||||
cleanup() {
|
||||
docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" down >/dev/null 2>&1 || true
|
||||
for wt in "${WORKTREES[@]}"; do
|
||||
git -C "$PROJECT_ROOT" worktree remove --force "$wt" >/dev/null 2>&1 || true
|
||||
done
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
slug_ref() {
|
||||
echo "$1" \
|
||||
| tr '[:upper:]' '[:lower:]' \
|
||||
| sed 's/[^a-z0-9_.-]/-/g; s/^-*//; s/-*$//' \
|
||||
| cut -c1-48
|
||||
}
|
||||
|
||||
image_tag() {
|
||||
local label="$1"
|
||||
local ref="$2"
|
||||
local slug
|
||||
slug="$(slug_ref "$ref")"
|
||||
[ -n "$slug" ] || slug="ref"
|
||||
printf 'fips-test:compare-%s-%s\n' "$label" "$slug"
|
||||
}
|
||||
|
||||
build_ref_image() {
|
||||
local label="$1"
|
||||
local ref="$2"
|
||||
local tag="$3"
|
||||
|
||||
local wt="$TMP_DIR/$label"
|
||||
local target_dir="$PROJECT_ROOT/target/iperf-compare-$label-$(slug_ref "$ref")"
|
||||
|
||||
echo ""
|
||||
echo "=== Building $label: $ref -> $tag ==="
|
||||
git -C "$PROJECT_ROOT" worktree add --detach "$wt" "$ref"
|
||||
WORKTREES+=("$wt")
|
||||
|
||||
mkdir -p "$target_dir"
|
||||
ln -s "$target_dir" "$wt/target"
|
||||
(
|
||||
cd "$wt"
|
||||
CARGO_TARGET_DIR="$wt/target" ./testing/scripts/build.sh
|
||||
)
|
||||
docker tag fips-test:latest "$tag"
|
||||
}
|
||||
|
||||
run_profile() {
|
||||
local label="$1"
|
||||
local image="$2"
|
||||
local run="$3"
|
||||
local log="$TMP_DIR/$label-run-$run.log"
|
||||
local duration="${DURATION:-10}"
|
||||
local parallel="${PARALLEL:-8}"
|
||||
local settle_seconds="${SETTLE_SECONDS:-3}"
|
||||
local iperf_timeout="${IPERF_TIMEOUT:-$((duration + 30))}"
|
||||
|
||||
echo ""
|
||||
echo "=== Running $label run $run/$RUNS with $image ==="
|
||||
FIPS_TEST_IMAGE="$image" docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" up -d --force-recreate
|
||||
if DURATION="$duration" PARALLEL="$parallel" \
|
||||
SETTLE_SECONDS="$settle_seconds" IPERF_TIMEOUT="$iperf_timeout" \
|
||||
"$SCRIPT_DIR/iperf-test.sh" "$PROFILE" | tee "$log"; then
|
||||
:
|
||||
else
|
||||
local status="$?"
|
||||
echo "=== $label exited with status $status ===" | tee -a "$log"
|
||||
FAILED_RUNS=1
|
||||
fi
|
||||
docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" down
|
||||
}
|
||||
|
||||
print_summary() {
|
||||
local label="$1"
|
||||
local run="$2"
|
||||
local log="$TMP_DIR/$label-run-$run.log"
|
||||
|
||||
[ -f "$log" ] || return 0
|
||||
|
||||
awk -v label="$label" -v run="$run" '
|
||||
/^=== .* ===$/ && $0 !~ /FIPS iperf3 Bandwidth Test/ && $0 !~ /Results:/ {
|
||||
test=$0
|
||||
sub(/^=== /, "", test)
|
||||
sub(/ ===$/, "", test)
|
||||
}
|
||||
/^Bandwidth:/ {
|
||||
print label "\t" run "\t" test "\t" $2 " " $3
|
||||
}
|
||||
' "$log"
|
||||
}
|
||||
|
||||
print_aggregate() {
|
||||
local summary="$1"
|
||||
|
||||
awk -F '\t' '
|
||||
NR == 1 { next }
|
||||
|
||||
function to_mbps(value, unit) {
|
||||
if (unit ~ /^Gbits/) return value * 1000
|
||||
if (unit ~ /^Mbits/) return value
|
||||
if (unit ~ /^Kbits/) return value / 1000
|
||||
return value
|
||||
}
|
||||
|
||||
{
|
||||
split($4, parts, " ")
|
||||
mbps = to_mbps(parts[1] + 0, parts[2])
|
||||
key = $1 SUBSEP $3
|
||||
if (!(key in seen)) {
|
||||
seen[key] = 1
|
||||
order[++order_len] = key
|
||||
refs[key] = $1
|
||||
tests[key] = $3
|
||||
min[key] = mbps
|
||||
max[key] = mbps
|
||||
}
|
||||
count[key]++
|
||||
sum[key] += mbps
|
||||
if (mbps < min[key]) min[key] = mbps
|
||||
if (mbps > max[key]) max[key] = mbps
|
||||
}
|
||||
|
||||
END {
|
||||
print "ref\ttest\truns\tavg_mbps\tmin_mbps\tmax_mbps"
|
||||
for (i = 1; i <= order_len; i++) {
|
||||
key = order[i]
|
||||
printf "%s\t%s\t%d\t%.0f\t%.0f\t%.0f\n",
|
||||
refs[key], tests[key], count[key],
|
||||
sum[key] / count[key], min[key], max[key]
|
||||
}
|
||||
}
|
||||
' "$summary"
|
||||
}
|
||||
|
||||
"$SCRIPT_DIR/generate-configs.sh" "$PROFILE"
|
||||
|
||||
BASE_IMAGE="$(image_tag base "$BASE_REF")"
|
||||
CANDIDATE_IMAGE="$(image_tag candidate "$CANDIDATE_REF")"
|
||||
|
||||
build_ref_image base "$BASE_REF" "$BASE_IMAGE"
|
||||
build_ref_image candidate "$CANDIDATE_REF" "$CANDIDATE_IMAGE"
|
||||
|
||||
for run in $(seq 1 "$RUNS"); do
|
||||
run_profile base "$BASE_IMAGE" "$run"
|
||||
run_profile candidate "$CANDIDATE_IMAGE" "$run"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
SUMMARY_FILE="$TMP_DIR/summary.tsv"
|
||||
{
|
||||
printf 'ref\trun\ttest\tbandwidth\n'
|
||||
for run in $(seq 1 "$RUNS"); do
|
||||
print_summary base "$run"
|
||||
print_summary candidate "$run"
|
||||
done
|
||||
} | tee "$SUMMARY_FILE"
|
||||
|
||||
echo ""
|
||||
echo "=== Aggregate ==="
|
||||
print_aggregate "$SUMMARY_FILE"
|
||||
|
||||
exit "$FAILED_RUNS"
|
||||
@@ -18,8 +18,10 @@ if [ "$2" = "--live" ] || [ "$1" = "--live" ]; then
|
||||
[ "$1" = "--live" ] && PROFILE="mesh"
|
||||
fi
|
||||
|
||||
DURATION=10
|
||||
PARALLEL=8
|
||||
DURATION="${DURATION:-10}"
|
||||
PARALLEL="${PARALLEL:-8}"
|
||||
SETTLE_SECONDS="${SETTLE_SECONDS:-3}"
|
||||
IPERF_TIMEOUT="${IPERF_TIMEOUT:-$((DURATION + 30))}"
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
@@ -47,7 +49,7 @@ iperf_test() {
|
||||
if [ "$LIVE_OUTPUT" = true ]; then
|
||||
# Show live output
|
||||
echo "Running iperf3 test (live output):"
|
||||
if docker exec "fips-$client_node" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL"; then
|
||||
if docker exec "fips-$client_node" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL"; then
|
||||
PASSED=$((PASSED + 1))
|
||||
else
|
||||
echo "FAIL"
|
||||
@@ -57,7 +59,7 @@ iperf_test() {
|
||||
# Capture and summarize output
|
||||
echo -n "Running iperf3 test... "
|
||||
local output
|
||||
if output=$(docker exec "fips-$client_node" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL" 2>&1); then
|
||||
if output=$(docker exec "fips-$client_node" timeout "$IPERF_TIMEOUT" iperf3 -c "${dest_npub}.fips" -t "$DURATION" -P "$PARALLEL" 2>&1); then
|
||||
# Check if we got valid results
|
||||
if echo "$output" | grep -q "sender"; then
|
||||
# Extract and display results (get SUM line for aggregate bandwidth)
|
||||
@@ -83,8 +85,8 @@ echo "=== FIPS iperf3 Bandwidth Test ($PROFILE topology) ==="
|
||||
echo ""
|
||||
|
||||
# Wait for nodes to converge
|
||||
echo "Waiting 3s for mesh convergence..."
|
||||
sleep 3
|
||||
echo "Waiting ${SETTLE_SECONDS}s for mesh convergence..."
|
||||
sleep "$SETTLE_SECONDS"
|
||||
|
||||
if [ "$PROFILE" = "mesh" ] || [ "$PROFILE" = "mesh-public" ]; then
|
||||
# Test key paths in mesh topology
|
||||
@@ -120,4 +122,4 @@ fi
|
||||
|
||||
echo ""
|
||||
echo "=== Results: $PASSED passed, $FAILED failed ==="
|
||||
[ "$FAILED" -eq 0 ] && exit 0 || exit 1
|
||||
[ "$FAILED" -eq 0 ] && exit 0 || exit 1
|
||||
|
||||
Reference in New Issue
Block a user