#!/bin/bash # Build two git refs into separate FIPS test images and run the same static # iperf3 topology against each one. This keeps performance PR evidence # repeatable: compare origin/master with a candidate branch using identical # configs, topology, duration, and parallelism. # # Usage: # ./testing/static/scripts/iperf-compare-refs.sh [mesh|chain] # # Environment: # DURATION=10 iperf3 duration passed through to iperf-test.sh # PARALLEL=8 iperf3 parallel streams passed through to iperf-test.sh # SETTLE_SECONDS=3 topology startup delay passed through to iperf-test.sh # IPERF_TIMEOUT per-path timeout, defaults to DURATION + 30 # RUNS=1 total measurement runs per ref set -euo pipefail if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then echo "Usage: $0 [mesh|chain]" >&2 exit 2 fi BASE_REF="$1" CANDIDATE_REF="$2" PROFILE="${3:-mesh}" RUNS="${RUNS:-1}" if ! [[ "$RUNS" =~ ^[1-9][0-9]*$ ]]; then echo "RUNS must be a positive integer" >&2 exit 2 fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" COMPOSE_FILE="$PROJECT_ROOT/testing/static/docker-compose.yml" TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/fips-iperf-compare.XXXXXX")" WORKTREES=() FAILED_RUNS=0 cleanup() { docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" down >/dev/null 2>&1 || true for wt in "${WORKTREES[@]}"; do git -C "$PROJECT_ROOT" worktree remove --force "$wt" >/dev/null 2>&1 || true done rm -rf "$TMP_DIR" } trap cleanup EXIT slug_ref() { echo "$1" \ | tr '[:upper:]' '[:lower:]' \ | sed 's/[^a-z0-9_.-]/-/g; s/^-*//; s/-*$//' \ | cut -c1-48 } image_tag() { local label="$1" local ref="$2" local slug slug="$(slug_ref "$ref")" [ -n "$slug" ] || slug="ref" printf 'fips-test:compare-%s-%s\n' "$label" "$slug" } build_ref_image() { local label="$1" local ref="$2" local tag="$3" local wt="$TMP_DIR/$label" local target_dir="$PROJECT_ROOT/target/iperf-compare-$label-$(slug_ref "$ref")" echo "" echo "=== Building $label: $ref -> $tag ===" git -C "$PROJECT_ROOT" worktree add --detach "$wt" "$ref" WORKTREES+=("$wt") mkdir -p "$target_dir" ln -s "$target_dir" "$wt/target" ( cd "$wt" CARGO_TARGET_DIR="$wt/target" ./testing/scripts/build.sh ) docker tag fips-test:latest "$tag" } run_profile() { local label="$1" local image="$2" local run="$3" local log="$TMP_DIR/$label-run-$run.log" local duration="${DURATION:-10}" local parallel="${PARALLEL:-8}" local settle_seconds="${SETTLE_SECONDS:-3}" local iperf_timeout="${IPERF_TIMEOUT:-$((duration + 30))}" echo "" echo "=== Running $label run $run/$RUNS with $image ===" FIPS_TEST_IMAGE="$image" docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" up -d --force-recreate if DURATION="$duration" PARALLEL="$parallel" \ SETTLE_SECONDS="$settle_seconds" IPERF_TIMEOUT="$iperf_timeout" \ "$SCRIPT_DIR/iperf-test.sh" "$PROFILE" | tee "$log"; then : else local status="$?" echo "=== $label exited with status $status ===" | tee -a "$log" FAILED_RUNS=1 fi docker compose -f "$COMPOSE_FILE" --profile "$PROFILE" down } print_summary() { local label="$1" local run="$2" local log="$TMP_DIR/$label-run-$run.log" [ -f "$log" ] || return 0 awk -v label="$label" -v run="$run" ' /^=== .* ===$/ && $0 !~ /FIPS iperf3 Bandwidth Test/ && $0 !~ /Results:/ { test=$0 sub(/^=== /, "", test) sub(/ ===$/, "", test) } /^Bandwidth:/ { print label "\t" run "\t" test "\t" $2 " " $3 } ' "$log" } print_aggregate() { local summary="$1" awk -F '\t' ' NR == 1 { next } function to_mbps(value, unit) { if (unit ~ /^Gbits/) return value * 1000 if (unit ~ /^Mbits/) return value if (unit ~ /^Kbits/) return value / 1000 return value } { split($4, parts, " ") mbps = to_mbps(parts[1] + 0, parts[2]) key = $1 SUBSEP $3 if (!(key in seen)) { seen[key] = 1 order[++order_len] = key refs[key] = $1 tests[key] = $3 min[key] = mbps max[key] = mbps } count[key]++ sum[key] += mbps if (mbps < min[key]) min[key] = mbps if (mbps > max[key]) max[key] = mbps } END { print "ref\ttest\truns\tavg_mbps\tmin_mbps\tmax_mbps" for (i = 1; i <= order_len; i++) { key = order[i] printf "%s\t%s\t%d\t%.0f\t%.0f\t%.0f\n", refs[key], tests[key], count[key], sum[key] / count[key], min[key], max[key] } } ' "$summary" } "$SCRIPT_DIR/generate-configs.sh" "$PROFILE" BASE_IMAGE="$(image_tag base "$BASE_REF")" CANDIDATE_IMAGE="$(image_tag candidate "$CANDIDATE_REF")" build_ref_image base "$BASE_REF" "$BASE_IMAGE" build_ref_image candidate "$CANDIDATE_REF" "$CANDIDATE_IMAGE" for run in $(seq 1 "$RUNS"); do run_profile base "$BASE_IMAGE" "$run" run_profile candidate "$CANDIDATE_IMAGE" "$run" done echo "" echo "=== Summary ===" SUMMARY_FILE="$TMP_DIR/summary.tsv" { printf 'ref\trun\ttest\tbandwidth\n' for run in $(seq 1 "$RUNS"); do print_summary base "$run" print_summary candidate "$run" done } | tee "$SUMMARY_FILE" echo "" echo "=== Aggregate ===" print_aggregate "$SUMMARY_FILE" exit "$FAILED_RUNS"