mirror of
https://github.com/jmcorgan/fips.git
synced 2026-07-30 19:46:15 +00:00
Johnathan Corgan
d4a2504f99
Found while fixing count_log_pattern: the gate reported that function clean even though it ends in echo and both callers consume its status. Its call-site patterns only matched direct forms -- if fn, while fn, fn ||, fn && -- so a status consumed through command substitution was invisible, because the line begins with the variable rather than the function name. That is not an exotic form. It is how a shell function returns a value, and it is precisely the shape of the swallowed-failure family this gate exists to catch, so the gate was blind to a large part of its own stated class. Three patterns added for the assignment, if-guarded and test-expression forms. The extension finds four real instances, all value-returning helpers whose trailing echo made their exit status unconditionally zero, and each now carries an explicit return 0. Also corrects the finding message, which described the trailing command as a log call; for these it is the return mechanism, and the hazard is that it fixes the status either way. Validated by breaking what it guards: a probe reintroducing the defect shape behind a command substitution is reported and exits 1, where before the extension it would have passed.
429 lines
16 KiB
Bash
Executable File
429 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
||
# Multi-run iperf3 + ping bench for FIPS mesh. Repeats each test N times
|
||
# (default 3), reports median + min/max + flags outliers >20% from
|
||
# median. Adds round-trip latency (ping) and a TCP-retransmit count
|
||
# (proxy for packet loss across the FIPS overlay) per path.
|
||
#
|
||
# Usage:
|
||
# ./bench-multirun.sh [mesh|chain]
|
||
#
|
||
# Environment:
|
||
# FIPS_BENCH_RUNS=3 Number of iperf3 repetitions per path
|
||
# FIPS_BENCH_DURATION=15 Seconds per iperf3 run
|
||
# FIPS_BENCH_PARALLEL=1 Parallel TCP streams (1 = single-stream)
|
||
# FIPS_BENCH_PING_COUNT=30 ICMP probes per path
|
||
# FIPS_BENCH_OUTLIER_PCT=20 Flag run as outlier if Δ from median > N%
|
||
# FIPS_BENCH_OUTPUT=json|text (default text; json emits NDJSON)
|
||
|
||
set -e
|
||
trap 'echo ""; echo "Bench interrupted"; exit 130' INT
|
||
|
||
PROFILE="${1:-mesh}"
|
||
RUNS="${FIPS_BENCH_RUNS:-5}"
|
||
DURATION="${FIPS_BENCH_DURATION:-15}"
|
||
PARALLEL="${FIPS_BENCH_PARALLEL:-1}"
|
||
PING_COUNT="${FIPS_BENCH_PING_COUNT:-30}"
|
||
OUTLIER_PCT="${FIPS_BENCH_OUTLIER_PCT:-20}"
|
||
OUTPUT="${FIPS_BENCH_OUTPUT:-text}"
|
||
|
||
if ! [[ "$RUNS" =~ ^[1-9][0-9]*$ ]] || [ "$RUNS" -lt 1 ]; then
|
||
echo "FIPS_BENCH_RUNS must be ≥ 1" >&2; exit 1
|
||
fi
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ENV_FILE="$SCRIPT_DIR/../generated-configs${FIPS_CI_NAME_SUFFIX:-}/npubs.env"
|
||
if [ ! -f "$ENV_FILE" ]; then
|
||
echo "Error: $ENV_FILE not found. Run generate-configs.sh first." >&2
|
||
exit 1
|
||
fi
|
||
# shellcheck source=../generated-configs/npubs.env
|
||
source "$ENV_FILE"
|
||
|
||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||
|
||
# Run iperf3 once, print "<mbps> <retransmits>" on success or "FAIL" on
|
||
# failure. Uses JSON output so we don't have to scrape human-readable
|
||
# SI prefixes.
|
||
iperf_once() {
|
||
local client="$1" dest_npub="$2"
|
||
local out
|
||
if ! out=$(docker exec "fips-${client}${FIPS_CI_NAME_SUFFIX:-}" iperf3 -c "${dest_npub}.fips" \
|
||
-t "$DURATION" -P "$PARALLEL" -J 2>&1); then
|
||
echo FAIL
|
||
return
|
||
fi
|
||
python3 - "$out" <<'PYEOF'
|
||
import json, sys
|
||
try:
|
||
d = json.loads(sys.argv[1])
|
||
bps = d['end']['sum_received']['bits_per_second']
|
||
# iperf3 reports TCP retransmits in 'sum_sent.retransmits' on
|
||
# the client side; missing on UDP or for the server-summary view.
|
||
retr = d.get('end', {}).get('sum_sent', {}).get('retransmits', 0)
|
||
print(f"{bps/1e6:.2f} {retr}")
|
||
except Exception as e:
|
||
print("FAIL")
|
||
PYEOF
|
||
}
|
||
|
||
# Median + min + max + CoV% + outlier indices over a whitespace-
|
||
# separated list of floats. CoV% = (stddev / mean) * 100, i.e. the
|
||
# coefficient of variation in percent — directly comparable across
|
||
# paths regardless of absolute throughput. Single-value runs print
|
||
# CoV=0.
|
||
# Prints: "<median> <min> <max> <cov_pct> <outliers_csv>"
|
||
stats() {
|
||
python3 - "$OUTLIER_PCT" "$@" <<'PYEOF'
|
||
import math
|
||
import sys
|
||
pct = float(sys.argv[1])
|
||
vals = [float(x) for x in sys.argv[2:]]
|
||
n = len(vals)
|
||
s = sorted(vals)
|
||
median = s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2
|
||
lo, hi = s[0], s[-1]
|
||
mean = sum(vals) / n if n else 0.0
|
||
if n > 1 and mean > 0:
|
||
variance = sum((v - mean) ** 2 for v in vals) / (n - 1) # sample stddev
|
||
cov_pct = math.sqrt(variance) / mean * 100
|
||
else:
|
||
cov_pct = 0.0
|
||
outliers = []
|
||
if median > 0:
|
||
for i, v in enumerate(vals):
|
||
if abs(v - median) / median * 100 > pct:
|
||
outliers.append(str(i + 1))
|
||
print(
|
||
f"{median:.2f} {lo:.2f} {hi:.2f} {cov_pct:.1f}% "
|
||
f"{','.join(outliers) if outliers else '-'}"
|
||
)
|
||
PYEOF
|
||
}
|
||
|
||
# Ping once, print "<min_ms> <avg_ms> <max_ms> <mdev_ms> <loss_pct>"
|
||
# or "FAIL".
|
||
ping_path() {
|
||
local client="$1" dest_npub="$2"
|
||
local out
|
||
if ! out=$(docker exec "fips-${client}${FIPS_CI_NAME_SUFFIX:-}" \
|
||
ping -c "$PING_COUNT" -i 0.2 -w "$((PING_COUNT * 2))" \
|
||
-q "${dest_npub}.fips" 2>&1); then
|
||
echo FAIL
|
||
return
|
||
fi
|
||
# "min/avg/max/mdev = 0.123/0.456/0.789/0.012 ms"
|
||
local rtt loss
|
||
rtt=$(echo "$out" | awk -F' = ' '/min\/avg\/max\/mdev/ {print $2}' | awk '{print $1}')
|
||
loss=$(echo "$out" | awk -F', ' '/packet loss/ {for (i=1;i<=NF;i++) if ($i ~ /packet loss/) print $i}' | awk '{print $1}')
|
||
if [ -z "$rtt" ]; then
|
||
echo FAIL
|
||
return
|
||
fi
|
||
echo "${rtt//\// } ${loss:-N/A}"
|
||
return 0
|
||
}
|
||
|
||
# ── Path definitions ───────────────────────────────────────────────────────
|
||
#
|
||
# Each entry is `<client-container> <dest-npub> <label>`. Labels are
|
||
# plain `client→dest` — the bench measures iperf3 throughput between
|
||
# the two named nodes. The static topology is printed at start so the
|
||
# operator can see the configured routing context.
|
||
#
|
||
# IMPORTANT — labels DO NOT encode hop count. With peer discovery
|
||
# enabled and all nodes on the same docker-bridge subnet, every node
|
||
# pair establishes a direct UDP path within a few ticks regardless of
|
||
# the static `peers:` list. The bench measures the post-convergence
|
||
# steady state, which is what real-world FIPS deployments see. To
|
||
# force on-wire multihop, isolate nodes on distinct docker networks.
|
||
case "$PROFILE" in
|
||
mesh|mesh-public)
|
||
# Static-peer paths only — without mDNS / a Nostr relay in the
|
||
# test container set, non-adjacent pairs (A↔B, A↔C) can't
|
||
# establish direct UDP and the bench would either fail
|
||
# convergence or measure multihop forwarding (apples-to-oranges
|
||
# vs builds that do have mDNS). Stick to pairs that are in
|
||
# each other's static `peers:` list so the measurement is
|
||
# deterministic and the same on every build.
|
||
PATHS=(
|
||
"node-a $NPUB_D A→D"
|
||
"node-a $NPUB_E A→E"
|
||
"node-e $NPUB_A E→A"
|
||
)
|
||
TOPOLOGY_LINES=(
|
||
"A peers with: D, E"
|
||
"D peers with: A, C, E"
|
||
"E peers with: A, C, D"
|
||
"(Static-peer pairs A↔D, A↔E only — non-adjacent pairs"
|
||
" are skipped because plain mesh.yaml has no discovery"
|
||
" transport that converges them onto direct UDP.)"
|
||
)
|
||
;;
|
||
chain)
|
||
# Chain topology forces intentional multihop forwarding for
|
||
# non-adjacent pairs: A peers only with B, B peers with A+C,
|
||
# etc. We bench just A→B (1 hop, direct static peer) so the
|
||
# comparison is again apples-to-apples without discovery.
|
||
PATHS=(
|
||
"node-a $NPUB_B A→B"
|
||
)
|
||
TOPOLOGY_LINES=(
|
||
"Static chain: A — B — C — D — E"
|
||
"(A↔B is the only static-peer pair we benchmark; multi-hop"
|
||
" forwarding to C/D/E needs reply-learned routing or mDNS"
|
||
" to settle, which is not deterministic on a fresh mesh.)"
|
||
)
|
||
;;
|
||
*)
|
||
echo "Unknown profile: $PROFILE" >&2; exit 1 ;;
|
||
esac
|
||
|
||
# ── Run ────────────────────────────────────────────────────────────────────
|
||
|
||
echo "=== FIPS multi-run bench ($PROFILE, ${RUNS}×${DURATION}s, P=${PARALLEL}) ==="
|
||
echo "Outlier flag: Δ from median > ${OUTLIER_PCT}%"
|
||
echo ""
|
||
echo "Topology (static \`peers:\` from $PROFILE.yaml):"
|
||
for line in "${TOPOLOGY_LINES[@]}"; do
|
||
echo " $line"
|
||
done
|
||
echo ""
|
||
|
||
# Wait for peer discovery to converge before measuring — otherwise
|
||
# the bench randomly measures either direct UDP (post-convergence
|
||
# steady state) or multihop forwarding through static peers (only
|
||
# until discovery converges). On the same docker-bridge subnet
|
||
# convergence should take seconds, not tens of seconds; this is the
|
||
# tight default. Bump FIPS_BENCH_CONVERGE_SECS for slower test setups
|
||
# (no Nostr relay, no mDNS, etc.).
|
||
#
|
||
# If any path hasn't converged by the deadline, the bench FAILS with
|
||
# a clear list of unconverged pairs — that's almost always a topology
|
||
# / discovery misconfiguration that would make the numbers noise.
|
||
CONVERGE_SECS="${FIPS_BENCH_CONVERGE_SECS:-15}"
|
||
|
||
peer_is_direct() {
|
||
local client="$1" dest_npub="$2"
|
||
docker exec "fips-${client}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show peers 2>/dev/null \
|
||
| python3 -c '
|
||
import json, sys
|
||
target = sys.argv[1]
|
||
try:
|
||
d = json.load(sys.stdin)
|
||
for p in d.get("peers", []):
|
||
if p.get("npub") == target:
|
||
print("yes" if p.get("connectivity") == "connected" else "no")
|
||
sys.exit(0)
|
||
print("no")
|
||
except Exception:
|
||
print("no")
|
||
' "$dest_npub" 2>/dev/null
|
||
}
|
||
|
||
# Per-peer `stats.bytes_sent` snapshot for the given client. Used to
|
||
# verify that the iperf3 traffic actually went out the intended next-
|
||
# hop (i.e. the routing protocol picked the static-peer link, not a
|
||
# multihop alternative).
|
||
peer_bytes_sent_snapshot() {
|
||
local client="$1"
|
||
docker exec "fips-${client}${FIPS_CI_NAME_SUFFIX:-}" fipsctl show peers 2>/dev/null \
|
||
| python3 -c '
|
||
import json, sys
|
||
try:
|
||
d = json.load(sys.stdin)
|
||
for p in d.get("peers", []):
|
||
npub = p.get("npub") or "?"
|
||
sent = (p.get("stats") or {}).get("bytes_sent", 0)
|
||
print(f"{npub} {sent}")
|
||
except Exception:
|
||
pass
|
||
'
|
||
}
|
||
|
||
# Compute deltas between two snapshots and identify which peer
|
||
# received the most bytes_sent growth, plus the absolute bytes for
|
||
# the requested target. Prints "<top_npub> <top_delta> <target_delta>".
|
||
peer_bytes_delta_winner() {
|
||
local target="$1"; shift
|
||
local before="$1"; shift
|
||
local after="$1"; shift
|
||
python3 - "$target" "$before" "$after" <<'PYEOF'
|
||
import sys
|
||
target, before, after = sys.argv[1], sys.argv[2], sys.argv[3]
|
||
b = {}
|
||
for line in before.splitlines():
|
||
if not line.strip(): continue
|
||
k, v = line.split()
|
||
b[k] = int(v)
|
||
a = {}
|
||
for line in after.splitlines():
|
||
if not line.strip(): continue
|
||
k, v = line.split()
|
||
a[k] = int(v)
|
||
deltas = {k: a.get(k, 0) - b.get(k, 0) for k in a}
|
||
target_delta = deltas.get(target, 0)
|
||
if deltas:
|
||
top_npub = max(deltas, key=deltas.get)
|
||
top_delta = deltas[top_npub]
|
||
else:
|
||
top_npub, top_delta = "-", 0
|
||
print(f"{top_npub} {top_delta} {target_delta}")
|
||
PYEOF
|
||
}
|
||
|
||
declare -A PEER_STATE
|
||
echo "Waiting for peer convergence (up to ${CONVERGE_SECS}s)…"
|
||
WAIT_START=$(date +%s)
|
||
while :; do
|
||
all_done=1
|
||
for path in "${PATHS[@]}"; do
|
||
read -r client npub label <<<"$path"
|
||
if [ "${PEER_STATE[$label]:-}" = "direct" ]; then
|
||
continue
|
||
fi
|
||
if [ "$(peer_is_direct "$client" "$npub")" = "yes" ]; then
|
||
PEER_STATE[$label]=direct
|
||
else
|
||
all_done=0
|
||
fi
|
||
done
|
||
if [ "$all_done" = 1 ]; then
|
||
break
|
||
fi
|
||
elapsed=$(( $(date +%s) - WAIT_START ))
|
||
if [ "$elapsed" -ge "$CONVERGE_SECS" ]; then
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
unconverged=()
|
||
for path in "${PATHS[@]}"; do
|
||
read -r client npub label <<<"$path"
|
||
state="${PEER_STATE[$label]:-via-forward}"
|
||
PEER_STATE[$label]="$state"
|
||
printf ' %-10s %s\n' "$label" "$state"
|
||
if [ "$state" != "direct" ]; then
|
||
unconverged+=("$label")
|
||
fi
|
||
done
|
||
echo ""
|
||
|
||
if [ "${#unconverged[@]}" -gt 0 ]; then
|
||
echo "ERROR: peer discovery did not converge for ${#unconverged[@]} of ${#PATHS[@]} paths within ${CONVERGE_SECS}s:" >&2
|
||
for p in "${unconverged[@]}"; do
|
||
echo " - $p" >&2
|
||
done
|
||
echo "" >&2
|
||
echo "On a single docker-bridge subnet convergence should take seconds." >&2
|
||
echo "Check that: (a) all node IPs are reachable peer-to-peer, (b) the" >&2
|
||
echo "test build includes a working discovery transport (Nostr relay /" >&2
|
||
echo "mDNS / etc.), (c) FIPS_BENCH_CONVERGE_SECS is high enough for" >&2
|
||
echo "this setup. Bench aborted to avoid reporting noisy mixed-state" >&2
|
||
echo "measurements." >&2
|
||
exit 2
|
||
fi
|
||
|
||
# Header for text output. All bandwidth columns are Mbits/sec
|
||
# (Mbps). CoV% = sample coefficient of variation (stddev/mean) over
|
||
# the N runs, in percent — directly comparable across paths.
|
||
if [ "$OUTPUT" = "text" ]; then
|
||
printf '%-10s %-12s %12s %12s %12s %8s %10s | %10s %8s %10s\n' \
|
||
path session 'median Mbps' 'min Mbps' 'max Mbps' 'CoV %' outliers \
|
||
'avg RTT ms' 'loss %' 'TCP retr'
|
||
printf '%-10s %-12s %12s %12s %12s %8s %10s | %10s %8s %10s\n' \
|
||
'------' '------' '------' '------' '------' '------' '------' \
|
||
'------' '------' '------'
|
||
fi
|
||
|
||
FAIL_COUNT=0
|
||
|
||
ROUTE_ERRORS=()
|
||
|
||
for path in "${PATHS[@]}"; do
|
||
read -r client npub label <<<"$path"
|
||
|
||
# Snapshot per-peer bytes_sent before iperf3 — used after the
|
||
# runs to verify the routing protocol actually pushed the test
|
||
# traffic out the intended static-peer link (and not, say,
|
||
# via an alternative multihop route the cost-based router could
|
||
# in principle pick). Same defensive check the iperf-test.sh
|
||
# path measurement assumes implicitly.
|
||
bytes_before=$(peer_bytes_sent_snapshot "$client")
|
||
|
||
# iperf3 runs
|
||
bw_runs=()
|
||
retr_runs=()
|
||
for i in $(seq 1 "$RUNS"); do
|
||
result=$(iperf_once "$client" "$npub")
|
||
if [ "$result" = "FAIL" ]; then
|
||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||
bw_runs+=("0")
|
||
retr_runs+=("0")
|
||
else
|
||
bw=$(echo "$result" | awk '{print $1}')
|
||
retr=$(echo "$result" | awk '{print $2}')
|
||
bw_runs+=("$bw")
|
||
retr_runs+=("$retr")
|
||
fi
|
||
done
|
||
|
||
bytes_after=$(peer_bytes_sent_snapshot "$client")
|
||
# `delta_winner` is "<npub_with_largest_bytes_sent_delta> <its_bytes> <target_dest_bytes>"
|
||
delta_winner=$(peer_bytes_delta_winner "$npub" "$bytes_before" "$bytes_after")
|
||
read -r winning_npub winning_delta target_delta <<<"$delta_winner"
|
||
if [ "$winning_npub" != "$npub" ]; then
|
||
ROUTE_ERRORS+=("$label: traffic exited via $winning_npub ($winning_delta B) instead of $npub ($target_delta B)")
|
||
fi
|
||
|
||
# Stats on bandwidth (median / min / max / coefficient-of-variation / outlier indices)
|
||
bw_stats=$(stats "${bw_runs[@]}")
|
||
read -r bw_median bw_min bw_max bw_cov bw_outliers <<<"$bw_stats"
|
||
# Sum retransmits across runs (a proxy for packet loss volume).
|
||
retr_sum=$(python3 -c "print(sum(int(x) for x in '${retr_runs[*]}'.split()))")
|
||
|
||
# Ping (single run; ping gives its own internal stats already).
|
||
ping_result=$(ping_path "$client" "$npub")
|
||
if [ "$ping_result" = "FAIL" ]; then
|
||
rtt_avg="N/A"
|
||
loss="N/A"
|
||
else
|
||
# min avg max mdev loss
|
||
read -r _rtt_min rtt_avg _rtt_max _rtt_mdev loss <<<"$ping_result"
|
||
fi
|
||
|
||
session_state="${PEER_STATE[$label]:-via-forward}"
|
||
if [ "$OUTPUT" = "json" ]; then
|
||
printf '{"path":"%s","client":"%s","dest_npub":"%s","session":"%s","mbps_runs":[%s],"mbps_median":%s,"mbps_min":%s,"mbps_max":%s,"mbps_cov_pct":"%s","outlier_runs":"%s","rtt_avg_ms":"%s","loss":"%s","tcp_retr_total":%s}\n' \
|
||
"$label" "$client" "$npub" "$session_state" \
|
||
"$(IFS=, ; echo "${bw_runs[*]}")" \
|
||
"$bw_median" "$bw_min" "$bw_max" "$bw_cov" "$bw_outliers" \
|
||
"$rtt_avg" "$loss" "$retr_sum"
|
||
else
|
||
printf '%-10s %-12s %12s %12s %12s %8s %10s | %10s %8s %10s\n' \
|
||
"$label" "$session_state" "$bw_median" "$bw_min" "$bw_max" \
|
||
"$bw_cov" "$bw_outliers" \
|
||
"$rtt_avg" "$loss" "$retr_sum"
|
||
if [ "$bw_outliers" != "-" ]; then
|
||
printf '%-10s runs (Mbps): %s\n' " ↑outliers" "${bw_runs[*]}"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
if [ "${#ROUTE_ERRORS[@]}" -gt 0 ]; then
|
||
echo "ERROR: traffic for ${#ROUTE_ERRORS[@]} of ${#PATHS[@]} paths did not exit via the intended static-peer link:" >&2
|
||
for e in "${ROUTE_ERRORS[@]}"; do
|
||
echo " - $e" >&2
|
||
done
|
||
echo "" >&2
|
||
echo "The numbers above measured a different route than the path label" >&2
|
||
echo "claims. Inspect peer cost / tree topology before trusting them." >&2
|
||
exit 3
|
||
fi
|
||
if [ "$FAIL_COUNT" -gt 0 ]; then
|
||
echo "WARN: $FAIL_COUNT iperf3 runs failed"
|
||
exit 1
|
||
fi
|
||
echo "OK"
|