From f512478bccbfe66adb7cb954e455e75176896293 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Fri, 15 May 2026 10:16:23 -0400 Subject: [PATCH] test(nests): sweep packet-loss rates + Linux docker harness for latency bench MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `AudioLatencyComparisonTest`: - Three new scenarios: 10 %, 20 %, 30 % loss in addition to the existing clean and 5 %. The whole benchmark now sweeps a useful arc for finding the breakdown point. - Frame-count floor now scales with loss instead of a fixed 60 % threshold. QUIC streams are reliable so loss surfaces mainly as latency, but at high rates the retransmit + congestion-control feedback can eat into the 10 s window, so the floor relaxes to `max(50, (1 - 3*lossRate) * EXPECTED_FRAMES)`. The 50-frame absolute floor (= 1 s of audio) catches a catastrophic publisher failure without flapping on the loaded loss-path runs. Linux benchmark harness (`nestsClient/tests/hang-interop/linux-bench/`): - `Dockerfile` for an arm64/amd64 image with JDK 21 + rustc 1.95 + cmake + the *-sys-crate build deps (libssl-dev, build-essential, pkg-config). Pins rustc 1.95 explicitly so moq-relay v0.10.25's Cargo.lock-requested `constant_time_eq@0.4.3` builds cleanly. - `run.sh` wrapper that: * builds the image, * unpacks a filtered tarball of the host repo into the container (mac host `nestsClient/build/` and `cargo target/` stay untouched), * symlinks `nestsClient/build` -> `/persistent-gradle-build` and `nestsClient/tests/hang-interop/target` -> `/persistent-cargo-target` so successive runs reuse compiled artefacts (~3-5 min cached vs ~7-15 min cold), * runs the gradle test invocation and surfaces the JUnit XML stats to `linux-bench/out/`. Linux numbers observed (arm64 Docker on a MacBook M2, 10 s window): | Loss | Stack | p50 | p95 | p99 | max | Frames | |------|--------|-------|-------|-------|-------|--------| | 0 % | Kotlin | 61.1 | 63.1 | 65.5 | 79.1 | 573 | | 0 % | Rust | 100.7 | 102.7 | 104.9 | 140.2 | 495 | | 5 % | Kotlin | 380.9 | 383.4 | 385.7 | 390.5 | 556 | | 5 % | Rust | 0.7 | 7.3 | 23.2 | 84.2 | 500 | | 10 % | Kotlin | 120.5 | 122.4 | 124.0 | 132.6 | 568 | | 10 % | Rust | 0.5 | 21.9 | 60.4 | 84.4 | 500 | | 20 % | Kotlin | 159.7 | 164.3 | 170.3 | 183.1 | 566 | | 20 % | Rust | 1.6 | 61.3 | 101.9 | 144.9 | 499 | | 30 % | Kotlin | 181.0 | 184.2 | 188.1 | 194.0 | 566 | | 30 % | Rust | 1.2 | 80.7 | 105.3 | 140.8 | 491 | Notes for future readers: Rust's p50 collapsing to ~1 ms under loss is the SUBSCRIBE-late-arrival signal, not "Rust is 100x faster" — by the time the SUBSCRIBE lands at the publisher, the in-flight encoder has moved past those early frames, so the listener starts receiving in near-real-time. Read p95/p99/max for the per-frame story (Rust expands 7 -> 22 -> 61 -> 80 ms p95 across loss rates). Frame delivery is stable for both stacks even at 30 % loss on Linux, unlike the macOS run which hits a 3.4 s Rust catastrophe at 30 %. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../native/AudioLatencyComparisonTest.kt | 36 ++++++-- .../tests/hang-interop/linux-bench/Dockerfile | 36 ++++++++ .../tests/hang-interop/linux-bench/run.sh | 90 +++++++++++++++++++ 3 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 nestsClient/tests/hang-interop/linux-bench/Dockerfile create mode 100755 nestsClient/tests/hang-interop/linux-bench/run.sh diff --git a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/interop/native/AudioLatencyComparisonTest.kt b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/interop/native/AudioLatencyComparisonTest.kt index 2297f69007..1370b88b75 100644 --- a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/interop/native/AudioLatencyComparisonTest.kt +++ b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/interop/native/AudioLatencyComparisonTest.kt @@ -148,6 +148,15 @@ class AudioLatencyComparisonTest { @Test fun under_5pct_packet_loss_pacing_and_one_way_latency() = runComparison(scenario = "loss-5pct", lossRate = 0.05f) + @Test + fun under_10pct_packet_loss_pacing_and_one_way_latency() = runComparison(scenario = "loss-10pct", lossRate = 0.10f) + + @Test + fun under_20pct_packet_loss_pacing_and_one_way_latency() = runComparison(scenario = "loss-20pct", lossRate = 0.20f) + + @Test + fun under_30pct_packet_loss_pacing_and_one_way_latency() = runComparison(scenario = "loss-30pct", lossRate = 0.30f) + /** * Single-scenario driver. Both `@Test` entry points are thin * wrappers so JUnit reports them as separate cases (and the @@ -350,8 +359,21 @@ class AudioLatencyComparisonTest { println(rustStats) println("=".repeat(70)) + // Frame-count floor scales with loss. QUIC streams are + // reliable so loss surfaces mainly as latency, but at high + // loss rates the retransmit + congestion-control feedback + // loop can eat into the test window, so the floor relaxes. + // The intent of the assertion is to catch a CATASTROPHIC + // failure (publisher never started, transport collapsed, + // listener subscribe never landed) — NOT to enforce a per- + // loss-rate delivery SLO. Reporting in the printed stats + // is the source of truth for delivery quality. val frameFloor = - if (lossRate > 0f) MIN_FRAMES_UNDER_LOSS else MIN_FRAMES_CLEAN_PATH + if (lossRate <= 0f) { + MIN_FRAMES_CLEAN_PATH + } else { + maxOf(MIN_FRAMES_HARD_FLOOR, ((1f - 3f * lossRate) * EXPECTED_FRAMES).toInt()) + } assertTrue( kotlinStats.frames >= frameFloor, "Kotlin speaker delivered only ${kotlinStats.frames} frames in the " + @@ -697,13 +719,13 @@ class AudioLatencyComparisonTest { private const val MIN_FRAMES_CLEAN_PATH = (EXPECTED_FRAMES * 0.8).toInt() /** - * Loss-path delivery floor: 60 % of expected. QUIC streams - * are reliable so the count should still be close to 100 % - * (loss surfaces as latency, not drops), but lowering the - * floor leaves headroom for a slow-start retransmit storm - * that eats into the test window. + * Absolute floor on delivered frames regardless of loss + * rate — anything under this is a catastrophic failure + * (publisher never started, transport collapsed). 50 frames + * ≈ 1 s of audio, which we should ALWAYS clear in a 10 s + * test even with aggressive loss. */ - private const val MIN_FRAMES_UNDER_LOSS = (EXPECTED_FRAMES * 0.6).toInt() + private const val MIN_FRAMES_HARD_FLOOR = 50 /** * Median inter-arrival should sit very close to [FRAME_MS] = 20 ms diff --git a/nestsClient/tests/hang-interop/linux-bench/Dockerfile b/nestsClient/tests/hang-interop/linux-bench/Dockerfile new file mode 100644 index 0000000000..b4117815e6 --- /dev/null +++ b/nestsClient/tests/hang-interop/linux-bench/Dockerfile @@ -0,0 +1,36 @@ +# Linux audio-latency benchmark image for AudioLatencyComparisonTest. +# Mirrors the macOS host toolchain (JDK 21, rustc 1.95, cmake 3) so the +# Kotlin vs Rust hang-publish comparison runs end-to-end against the +# same set of sidecars, just on a different OS. + +FROM eclipse-temurin:21-jdk-jammy + +# rustup pulls 1.95.0 explicitly. cmake/build-essential/pkg-config/libssl-dev +# are needed by transitive *-sys crates (audiopus_sys bundles libopus and +# builds it via cmake; rustls→aws-lc-sys also uses cmake). +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates git \ + build-essential pkg-config cmake \ + libssl-dev \ + bash \ + && rm -rf /var/lib/apt/lists/* + +ENV RUSTUP_HOME=/opt/rustup \ + CARGO_HOME=/opt/cargo \ + PATH=/opt/cargo/bin:$PATH + +# Pin rustc 1.95.0 to match the host. moq-relay v0.10.25's transitive +# Cargo.lock requires rustc >= 1.95 (constant_time_eq@0.4.3). +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o /tmp/rustup.sh \ + && sh /tmp/rustup.sh -y --default-toolchain 1.95.0 --profile minimal \ + && rm /tmp/rustup.sh \ + && rustc --version && cargo --version + +# CMake 4.x stripped support for very old `cmake_minimum_required` floors, +# which audiopus_sys's bundled libopus CMakeLists trips over. Jammy ships +# cmake 3.22 (still affected by some old crate `*-sys` requirements but +# tolerates the policy). Set the env var defensively to match the +# gradle Exec tasks. +ENV CMAKE_POLICY_VERSION_MINIMUM=3.5 + +WORKDIR /repo diff --git a/nestsClient/tests/hang-interop/linux-bench/run.sh b/nestsClient/tests/hang-interop/linux-bench/run.sh new file mode 100755 index 0000000000..a06fda8f8c --- /dev/null +++ b/nestsClient/tests/hang-interop/linux-bench/run.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Run AudioLatencyComparisonTest inside a Linux container so the numbers +# match what a Linux production host would see — different OS, same JDK, +# same rustc, same sidecar binaries. The mac host's `nestsClient/build` +# and cargo `target/` are NOT touched: a fresh tarball of the repo is +# unpacked into the container's filesystem, with two symlinks pointing +# the gradle and cargo output dirs at named volumes so successive runs +# don't pay full rebuild cost. +# +# Prereqs: +# - docker desktop (or any docker engine) on the host +# - the test itself is gated by -DnestsHangInterop=true; this script +# passes that flag. +# +# Usage from repo root: +# ./nestsClient/tests/hang-interop/linux-bench/run.sh +# +# Output: stats table on stdout. JUnit XML lands in +# nestsClient/tests/hang-interop/linux-bench/out/. +# +# First-run cost: ~7-15 min (image build + full cargo + moq-relay +# install + gradle compile). Cached runs: ~3-5 min. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "$SCRIPT_DIR/../../../.." && pwd) +PLATFORM="${PLATFORM:-linux/arm64}" +IMAGE_TAG="amethyst-linux-bench:$(echo "$PLATFORM" | tr / -)" +OUT_DIR="$SCRIPT_DIR/out" + +# Cache volumes — named per-platform so arm64 and amd64 don't share +# binaries that wouldn't run. Stored under the user cache dir so they +# survive across repo clones / git clean. +CACHE_ROOT="${LINUX_BENCH_CACHE:-$HOME/.cache/amethyst-linux-bench}" +CARGO_REGISTRY_VOL="$CACHE_ROOT/$PLATFORM/cargo-registry" +CARGO_TARGET_VOL="$CACHE_ROOT/$PLATFORM/cargo-target" +GRADLE_BUILD_VOL="$CACHE_ROOT/$PLATFORM/gradle-build" +GRADLE_CACHE_VOL="$CACHE_ROOT/$PLATFORM/gradle-cache" +mkdir -p "$CARGO_REGISTRY_VOL" "$CARGO_TARGET_VOL" "$GRADLE_BUILD_VOL" "$GRADLE_CACHE_VOL" "$OUT_DIR" + +echo ">>> building image $IMAGE_TAG ($PLATFORM)" +docker build --platform "$PLATFORM" -t "$IMAGE_TAG" "$SCRIPT_DIR" + +echo ">>> running benchmark" +docker run --rm --platform "$PLATFORM" \ + -v "$REPO_ROOT":/src:ro \ + -v "$OUT_DIR":/out \ + -v "$CARGO_REGISTRY_VOL":/opt/cargo/registry \ + -v "$CARGO_TARGET_VOL":/persistent-cargo-target \ + -v "$GRADLE_BUILD_VOL":/persistent-gradle-build \ + -v "$GRADLE_CACHE_VOL":/root/.gradle \ + -e GRADLE_OPTS="-Xmx4g" \ + "$IMAGE_TAG" \ + bash -c ' + set -e + rm -rf /repo; mkdir /repo + # Filter out build / cache dirs at copy time so the macOS-host + # state never lands in the Linux container. + tar -C /src --exclude=./.git --exclude="*/build" --exclude="*/.gradle" --exclude="*/target" --exclude="*/node_modules" -cf - . \ + | tar -C /repo -xf - + # Redirect both gradle build/ and cargo target/ at named + # volumes so repeated runs reuse compiled artefacts. The + # default repo-relative paths still resolve via the symlinks, + # so the existing NativeMoqRelayHarness sidecar lookups stay + # unchanged. + rm -rf /repo/nestsClient/tests/hang-interop/target + ln -s /persistent-cargo-target /repo/nestsClient/tests/hang-interop/target + rm -rf /repo/nestsClient/build + ln -s /persistent-gradle-build /repo/nestsClient/build + cd /repo + echo "=== toolchain ===" + rustc --version; cmake --version | head -1; java -version 2>&1 | head -1 + ./gradlew :nestsClient:jvmTest -DnestsHangInterop=true \ + --tests "com.vitorpamplona.nestsclient.interop.native.AudioLatencyComparisonTest" \ + 2>&1 | tail -10 + # Surface the test report XML to the host so we can grep + # printed stats without rebuilding the image. + rm -f /out/*.xml + cp /repo/nestsClient/build/test-results/jvmTest/TEST-*AudioLatencyComparisonTest*.xml /out/ + ' + +echo "" +echo ">>> stats" +for f in "$OUT_DIR"/TEST-*.xml; do + # Pull the CDATA-wrapped stdout block out without needing python on + # the host — sed range, then strip the CDATA wrappers. + sed -n '/<\/system-out>/p' "$f" \ + | sed -e 's/^<\/system-out>$//' +done