diff --git a/quic/interop/Dockerfile b/quic/interop/Dockerfile new file mode 100644 index 0000000000..d556b2d541 --- /dev/null +++ b/quic/interop/Dockerfile @@ -0,0 +1,23 @@ +# quic-interop-runner endpoint image for :quic. +# +# Build (from repo root): +# ./gradlew :quic-interop:installDist +# docker build -t amethyst-quic-interop -f quic/interop/Dockerfile . +# +# Or run `make build` inside quic/interop/. +FROM martenseemann/quic-network-simulator-endpoint:latest + +RUN apt-get update \ + && apt-get install -y --no-install-recommends openjdk-21-jre-headless \ + && rm -rf /var/lib/apt/lists/* + +COPY quic/interop/build/install/quic-interop /opt/quic-interop +COPY quic/interop/run_endpoint.sh /run_endpoint.sh +RUN chmod +x /run_endpoint.sh /opt/quic-interop/bin/quic-interop + +# Build-time toggle for verbose tracing. Pass --build-arg DEBUG=1 (or +# `make build DEBUG=1` via the Makefile) to bake QUIC_INTEROP_DEBUG into +# the image so the InteropClient + writer emit per-drain stats to stderr. +# Off by default — production matrix runs stay quiet. +ARG DEBUG=0 +ENV QUIC_INTEROP_DEBUG=${DEBUG} diff --git a/quic/interop/Makefile b/quic/interop/Makefile new file mode 100644 index 0000000000..c49c901694 --- /dev/null +++ b/quic/interop/Makefile @@ -0,0 +1,33 @@ +# Local iteration loop for the quic-interop-runner endpoint image. +# +# make build # compile JVM dist + build Docker image +# make image-name # print the image tag (for use in implementations_quic.json) +# make clean # nuke the dist + image +# +# Real testing happens via the runner: clone quic-interop-runner separately +# and drive it with `quic/interop/run-matrix.sh -s -t `. +IMAGE ?= amethyst-quic-interop:latest +REPO_ROOT := $(shell git rev-parse --show-toplevel) +DIST_DIR := build/install/quic-interop + +# Build-time toggle: `make build DEBUG=1` bakes QUIC_INTEROP_DEBUG=1 into +# the image so the InteropClient + writer emit per-drain diagnostics to +# stderr. Off by default — production matrix runs stay silent. +DEBUG ?= 0 + +.PHONY: build dist docker image-name clean + +build: docker + +dist: + cd $(REPO_ROOT) && ./gradlew :quic-interop:installDist + +docker: dist + cd $(REPO_ROOT) && docker build -t $(IMAGE) --build-arg DEBUG=$(DEBUG) -f quic/interop/Dockerfile . + +image-name: + @echo $(IMAGE) + +clean: + cd $(REPO_ROOT) && ./gradlew :quic-interop:clean + docker image rm -f $(IMAGE) 2>/dev/null || true diff --git a/quic/interop/build.gradle.kts b/quic/interop/build.gradle.kts new file mode 100644 index 0000000000..09cae4e32a --- /dev/null +++ b/quic/interop/build.gradle.kts @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +plugins { + alias(libs.plugins.jetbrainsKotlinJvm) + application +} + +kotlin { + jvmToolchain(21) + compilerOptions { + jvmTarget.set(JvmTarget.JVM_21) + } +} + +sourceSets { + main { + kotlin.srcDir("src/main/kotlin") + } + test { + kotlin.srcDir("src/test/kotlin") + } +} + +dependencies { + implementation(project(":quic")) + implementation(libs.kotlinx.coroutines.core) + implementation(libs.jackson.module.kotlin) + + testImplementation(libs.kotlin.test) +} + +application { + mainClass.set("com.vitorpamplona.quic.interop.runner.InteropClientKt") + applicationName = "quic-interop" +} diff --git a/quic/interop/inspect-multiplexing.sh b/quic/interop/inspect-multiplexing.sh new file mode 100755 index 0000000000..02481cadbf --- /dev/null +++ b/quic/interop/inspect-multiplexing.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Pull the post-mortem diagnostics for the most recent multiplexing run. +# Runs from anywhere; resolves logs relative to the runner clone path +# (../quic-interop-runner from this repo). +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +RUNNER_LOGS="${REPO_ROOT}/../quic-interop-runner/logs" + +if [[ ! -d "$RUNNER_LOGS" ]]; then + echo "no runner logs at $RUNNER_LOGS" >&2 + exit 1 +fi + +RUN_DIR="$(ls -1dt "$RUNNER_LOGS"/run-* 2>/dev/null | head -n 1 || true)" +if [[ -z "$RUN_DIR" ]]; then + echo "no run-* dirs under $RUNNER_LOGS" >&2 + exit 1 +fi +echo "==> run dir: $RUN_DIR" + +# Layout: /_//{client,server,sim}/ +CASE_DIR="$(ls -1d "$RUN_DIR"/*amethyst*/multiplexing 2>/dev/null | head -n 1 || true)" +if [[ -z "$CASE_DIR" ]]; then + echo "no /multiplexing dir; tree under run:" >&2 + find "$RUN_DIR" -maxdepth 3 -type d >&2 + exit 1 +fi +echo "==> case dir: $CASE_DIR" + +echo +echo "=============== writer-side debug traces (DEBUG=1 build only) ===============" +# Per-drain frame/budget stats from buildApplicationPacket. The +# smoking-gun section for "writer is iterating N active streams but +# emits only 1 STREAM frame per packet". Empty unless the image was +# built with `make build DEBUG=1`. +# +# `grep -c` exits 1 on no matches AND prints "0", so a naive +# `grep -c ... || echo 0` doubles up. Suppress the exit code +# instead. +echo +echo "=============== boot line (verifies image has latest debug build) ===============" +grep '\[boot\]' "$CASE_DIR/output.txt" 2>/dev/null | head -n 5 \ + || echo "(no [boot] lines — image is older than the boot log; rebuild with: DEBUG=1 ./quic/interop/run-matrix.sh ...)" + +WRITER_LINES=$(grep -cE '\[(writer|batch|interop|boot)' "$CASE_DIR/output.txt" 2>/dev/null) || WRITER_LINES=0 +if [[ "$WRITER_LINES" -gt 0 ]]; then + echo "($WRITER_LINES diagnostic lines; [batch] / [interop] entries first then first 30 [writer.app]:)" + # Use `|| true` because grep returns 1 on no matches and the + # script runs under `set -euo pipefail` — otherwise an empty + # [batch]/[interop] subset (run from a build without those logs) + # would abort the whole script. + grep -E '\[(batch|interop)' "$CASE_DIR/output.txt" | head -n 20 || true + echo "..." + grep '\[writer' "$CASE_DIR/output.txt" | head -n 30 || true + echo + echo "stream_frames histogram (writer-reported):" + grep -oE 'stream_frames=[0-9]+' "$CASE_DIR/output.txt" | sort | uniq -c | sort -rn || true + echo + echo "active histogram (active stream count at drain time):" + grep -oE 'active=[0-9]+' "$CASE_DIR/output.txt" | sort | uniq -c | sort -rn || true +else + echo "(no diagnostic lines yet — to enable, REBUILD the image with DEBUG=1:" + echo " DEBUG=1 ./quic/interop/run-matrix.sh -s aioquic -t multiplexing" + echo " then re-run this script)" +fi + +echo +echo "=============== server stderr (last 50 lines) ===============" +# Peer's CONNECTION_CLOSE reason if any; processing pace via stream +# create/discard cadence. +tail -n 50 "$CASE_DIR/server/stderr.log" 2>/dev/null \ + || echo "(no server/stderr.log)" + +QLOG="$(ls -1 "$CASE_DIR"/client/qlog/*.sqlog \ + "$CASE_DIR"/client/qlog/*.qlog \ + 2>/dev/null | head -n 1 || true)" +if [[ -z "$QLOG" ]]; then + echo + echo "(no qlog under $CASE_DIR/client/qlog — skipping qlog sections)" + exit 0 +fi + +echo +echo "=============== qlog ($QLOG, $(wc -l <"$QLOG" | tr -d ' ') lines) ===============" + +echo +echo "stream-frames-per-sent-packet histogram:" +# Many `0` = ack-only; many `1` = the bug; many `>=4` = writer +# coalescing as intended. +grep '"name":"transport:packet_sent"' "$QLOG" \ + | awk -F'"frame_type":"stream"' '{print NF - 1}' \ + | sort -n | uniq -c + +echo +echo "transport_parameters (local + remote):" +grep '"name":"transport:parameters_set"' "$QLOG" + +echo +echo "first 10 packet_sent (full) — burst shape after handshake:" +grep '"name":"transport:packet_sent"' "$QLOG" | head -n 10 + +echo +echo "connection_closed events (peer CCs are the smoking gun for spec violations):" +grep '"name":"transport:connection_closed"' "$QLOG" || echo "(none)" + +echo +echo "packet_dropped events (last 10):" +grep '"name":"transport:packet_dropped"' "$QLOG" | tail -n 10 \ + || echo "(none)" diff --git a/quic/interop/inspect-testcase.sh b/quic/interop/inspect-testcase.sh new file mode 100755 index 0000000000..2de0f3da8e --- /dev/null +++ b/quic/interop/inspect-testcase.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Per-testcase deep-dive diagnostic. Use after running any single +# testcase that failed: +# ./quic/interop/inspect-testcase.sh longrtt +# +# Auto-finds the most recent run dir that has the named testcase and +# reports the runner status line + qlog summary + frame histograms +# (sent/received) + connection-close events. Designed to fit in one +# screen of output. +set -o pipefail + +TC="${1:-}" +if [[ -z "$TC" ]]; then + echo "usage: $0 (e.g. longrtt, retry, http3)" >&2 + exit 2 +fi + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +RUNNER_LOGS="${REPO_ROOT}/../quic-interop-runner/logs" + +# Most recent run dir that has the named testcase. +RUN_DIR="" +# Filter to actual directories — zsh's glob also matches the +# sibling .stdout.log files that run-matrix.sh tees, which start +# with the same 'run-' prefix and would resolve as not-a-dir. +for d in $(ls -1dt "$RUNNER_LOGS"/run-* 2>/dev/null); do + [[ -d "$d" ]] || continue + if ls -d "$d"/*amethyst*/"$TC" >/dev/null 2>&1; then + RUN_DIR="$d" + break + fi +done +if [[ -z "$RUN_DIR" ]]; then + echo "no run dir with testcase '$TC' under $RUNNER_LOGS" >&2 + exit 1 +fi +echo "==> run dir: $RUN_DIR" +TC_DIR="$(ls -1d "$RUN_DIR"/*amethyst*/"$TC" 2>/dev/null | head -n 1)" +echo "==> testcase dir: $TC_DIR" + +echo +echo "=============== runner status ===============" +if [[ -f "${RUN_DIR}.stdout.log" ]]; then + grep -E "Test: $TC took" "${RUN_DIR}.stdout.log" | tail -n 5 +else + echo "(no .stdout.log — run-matrix.sh predates the tee, status not saved)" +fi + +echo +echo "=============== client diagnostic traces (DEBUG=1) ===============" +# Three places to look, in priority order: +# 1. Per-testcase client output (testcases that write to file) +# 2. Per-testcase output.txt (some runner versions) +# 3. The runner's tee'd .stdout.log narrowed to this testcase's +# window — but only works for short tests, since matrix runs +# restart containers and longer tests' stderr can be lost +# when the container is killed mid-test. +CLIENT_TRACES=() +[[ -f "$TC_DIR/client/output.txt" ]] && CLIENT_TRACES+=("$TC_DIR/client/output.txt") +[[ -f "$TC_DIR/output.txt" ]] && CLIENT_TRACES+=("$TC_DIR/output.txt") +[[ -f "${RUN_DIR}.stdout.log" ]] && CLIENT_TRACES+=("${RUN_DIR}.stdout.log") + +found=0 +for f in "${CLIENT_TRACES[@]}"; do + n=$(grep -cE '\[(boot|interop|batch|writer\.)' "$f" 2>/dev/null || echo 0) + if [[ "$n" -gt 0 ]]; then + echo "(found $n diagnostic lines in $f)" + if [[ "$f" == *.stdout.log ]]; then + # Narrow to this testcase's window. + awk -v tc="$TC" ' + $0 ~ "Running test case: " tc { in_tc = 1; next } + in_tc && /Running test case:/ { exit } + in_tc && /\[(boot|interop|batch|writer\.)/ { print } + ' "$f" | head -n 50 || true + else + grep -E '\[(boot|interop|batch|writer\.)' "$f" | head -n 50 || true + fi + found=1 + break + fi +done + +if [[ "$found" -eq 0 ]]; then + echo "(no diagnostic lines for this testcase)" + echo + echo "Possible causes:" + echo " - The image was built without DEBUG=1: use 'DEBUG=1 ./run-matrix.sh ...'" + echo " - The container was killed mid-test before flushing stderr" + echo " - This testcase ran in a matrix and only the FIRST testcase's" + echo " traces were captured. Run this single testcase in isolation:" + echo " DEBUG=1 ./quic/interop/run-matrix.sh -s -t $TC" +fi + +echo +echo "=============== file sizes generated for this testcase ===============" +if [[ -f "${RUN_DIR}.stdout.log" ]]; then + # 'Generated random file: NAME of size: N' lines printed before + # each test run. Extract the ones that came RIGHT BEFORE this + # testcase's "Running test case" line. + awk -v tc="$TC" ' + /^Running test case: / { current = $0 } + /^Generated random file:/ { last_files = last_files "\n" $0 } + $0 ~ ("Running test case: " tc) { + print last_files + last_files = "" + exit + } + ' "${RUN_DIR}.stdout.log" | tail -n 10 +else + echo "(no .stdout.log)" +fi + +echo +echo "=============== qlog event-type histogram ===============" +QLOG="$(ls -1 "$TC_DIR"/client/qlog/*.sqlog "$TC_DIR"/client/qlog/*.qlog 2>/dev/null | head -n 1 || true)" +if [[ -z "$QLOG" ]]; then + echo "(no qlog — connection probably never made it past TLS)" +else + echo "(qlog: $QLOG, $(wc -l <"$QLOG" | tr -d ' ') lines)" + grep -oE '"name":"[^"]+"' "$QLOG" | sort | uniq -c | sort -rn | head +fi + +if [[ -n "$QLOG" ]]; then + echo + echo "=============== transport_parameters (peer's flow-control budget) ===============" + grep '"name":"transport:parameters_set"' "$QLOG" + + echo + echo "=============== connection_closed (smoking gun for spec violations) ===============" + grep '"name":"transport:connection_closed"' "$QLOG" | head -n 5 \ + || echo "(none — connection didn't formally close; ran out of time?)" + + echo + echo "=============== last 10 sent / received packets (steady-state shape) ===============" + echo "-- received --" + grep '"name":"transport:packet_received"' "$QLOG" | tail -n 10 + echo "-- sent --" + grep '"name":"transport:packet_sent"' "$QLOG" | tail -n 10 + + echo + echo "=============== first/last received packet timestamps (transfer rate hint) ===============" + FIRST=$(grep '"name":"transport:packet_received"' "$QLOG" | head -n 1 | grep -oE '"time":[0-9]+' | head -n 1) + LAST=$(grep '"name":"transport:packet_received"' "$QLOG" | tail -n 1 | grep -oE '"time":[0-9]+' | head -n 1) + PKTS=$(grep -c '"name":"transport:packet_received"' "$QLOG") + echo "first=$FIRST last=$LAST pkts=$PKTS" +fi + +echo +echo "=============== server stderr (last 30 lines) ===============" +tail -n 30 "$TC_DIR/server/stderr.log" 2>/dev/null \ + || echo "(no server/stderr.log)" diff --git a/quic/interop/plans/2026-05-06-interop-runner.md b/quic/interop/plans/2026-05-06-interop-runner.md new file mode 100644 index 0000000000..3d07f186ec --- /dev/null +++ b/quic/interop/plans/2026-05-06-interop-runner.md @@ -0,0 +1,257 @@ +# quic-interop-runner endpoint — Phase 0 scaffolding + +Date: 2026-05-06 + +## Why + +We want the [`quic-interop-runner`](https://github.com/quic-interop/quic-interop-runner) +matrix as a **bug-finding harness**, not a vanity scoreboard. Each peer impl +(quiche, aioquic, picoquic, ngtcp2, msquic, mvfst, lsquic, neqo, kwik) enforces +different parts of RFC 9000/9001/9002/9114 strictly, so failures triangulate +to specific bugs in `:quic`. The runner's ns-3 sim also exposes loss / +reorder / migration scenarios that are awkward to reproduce in unit tests. + +## What's in Phase 0 + +- `:quic-interop` Gradle module (JVM-only application, registered at + `quic/interop/` via `settings.gradle`). +- `InteropClient.kt` reads the runner's env-var contract (`ROLE`, `TESTCASE`, + `REQUESTS`, …) and dispatches by testcase. Phase 0 implements only + `handshake`; everything else returns `127` (runner-skip). +- `Dockerfile` based on `martenseemann/quic-network-simulator-endpoint` + + OpenJDK 21 runtime, copies the `installDist` output. +- `run_endpoint.sh` sources the base image's `/setup.sh` then execs our JVM + binary. +- `Makefile` wrappers: `make build`, `make clean`. (A `make smoke` + target previously stood up picoquic + our endpoint on a private Docker + bridge to bisect runner failures from impl failures; dropped once + the runner reliably exercised both paths.) + +## Local iteration loop + +The fast path: `quic/interop/run-matrix.sh` clones the runner alongside +this repo, sets up a venv, merges our `implementations_quic.json` snippet, +builds the endpoint image, and invokes `run.py`. All steps are +idempotent so repeated invocations just iterate. + +``` +# Single test against the most permissive peer: +quic/interop/run-matrix.sh -s aioquic -t handshake + +# A focused triangulation: +quic/interop/run-matrix.sh -s aioquic -t handshake,chacha20 +quic/interop/run-matrix.sh -s quic-go -t handshake,chacha20 +quic/interop/run-matrix.sh -s picoquic -t handshake,chacha20 + +# Tight inner loop — skip the image rebuild between test selections: +SKIP_BUILD=1 quic/interop/run-matrix.sh -s aioquic -t transfer +``` + +Manual flow (if `run-matrix.sh` doesn't fit): + +``` +make -C quic/interop build +# Then in a sibling clone of quic-interop-runner, merge our snippet: +jq -s '.[0] * .[1]' implementations_quic.json \ + ../amethyst/quic/interop/quic-interop-runner-snippet.json \ + > implementations_quic.json.new && mv implementations_quic.json.new implementations_quic.json +python run.py -d -i amethyst -s aioquic -t handshake,chacha20 --log-dir ./logs +``` + +Inspect `./logs//client_qlog/*.qlog` in qvis when something breaks. + +## Phase ladder (excerpt — full plan in conversation) + +| Phase | Goal | Tests | Exit criterion | +|---|---|---|---| +| 0 | Minimum harness | `handshake` | one test reproducible end-to-end ✅ | +| 1 | Triangulate handshake bugs | + `versionnegotiation`, `chacha20` | green vs aioquic + quiche + picoquic | +| 2 | Streams + loss + multiplexing | + `transfer`, `multiplexing`, `*loss`, `http3` | `transfer` / `multiplexing` / `http3` ✅ landed; loss tests pending | +| 3 | Edge cases | `retry`, `resumption`, `zerortt`, `keyupdate`, `rebinding-*`, `blackhole`, `amplificationlimit` | every test green or unsupported-127 with a written reason | +| 4 | CI gate | nightly Phases 1–2; PR-blocking subset on every push | qlogs uploaded as artifacts on red | + +## Phase 2 — landed 2026-05-06 + +- Minimal `Http3GetClient` (in `:quic-interop`, NOT `:quic` — interop-test + surface, not a production HTTP/3 client). Opens the three required + client uni streams (control + QPACK encoder + QPACK decoder), sends + empty SETTINGS, then per request opens a bidi stream, encodes a HEADERS + frame with the four pseudo-headers using the existing literal-only + `QpackEncoder`, FINs, and reassembles HEADERS+DATA frames from the + response. Out-of-scope: GOAWAY, PUSH_PROMISE, dynamic QPACK table, + trailers, priority. +- `transfer` + `http3` testcases: GET each URL in `REQUESTS` sequentially, + write each body to `$DOWNLOADS/`. Status != 200 fails. +- `multiplexing` testcase: same as `transfer` but issues each GET in a + parallel `coroutineScope { async { … } }` so the request streams + genuinely overlap on the wire (what tshark verifies). +- Aliased sim-driven testcases — these reuse the same client code paths; + the runner injects the network condition via the ns-3 sim. Failures + here are exactly the bug-finding signal we want, since they exercise + loss recovery / RTT estimator / congestion behaviour against real peers: + - `transferloss` → transfer (random packet loss) + - `transfercorruption` → transfer (random bit-flip; AEAD AUTH FAIL → drop + retransmit) + - `longrtt` → transfer (emulated high-latency link) + - `goodput` → transfer (throughput floor) + - `crosstraffic` → transfer (competing UDP flows on the same link) + - `handshakeloss` → handshake (loss during handshake — tests CRYPTO retransmit) + +## Phase 3 — landed 2026-05-07 (post-quic-go interop) + +- ALPN per testcase (`:quic-interop`'s `Alpn` enum + per-test switch in + `InteropClient.main`). quic-go enforces strictly with TLS + `no_application_protocol` (CRYPTO_ERROR 0x178); aioquic / picoquic accept + either. Convention: `h3` for `http3`/`multiplexing`, `hq-interop` + everywhere else. +- `HqInteropGetClient` (HTTP/0.9 over QUIC) — open bidi, send `GET /path\r\n`, + FIN, read body until server FINs. No framing, no QPACK. ~30 lines. +- Multi-stream FIN delivery fix in `QuicConnection.closeAllSignals()`: pre-fix + iterated only the connection-wide signal channels, leaving every per-stream + `incomingChannel` open after teardown — coroutines suspended on + `stream.incoming.collect { ... }` hung forever. Fix iterates `streamsList` + on close. Three regression tests in `MultiStreamFinDeliveryTest`. +- `retry` + `ipv6` testcases enabled in dispatch. `retry` rides on agent 3's + RFC 9000 §17.2.5 + RFC 9001 §5.8 implementation. `ipv6` should "just work" + via JDK's `DatagramChannel` v6 support; runner-validated when run. + +## Validated against (as of 2026-05-07 evening) + +Latest run results before pushing the multi-ALPN fix `acfe815e1`: + +| Peer | handshake | chacha20 | transfer | http3 | multiplexing | +|---|---|---|---|---|---| +| aioquic | ✓ | ✓ | ✓ | ✓ | ✕ (channel-saturation, agent C investigating) | +| picoquic | ✕ (alpn=hq-interop unsupported, fixed in `acfe815e1`) | ✕ | ✕ | ✓ | ✕ | +| quic-go | (untested post-fix) | | | | | + +After `acfe815e1`'s multi-ALPN offer, predictions: +- picoquic returns to 4/4 (server picks `h3`, we run Http3GetClient). +- quic-go: handshake / chacha20 / transfer / http3 should green (server picks `hq-interop` for non-h3 tests). +- aioquic: still 4/5; multiplexing held back by channel-saturation bug. + +## Phase 4 — landed 2026-05-07 (overnight agents) + +All three agents merged onto the branch with three rounds of fixup +(each agent's worktree was based on `main`, not the branch HEAD, so +they clobbered each other's changes during merge): + +- **Agent A — VN-handling defense in `:quic`**: configurable + `initialVersion` on `QuicConnection`, `applyVersionNegotiation` + state machine, `vnConsumed` latch, downgrade defenses. *NOTE*: the + runner does not have a `versionnegotiation` testcase (it has `v2`, + which tests QUIC v2 — we're v1-only). Code stays as defensive + support for any server that throws a VN at us, but unused by the + matrix. +- **Agent B — qlog observer**: `QlogObserver` interface in `:quic` + with NoOp default + 12 event hooks (packet-sent/received/dropped, + key-updated, conn-started/closed, loss-detected, PTO-fired, + transport-params, ALPN, version). `QlogWriter` (Jackson-backed + JSON-NDJSON) in `:quic-interop`. `InteropClient` reads `$QLOGDIR` + the runner already sets and writes `client.sqlog` per testcase. + Drag straight into qvis.quictools.info to see the trace. +- **Agent C — peer-uni-stream drainer**: `drainPeerInitiatedUniStreamsIntoBlackHole` + helper on `QuicConnection`, wired into `Http3GetClient.init(scope)`. + Fixes the multiplexing channel-saturation symptom (server's uni + streams accumulated bytes in 64-chunk per-stream channels until + parser tore down with INTERNAL_ERROR ~4.5s into a multi-stream run). + +## Phase 5 — landed 2026-05-07 (post-overnight bug-hunt) + +The full overnight session pulled hard on the bugs the matrix exposed. +qlog turned out to be the unblocking tool — every fix in this phase +came from staring at a `client.sqlog` and matching it against the RFC +pages it referenced. + +| Commit | Diagnosis pattern | +|---|---| +| `bd9d717df` | `IOException: Stream closed` from QlogWriter mid-test → observer threw into the send loop, killed connections that had already completed transfer. | +| `99a1a91de` | qlog stops at t=400ms, no inbound packets → per-event `writer.flush()` stalled the connection lock on macOS Docker filesystem virtualization. | +| `c0d7b6031` | aioquic `CONNECTION_CLOSE: Packet contains no CRYPTO frame` after PTO probe → our PTO emitted bare PING, not CRYPTO retransmit. Restored agent 2's wiring lost in the qlog merge. | +| `17b80270d` | runner's retry verdict `Client reset the packet number. Check failed for PN 0` → `applyRetry` reset Initial PN to 0; RFC 9001 §5.7 says PN namespace continues across Retry. New `LevelState.resetForRetry` helper. | +| `9a74d1d5d` | multiplexing qlog showed STREAM frames still arriving at t=31s when local timeout fired → bumped `TRANSFER_TIMEOUT_SEC` 30→60. Doesn't fix throughput, just lets the test budget match the workload. | +| `32ccbd2b2` | `coroutineScope { urls.map { async }.map { it.await() } }` blocks on slowest hung stream → per-stream `withTimeoutOrNull` so a single bad stream surfaces as status=0 instead of hanging the matrix. | + +After all of these, expectation is aioquic / picoquic ≥6/7 (M +might still be flaky). retry green via qlog-driven debugging is +the marquee result. + +## Still open + +- **`v2`** — server demands QUIC v2 (RFC 9369). We're v1-only. + Implementing v2 is its own project (different Initial-secret + derivation, different transport-parameter encoding, different + long-header type bits — not just a version-number swap). +- **Multiplexing throughput on Mac+Rosetta** — measured at ~23 streams/sec + (aioquic processed 1359 GETs in 58s). The runner's multiplexing test + generates 1999 files; even with a 60s budget we only get 70% of them + open before the test ends, with most coroutines failing to surface a + status=200 in time for the file-write loop. Throughput is hard-bottlenecked + by `:quic`'s single `conn.lock` serializing the send loop, the read + loop, and `openBidiStream` across all 1999 awaiting coroutines. Under + this lock contention pattern, parallelism doesn't help — coroutines + queue on the lock anyway. + Possible fixes (non-trivial): + - Per-level / per-stream lock split (big refactor; touches almost + everything in `:quic/connection`). + - Batch concurrency via a `Semaphore` capping in-flight `client.get()` + calls to e.g. 64 at a time. Doesn't lift the throughput ceiling but + reduces lock thrash and probably lets MORE files complete in 60s. + - QPACK dynamic-table support so aioquic doesn't need to fall back + to literal encoding (would help on the response-decode side). + None are urgent; this is a stress-test scenario, not a normal-traffic + one. Filed for follow-up. +- **Server role** — entire stack is client-only. Implementing the + server side would unlock `handshakeloss` against aioquic + (currently `?` because aioquic-server doesn't support it), and + would make us a self-validating peer. + +## Concurrency + +`run-matrix.sh` is NOT safe to run in parallel. The runner's +docker-compose.yml hardcodes `container_name: sim/server/client` — +Docker enforces those globally. Use a sequential loop: + +``` +for peer in aioquic picoquic quic-go; do + quic/interop/run-matrix.sh -s $peer -t handshake,chacha20,... +done +``` + +## Explicitly unsupported testcases (return 127, runner skips) + +| Testcase | Reason | +|---|---| +| `versionnegotiation` | `QuicConnectionWriter` hard-codes `QuicVersion.V1`; needs a configurable initial version + VN-response retry path. **In progress (background agent)**. | +| `resumption` | session ticket parsing + persistence not yet implemented | +| `zerortt` | depends on `resumption` + early-data path | +| `keyupdate` | KEY_PHASE bit handling not yet implemented in 1-RTT | +| `rebinding-port`, `rebinding-addr` | client-side connection migration (re-bind UDP socket, NEW_CONNECTION_ID rotation) not implemented | +| `amplificationlimit` | server-side test, N/A for client role | +| `blackhole` | inverse test (verifies we *fail* on dead network in bounded time); needs special handling | +| `ipv6` | `UdpSocket` IPv6 path not exercised; risky to claim without testing | + +## Phase 1a — landed 2026-05-06 + +- `SSLKEYLOGFILE` writer in `InteropClient` (NSS Key Log Format) so Wireshark + decrypts the sim's pcap captures. Backed by: + - `TlsClient.clientRandom` (new public read-only property, captured in + `start()` before sending ClientHello). + - `QuicConnection.extraSecretsListener` (new optional constructor param, + chained after the connection's own key-installation listener; no-op + default, so production callers are unaffected). +- `chacha20` testcase: forces ChaCha20-Poly1305 only via new + `QuicConnection.cipherSuites` knob (threaded through `TlsClient` → + `buildQuicClientHello` → `TlsClientHello`). + +## Phase 1b — open + +- `QLOGDIR`: `:quic` has no qlog observer infrastructure yet. Wiring needs + hooks at packet send/recv, frame dispatch, recovery, and TLS state + transitions, plus the qlog JSON-NDJSON schema. Sized as its own design + doc before implementation. +- `versionnegotiation`: `QuicConnectionWriter` hard-codes `QuicVersion.V1` + in two call sites; threading a configurable initial-version through and + wiring the response-handling path is non-trivial. Defer. +- Server role: we are client-first. Reassess after Phase 3. +- WebTransport is **not** part of the standard interop matrix; it needs a + separate harness against `moq-rs` / chrome-headless. diff --git a/quic/interop/quic-interop-runner-snippet.json b/quic/interop/quic-interop-runner-snippet.json new file mode 100644 index 0000000000..6af700f44c --- /dev/null +++ b/quic/interop/quic-interop-runner-snippet.json @@ -0,0 +1,7 @@ +{ + "amethyst": { + "image": "amethyst-quic-interop:latest", + "url": "https://github.com/vitorpamplona/amethyst", + "role": "client" + } +} diff --git a/quic/interop/run-matrix.sh b/quic/interop/run-matrix.sh new file mode 100755 index 0000000000..f6e10f8ab3 --- /dev/null +++ b/quic/interop/run-matrix.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Drive the quic-interop-runner against the :quic-interop endpoint image. +# +# Idempotent: clones the runner alongside this repo if missing, sets up its +# venv, registers `amethyst` in implementations_quic.json, builds our endpoint +# image, then invokes run.py with the user's args. +# +# Usage: +# quic/interop/run-matrix.sh # full matrix vs aioquic +# quic/interop/run-matrix.sh -s aioquic -t handshake # one test +# quic/interop/run-matrix.sh -s quic-go -t handshake,chacha20 # different peer +# +# Env overrides: +# RUNNER_DIR — where to clone / find the runner (default: ../quic-interop-runner) +# LOG_DIR — qlog / pcap output (default: $RUNNER_DIR/logs) +# SKIP_BUILD=1 — skip `make build` (image already current) +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd) +RUNNER_DIR="${RUNNER_DIR:-$REPO_ROOT/../quic-interop-runner}" +LOG_DIR="${LOG_DIR:-$RUNNER_DIR/logs}" + +is_macos=false +case "${OSTYPE:-}" in + darwin*) is_macos=true ;; +esac + +install_hint() { + if $is_macos; then + echo " brew install $1" + else + echo " sudo apt-get install -y $1" + fi +} + +need() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: '$1' not found in PATH. Install with:" >&2 + install_hint "$2" >&2 + exit 1 + fi +} + +need docker docker.io +need python3 python3 +need jq jq +need git git +need make make + +# Docker daemon must be reachable, not just installed. On macOS this is the +# usual gotcha — Docker Desktop installed but not running. +if ! docker info >/dev/null 2>&1; then + echo "error: docker daemon not reachable." >&2 + if $is_macos; then + echo " → launch Docker Desktop and retry." >&2 + else + echo " → 'sudo systemctl start docker' (and add yourself to the docker group)." >&2 + fi + exit 1 +fi + +# 1. Clone the runner if missing. +if [ ! -d "$RUNNER_DIR" ]; then + echo "==> cloning quic-interop-runner into $RUNNER_DIR" + git clone --depth 1 https://github.com/quic-interop/quic-interop-runner.git "$RUNNER_DIR" +fi + +# 2. venv + Python deps. Prefer 3.13 — pyshark (the runner's pcap parser) +# trips on 3.14's asyncio.get_event_loop() removal. Fall back to whatever +# `python3` resolves to if 3.13 isn't installed. +PYTHON_BIN="" +for candidate in python3.13 python3.12 python3.11 python3; do + if command -v "$candidate" >/dev/null 2>&1; then + if "$candidate" -c 'import sys; sys.exit(0 if sys.version_info < (3,14) else 1)' 2>/dev/null; then + PYTHON_BIN="$candidate" + break + fi + fi +done +if [ -z "$PYTHON_BIN" ]; then + echo "warning: no Python < 3.14 found; pyshark will likely crash on pcap validation." >&2 + install_hint python3.13 + PYTHON_BIN="python3" +fi +if [ ! -d "$RUNNER_DIR/.venv" ]; then + echo "==> creating venv at $RUNNER_DIR/.venv (using $PYTHON_BIN)" + "$PYTHON_BIN" -m venv "$RUNNER_DIR/.venv" +fi +"$RUNNER_DIR/.venv/bin/pip" install -q -r "$RUNNER_DIR/requirements.txt" + +# 3. Register our endpoint in implementations_quic.json (idempotent). +if ! jq -e '.amethyst' "$RUNNER_DIR/implementations_quic.json" >/dev/null 2>&1; then + echo "==> registering 'amethyst' in implementations_quic.json" + tmp=$(mktemp) + jq -s '.[0] * .[1]' \ + "$RUNNER_DIR/implementations_quic.json" \ + "$SCRIPT_DIR/quic-interop-runner-snippet.json" \ + > "$tmp" + mv "$tmp" "$RUNNER_DIR/implementations_quic.json" +fi + +# 4. Build the endpoint image (skippable for tight loops). Pass DEBUG=1 +# to bake QUIC_INTEROP_DEBUG=1 into the image so the InteropClient + +# writer emit per-drain stats to stderr (visible via +# inspect-multiplexing.sh after the run). Off by default. +if [ "${SKIP_BUILD:-0}" != "1" ]; then + if [ "${DEBUG:-0}" = "1" ]; then + echo "==> building amethyst-quic-interop image (DEBUG=1)" + make -C "$SCRIPT_DIR" build DEBUG=1 + else + echo "==> building amethyst-quic-interop image" + make -C "$SCRIPT_DIR" build + fi +fi + +# 5. Drive the runner. +# +# run.py hard-exits if --log-dir already exists, so we use a fresh +# per-invocation subdirectory under $LOG_DIR. The parent must exist; the +# child must not. Tail of `ls -t "$LOG_DIR" | head -1` finds the latest. +# +# NOTE: this script is NOT safe to run concurrently against itself. +# quic-interop-runner's docker-compose.yml hardcodes `container_name: +# sim/server/client`, which Docker enforces globally regardless of +# COMPOSE_PROJECT_NAME. Two simultaneous invocations collide on +# `docker create container "sim"`. Run sequentially: +# for peer in aioquic picoquic quic-go; do +# quic/interop/run-matrix.sh -s $peer -t handshake,chacha20,... +# done +# +# Output filter: by default we drop the runner / container boilerplate +# (interface checksum offload toggles, route setup, container lifecycle, +# the long Command: WAITFORSERVER=... line, the platform-mismatch warning +# that fires once per test on Apple Silicon). Set VERBOSE=1 to bypass. +mkdir -p "$LOG_DIR" +RUN_LOG_DIR="$LOG_DIR/run-$(date +%Y%m%d-%H%M%S)" +echo "==> running matrix (args: $* | logs: $RUN_LOG_DIR)" +cd "$RUNNER_DIR" + +if [ "${VERBOSE:-0}" = "1" ]; then + exec "$RUNNER_DIR/.venv/bin/python" run.py \ + -d -i amethyst --log-dir "$RUN_LOG_DIR" "$@" +else + # Tee unfiltered stdout to a sibling file so summarize-matrix.sh + # can find the 'Test: X took Y, status:' lines later — the runner + # only writes those to its own stdout, not into per-testcase + # output.txt. Sibling rather than inside RUN_LOG_DIR because + # run.py refuses to start if its --log-dir already exists. + RUNNER_STDOUT="${RUN_LOG_DIR}.stdout.log" + "$RUNNER_DIR/.venv/bin/python" run.py \ + -d -i amethyst --log-dir "$RUN_LOG_DIR" "$@" 2>&1 \ + | tee "$RUNNER_STDOUT" \ + | grep -Ev \ + -e '^(client|server|sim) +\| +(Setting up routes|Actual changes:|tx-[a-z0-9-]+:|Endpoint'\''s IPv[46] address is)' \ + -e '^ Container [a-z]+ +(Recreate|Recreated|Stopping|Stopped|Starting|Started)( [0-9.]+s)?$' \ + -e '^ server The requested image'\''s platform' \ + -e '^Attaching to client, server, sim$' \ + -e '^Aborting on container exit\.\.\.$' \ + -e '^(client|server|sim) exited with code [0-9]+$' \ + -e '^Packets captured: [0-9]+$' \ + -e '^sim +\| +Packets received/dropped on interface ' \ + -e '^sim +\| +(Received signal:|msg=|NS_FATAL)' \ + -e '^Using the client'\''s key log file\.$' \ + -e '^Command: WAITFORSERVER=' \ + -e '^[0-9-]+ [0-9:,]+ Generated random file: ' \ + -e '^[0-9-]+ [0-9:,]+ Requests: \[' \ + -e '^==> ' \ + -e '^$' + exit "${PIPESTATUS[0]}" +fi diff --git a/quic/interop/run_endpoint.sh b/quic/interop/run_endpoint.sh new file mode 100755 index 0000000000..8d19e99fd2 --- /dev/null +++ b/quic/interop/run_endpoint.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# quic-interop-runner endpoint entry point. +# +# The base image (martenseemann/quic-network-simulator-endpoint) provides +# /setup.sh, which configures routing for the runner's ns-3 sim. Source it +# inside the runner; tolerate failure (e.g. missing NET_ADMIN cap) so the +# JVM client still launches if a caller invokes the image outside the +# runner — they get default container networking instead. +set -uo pipefail + +# shellcheck disable=SC1091 +source /setup.sh 2>/dev/null || echo "(setup.sh skipped — not under the runner sim)" >&2 + +set -e + +case "${ROLE:-client}" in + client) + exec /opt/quic-interop/bin/quic-interop + ;; + server) + # Phase 0: server role unimplemented. Exit 127 so the runner skips. + echo "server role not implemented" >&2 + exit 127 + ;; + *) + echo "unknown ROLE=${ROLE}" >&2 + exit 127 + ;; +esac diff --git a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/HqInteropGetClient.kt b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/HqInteropGetClient.kt new file mode 100644 index 0000000000..027bf73bce --- /dev/null +++ b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/HqInteropGetClient.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop.runner + +import com.vitorpamplona.quic.connection.QuicConnection +import com.vitorpamplona.quic.connection.QuicConnectionDriver +import kotlinx.coroutines.flow.toList + +/** + * HQ-interop (HTTP/0.9 over QUIC) GET client. quic-interop-runner convention + * for the non-`http3` testcases (handshake / chacha20 / transfer / loss + * variants): the client opens a bidi stream, sends `GET /path\r\n` (raw + * ASCII, no framing), FINs the send side, and reads the response body + * verbatim until the server FINs. There is no status code, no headers, no + * QPACK, no control stream — the body bytes ARE the response. + * + * Per the runner's testcase validator, an empty body means failure, a + * non-empty body that matches what the server staged at $WWW/ means + * success. We surface non-empty as status=200, empty as status=0, so the + * caller's `if (resp.status != 200) anyFailed = true` check keeps working. + */ +class HqInteropGetClient( + private val conn: QuicConnection, + private val driver: QuicConnectionDriver, +) : GetClient { + override suspend fun prepareRequest( + @Suppress("UNUSED_PARAMETER") authority: String, + path: String, + ): RequestHandle { + val stream = conn.openBidiStream() + val request = "GET $path\r\n".encodeToByteArray() + stream.send.enqueue(request) + stream.send.finish() + // Wake the send loop — same reasoning as Http3GetClient. + driver.wakeup() + return HqRequestHandle(stream) + } + + override suspend fun prepareRequests( + @Suppress("UNUSED_PARAMETER") authority: String, + paths: List, + ): List { + // Pre-format outside the lock — see Http3GetClient. Then + // openBidiStreamsBatch atomically opens all N streams under + // streamsLock so the writer's next drain coalesces them. + val encoded = paths.map { "GET $it\r\n".encodeToByteArray() } + return conn.openBidiStreamsBatch(encoded) { stream, request -> + stream.send.enqueue(request) + stream.send.finish() + HqRequestHandle(stream) + } + } + + override suspend fun awaitResponse(handle: RequestHandle): GetResponse { + val stream = (handle as HqRequestHandle).stream + val chunks = stream.incoming.toList() + val total = chunks.sumOf { it.size } + val body = ByteArray(total) + var off = 0 + for (c in chunks) { + c.copyInto(body, off) + off += c.size + } + return GetResponse(status = if (body.isEmpty()) 0 else 200, body = body) + } +} + +private class HqRequestHandle( + val stream: com.vitorpamplona.quic.stream.QuicStream, +) : RequestHandle diff --git a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClient.kt b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClient.kt new file mode 100644 index 0000000000..c1a20bed55 --- /dev/null +++ b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClient.kt @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop.runner + +import com.vitorpamplona.quic.QuicWriter +import com.vitorpamplona.quic.connection.QuicConnection +import com.vitorpamplona.quic.connection.QuicConnectionDriver +import com.vitorpamplona.quic.connection.drainPeerInitiatedUniStreamsIntoBlackHole +import com.vitorpamplona.quic.http3.Http3Frame +import com.vitorpamplona.quic.http3.Http3FrameReader +import com.vitorpamplona.quic.http3.Http3FrameType +import com.vitorpamplona.quic.http3.Http3Settings +import com.vitorpamplona.quic.http3.Http3StreamType +import com.vitorpamplona.quic.qpack.QpackDecoder +import com.vitorpamplona.quic.qpack.QpackEncoder +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.collect + +/** Common shape for the two interop GET clients (HTTP/3 and HQ-interop). + * + * The interface is split into two phases so the parallel multiplexing + * path can BATCH enqueues (synchronous, serial) and SINGLE-wakeup the + * send loop, vs. waking on every individual request which produces + * one tiny packet per stream instead of coalesced packets per drain. */ +interface GetClient { + /** Open a stream + enqueue the request bytes + FIN. Does NOT wake the + * send loop — caller is responsible for batching wakes. Returns an + * opaque handle the caller passes to [awaitResponse]. */ + suspend fun prepareRequest( + authority: String, + path: String, + ): RequestHandle + + /** + * Atomically open + enqueue + FIN N streams under a single hold of + * the connection lock. The send loop cannot interject between + * opens, so when it next drains it sees ALL N streams' data ready + * and packs them into coalesced packets instead of emitting one + * tiny packet per stream. + * + * Without this, the equivalent serial loop of [prepareRequest] + * yields between calls (lock release → send loop wakes → drains + * one stream → next prepareRequest acquires...) and we send one + * stream per packet — what cratered the multiplexing testcase. + */ + suspend fun prepareRequests( + authority: String, + paths: List, + ): List + + /** Suspend until the server FINs the response stream associated with + * [handle]. Returns the assembled response. */ + suspend fun awaitResponse(handle: RequestHandle): GetResponse + + /** Convenience shortcut for the sequential / single-request paths. */ + suspend fun get( + authority: String, + path: String, + ): GetResponse { + val h = prepareRequest(authority, path) + return awaitResponse(h) + } +} + +/** Opaque handle returned by [GetClient.prepareRequest]. Implementations + * cast it back to their internal stream representation. */ +interface RequestHandle + +data class GetResponse( + val status: Int, + val body: ByteArray, +) + +/** + * Minimal HTTP/3 GET client used by the interop endpoint to satisfy the + * `http3` and `multiplexing` testcases. + * + * Opens the three required client-side unidirectional streams (control, + * QPACK encoder, QPACK decoder) per RFC 9114 §6.2.1. Encodes requests with + * the literal-only [QpackEncoder] (no dynamic table — RFC 9204 Required + * Insert Count = 0) so we don't need to push QPACK encoder instructions. + * + * Not a production HTTP/3 client. Specifically: no GOAWAY handling, no + * priority, no push-promise, no trailers, no dynamic QPACK table. + */ +class Http3GetClient( + private val conn: QuicConnection, + private val driver: QuicConnectionDriver, +) : GetClient { + suspend fun init(scope: CoroutineScope) { + // Control stream: type-0x00 prefix followed by a SETTINGS frame + // (empty body is legal — RFC 9114 §7.2.4). Empty body = spec + // defaults: QPACK_MAX_TABLE_CAPACITY=0, QPACK_BLOCKED_STREAMS=0, + // MAX_FIELD_SECTION_SIZE=unlimited. We deliberately do NOT + // explicitly send QPACK_MAX_TABLE_CAPACITY=0 — empirically + // (aioquic 2026-05-07) that worsened the multiplexing case + // from 1 file to 0 files. + val control = conn.openUniStream() + val w = QuicWriter() + w.writeVarint(Http3StreamType.CONTROL) + w.writeBytes(Http3Settings(emptyMap()).encodeFrame()) + control.send.enqueue(w.toByteArray()) + // Control stream stays open for the lifetime of the H3 connection; + // do NOT call finish() — peers treat that as H3_CLOSED_CRITICAL_STREAM. + + // Required: open QPACK encoder + decoder streams, even though we + // never insert into the dynamic table. Just send the type prefix. + val qpackEnc = conn.openUniStream() + val w2 = QuicWriter() + w2.writeVarint(Http3StreamType.QPACK_ENCODER) + qpackEnc.send.enqueue(w2.toByteArray()) + + val qpackDec = conn.openUniStream() + val w3 = QuicWriter() + w3.writeVarint(Http3StreamType.QPACK_DECODER) + qpackDec.send.enqueue(w3.toByteArray()) + + // RFC 9114 §6.2: the server opens its own three uni streams + // (control, qpack encoder, qpack decoder). Their bytes accumulate + // in per-stream `incomingChannel`s (capacity 64); without an + // active consumer the channel saturates and `:quic` tears down + // the connection with INTERNAL_ERROR. We don't actually use the + // dynamic table or care about the server's settings, so drain + // and discard. Without this, multiplexing testcase fails after + // ~4.5s with "consumer overflowed" tear-down. + conn.drainPeerInitiatedUniStreamsIntoBlackHole(scope) + } + + override suspend fun prepareRequest( + authority: String, + path: String, + ): RequestHandle { + val stream = conn.openBidiStream() + stream.send.enqueue(encodeRequest(authority, path)) + stream.send.finish() + // Without this, the data sits in the queue until the PTO + // timer fires (~1 s later). On the longrtt scenario (750 ms + // one-way, 1.5 s RTT) that's a fatal delay — the runner's + // 8 s timeout doesn't leave room for the PTO + RTT + RTT + // dance. The parallel path wakes after the chunk; the + // serial path was missing the same nudge. + driver.wakeup() + return Http3RequestHandle(stream) + } + + override suspend fun prepareRequests( + authority: String, + paths: List, + ): List { + // Pre-encode all requests OUTSIDE the lock so QPACK encoding + // (allocations, hashing, varint emit) doesn't extend the + // critical section. With 1999 paths the encoding cost is + // non-trivial; doing it under streamsLock would stall the + // send loop for the full encode time across every chunk. + val encoded = paths.map { encodeRequest(authority, it) } + // openBidiStreamsBatch holds streamsLock for the entire + // batch — the send loop can't interject between opens so + // the writer's next drain finds all N streams' frames ready + // and packs them into coalesced packets (vs. the regressed + // shape that emitted one STREAM per packet against aioquic + // on 2026-05-06). + return conn.openBidiStreamsBatch(encoded) { stream, request -> + stream.send.enqueue(request) + stream.send.finish() + Http3RequestHandle(stream) + } + } + + override suspend fun awaitResponse(handle: RequestHandle): GetResponse { + val stream = (handle as Http3RequestHandle).stream + val reader = Http3FrameReader() + var status = 0 + val body = mutableListOf() + stream.incoming.collect { chunk -> + reader.push(chunk) + while (true) { + val frame = reader.next() ?: break + when (frame) { + is Http3Frame.Headers -> { + val fields = QpackDecoder().decodeFieldSection(frame.qpackPayload) + status = fields.firstOrNull { it.first == ":status" }?.second?.toIntOrNull() ?: 0 + } + + is Http3Frame.Data -> { + body += frame.body + } + + else -> { + Unit + } + } + } + } + return GetResponse(status = status, body = concat(body)) + } +} + +private class Http3RequestHandle( + val stream: com.vitorpamplona.quic.stream.QuicStream, +) : RequestHandle + +/** + * Serialize a GET request as a single HEADERS frame ready to be enqueued + * onto a fresh bidi stream. Exposed for unit-testing the wire format + * without spinning up a QUIC connection. + */ +internal fun encodeRequest( + authority: String, + path: String, +): ByteArray { + val headers = + listOf( + ":method" to "GET", + ":scheme" to "https", + ":authority" to authority, + ":path" to path, + ) + val qpack = QpackEncoder().encodeFieldSection(headers) + val w = QuicWriter() + w.writeVarint(Http3FrameType.HEADERS) + w.writeVarint(qpack.size.toLong()) + w.writeBytes(qpack) + return w.toByteArray() +} + +private fun concat(parts: List): ByteArray { + val total = parts.sumOf { it.size } + val out = ByteArray(total) + var off = 0 + for (p in parts) { + p.copyInto(out, off) + off += p.size + } + return out +} diff --git a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/InteropClient.kt b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/InteropClient.kt new file mode 100644 index 0000000000..30759e9e5f --- /dev/null +++ b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/InteropClient.kt @@ -0,0 +1,553 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop.runner + +import com.vitorpamplona.quic.connection.QuicConnection +import com.vitorpamplona.quic.connection.QuicConnectionConfig +import com.vitorpamplona.quic.connection.QuicConnectionDriver +import com.vitorpamplona.quic.tls.PermissiveCertificateValidator +import com.vitorpamplona.quic.tls.TlsConstants +import com.vitorpamplona.quic.tls.TlsSecretsListener +import com.vitorpamplona.quic.transport.UdpSocket +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.cancel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import java.io.File +import java.net.URI +import kotlin.system.exitProcess + +/** + * Endpoint that implements the [quic-interop-runner](https://github.com/quic-interop/quic-interop-runner) + * Docker contract. Reads the env-var protocol the runner pushes into each + * client container and dispatches by [TESTCASE]. + * + * Exit codes: + * - `0` — testcase passed + * - `1` — testcase failed (bug in our impl, OR partner) + * - `127` — testcase not implemented (runner skips, doesn't fail) + */ +private fun nowMs(): Long = System.currentTimeMillis() + +private const val EXIT_OK = 0 +private const val EXIT_FAIL = 1 +private const val EXIT_UNSUPPORTED = 127 + +private const val HANDSHAKE_TIMEOUT_SEC = 10L + +// Multiplexing generates ~hundreds-to-thousands of small files; download +// throughput on Mac+Rosetta is dominated by Docker filesystem overhead +// per-write. 30s wasn't enough for the larger file counts; the qlog +// against aioquic showed us still actively receiving STREAM frames at +// t=31s when our local timeout fired. 60s gives more headroom without +// inflating turnaround for the cases that actually complete fast. +private const val TRANSFER_TIMEOUT_SEC = 60L + +// Per-stream timeout in the parallel-multiplexing path. If any single +// GET hangs past this (e.g. its FIN was lost in the shuffle of +// hundreds of concurrent streams), the await on that future returns +// a status=0 response and the others continue. Without this, a single +// stuck stream would block the whole .map { it.await() } chain. +private const val PER_STREAM_TIMEOUT_SEC = 20L + +// Concurrency cap for the parallel-multiplexing path. Each chunk of this +// many requests fires fully in parallel; chunks process sequentially. +// Sized so that conn.lock contention stays manageable while still +// satisfying the runner's "streams overlap on the wire" check (which +// only needs a handful of streams concurrent at any given moment, not +// all of them simultaneously). Empirically validated against aioquic + +// picoquic at this value. +private const val MULTIPLEX_PARALLELISM = 64 + +fun main() { + // Single env-var check, propagated to library code that opts into + // verbose tracing only when this is set. + val debugEnv = System.getenv("QUIC_INTEROP_DEBUG") + if (debugEnv == "1") { + com.vitorpamplona.quic.connection.writerDebugEnabled = true + System.err.println( + "[boot] DEBUG=1; writerDebugEnabled=true; build_id=" + + "${com.vitorpamplona.quic.connection.WRITER_DEBUG_BUILD_ID}; " + + "TESTCASE=${System.getenv("TESTCASE") ?: "(unset)"}; " + + "ROLE=${System.getenv("ROLE") ?: "(unset)"}", + ) + } else { + System.err.println("[boot] DEBUG=${debugEnv ?: "(unset)"} writerDebugEnabled=false") + } + + val role = System.getenv("ROLE") ?: "client" + if (role != "client") { + System.err.println("server role not implemented") + exitProcess(EXIT_UNSUPPORTED) + } + + val testcase = System.getenv("TESTCASE")?.trim().orEmpty() + val requests = System.getenv("REQUESTS")?.trim().orEmpty() + // The runner mounts $CLIENT_DOWNLOADS to /downloads as a Docker volume + // (see quic-interop-runner's docker-compose.yml `client.volumes`). It + // does NOT export a DOWNLOADS env var. Hard-code the mount path. + val downloadsDir = File("/downloads") + val keyLogPath = System.getenv("SSLKEYLOGFILE")?.takeIf { it.isNotBlank() } + val qlogDir = System.getenv("QLOGDIR")?.takeIf { it.isNotBlank() }?.let { File(it) } + + // One-line context header. Verbose per-field dump deferred to debug + // mode (env var QUIC_INTEROP_DEBUG=1) so the runner's aggregated output + // stays readable across a full matrix run. + if (System.getenv("QUIC_INTEROP_DEBUG") == "1") { + System.err.println("== quic-interop client ==") + System.err.println("testcase: $testcase") + System.err.println("requests: $requests") + System.err.println("downloads dir: ${downloadsDir.absolutePath} (exists=${downloadsDir.isDirectory})") + System.err.println("sslkeylogfile: ${keyLogPath ?: "(unset)"}") + System.err.println("qlogdir: ${qlogDir?.absolutePath ?: "(unset)"}") + } + + val cipherSuites = + when (testcase) { + "chacha20" -> intArrayOf(TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256) + else -> null + } + + // For the versionnegotiation testcase the runner expects us to send + // an Initial advertising a version the server doesn't support, then + // honor its VN response by retrying with v1. agent A's + // QuicVersion.FORCE_VERSION_NEGOTIATION drives that flow. + val initialVersion = + when (testcase) { + "versionnegotiation" -> com.vitorpamplona.quic.packet.QuicVersion.FORCE_VERSION_NEGOTIATION + else -> com.vitorpamplona.quic.packet.QuicVersion.V1 + } + + // ALPN selection. Different servers configure different ALPNs PER + // testcase, with no consistent convention: + // - quic-go-qns — strictly hq-interop for non-http3 tests + // - aioquic-qns — accepts either + // - picoquic-qns — strictly h3 for ALL testcases + // + // Solution: offer BOTH `h3` and `hq-interop` in the ClientHello. TLS + // ALPN negotiation lets the server pick whichever matches its config. + // For testcases that REQUIRE H3 framing (http3, multiplexing) we + // restrict to h3 so any server that picks hq-interop fails fast. + val offeredAlpns = + when (testcase) { + "http3", "multiplexing" -> listOf(Alpn.H3) + else -> listOf(Alpn.HQ_INTEROP, Alpn.H3) + } + + val code = + when (testcase) { + // All these testcases require: successful handshake + file + // transferred to /downloads. Per-testcase notes: + // chacha20 — runner verifies the negotiated cipher + // was ChaCha20-Poly1305 via tshark on the + // pcap (decrypted with SSLKEYLOGFILE). + // handshakeloss/ — same client behaviour against the + // transferloss runner's sim with random packet loss. + // transfercorruption — random bit-flips (recovery via + // AEAD AUTH FAIL → drop + retransmit). + // longrtt — emulated high-latency link. + // goodput / crosstraffic — throughput-floor / competing-flow + // scenarios. + // multiplexing — H3 GETs issued in parallel; runner + // verifies overlap on the wire via tshark. + // retry — server sends a Retry packet first; our + // applyRetry path (RFC 9000 §17.2.5 + + // RFC 9001 §5.8) handles DCID swap + + // token threading + key re-derivation. + // ipv6 — same flow over an IPv6 socket; + // JDK DatagramChannel.connect handles + // the v6 address resolution natively. + "handshake", "chacha20", "handshakeloss", + "transfer", "http3", "multiplexing", + "transferloss", "transfercorruption", "longrtt", "goodput", "crosstraffic", + "retry", "ipv6", + // NOTE: the runner does NOT have a `versionnegotiation` testcase + // (its Available list excludes it). The :quic VN-handling code + // (applyVersionNegotiation, FORCE_VERSION_NEGOTIATION constant) + // stays as defensive support for any server that decides to + // send a VN packet at us, but no testcase exercises it directly. + -> { + runTransferTest( + requests = requests, + downloadsDir = downloadsDir, + cipherSuites = cipherSuites, + offeredAlpns = offeredAlpns, + initialVersion = initialVersion, + keyLogPath = keyLogPath, + qlogDir = qlogDir, + // The runner reuses TESTCASE_CLIENT=transfer for the + // multiplexing testcase — discrimination is by URL + // count, not testcase name. We were checking + // testcase == "multiplexing" which is NEVER true + // (we'd see TESTCASE=multiplexing only on a + // hypothetical client where the runner explicitly + // sets it). Symptom: 60s timeout with 1421/2000 + // files at 1 stream / RTT — exactly the serial + // client.get(...) path. Confirmed via the boot log: + // [boot] TESTCASE=transfer; transfer mode: + // parallel=false urls=1999 + // + // Cheap whitespace tokenization here just counts; + // runTransferTest re-parses into URI[] inside. + parallel = requests.split(Regex("\\s+")).count { it.isNotBlank() } > 1, + ) + } + + else -> { + EXIT_UNSUPPORTED + } + } + exitProcess(code) +} + +internal enum class Alpn( + val wireBytes: ByteArray, +) { + /** RFC 9114 — full HTTP/3 + QPACK + H3 framing. */ + H3("h3".encodeToByteArray()), + + /** quic-interop-runner convention — HTTP/0.9 over QUIC. Just `GET /path\r\n` + * on a fresh bidi stream, server returns the body, FIN both sides. No + * control stream, no QPACK, no SETTINGS. Used for handshake / chacha20 / + * transfer / loss-variant testcases. */ + HQ_INTEROP("hq-interop".encodeToByteArray()), +} + +private fun runTransferTest( + requests: String, + downloadsDir: File, + cipherSuites: IntArray?, + offeredAlpns: List, + initialVersion: Int, + keyLogPath: String?, + qlogDir: File?, + parallel: Boolean, +): Int { + val urls = + requests + .split(Regex("\\s+")) + .filter { it.isNotBlank() } + .map { runCatching { URI(it) }.getOrNull() } + .filter { it != null && it.host != null } + .map { it!! } + if (urls.isEmpty()) { + System.err.println("no parseable URL in REQUESTS") + return EXIT_FAIL + } + val first = urls[0] + val host = first.host + val port = first.port.takeIf { it > 0 } ?: 443 + + downloadsDir.mkdirs() + + val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + val outcome = + runBlocking { + val socket = + try { + UdpSocket.connect(host, port) + } catch (t: Throwable) { + return@runBlocking "udp_failed: ${t.message ?: t::class.simpleName}" + } + val keyLogger = keyLogPath?.let { SslKeyLogger(File(it)) } + val qlogWriter = + qlogDir?.let { dir -> + dir.mkdirs() + // ODCID is unknown until the connection generates one in + // its init block; we'd need to plumb through, but for the + // header it's fine to start with a placeholder and the + // packet-sent events will carry SCID/DCID anyway. + QlogWriter(file = File(dir, "client.sqlog"), odcidHex = "client") + } + val conn = + QuicConnection( + serverName = host, + config = + QuicConnectionConfig( + // Interop stress sizes — push the receive + // window past the largest single-file transfer + // any testcase exercises (longrtt does 5 MB, + // transferloss / transfercorruption do up to + // a few MB). Without this, the peer sends + // up to initialMaxStreamDataBidiLocal then + // stalls until our parser sends a + // MAX_STREAM_DATA — at high RTT (longrtt + // is 750 ms one-way) each stall is ~1.5 s + // round-trip lost. Setting both connection- + // and stream-level windows to 32 MB lets + // the peer's CC alone determine throughput. + initialMaxData = 32L * 1024 * 1024, + initialMaxStreamDataBidiLocal = 32L * 1024 * 1024, + initialMaxStreamDataBidiRemote = 32L * 1024 * 1024, + initialMaxStreamDataUni = 32L * 1024 * 1024, + ), + tlsCertificateValidator = PermissiveCertificateValidator(), + alpnList = offeredAlpns.map { it.wireBytes }, + initialVersion = initialVersion, + cipherSuites = + cipherSuites + ?: intArrayOf( + TlsConstants.CIPHER_TLS_AES_128_GCM_SHA256, + TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256, + ), + extraSecretsListener = keyLogger?.listener, + qlogObserver = qlogWriter ?: com.vitorpamplona.quic.observability.QlogObserver.NoOp, + ) + val driver = QuicConnectionDriver(conn, socket, scope) + driver.start() + + val handshake = + withTimeoutOrNull(HANDSHAKE_TIMEOUT_SEC * 1_000L) { + runCatching { conn.awaitHandshake() } + } + if (handshake == null || handshake.isFailure) { + runCatching { driver.close() } + conn.tls.clientRandom?.let { keyLogger?.flush(it) } + runCatching { qlogWriter?.close() } + return@runBlocking "handshake_failed" + } + + // Dispatch the GET client by what the server actually picked. + // If the server didn't pick or picked something unrecognized, + // fall back to HQ-interop (simpler, more permissive). + val negotiated = + conn.tls.negotiatedAlpn + ?.decodeToString() + .orEmpty() + val client: GetClient = + when (negotiated) { + "h3" -> { + Http3GetClient(conn, driver).also { it.init(scope) } + } + + "hq-interop" -> { + HqInteropGetClient(conn, driver) + } + + else -> { + System.err.println("unrecognized negotiated ALPN '$negotiated'; defaulting to hq-interop") + HqInteropGetClient(conn, driver) + } + } + + val authority = if (port == 443) host else "$host:$port" + // Unconditional one-shot log so we can confirm which branch + // runs even when DEBUG=0 — this is a control-flow boundary, + // not a hot-path trace. + System.err.println("[boot] transfer mode: parallel=$parallel urls=${urls.size}") + val outcome = + withTimeoutOrNull(TRANSFER_TIMEOUT_SEC * 1_000L) { + val responses = + if (parallel) { + // Multiplexing throughput note. Spawning 1999 + // simultaneous coroutines all racing the same + // conn.lock cratered throughput (~23 streams/sec + // on Mac+Rosetta, qlog-measured against aioquic). + // Lock contention scales superlinearly with + // suspended coroutines: every drainOutbound + // walks streamsList O(N), every openBidiStream + // queues behind every other waiter, and the + // dispatcher thrashes context-switching. + // + // Bound concurrency: process in chunks of + // [MULTIPLEX_PARALLELISM]. Each chunk is + // batched in two phases: + // 1. SERIAL prepareRequest for every URL + // in the chunk — opens the bidi + // stream, encodes the request, FINs. + // Synchronous; no async / no per-call + // wakeup. + // 2. SINGLE driver.wakeup() so the send + // loop drains all 64 enqueued requests + // in coalesced packets (multi-stream + // framing per drain) instead of one + // tiny packet per stream. + // 3. PARALLEL await — one async per + // stream collects its response with + // a per-stream timeout so a hung + // stream surfaces as status=0 instead + // of blocking its peers. + // + // Earlier shape (per-call wakeup inside + // client.get()) produced ~23 streams/sec + // because each individual enqueue tripped + // the send loop, which then drained alone + // (the other 63 coroutines hadn't queued + // yet on the dispatcher). Result: one + // ~80-byte packet per stream instead of + // ~10 streams/packet. Coalescing recovered + // by batching enqueues + single wake. + val collected = mutableListOf>() + // Quiet by default. QUIC_INTEROP_DEBUG=1 emits one + // line per chunk to stderr — wall-clock split between + // "all enqueued" and "all responded" lets us see + // whether time is spent in the writer (lots of ms + // before responses start arriving) or the server + // (responses dribble in over a long stretch). + val debug = System.getenv("QUIC_INTEROP_DEBUG") == "1" + val transferStartMs = nowMs() + if (debug) { + System.err.println( + "[interop] multiplex start: total_urls=${urls.size} " + + "MULTIPLEX_PARALLELISM=$MULTIPLEX_PARALLELISM " + + "expected_chunks=${(urls.size + MULTIPLEX_PARALLELISM - 1) / MULTIPLEX_PARALLELISM}", + ) + } + urls.chunked(MULTIPLEX_PARALLELISM).forEachIndexed { chunkIdx, chunk -> + val chunkStartMs = nowMs() + if (debug && chunkIdx < 3) { + System.err.println( + "[interop] chunk=$chunkIdx size=${chunk.size} starting prepareRequests", + ) + } + // Single lock-held batch open + enqueue. + // Without this, openBidiStream's per-call + // lock acquire / release lets the send loop + // interject between opens and drain one + // stream per packet. + val handles = client.prepareRequests(authority, chunk.map { it.path }) + driver.wakeup() + val enqueuedMs = nowMs() + coroutineScope { + val deferreds = + chunk.zip(handles).map { (url, handle) -> + async { + val resp = + withTimeoutOrNull(PER_STREAM_TIMEOUT_SEC * 1_000L) { + client.awaitResponse(handle) + } + url to (resp ?: GetResponse(status = 0, body = ByteArray(0))) + } + } + deferreds.forEach { collected += it.await() } + } + if (debug) { + val doneMs = nowMs() + System.err.println( + "[interop] chunk=$chunkIdx size=${chunk.size} " + + "enqueue=${enqueuedMs - chunkStartMs}ms " + + "responses=${doneMs - enqueuedMs}ms " + + "cumulative=${doneMs - transferStartMs}ms", + ) + } + } + collected + } else { + urls.map { url -> url to client.get(authority, url.path) } + } + var anyFailed = false + for ((url, resp) in responses) { + if (resp.status != 200) { + System.err.println("GET ${url.path} → status ${resp.status}") + anyFailed = true + continue + } + val name = url.path.substringAfterLast('/').ifBlank { "index" } + File(downloadsDir, name).writeBytes(resp.body) + } + if (anyFailed) "request_failed" else "ok" + } ?: "transfer_timeout" + + runCatching { driver.close() } + conn.tls.clientRandom?.let { keyLogger?.flush(it) } + runCatching { qlogWriter?.close() } + delay(50) + outcome + } + scope.cancel() + + return if (outcome == "ok") { + EXIT_OK + } else { + System.err.println("transfer $outcome") + EXIT_FAIL + } +} + +/** + * Writes [NSS Key Log Format](https://firefox-source-docs.mozilla.org/security/nss/legacy/key_log_format/index.html) + * lines so Wireshark can decrypt the sim's captured pcap. + * + * CLIENT_HANDSHAKE_TRAFFIC_SECRET + * SERVER_HANDSHAKE_TRAFFIC_SECRET + * CLIENT_TRAFFIC_SECRET_0 + * SERVER_TRAFFIC_SECRET_0 + */ +internal class SslKeyLogger( + private val file: File, +) { + private val pending = mutableListOf>() + + val listener: TlsSecretsListener = + object : TlsSecretsListener { + override fun onHandshakeKeysReady( + cipherSuite: Int, + clientSecret: ByteArray, + serverSecret: ByteArray, + ) { + pending += "CLIENT_HANDSHAKE_TRAFFIC_SECRET" to clientSecret + pending += "SERVER_HANDSHAKE_TRAFFIC_SECRET" to serverSecret + } + + override fun onApplicationKeysReady( + cipherSuite: Int, + clientSecret: ByteArray, + serverSecret: ByteArray, + ) { + pending += "CLIENT_TRAFFIC_SECRET_0" to clientSecret + pending += "SERVER_TRAFFIC_SECRET_0" to serverSecret + } + + override fun onHandshakeComplete() = Unit + } + + fun flush(clientRandom: ByteArray) { + val randomHex = clientRandom.toHex() + file.parentFile?.mkdirs() + file.appendText( + buildString { + for ((label, secret) in pending) { + append(label) + .append(' ') + .append(randomHex) + .append(' ') + .append(secret.toHex()) + .append('\n') + } + }, + ) + pending.clear() + } +} + +internal fun ByteArray.toHex(): String = + buildString(size * 2) { + for (b in this@toHex) { + val v = b.toInt() and 0xff + append("0123456789abcdef"[v ushr 4]) + append("0123456789abcdef"[v and 0xf]) + } + } diff --git a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/QlogWriter.kt b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/QlogWriter.kt new file mode 100644 index 0000000000..59f4f5afc1 --- /dev/null +++ b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/QlogWriter.kt @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop.runner + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.vitorpamplona.quic.connection.EncryptionLevel +import com.vitorpamplona.quic.observability.QlogObserver +import java.io.BufferedWriter +import java.io.Closeable +import java.io.File +import java.io.FileWriter +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +/** + * JSON-NDJSON qlog writer (qlog 0.3 / JSON-SEQ format) used by the + * `:quic` interop runner. One JSON object per line; first line is the + * qlog header, subsequent lines are events. + * + * Tools like qvis (https://qvis.quictools.info/) consume the resulting + * `.sqlog` file to render sequence diagrams + RTT graphs + recovery + * timelines. + * + * **Goal: every interop-runner test failure produces a qlog file the + * caller can drop into qvis to see exactly what we did differently + * from the spec.** + * + * Threading: [java.io.BufferedWriter] is not safe for concurrent + * writers; we hold a [ReentrantLock] around each line emit so the + * read + send loops can fire events concurrently without interleaving + * partial JSON. + */ +class QlogWriter( + file: File, + private val odcidHex: String, + private val mapper: ObjectMapper = DEFAULT_MAPPER, + /** Wall-clock provider; tests inject a deterministic source. */ + private val nowMillis: () -> Long = { System.currentTimeMillis() }, +) : QlogObserver, + Closeable { + // append=false → truncate any prior trace at this path so a reused + // QLOGDIR doesn't accumulate stale events from a previous run. + private val writer: BufferedWriter = BufferedWriter(FileWriter(file, false)) + + /** Latched true once close() runs OR once a write throws IOException. + * Subsequent emits short-circuit instead of throwing into the + * send loop and tearing down an otherwise-healthy connection. */ + @Volatile + private var closed: Boolean = false + private val lock = ReentrantLock() + private val startMillis: Long = nowMillis() + + init { + // qlog 0.3 JSON-SEQ header. qvis tolerates both `qlog_format` + // values "JSON-SEQ" and "NDJSON"; we use JSON-SEQ to match the + // most-common production qlog files (Chromium, mvfst). + val header = + mapOf( + "qlog_version" to "0.3", + "qlog_format" to "JSON-SEQ", + "title" to "amethyst :quic client trace", + "trace" to + mapOf( + "vantage_point" to mapOf("type" to "client", "name" to "amethyst-quic"), + "common_fields" to + mapOf( + "ODCID" to odcidHex, + "reference_time" to startMillis, + "time_format" to "relative", + ), + ), + ) + writeLineLocked(mapper.writeValueAsString(header)) + } + + override fun onConnectionStarted( + serverName: String, + dcid: ByteArray, + scid: ByteArray, + ) { + emit( + "transport:connection_started", + mapOf( + "ip_version" to "v4_or_v6", + "server_name" to serverName, + "dst_cid" to hex(dcid), + "src_cid" to hex(scid), + ), + ) + } + + override fun onConnectionClosed( + initiator: String, + errorCode: Long, + reason: String, + ) { + emit( + "transport:connection_closed", + mapOf( + "owner" to initiator, + "application_code" to errorCode, + "reason" to reason, + ), + ) + } + + override fun onPacketSent( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) { + emit( + "transport:packet_sent", + mapOf( + "header" to + mapOf( + "packet_type" to packetTypeFor(level), + "packet_number" to packetNumber, + ), + "raw" to mapOf("length" to sizeBytes), + "frames" to frames.map { mapOf("frame_type" to it) }, + ), + ) + } + + override fun onPacketReceived( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) { + emit( + "transport:packet_received", + mapOf( + "header" to + mapOf( + "packet_type" to packetTypeFor(level), + "packet_number" to packetNumber, + ), + "raw" to mapOf("length" to sizeBytes), + "frames" to frames.map { mapOf("frame_type" to it) }, + ), + ) + } + + override fun onPacketDropped( + reason: String, + sizeBytes: Int, + ) { + emit( + "transport:packet_dropped", + mapOf( + "trigger" to reason, + "raw" to mapOf("length" to sizeBytes), + ), + ) + } + + override fun onKeyUpdated( + keyType: String, + level: EncryptionLevel, + ) { + emit( + "security:key_updated", + mapOf( + "key_type" to "${keyType}_${packetTypeFor(level)}_secret", + "trigger" to "tls", + ), + ) + } + + override fun onLossDetected( + level: EncryptionLevel, + lostPacketNumbers: List, + ) { + for (pn in lostPacketNumbers) { + emit( + "recovery:packet_lost", + mapOf( + "header" to + mapOf( + "packet_type" to packetTypeFor(level), + "packet_number" to pn, + ), + "trigger" to "reordering_threshold_or_time_threshold", + ), + ) + } + } + + override fun onPtoFired( + consecutivePtoCount: Int, + ptoMillis: Long, + ) { + emit( + "recovery:loss_timer_updated", + mapOf( + "event_type" to "expired", + "timer_type" to "pto", + "pto_count" to consecutivePtoCount, + "delta" to ptoMillis, + ), + ) + } + + override fun onCongestionStateUpdated(newState: String) { + emit( + "recovery:congestion_state_updated", + mapOf("new" to newState), + ) + } + + override fun onTransportParametersSet( + initiator: String, + params: Map, + ) { + emit( + "transport:parameters_set", + mapOf( + "owner" to initiator, + "params" to params, + ), + ) + } + + override fun onAlpnNegotiated(alpn: String) { + emit( + "transport:alpn_information", + mapOf("chosen_alpn" to alpn), + ) + } + + override fun onVersionInformation( + chosenVersion: String, + otherVersionsOffered: List, + ) { + emit( + "transport:version_information", + mapOf( + "chosen_version" to chosenVersion, + "client_versions" to otherVersionsOffered, + ), + ) + } + + override fun close() { + lock.withLock { + closed = true + try { + writer.flush() + } finally { + writer.close() + } + } + } + + private fun emit( + name: String, + data: Map, + ) { + val event = + linkedMapOf( + "time" to (nowMillis() - startMillis), + "name" to name, + "data" to data, + ) + // Serialize OUTSIDE the lock so concurrent emitters don't + // serialize their JSON serially. The lock is only held while + // appending the line to the file. + val line = mapper.writeValueAsString(event) + writeLineLocked(line) + } + + private fun writeLineLocked(line: String) { + lock.withLock { + // The send loop may still emit packet_sent events for the + // CONNECTION_CLOSE packet after the application calls close() + // — observers MUST NOT break the connection. Silently swallow + // post-close IOExceptions; the trace is what we have. + if (closed) return + try { + writer.write(line) + writer.write("\n") + // Deliberately NOT flushing per event: this method runs + // inside conn.lock.withLock { drainOutbound(...) } on the + // hot send path. On macOS Docker Desktop the filesystem + // is virtualized and per-event flush is multi-millisecond. + // Every flush stalls the connection lock, blocks the + // receive loop, and the connection silently dies + // mid-handshake. close() does the only flush we need + // for normal completion. A hard-killed JVM loses recent + // events but that's an acceptable trade — the alternative + // is the connection never completing in the first place. + } catch (_: java.io.IOException) { + // Stream closed under us. Latch closed so subsequent emits + // skip the lock and return immediately. + closed = true + } + } + } + + companion object { + private val DEFAULT_MAPPER: ObjectMapper = jacksonObjectMapper() + + private fun packetTypeFor(level: EncryptionLevel): String = + when (level) { + EncryptionLevel.INITIAL -> "initial" + EncryptionLevel.HANDSHAKE -> "handshake" + EncryptionLevel.APPLICATION -> "1RTT" + } + + private val HEX_CHARS = "0123456789abcdef".toCharArray() + + fun hex(bytes: ByteArray): String { + val sb = StringBuilder(bytes.size * 2) + for (b in bytes) { + val v = b.toInt() and 0xFF + sb.append(HEX_CHARS[v ushr 4]) + sb.append(HEX_CHARS[v and 0x0F]) + } + return sb.toString() + } + } +} diff --git a/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClientTest.kt b/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClientTest.kt new file mode 100644 index 0000000000..6a06e8fa2c --- /dev/null +++ b/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClientTest.kt @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop.runner + +import com.vitorpamplona.quic.http3.Http3Frame +import com.vitorpamplona.quic.http3.Http3FrameReader +import com.vitorpamplona.quic.qpack.QpackDecoder +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class Http3GetClientTest { + @Test + fun `encodeRequest produces a single HEADERS frame with the four pseudo-headers`() { + val bytes = encodeRequest(authority = "example.com", path = "/file.bin") + + val reader = Http3FrameReader().apply { push(bytes) } + val frame = reader.next() + assertTrue(frame is Http3Frame.Headers, "first frame must be HEADERS, got $frame") + assertEquals(null, reader.next(), "should be exactly one frame") + + val fields = QpackDecoder().decodeFieldSection(frame.qpackPayload) + // QPACK emits headers in order; confirm pseudo-headers come first + // and carry the expected values. + val map = fields.associate { it.first to it.second } + assertEquals("GET", map[":method"]) + assertEquals("https", map[":scheme"]) + assertEquals("example.com", map[":authority"]) + assertEquals("/file.bin", map[":path"]) + } + + @Test + fun `encodeRequest survives an authority with a non-default port`() { + val bytes = encodeRequest(authority = "example.com:8443", path = "/") + val frame = Http3FrameReader().apply { push(bytes) }.next() + require(frame is Http3Frame.Headers) + val map = QpackDecoder().decodeFieldSection(frame.qpackPayload).associate { it.first to it.second } + assertEquals("example.com:8443", map[":authority"]) + assertEquals("/", map[":path"]) + } +} diff --git a/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/QlogWriterTest.kt b/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/QlogWriterTest.kt new file mode 100644 index 0000000000..340be74cca --- /dev/null +++ b/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/QlogWriterTest.kt @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop.runner + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.vitorpamplona.quic.connection.EncryptionLevel +import org.junit.Test +import java.io.File +import java.nio.file.Files +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * Validates the JSON-NDJSON shape qvis (https://qvis.quictools.info/) + * expects: header line + one event per line, every line independently + * parseable as JSON. + * + * Drives [QlogWriter] through one event of each type to confirm we + * don't emit anything that breaks the format. + */ +class QlogWriterTest { + @Test + fun headerThenOneEventPerLine_allParseable() { + val tmp = Files.createTempFile("amethyst-qlog-test", ".sqlog").toFile() + tmp.deleteOnExit() + var clock = 0L + QlogWriter(tmp, odcidHex = "deadbeef", nowMillis = { clock }).use { w -> + clock = 5L + w.onConnectionStarted("example.test", byteArrayOf(1, 2, 3), byteArrayOf(4, 5, 6)) + clock = 7L + w.onTransportParametersSet("local", mapOf("initial_max_data" to "1000000")) + clock = 10L + w.onPacketSent(EncryptionLevel.INITIAL, packetNumber = 0, sizeBytes = 1200, frames = listOf("crypto")) + clock = 15L + w.onPacketReceived(EncryptionLevel.INITIAL, packetNumber = 0, sizeBytes = 800, frames = listOf("crypto", "ack")) + clock = 20L + w.onPacketDropped("AEAD auth failed", sizeBytes = 80) + clock = 25L + w.onKeyUpdated("server", EncryptionLevel.HANDSHAKE) + clock = 30L + w.onLossDetected(EncryptionLevel.APPLICATION, lostPacketNumbers = listOf(3L, 5L)) + clock = 35L + w.onPtoFired(consecutivePtoCount = 1, ptoMillis = 333) + clock = 40L + w.onCongestionStateUpdated("recovery") + clock = 45L + w.onAlpnNegotiated("h3") + clock = 50L + w.onVersionInformation("v1", emptyList()) + clock = 55L + w.onConnectionClosed("local", errorCode = 0, reason = "done") + } + + val lines = tmp.readLines().filter { it.isNotBlank() } + assertTrue(lines.size >= 12, "expected >= 12 lines (header + at least 11 events) but got ${lines.size}") + + val mapper = jacksonObjectMapper() + + // Line 1: qlog header. + val header = mapper.readTree(lines[0]) + assertEquals("0.3", header.get("qlog_version").asText(), "qlog_version must be 0.3") + assertEquals("JSON-SEQ", header.get("qlog_format").asText(), "qlog_format must be JSON-SEQ") + val vp = header.get("trace").get("vantage_point") + assertEquals("client", vp.get("type").asText()) + assertEquals( + "deadbeef", + header + .get("trace") + .get("common_fields") + .get("ODCID") + .asText(), + ) + + // Lines 2..N: event objects with `time`, `name`, `data`. + for (i in 1 until lines.size) { + val node = mapper.readTree(lines[i]) + assertNotNull(node.get("time"), "line $i missing 'time': ${lines[i]}") + val name = node.get("name") + assertNotNull(name, "line $i missing 'name': ${lines[i]}") + assertTrue( + name.asText().contains(":"), + "name '${name.asText()}' must be in ':' form", + ) + assertNotNull(node.get("data"), "line $i missing 'data': ${lines[i]}") + } + + // Spot-check specific events made it through. + val names = lines.drop(1).map { mapper.readTree(it).get("name").asText() } + assertTrue(names.contains("transport:connection_started"), names.toString()) + assertTrue(names.contains("transport:packet_sent"), names.toString()) + assertTrue(names.contains("transport:packet_received"), names.toString()) + assertTrue(names.contains("transport:packet_dropped"), names.toString()) + assertTrue(names.contains("security:key_updated"), names.toString()) + assertTrue(names.contains("recovery:packet_lost"), names.toString()) + assertTrue(names.contains("recovery:loss_timer_updated"), names.toString()) + assertTrue(names.contains("transport:parameters_set"), names.toString()) + assertTrue(names.contains("transport:alpn_information"), names.toString()) + assertTrue(names.contains("transport:version_information"), names.toString()) + assertTrue(names.contains("transport:connection_closed"), names.toString()) + } + + @Test + fun timesAreRelativeToConstructorTime() { + val tmp = Files.createTempFile("amethyst-qlog-rel", ".sqlog").toFile() + tmp.deleteOnExit() + var clock = 1_000L + QlogWriter(tmp, odcidHex = "00", nowMillis = { clock }).use { w -> + clock = 1_050L + w.onAlpnNegotiated("h3") + } + val lines = tmp.readLines().filter { it.isNotBlank() } + val mapper = jacksonObjectMapper() + val event = mapper.readTree(lines[1]) + assertEquals(50L, event.get("time").asLong(), "time must be relative to constructor (1050 - 1000)") + } + + @Test + fun fileEndsWithNewline_qvisCompatible() { + val tmp = Files.createTempFile("amethyst-qlog-nl", ".sqlog").toFile() + tmp.deleteOnExit() + QlogWriter(tmp, odcidHex = "00").use { w -> + w.onAlpnNegotiated("h3") + } + val bytes = tmp.readBytes() + assertTrue(bytes.isNotEmpty(), "file must not be empty") + assertEquals('\n'.code.toByte(), bytes.last(), "file must end with '\\n' so trailing event parses") + } + + @Test + fun handlesEmptyFramesList(): Unit = + File.createTempFile("amethyst-qlog-empty", ".sqlog").let { tmp -> + tmp.deleteOnExit() + QlogWriter(tmp, odcidHex = "00").use { w -> + w.onPacketSent(EncryptionLevel.INITIAL, 0, 1200, emptyList()) + } + val mapper = jacksonObjectMapper() + val lines = tmp.readLines().filter { it.isNotBlank() } + val frames = mapper.readTree(lines[1]).get("data").get("frames") + assertTrue(frames.isArray, "frames must be an array even when empty") + assertEquals(0, frames.size()) + } +} diff --git a/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/SslKeyLoggerTest.kt b/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/SslKeyLoggerTest.kt new file mode 100644 index 0000000000..e9688d0d31 --- /dev/null +++ b/quic/interop/src/test/kotlin/com/vitorpamplona/quic/interop/runner/SslKeyLoggerTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop.runner + +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class SslKeyLoggerTest { + @Test + fun `emits NSS Key Log lines for handshake and application secrets`() { + val tmp = File.createTempFile("ssl-keylog-test", ".log").also { it.deleteOnExit() } + val logger = SslKeyLogger(tmp) + + logger.listener.onHandshakeKeysReady( + cipherSuite = 0x1301, + clientSecret = ByteArray(32) { 0xAA.toByte() }, + serverSecret = ByteArray(32) { 0xBB.toByte() }, + ) + logger.listener.onApplicationKeysReady( + cipherSuite = 0x1301, + clientSecret = ByteArray(32) { 0xCC.toByte() }, + serverSecret = ByteArray(32) { 0xDD.toByte() }, + ) + + val clientRandom = ByteArray(32) { it.toByte() } + logger.flush(clientRandom) + + val lines = + tmp + .readText() + .lineSequence() + .filter { it.isNotEmpty() } + .toList() + assertEquals(4, lines.size, "one line per secret") + + val randomHex = clientRandom.toHex() + val expectedClientHs = "CLIENT_HANDSHAKE_TRAFFIC_SECRET $randomHex ${ByteArray(32) { 0xAA.toByte() }.toHex()}" + val expectedServerHs = "SERVER_HANDSHAKE_TRAFFIC_SECRET $randomHex ${ByteArray(32) { 0xBB.toByte() }.toHex()}" + val expectedClientApp = "CLIENT_TRAFFIC_SECRET_0 $randomHex ${ByteArray(32) { 0xCC.toByte() }.toHex()}" + val expectedServerApp = "SERVER_TRAFFIC_SECRET_0 $randomHex ${ByteArray(32) { 0xDD.toByte() }.toHex()}" + + assertEquals(expectedClientHs, lines[0]) + assertEquals(expectedServerHs, lines[1]) + assertEquals(expectedClientApp, lines[2]) + assertEquals(expectedServerApp, lines[3]) + } + + @Test + fun `flush is idempotent — second flush emits nothing`() { + val tmp = File.createTempFile("ssl-keylog-test", ".log").also { it.deleteOnExit() } + val logger = SslKeyLogger(tmp) + + logger.listener.onHandshakeKeysReady( + cipherSuite = 0x1301, + clientSecret = ByteArray(32) { 1 }, + serverSecret = ByteArray(32) { 2 }, + ) + logger.flush(ByteArray(32)) + val firstLen = tmp.length() + + logger.flush(ByteArray(32)) + assertEquals(firstLen, tmp.length(), "second flush must not append") + } + + @Test + fun `toHex round-trips lowercase`() { + val bytes = byteArrayOf(0x00, 0x0f, 0x10.toByte(), 0xff.toByte()) + assertEquals("000f10ff", bytes.toHex()) + assertTrue(bytes.toHex().all { it.isDigit() || it in 'a'..'f' }) + } +} diff --git a/quic/interop/summarize-matrix.sh b/quic/interop/summarize-matrix.sh new file mode 100755 index 0000000000..56a7b901a3 --- /dev/null +++ b/quic/interop/summarize-matrix.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Summarize per-testcase results for the most recent matrix run. +# Useful when run-matrix.sh terminated early — the per-testcase +# output.txt files have status lines we can extract. +set -o pipefail +# NOT set -u: bash 3.2 (macOS default) treats `${arr[@]}` on an +# empty array as an unbound-variable error, which we hit on runs +# that have no log files (the artifact-inspection path is still +# valid). + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +RUNNER_LOGS="${REPO_ROOT}/../quic-interop-runner/logs" + +if [[ ! -d "$RUNNER_LOGS" ]]; then + echo "no runner logs at $RUNNER_LOGS" >&2 + exit 1 +fi + +RUN_DIR="$(ls -1dt "$RUNNER_LOGS"/run-* 2>/dev/null | head -n 1 || true)" +if [[ -z "$RUN_DIR" ]]; then + echo "no run-* dirs under $RUNNER_LOGS" >&2 + exit 1 +fi +echo "==> run dir: $RUN_DIR" +echo + +# Layout: ///output.txt +# The runner writes a "Test: took X, status: TestResult.{SUCCEEDED|FAILED|UNSUPPORTED}" +# line at the end of each test. +PAIR_DIR="$(ls -1d "$RUN_DIR"/*amethyst* 2>/dev/null | head -n 1 || true)" +if [[ -z "$PAIR_DIR" ]]; then + echo "no dir under $RUN_DIR" >&2 + exit 1 +fi + +# Some runner versions don't write status to per-testcase output.txt. +# Fall back to inferring from artifacts (qlog connection_closed events, +# pcap presence, etc.). We tag these with [inf] in the result column. +infer_status_from_artifacts() { + local tc_dir="$1" + local tc="$2" + # Did the connection formally close with an error? + local qlog + qlog=$(ls -1 "$tc_dir"/client/qlog/*.sqlog "$tc_dir"/client/qlog/*.qlog 2>/dev/null | head -n 1 || true) + if [[ -n "$qlog" ]]; then + # Did the server close us with an error code? + local cc + cc=$(grep '"name":"transport:connection_closed"' "$qlog" 2>/dev/null | tail -n 1) + if [[ -n "$cc" ]]; then + local reason + reason=$(echo "$cc" | sed -nE 's/.*"reason":"([^"]+)".*/\1/p') + echo "FAILED: $reason" + return + fi + # No close → did we get >0 packets received? If so, infer + # something happened. Status is genuinely uncertain. + local rx + rx=$(grep -c '"name":"transport:packet_received"' "$qlog" 2>/dev/null || echo 0) + if [[ "$rx" -gt 0 ]]; then + echo "RAN ($rx pkts rx; no formal close)" + return + fi + fi + echo "UNKNOWN (no qlog)" +} + +# The "Test: X took Y, status: TestResult.Z" line is written by run.py +# to its own stdout, not into the per-testcase output.txt. Search a few +# likely locations: the per-testcase output.txt (in case the runner +# version we're using writes there), the run dir, the runner-logs root, +# and the working dir we were invoked from. +# Build list of files to grep, in priority order. Bash 3.2 (macOS +# default) chokes on multiline process-substitution with comments, +# so we just push to an array imperatively. +SEARCH_FILES=() +# 1. run-matrix.sh tees the runner stdout to a sibling .stdout.log +# file — that has the authoritative "Test: X took Y, status:" lines. +if [[ -f "${RUN_DIR}.stdout.log" ]]; then + SEARCH_FILES+=("${RUN_DIR}.stdout.log") +fi +# 2. Per-testcase output.txt — older runner versions wrote status here. +while IFS= read -r f; do + SEARCH_FILES+=("$f") +done < <(find "$PAIR_DIR" -maxdepth 3 -name 'output.txt' -type f 2>/dev/null) +# 3. Run-dir / runner-logs *.log — catch-all. +while IFS= read -r f; do + SEARCH_FILES+=("$f") +done < <(find "$RUN_DIR" -maxdepth 1 -name '*.log' -type f 2>/dev/null) +while IFS= read -r f; do + SEARCH_FILES+=("$f") +done < <(find "$RUNNER_LOGS" -maxdepth 1 -name 'run-*.log' -type f 2>/dev/null) + +printf "%-22s %-15s %-10s\n" "TESTCASE" "RESULT" "TIME" +printf "%-22s %-15s %-10s\n" "----------------------" "---------------" "----------" +for tc_dir in "$PAIR_DIR"/*/; do + tc=$(basename "$tc_dir") + # Search every candidate file for a status line matching this + # testcase. Allow optional leading timestamp from the runner's + # logging format (`2026-05-07 12:34:56,789 Test: ...`). + line="" + for f in "${SEARCH_FILES[@]}"; do + [[ -f "$f" ]] || continue + match=$(grep -E "Test: $tc took [0-9.]+s, status: TestResult\." "$f" 2>/dev/null | tail -n 1) + if [[ -n "$match" ]]; then + line="$match" + break + fi + done + if [[ -n "$line" ]]; then + status=$(echo "$line" | sed -nE 's/.*TestResult\.([A-Z_]+).*/\1/p') + time=$(echo "$line" | sed -nE 's/.*took ([0-9.]+s).*/\1/p') + case "$status" in + SUCCEEDED) marker="✓" ;; + UNSUPPORTED) marker="?" ;; + FAILED) marker="✕" ;; + *) marker="·" ;; + esac + printf "%-22s %s %-13s %-10s\n" "$tc" "$marker" "$status" "$time" + else + # Fall back to artifact inspection. + inferred=$(infer_status_from_artifacts "$tc_dir" "$tc") + printf "%-22s %s [inf] %s\n" "$tc" "·" "$inferred" + fi +done diff --git a/quic/plans/2026-05-08-lock-split-design.md b/quic/plans/2026-05-08-lock-split-design.md new file mode 100644 index 0000000000..64cd0cb7fa --- /dev/null +++ b/quic/plans/2026-05-08-lock-split-design.md @@ -0,0 +1,223 @@ +# QuicConnection Lock Split — Design Note + +Date: 2026-05-08 + +## Problem + +`QuicConnection.lock: Mutex` serialises every meaningful operation: + + - `drainOutbound` (send loop, holds lock during a full datagram build — + iterates every stream, allocates packet numbers, encrypts). + - `feedDatagram` (read loop, holds lock during decrypt + frame dispatch + + per-stream insert). + - `openBidiStream` / `openUniStream` (app code, holds lock for stream + allocation + map insert). + - `getOrCreatePeerStreamLocked` (parser path on the read loop's + critical section, but app code can also call it from tests). + +Multiplexing test against aioquic measures ~25 streams/sec — every +coroutine fights this single mutex. + +## Goal + +Split the mutex into per-domain mutexes so the read loop, send loop, and +app code can mostly progress concurrently. Per-stream `synchronized(this)` +inside `SendBuffer`/`ReceiveBuffer` already handles per-stream +serialisation; we don't touch those. + +## Domain Map + +### Domain A — `streamsLock: Mutex` (the streams registry) + +Fields: + + - `streams: MutableMap` + - `streamsList: MutableList` (insertion-ordered list parallel + to `streams`, used by writer round-robin) + - `nextLocalBidiIndex`, `nextLocalUniIndex` + - `streamRoundRobinStart` — read+written by writer; used in the + same critical section it holds `streamsLock` for the iteration + - `peerInitiatedUniCount`, `peerInitiatedBidiCount` + - `advertisedMaxStreamsUni`, `advertisedMaxStreamsBidi`, + `advertisedMaxData` + - `pendingMaxStreamsUni`, `pendingMaxStreamsBidi`, `pendingMaxData` + - `pendingMaxStreamData: MutableMap` + - `pendingNewConnectionId: MutableMap` + - `newPeerStreams: ArrayDeque` + - `pendingDatagrams: ArrayDeque` — outbound DATAGRAMs + - `incomingDatagrams: ArrayDeque` — inbound DATAGRAMs + - `sendConnectionFlowCredit`, `sendConnectionFlowConsumed` + - `receiveConnectionFlowLimit` + +Rationale: the writer needs an atomic snapshot of "all streams + all +pending control-frame retransmits + datagram queues + flow-control +counters" in one critical section to assemble a packet. The parser needs +the same coverage when delivering a STREAM frame (look up or create +the stream + queue receive bytes + bump pending* fields). Splitting +these into multiple sub-locks would force the writer/parser to acquire +several locks per pass — same contention, more deadlock risk. + +`peerMaxStreamsBidi`, `peerMaxStreamsUni` stay `@Volatile` (already are): +the writer reads them once at the top of a stream open; the parser +writes once on inbound MAX_STREAMS. Atomic long write is sufficient on +all supported platforms. + +### Domain B — `LevelState.levelLock: Mutex` (one per level: initial / handshake / application) + +Fields per `LevelState`: + + - `pnSpace: PacketNumberSpaceState` + - `sentPackets: MutableMap` + - `ackTracker` + - `cryptoSend: SendBuffer`, `cryptoReceive: ReceiveBuffer` + - `sendProtection: PacketProtection?`, `receiveProtection: PacketProtection?` + - `keysDiscarded` + - `largestAckedPn`, `largestAckedSentTimeMs` + +The writer iterates through levels in order (initial → handshake → +application) when building a coalesced datagram. Each level's critical +section is independent, so the lock is held only for the duration of +build at that level (which doesn't touch the streams registry except +to read `streamsListLocked()` for stream frames inside the application +build — that read transitions through `streamsLock`). + +### Domain C — `lifecycleLock: Mutex` (status + handshake metadata) + +Fields: + + - `status: Status` + - `closeReason: String?`, `closeErrorCode: Long` + - `peerTransportParameters: TransportParameters?` — read-mostly after + handshake; using `@Volatile` reference + write-once-after-handshake + is sufficient here. Promoted to `@Volatile` so writer/parser can + snapshot without a lock. + - `handshakeComplete: Boolean` + - `closeAllSignals` (the channels are themselves thread-safe; lock is + only required to serialise the status transition) + +### Domain D — Atomic / `@Volatile` (no lock) + +Fields: + + - `pendingPing` — toggled by driver under PTO; observed by writer. + Promote to `@Volatile`. + - `consecutivePtoCount` — already `@Volatile`. Driver writes it under + its own logic; no further protection needed because it's only read + inside the same loop iteration that wrote it. + - `destinationConnectionId` — already has volatile semantics + (`internal set` on a `@Volatile var`). Stays as is. + - `udpStatsSupplier` — already `@Volatile`. + - `peerMaxStreamsBidi`, `peerMaxStreamsUni` — already `@Volatile`. + - `handshakeDoneSignal: CompletableDeferred` — coroutines + primitive, thread-safe. + +### Domain E — Per-stream (UNCHANGED) + +`QuicStream` already protects its `SendBuffer` / `ReceiveBuffer` with +internal `synchronized(this)` blocks. Nothing changes here. + +## Lock Acquisition Order + +To prevent deadlock, document and enforce: + +``` +lifecycleLock < streamsLock < (any LevelState.levelLock) +``` + +Per-stream `synchronized(...)` blocks inside `SendBuffer`/`ReceiveBuffer` +remain at the leaf — never acquire any QuicConnection mutex while +holding a per-stream lock. + +In practice the only nesting that happens is: + + - `drainOutbound` acquires `streamsLock` (for the streams loop + + stream-frame creation) but the per-level builds happen *outside* + that block — each level acquires its own `levelLock` separately. + No nested `streamsLock` ⊃ `levelLock` chain. + - Actually re-checking the design: the writer needs to allocate a + PN at the chosen level *while* it has decided which streams to + flush. Two options: + (1) acquire streamsLock, snapshot streams + frames, release; + acquire each levelLock to encode + record. + (2) hold streamsLock during level encode for the application + packet (because stream-frame retransmit tokens get recorded + into level.sentPackets in the same operation). + We take option (2) — encode under both locks, with strict order + `streamsLock` → `levelLock`. The other levels (initial/handshake) + don't touch streamsLock at all, so they only acquire `levelLock`. + +## Public API Compatibility + +`QuicConnection.lock: Mutex` is `val`-public. External callers exist +(tests + InMemoryQuicPipe-driven harnesses). To avoid breaking those: + + - Keep the `lock: Mutex` field as a deprecated forwarder. It now + *also* exists, but it is an alias for `lifecycleLock`. New code + must NOT use it. Existing tests that lock it before mutating + state used to cover all domains; we update them in place to use + the appropriate lock(s). + +Actually simpler: keep `lock: Mutex` as a *no-op* lock (still a +`Mutex` so external code compiles), document that it no longer +guards anything, update the tests that lock it. + +After review: tests use `conn.lock` to serialise their direct calls to +`onTokensAcked`/`onTokensLost`/`getOrCreatePeerStreamLocked`. We update +those tests to acquire `streamsLock` instead (since those routines +mutate stream-domain state). The `lock` field is kept as deprecated +for source compatibility but is functionally a leaf no-op. + +## Migration Plan + +1. Add `streamsLock`, `lifecycleLock` fields. Keep `lock` as alias of + `lifecycleLock`. +2. Add `levelLock` to `LevelState`. +3. Convert `getOrCreatePeerStreamLocked` → `getOrCreatePeerStream` doing + its own `streamsLock` acquisition. Keep the old name as a forwarder + for backwards compat. +4. Update `openBidiStream`, `openUniStream`, `streamById`, `pollIncomingPeerStream`, + `awaitIncomingPeerStream`, `pollIncomingDatagram`, `awaitIncomingDatagram`, + `queueDatagram`, `flowControlSnapshot` to acquire `streamsLock`. +5. Update `close`, `markClosedExternally` to use `lifecycleLock`. +6. Update driver's `readLoop`/`sendLoop`: + - `feedDatagram` no longer wraps in conn-wide lock. Instead the + parser acquires `streamsLock` around stream-touching code, + and `levelLock` around level-touching code. + - `drainOutbound` is restructured similarly. +7. Update tests that hold `conn.lock` to use the relevant new lock. + +## Risk + Mitigation + + - **Deadlock**: enforce order via code review + (where practical) + inline comments at each acquisition site. Keep nesting shallow. + - **Missed coverage**: enumerate every field in this doc; if a field + can be mutated from two domains we either move it to a single domain + or annotate it as @Volatile. + - **Performance regression**: more mutex acquisitions overall; but + the critical path (multiplexing test) sees parallel execution + instead of serial, which more than compensates. + +## Implementation Phases + +This commit implements **phase 1** — separate domain locks but +`drainOutbound` and `feedDatagram` still hold `streamsLock` for the +entire pass. The wins from phase 1 alone: + + - App code (`openBidiStream`, `streamById`, `flowControlSnapshot`) no + longer contends with `lifecycleLock`-only operations. + - The PTO timer path stops touching any mutex (volatile fields). + - `markClosedExternally` no longer needs a lock. + - `close()` only takes lifecycleLock — opens the path for in-progress + drain to finish without status-write contention. + +Phase 2 (deferred follow-up): split `buildApplicationPacket` into a +"collect frames under streamsLock" stage and an "encrypt + record +under levelLock" stage so app coroutines can intersperse during the +encrypt window. That requires more invasive surgery on the writer's +internals; phase 1 ships first to lock in the safer subset. + +## Verification + + - `:quic:jvmTest` — full suite must pass. + - `MultiplexingThroughputTest` (new): 1000 streams in <500 ms on + InMemoryQuicPipe. diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/LevelState.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/LevelState.kt index b3ba89e64a..87fe408ccc 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/LevelState.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/LevelState.kt @@ -24,7 +24,18 @@ import com.vitorpamplona.quic.connection.recovery.SentPacket import com.vitorpamplona.quic.stream.ReceiveBuffer import com.vitorpamplona.quic.stream.SendBuffer -/** Per-encryption-level state owned by [QuicConnection]. */ +/** + * Per-encryption-level state owned by [QuicConnection]. + * + * Concurrency: [cryptoSend] / [cryptoReceive] use their internal + * [SendBuffer] / [ReceiveBuffer] `synchronized(this)` blocks for + * thread safety — the writer's `takeChunk`, the parser's `markAcked`, + * and PTO-driven `requeueAllInflight` are all serialized through + * those leaf locks. [sentPackets] is currently mutated by the writer + * (under [QuicConnection.streamsLock]) and read by the parser without + * synchronization; that race is pre-existing audit-tracked and not + * fixed by [LevelState] today. + */ class LevelState { val pnSpace = PacketNumberSpaceState() @@ -122,4 +133,79 @@ class LevelState { largestAckedSentTimeMs = null keysDiscarded = true } + + /** + * RFC 9000 §6: reset every per-level field to a constructor-fresh + * state, then install [sendProtection] / [receiveProtection] keyed + * to the post-VN destination CID. + * + * Differs from [discardKeys] in that this re-arms the level for + * a fresh handshake — caller ( + * [QuicConnection.applyVersionNegotiation]) re-enqueues the cached + * ClientHello onto [cryptoSend] immediately afterwards, so the + * next outbound drain emits a v1 Initial with PN=0 and the same + * TLS bytes the original Initial carried. + */ + internal fun resetForVersionNegotiation( + sendProtection: PacketProtection, + receiveProtection: PacketProtection, + ) { + // pnSpace is val (lock-split refactor); reset its fields in place. + // The PacketNumberSpaceState.resetForRetry() name is historical but + // the semantics are right for VN too: zero out the PN counter + + // received-side state. + pnSpace.resetForRetry() + ackTracker = + com.vitorpamplona.quic.recovery + .AckTracker() + cryptoSend = SendBuffer() + cryptoReceive = ReceiveBuffer() + sentPackets.clear() + largestAckedPn = null + largestAckedSentTimeMs = null + keysDiscarded = false + this.sendProtection = sendProtection + this.receiveProtection = receiveProtection + } + + /** + * Reset the Initial-level state for a successful Retry. + * + * Differs from [resetForVersionNegotiation] in that the packet number + * namespace is **preserved**: per RFC 9001 §5.7 + RFC 9000 §17.2.5, + * the Initial PN space spans the original AND the post-Retry Initials. + * The client MUST NOT reuse a packet number across the boundary — + * the next Initial sent after Retry uses PN = (last-used) + 1. + * + * The runner's retry test specifically checks for this: a client that + * resets the PN gets "Client reset the packet number. Check failed + * for PN 0". + * + * Everything else resets to mirror VN semantics: + * - cryptoSend cleared so the caller can re-enqueue the cached + * ClientHello on top of fresh empty buffer state. + * - sentPackets cleared because the pre-Retry Initial's loss-recovery + * state references PNs the server will never ACK (the server + * discarded that packet when it sent the Retry). + * - keys re-derived from the new DCID and reinstalled. + * - keysDiscarded latch reset (the pre-Retry keys are gone, but the + * level itself is still alive with the new keys). + */ + internal fun resetForRetry( + sendProtection: PacketProtection, + receiveProtection: PacketProtection, + ) { + // pnSpace is INTENTIONALLY preserved — see kdoc above. + ackTracker = + com.vitorpamplona.quic.recovery + .AckTracker() + cryptoSend = SendBuffer() + cryptoReceive = ReceiveBuffer() + sentPackets.clear() + largestAckedPn = null + largestAckedSentTimeMs = null + keysDiscarded = false + this.sendProtection = sendProtection + this.receiveProtection = receiveProtection + } } diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/PacketNumberSpace.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/PacketNumberSpace.kt index b187dc512a..b6a48a2362 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/PacketNumberSpace.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/PacketNumberSpace.kt @@ -64,6 +64,20 @@ class PacketNumberSpaceState { if (nextPacketNumber > 0) nextPacketNumber-- } + /** + * RFC 9000 §17.2.5.2: reset the Initial-level packet-number space + * after a Retry packet rolls our DCID. The new Initial keys are + * derived from a different secret, so the (PN, key) pair the AEAD + * nonce relies on is unique even with the PN going back to 0. + * Inbound state is reset because no Initial packet has been + * received yet on the new keys. + */ + internal fun resetForRetry() { + nextPacketNumber = 0L + largestReceived = -1L + largestReceivedTime = 0L + } + /** Note that an inbound packet was successfully decrypted. */ fun observeInbound( packetNumber: Long, diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/PeerStreamDrain.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/PeerStreamDrain.kt new file mode 100644 index 0000000000..607d890dad --- /dev/null +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/PeerStreamDrain.kt @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.stream.StreamId +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.launch + +/** + * Spawn a long-lived coroutine that accepts every peer-initiated + * unidirectional stream this connection surfaces and drains it to + * `/dev/null`. Returns the [Job] of the launched dispatcher so the + * caller can join / cancel it. + * + * Why this exists: RFC 9114 §6.2.1 mandates that an HTTP/3 server + * opens at least three peer-initiated uni streams (CONTROL + + * QPACK_ENCODER + QPACK_DECODER) immediately after the handshake. + * [QuicConnectionParser] routes their bytes into each stream's + * bounded `incomingChannel` (capacity 64 chunks) — if no consumer + * reads them, the next chunk delivery overflows and the connection + * tears down with `INTERNAL_ERROR: stream … consumer overflowed` + * (audit-4 #3). The symptom for the multiplexing interop test was + * a zero-request connection that died after ~5 seconds the moment + * the server's QPACK encoder pushed dynamic-table inserts. + * + * This helper is the explicit "I do not care about these particular + * peer streams" knob for callers (e.g. an H3 GET client that runs + * with QPACK dynamic-table off) that don't need to interpret the + * SETTINGS / QPACK bytes. The :quic library does NOT default to + * silently dropping app bytes — apps that DO care about peer-uni + * streams (WebTransport, MoQ-over-WT) call + * [QuicConnection.awaitIncomingPeerStream] directly and route each + * stream by inspecting its leading varint. + * + * Bidi peer streams are deliberately re-queued back to + * [QuicConnection.newPeerStreams] (well — left untouched on the + * head when surfaced; we just don't consume them here) so that an + * application that opts in to draining uni streams doesn't + * accidentally swallow peer-initiated bidi requests. In the + * H3-client multiplexing case we never expect the server to open + * a bidi stream against us, but if it does the connection-level + * handling stays correct. + * + * Usage from an integrator (sketch — `Http3GetClient` is on a + * different branch on this repo today): + * + * ``` + * suspend fun init(scope: CoroutineScope) { + * // Open our own H3 control + QPACK uni streams. + * openH3ControlStream() + * openQpackEncoderStream() + * openQpackDecoderStream() + * + * // Accept the server's three counterparts and discard their bytes. + * conn.drainPeerInitiatedUniStreamsIntoBlackHole(scope) + * } + * ``` + * + * Lifecycle: the launched coroutine exits cleanly when + * [QuicConnection.awaitIncomingPeerStream] returns null (the + * connection has reached `CLOSED`). Cancelling the [scope] also + * tears it down. + */ +fun QuicConnection.drainPeerInitiatedUniStreamsIntoBlackHole(scope: CoroutineScope): Job = + scope.launch { + while (true) { + val stream = awaitIncomingPeerStream() ?: return@launch + // Only drain peer-initiated UNI streams. Peer bidi streams are + // returned to whatever else the application wants to do with + // them — but we have to put them somewhere because + // awaitIncomingPeerStream removed them from the queue. The + // pragmatic choice on a connection that uses this helper: + // log + ignore. If the application ALSO cares about peer + // bidi streams, it should NOT use this helper and instead + // implement its own routing dispatcher. + if (StreamId.kindOf(stream.streamId) != StreamId.Kind.SERVER_UNI) { + // Drain the bidi too — silently dropping bytes is bad + // policy, but tearing down the connection because the + // server opened an unexpected bidi is worse. The uni + // case is the documented one. + launch { drainStreamSilently(stream) } + continue + } + launch { drainStreamSilently(stream) } + } + } + +private suspend fun drainStreamSilently(stream: com.vitorpamplona.quic.stream.QuicStream) { + @Suppress("UNUSED_VARIABLE") + stream.incoming.collect { _ -> + // intentionally discarded; this stream is one the caller has + // declared it does not care about (typically the server's H3 + // CONTROL / QPACK_ENCODER / QPACK_DECODER streams). + } +} diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnection.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnection.kt index cbdbe2e9dc..e23cabdafe 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnection.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnection.kt @@ -24,6 +24,8 @@ import com.vitorpamplona.quic.crypto.AesEcbHeaderProtection import com.vitorpamplona.quic.crypto.InitialSecrets import com.vitorpamplona.quic.crypto.PlatformAesOneBlock import com.vitorpamplona.quic.crypto.bestAes128GcmAead +import com.vitorpamplona.quic.observability.QlogObserver +import com.vitorpamplona.quic.packet.QuicVersion import com.vitorpamplona.quic.stream.QuicStream import com.vitorpamplona.quic.stream.StreamId import com.vitorpamplona.quic.tls.TlsClient @@ -73,24 +75,119 @@ class QuicConnection( .toEpochMilliseconds() }, val alpnList: List = listOf(TlsConstants.ALPN_H3), + /** + * Optional second listener invoked after the connection's own + * key-installation listener. Used by the interop runner endpoint to + * dump SSLKEYLOG lines so Wireshark can decrypt captured pcaps. + * Default `null` keeps production callers unaffected. + */ + val extraSecretsListener: TlsSecretsListener? = null, + /** + * TLS cipher suites to offer in the ClientHello. Override to e.g. + * `intArrayOf(TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256)` for the + * `chacha20` interop testcase. Default matches [TlsClient]'s default. + */ + val cipherSuites: IntArray = + intArrayOf( + TlsConstants.CIPHER_TLS_AES_128_GCM_SHA256, + TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256, + ), + /** + * Version this connection puts in the FIRST Initial it sends. Defaults + * to [QuicVersion.V1]; the interop runner sets it to + * [QuicVersion.FORCE_VERSION_NEGOTIATION] for the `versionnegotiation` + * testcase, which drives the client through the RFC 9000 §6 VN flow. + */ + val initialVersion: Int = QuicVersion.V1, + /** + * Optional qlog observer (draft-marx-qlog). Production callers + * leave this at [QlogObserver.NoOp] (zero overhead). Interop / + * test runners attach a JSON-NDJSON writer so a failed run + * produces a `client.sqlog` consumable by qvis. + */ + val qlogObserver: QlogObserver = QlogObserver.NoOp, ) { val sourceConnectionId: ConnectionId = ConnectionId.random(8) var destinationConnectionId: ConnectionId = ConnectionId.random(8) internal set val originalDestinationConnectionId: ConnectionId = destinationConnectionId + /** + * Version the writer stamps into the long-header version field on the + * NEXT outbound Initial / Handshake packet. Initialised to + * [initialVersion]; switched to [QuicVersion.V1] by + * [applyVersionNegotiation] after a successful VN exchange. + */ + @Volatile + var currentVersion: Int = initialVersion + internal set + + /** + * RFC 9000 §6.2: a client MUST consume at most one VN response per + * connection. After [applyVersionNegotiation] runs once, any further + * inbound VN packet is dropped silently — the latch defends against a + * mid-handshake attacker who replays an old VN datagram to wedge us + * into an endless re-negotiation loop. + */ + @Volatile + var vnConsumed: Boolean = false + internal set + + /** + * RFC 9000 §17.2.5.1: the Retry token the server handed us in a Retry + * packet, which we must echo verbatim in the Token field of every + * subsequent Initial we send. Null until [applyRetry] runs. + */ + @Volatile + var retryToken: ByteArray? = null + internal set + + /** + * RFC 9000 §17.2.5.2: a client MUST NOT process more than one Retry + * packet per connection. Any subsequent Retry is silently dropped. + * Latched true by [applyRetry] on a successfully-verified Retry. + */ + @Volatile + var retryConsumed: Boolean = false + internal set + + /** + * Cached ClientHello bytes captured by [start]. Re-enqueued onto the + * fresh Initial-level [LevelState.cryptoSend] when + * [applyVersionNegotiation] or [applyRetry] resets the encryption + * level so the new Initial datagram still carries a valid TLS handshake. + * Without this the reset wipes the bytes that [TlsClient] already + * enqueued and the post-VN/post-Retry Initial would carry an empty + * CRYPTO frame. + */ + private var originalClientHello: ByteArray? = null + val initial = LevelState() val handshake = LevelState() val application = LevelState() + @Volatile var handshakeComplete: Boolean = false private set + /** + * Lock-split refactor (2026-05-08): @Volatile because the writer/parser + * read this without acquiring [lifecycleLock] (the field is written + * once at handshake completion, then immutable). + */ + @Volatile var peerTransportParameters: TransportParameters? = null private set enum class Status { HANDSHAKING, CONNECTED, CLOSING, CLOSED } + /** + * Lock-split refactor (2026-05-08): @Volatile so concurrent loops can + * read the status without a lock — coarse "are we still alive?" checks. + * Mutating transitions still go through [lifecycleLock] for atomicity + * with [closeReason]/[closeErrorCode] updates. + */ + @Volatile var status: Status = Status.HANDSHAKING internal set @@ -234,7 +331,11 @@ class QuicConnection( * emits a PING frame on the next drain. The PING elicits an * ACK from the peer; that ACK runs through loss detection and * declares any in-flight packets lost, triggering retransmit. + * + * Lock-split refactor (2026-05-08): @Volatile so the driver + * sets it without acquiring any mutex. */ + @Volatile internal var pendingPing: Boolean = false /** @@ -317,6 +418,8 @@ class QuicConnection( ) { handshake.sendProtection = packetProtectionFromSecret(cipherSuite, clientSecret) handshake.receiveProtection = packetProtectionFromSecret(cipherSuite, serverSecret) + qlogObserver.onKeyUpdated("client", EncryptionLevel.HANDSHAKE) + qlogObserver.onKeyUpdated("server", EncryptionLevel.HANDSHAKE) } override fun onApplicationKeysReady( @@ -326,13 +429,17 @@ class QuicConnection( ) { application.sendProtection = packetProtectionFromSecret(cipherSuite, clientSecret) application.receiveProtection = packetProtectionFromSecret(cipherSuite, serverSecret) + qlogObserver.onKeyUpdated("client", EncryptionLevel.APPLICATION) + qlogObserver.onKeyUpdated("server", EncryptionLevel.APPLICATION) } override fun onHandshakeComplete() { handshakeComplete = true if (status == Status.HANDSHAKING) status = Status.CONNECTED applyPeerTransportParameters() + tls.negotiatedAlpn?.let { qlogObserver.onAlpnNegotiated(it.decodeToString()) } handshakeDoneSignal.complete(Unit) + extraSecretsListener?.onHandshakeComplete() } } @@ -358,6 +465,7 @@ class QuicConnection( secretsListener = tlsListener, certificateValidator = tlsCertificateValidator, offeredAlpns = alpnList, + cipherSuites = cipherSuites, ) init { @@ -375,9 +483,165 @@ class QuicConnection( /** Begin the handshake — emits ClientHello into Initial CRYPTO. */ fun start() { + // qlog: emit connection_started + initial transport_parameters_set + // before any wire traffic so the trace makes chronological sense + // when handed to qvis. + qlogObserver.onConnectionStarted( + serverName = serverName, + dcid = destinationConnectionId.bytes, + scid = sourceConnectionId.bytes, + ) + qlogObserver.onTransportParametersSet("local", localTransportParametersSummary()) + // RFC 9000 §6: we're not doing version negotiation, so the + // chosen version is unconditional. + qlogObserver.onVersionInformation("v1", emptyList()) tls.start() // Drain ClientHello bytes into the Initial-level CRYPTO send buffer. - tls.pollOutbound(TlsClient.Level.INITIAL)?.let { initial.cryptoSend.enqueue(it) } + // Cache the bytes so [applyVersionNegotiation] can re-enqueue them + // onto a fresh cryptoSend after resetting Initial-level state. + // Cannot re-pollOutbound — the queue is destructive. + tls.pollOutbound(TlsClient.Level.INITIAL)?.let { + originalClientHello = it + initial.cryptoSend.enqueue(it) + } + } + + /** + * Apply a Version Negotiation packet (RFC 9000 §6) received from the + * server. The client offered [initialVersion]; the server replies with + * a list of versions it supports. We pick [QuicVersion.V1] from the + * list, regenerate the destination CID + Initial keys, reset the + * Initial encryption level, and re-emit the cached ClientHello so the + * next drain produces a valid v1 Initial packet. + * + * RFC 9000 §6.2 invariants enforced here: + * - The supported_versions list MUST NOT contain + * [initialVersion] — including it would mean the server received + * our offer and STILL replied with VN, which is a downgrade signal. + * Drop the packet (treat as no-op). + * - At most one VN per connection (latched via [vnConsumed]). + * - If we cannot speak any of the offered versions, fail the + * handshake. + * + * Caller MUST hold [lock] (the parser already does). + */ + internal fun applyVersionNegotiation(supportedVersions: List) { + // RFC 9000 §6.2: a second VN must be ignored. + if (vnConsumed) return + // Anti-downgrade: server-claimed support for the version we + // already offered indicates VN replay / spoof. Drop silently. + if (supportedVersions.contains(initialVersion)) return + // Pick a version we can speak. Today that's only v1. + if (!supportedVersions.contains(QuicVersion.V1)) { + signalHandshakeFailed( + QuicVersionNegotiationException( + "VERSION_NEGOTIATION: server offered ${supportedVersions.map { v -> "0x" + v.toUInt().toString(16) }}, " + + "client only supports 0x" + QuicVersion.V1.toUInt().toString(16), + ), + ) + markClosedExternally("VERSION_NEGOTIATION: no mutually supported version") + return + } + + // Latch BEFORE reset so a re-entrant inbound VN during the reset + // window is rejected by the early-return at the top. + vnConsumed = true + + // Generate a fresh destination CID. RFC 9000 §6.2 doesn't strictly + // require this (the server hasn't indexed our CID with any state + // since its only response was VN), but it matches what reference + // implementations do and keeps the post-VN connection + // cryptographically isolated from the pre-VN exchange. + val newDcid = ConnectionId.random(8) + destinationConnectionId = newDcid + + // Reset Initial-level state in place: fresh PN space (next + // allocateOutbound returns 0), fresh ackTracker, fresh + // cryptoSend / cryptoReceive, fresh sentPackets retention. + // Writer + parser only ever reach the level via [conn.initial], + // so we mutate the fields rather than swap the instance. + val proto = InitialSecrets.derive(newDcid.bytes) + val hp = AesEcbHeaderProtection(PlatformAesOneBlock) + val newSend = + PacketProtection(bestAes128GcmAead(proto.clientKey), proto.clientKey, proto.clientIv, hp, proto.clientHp) + val newReceive = + PacketProtection(bestAes128GcmAead(proto.serverKey), proto.serverKey, proto.serverIv, hp, proto.serverHp) + initial.resetForVersionNegotiation(sendProtection = newSend, receiveProtection = newReceive) + // Re-enqueue the ClientHello so the next drainOutbound emits a v1 + // Initial datagram with the same TLS handshake the original carried. + originalClientHello?.let { initial.cryptoSend.enqueue(it) } + + // Switch the writer's stamp to v1 so the next Initial / Handshake + // long-header carries the right version. + currentVersion = QuicVersion.V1 + } + + /** + * Apply a verified Retry packet per RFC 9000 §17.2.5 + RFC 9001 §5.8. + * Validates the retry-integrity tag, swaps DCID, re-derives Initial keys, + * resets the Initial PN space, and re-enqueues the cached ClientHello so + * the next outbound Initial carries `Token = retryPacket.retryToken`. + * + * Returns false on bad tag, second Retry (RFC 9000 §17.2.5.2), or + * pre-start (no original ClientHello captured) — all silently dropped. + */ + internal fun applyRetry( + retryPacket: com.vitorpamplona.quic.packet.RetryPacket, + originalPacketBytes: ByteArray, + ): Boolean { + if (retryConsumed) return false + if (!retryPacket.verifyIntegrityTag(originalPacketBytes, originalDestinationConnectionId.bytes)) { + return false + } + val savedClientHello = originalClientHello ?: return false + + destinationConnectionId = retryPacket.scid + val proto = InitialSecrets.derive(destinationConnectionId.bytes) + val hp = AesEcbHeaderProtection(PlatformAesOneBlock) + // Use resetForRetry, NOT resetForVersionNegotiation: RFC 9001 §5.7 + // requires the Initial PN namespace to continue across the Retry + // boundary. Resetting PN to 0 caused the runner to flag + // "Client reset the packet number. Check failed for PN 0". + initial.resetForRetry( + sendProtection = + PacketProtection(bestAes128GcmAead(proto.clientKey), proto.clientKey, proto.clientIv, hp, proto.clientHp), + receiveProtection = + PacketProtection(bestAes128GcmAead(proto.serverKey), proto.serverKey, proto.serverIv, hp, proto.serverHp), + ) + initial.cryptoSend.enqueue(savedClientHello) + + retryToken = retryPacket.retryToken + retryConsumed = true + return true + } + + private fun localTransportParametersSummary(): Map { + val out = LinkedHashMap(8) + out["initial_max_data"] = config.initialMaxData.toString() + out["initial_max_stream_data_bidi_local"] = config.initialMaxStreamDataBidiLocal.toString() + out["initial_max_stream_data_bidi_remote"] = config.initialMaxStreamDataBidiRemote.toString() + out["initial_max_stream_data_uni"] = config.initialMaxStreamDataUni.toString() + out["initial_max_streams_bidi"] = config.initialMaxStreamsBidi.toString() + out["initial_max_streams_uni"] = config.initialMaxStreamsUni.toString() + out["max_idle_timeout"] = config.maxIdleTimeoutMillis.toString() + out["max_udp_payload_size"] = config.maxUdpPayloadSize.toString() + out["max_datagram_frame_size"] = config.maxDatagramFrameSize.toString() + return out + } + + private fun peerTransportParametersSummary(tp: TransportParameters): Map { + val out = LinkedHashMap(10) + tp.initialMaxData?.let { out["initial_max_data"] = it.toString() } + tp.initialMaxStreamDataBidiLocal?.let { out["initial_max_stream_data_bidi_local"] = it.toString() } + tp.initialMaxStreamDataBidiRemote?.let { out["initial_max_stream_data_bidi_remote"] = it.toString() } + tp.initialMaxStreamDataUni?.let { out["initial_max_stream_data_uni"] = it.toString() } + tp.initialMaxStreamsBidi?.let { out["initial_max_streams_bidi"] = it.toString() } + tp.initialMaxStreamsUni?.let { out["initial_max_streams_uni"] = it.toString() } + tp.maxIdleTimeoutMillis?.let { out["max_idle_timeout"] = it.toString() } + tp.maxUdpPayloadSize?.let { out["max_udp_payload_size"] = it.toString() } + tp.maxDatagramFrameSize?.let { out["max_datagram_frame_size"] = it.toString() } + tp.maxAckDelay?.let { out["max_ack_delay"] = it.toString() } + return out } private fun buildLocalTransportParameters(): TransportParameters = @@ -397,6 +661,13 @@ class QuicConnection( maxDatagramFrameSize = config.maxDatagramFrameSize, ) + /** + * Lock-split refactor (2026-05-08): caller must hold [streamsLock] + * because we mutate [streams], [peerMaxStreamsBidi]/Uni, and + * [sendConnectionFlowCredit]. Invoked from the TLS listener inside + * [QuicConnectionParser.feedDatagram] which acquires [streamsLock] + * around CRYPTO-frame handling. + */ private fun applyPeerTransportParameters() { val raw = tls.peerTransportParameters ?: return val tp = TransportParameters.decode(raw) @@ -425,6 +696,7 @@ class QuicConnection( return } peerTransportParameters = tp + qlogObserver.onTransportParametersSet("remote", peerTransportParametersSummary(tp)) sendConnectionFlowCredit = tp.initialMaxData ?: 0L peerMaxStreamsBidi = tp.initialMaxStreamsBidi ?: 0L peerMaxStreamsUni = tp.initialMaxStreamsUni ?: 0L @@ -444,13 +716,41 @@ class QuicConnection( } /** - * Single mutex protecting connection-wide mutable state: streams map, - * datagram queues, stream-id counters, status. The driver acquires this - * around its read/send loops; public API methods listed below acquire it - * before mutating. Internal-only methods (used only from inside the - * driver loops) do NOT lock — caller must hold the lock. + * Lock-split refactor (2026-05-08): split the previous single + * `lock` into two independent mutexes so the read loop, send + * loop, and app coroutines can mostly progress in parallel. + * + * - [streamsLock] guards the streams registry, datagram queues, + * stream-id counters, connection-level flow-control bookkeeping, + * packet-number space + sentPackets retention + CRYPTO buffer + * mutations at every encryption level. The writer's drain and + * the parser's feed both take it. + * - [lifecycleLock] guards [status] / [closeReason] / + * [closeErrorCode] transitions. + * + * Per-stream and per-level buffer mutations serialize through + * `synchronized(this)` inside `SendBuffer` / `ReceiveBuffer` / + * `AckTracker` — those leaf locks are safe to take with or + * without an outer mutex held. + * + * Acquisition order to prevent deadlock: + * `lifecycleLock` → `streamsLock`. Never go the other way. + * + * The historical `lock` field is retained as an alias of + * [lifecycleLock] for source-compatibility with external callers + * (tests, harnesses, in-process bridges). New code MUST NOT use it + * — it no longer protects streams or level state. */ - val lock: Mutex = Mutex() + val streamsLock: Mutex = Mutex() + + val lifecycleLock: Mutex = Mutex() + + @Deprecated( + "Use streamsLock or lifecycleLock as appropriate. Lock-split refactor 2026-05-08.", + replaceWith = ReplaceWith("streamsLock"), + ) + val lock: Mutex + get() = lifecycleLock /** * Allocate a new client-initiated bidirectional stream. Locked. @@ -460,22 +760,97 @@ class QuicConnection( * check capacity proactively if the caller wants to back-pressure rather * than throw. */ - suspend fun openBidiStream(): QuicStream = - lock.withLock { - if (nextLocalBidiIndex >= peerMaxStreamsBidi) { - throw QuicStreamLimitException( - "peer-granted bidi stream cap reached " + - "(used=$nextLocalBidiIndex limit=$peerMaxStreamsBidi)", - ) + suspend fun openBidiStream(): QuicStream = streamsLock.withLock { openBidiStreamLocked() } + + /** + * Atomically open one bidi stream per [items] entry under a single + * [streamsLock] hold and run [init] for each (stream, item) inside + * the lock. The send loop cannot interject between opens — when it + * next drains it sees ALL N streams' frames ready and packs them + * into coalesced packets instead of emitting one tiny packet per + * stream. + * + * **`init` runs under `streamsLock`.** It must not suspend + * (the type signature enforces this) and SHOULD be fast — any + * expensive work (encoding, allocation-heavy formatting) belongs + * outside the call so it doesn't extend the lock-hold time. The + * intended shape per caller: + * + * val encoded = items.map { encode(it) } // outside + * conn.openBidiStreamsBatch(encoded) { stream, payload -> // under lock + * stream.send.enqueue(payload) + * stream.send.finish() + * Handle(stream) + * } + * + * This is the bug-resistant API for the prepareRequests pattern. + * The previous shape (caller manually wraps `streamsLock.withLock` + * around a loop of [openBidiStreamLocked]) regressed twice: once + * by holding the wrong lock, and once by skipping the wrapper + * entirely. Both shapes failed silently as "one STREAM per packet" + * under multiplex load, while the unit tests passed. + * + * Callers that just need a single stream should still use + * [openBidiStream]. [openBidiStreamLocked] remains public for the + * rare custom-batching scenarios that need finer control, but + * those callers should generally migrate to this API. + */ + suspend fun openBidiStreamsBatch( + items: List, + init: (QuicStream, I) -> R, + ): List { + if (items.isEmpty()) return emptyList() + val streamsBefore = if (writerDebugEnabled) streams.size else 0 + val result = + streamsLock.withLock { + items.map { init(openBidiStreamLocked(), it) } } - val id = StreamId.build(StreamId.Kind.CLIENT_BIDI, nextLocalBidiIndex++) - val stream = QuicStream(id, QuicStream.Direction.BIDIRECTIONAL) - stream.sendCredit = peerTransportParameters?.initialMaxStreamDataBidiRemote ?: config.initialMaxStreamDataBidiRemote - stream.receiveLimit = config.initialMaxStreamDataBidiLocal - streams[id] = stream - streamsList += stream - stream + if (writerDebugEnabled) { + System.err.println( + "[batch] openBidiStreamsBatch items=${items.size} returned=${result.size} " + + "streamsList_before=$streamsBefore streamsList_after=${streams.size}", + ) } + return result + } + + /** + * The streamsLock-holding primitive used by [openBidiStream] and + * [openBidiStreamsBatch]. Public so callers that need a custom + * batching shape (e.g. mixed bidi+uni opens) can compose it under + * a manual [streamsLock] hold. Caller MUST hold [streamsLock]. + */ + fun openBidiStreamLocked(): QuicStream { + // Mutex.isLocked is the only check we have — kotlinx.coroutines + // Mutex doesn't expose ownership without an `owner` argument, + // and we don't pass one in production. So this catches the + // common bug — caller used the wrong lock or no lock — but + // not the rarer case of "caller held a DIFFERENT lock that + // happens to be locked too." The interop runner's multiplexing + // failure on 2026-05-06 was precisely this: prepareRequests + // held lifecycleLock (`conn.lock`) and called this fn, the + // send loop's drainOutbound interleaved between opens, and + // we emitted one STREAM per packet (1421/2000 files in 60s). + check(streamsLock.isLocked) { + "openBidiStreamLocked requires streamsLock to be held — caller " + + "must wrap with streamsLock.withLock { ... }. Without that, " + + "drainOutbound can race the streams mutation and emit one " + + "STREAM per packet under multiplex load." + } + if (nextLocalBidiIndex >= peerMaxStreamsBidi) { + throw QuicStreamLimitException( + "peer-granted bidi stream cap reached " + + "(used=$nextLocalBidiIndex limit=$peerMaxStreamsBidi)", + ) + } + val id = StreamId.build(StreamId.Kind.CLIENT_BIDI, nextLocalBidiIndex++) + val stream = QuicStream(id, QuicStream.Direction.BIDIRECTIONAL) + stream.sendCredit = peerTransportParameters?.initialMaxStreamDataBidiRemote ?: config.initialMaxStreamDataBidiRemote + stream.receiveLimit = config.initialMaxStreamDataBidiLocal + streams[id] = stream + streamsList += stream + return stream + } /** * Allocate a new client-initiated unidirectional (write-only) stream. @@ -486,22 +861,57 @@ class QuicConnection( * [QuicStream.bestEffort]). Used for moq-lite group streams * carrying real-time Opus audio. */ - suspend fun openUniStream(bestEffort: Boolean = false): QuicStream = - lock.withLock { - if (nextLocalUniIndex >= peerMaxStreamsUni) { - throw QuicStreamLimitException( - "peer-granted uni stream cap reached " + - "(used=$nextLocalUniIndex limit=$peerMaxStreamsUni)", - ) - } - val id = StreamId.build(StreamId.Kind.CLIENT_UNI, nextLocalUniIndex++) - val stream = QuicStream(id, QuicStream.Direction.UNIDIRECTIONAL_LOCAL_TO_REMOTE, bestEffort = bestEffort) - stream.sendCredit = peerTransportParameters?.initialMaxStreamDataUni ?: config.initialMaxStreamDataUni - stream.receiveLimit = 0L // can't receive - streams[id] = stream - streamsList += stream - stream + suspend fun openUniStream(bestEffort: Boolean = false): QuicStream = streamsLock.withLock { openUniStreamLocked(bestEffort) } + + /** + * The streamsLock-holding primitive used by [openUniStream] and + * [openUniStreamsBatch]. Caller MUST hold [streamsLock]. + */ + fun openUniStreamLocked(bestEffort: Boolean = false): QuicStream { + check(streamsLock.isLocked) { + "openUniStreamLocked requires streamsLock to be held" } + if (nextLocalUniIndex >= peerMaxStreamsUni) { + throw QuicStreamLimitException( + "peer-granted uni stream cap reached " + + "(used=$nextLocalUniIndex limit=$peerMaxStreamsUni)", + ) + } + val id = StreamId.build(StreamId.Kind.CLIENT_UNI, nextLocalUniIndex++) + val stream = QuicStream(id, QuicStream.Direction.UNIDIRECTIONAL_LOCAL_TO_REMOTE, bestEffort = bestEffort) + stream.sendCredit = peerTransportParameters?.initialMaxStreamDataUni ?: config.initialMaxStreamDataUni + stream.receiveLimit = 0L // can't receive + streams[id] = stream + streamsList += stream + return stream + } + + /** + * Bug-resistant counterpart to [openBidiStreamsBatch] for uni + * streams. Atomically open one client-uni stream per [items] + * entry under a single [streamsLock] hold and run [init] for + * each (stream, item). + * + * **`init` runs under `streamsLock`** — same caveat as + * [openBidiStreamsBatch]: keep it fast, encode outside the call. + * + * Use this for moq audio-rooms and any other path that opens many + * uni streams in burst — without batching, each open releases the + * lock and the send loop can interject, emitting one stream per + * packet (the same shape that broke bidi multiplexing on + * 2026-05-06). [bestEffort] applies uniformly to every stream + * in the batch; mixed-mode batches need separate calls. + */ + suspend fun openUniStreamsBatch( + items: List, + bestEffort: Boolean = false, + init: (QuicStream, I) -> R, + ): List { + if (items.isEmpty()) return emptyList() + return streamsLock.withLock { + items.map { init(openUniStreamLocked(bestEffort), it) } + } + } /** Snapshot of peer-granted bidi cap. Reads do not need the lock — long writes are atomic on every supported platform. */ fun peerMaxStreamsBidiSnapshot(): Long = peerMaxStreamsBidi @@ -529,7 +939,7 @@ class QuicConnection( * See `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`. */ suspend fun flowControlSnapshot(): QuicFlowControlSnapshot = - lock.withLock { + streamsLock.withLock { val tp = peerTransportParameters // Sum bytes the application has enqueued but the writer // hasn't yet handed to a STREAM frame. A non-zero value @@ -568,17 +978,32 @@ class QuicConnection( ) } - suspend fun pollIncomingPeerStream(): QuicStream? = lock.withLock { newPeerStreams.removeFirstOrNull() } + suspend fun pollIncomingPeerStream(): QuicStream? = streamsLock.withLock { newPeerStreams.removeFirstOrNull() } /** * Suspends until a peer-initiated stream is queued OR the connection * closes. Returns null on close. Replaces the older `pollIncomingPeerStream * + delay(5)` busy-loop — this version wakes within microseconds of the * parser appending a stream and parks the coroutine the rest of the time. + * + * **An H3 application MUST consume peer-initiated streams.** RFC 9114 + * §6.2.1 mandates that the server opens at least three peer-initiated + * uni streams (CONTROL + QPACK_ENCODER + QPACK_DECODER). The parser + * routes their bytes into the per-[QuicStream] `incomingChannel` + * (capacity 64 chunks); if nothing accepts and reads them, the channel + * fills and the next inbound chunk trips the audit-4 #3 "slow consumer" + * tear-down at [QuicConnectionParser] (`INTERNAL_ERROR: stream … + * consumer overflowed`). Symptoms: under H3 multiplexing of many bidi + * request streams, the server's QPACK encoder issues a burst of + * dynamic-table inserts on its uni stream and the connection dies + * after ~5 s with zero requests completed. See + * [drainPeerInitiatedUniStreamsIntoBlackHole] for a one-line opt-in + * drainer that satisfies the contract when the H3 layer doesn't + * actually need the SETTINGS / QPACK bytes. */ suspend fun awaitIncomingPeerStream(): QuicStream? { while (true) { - lock.withLock { newPeerStreams.removeFirstOrNull() }?.let { return it } + streamsLock.withLock { newPeerStreams.removeFirstOrNull() }?.let { return it } if (status == Status.CLOSED) return null // select between "wakeup" and "closed" so neither path can hang. val keepWaiting = @@ -593,17 +1018,17 @@ class QuicConnection( if (!keepWaiting) { // After a close-wake, drain one more time to surface any // streams added between the last drain and the close. - lock.withLock { newPeerStreams.removeFirstOrNull() }?.let { return it } + streamsLock.withLock { newPeerStreams.removeFirstOrNull() }?.let { return it } return null } } } - suspend fun streamById(id: Long): QuicStream? = lock.withLock { streams[id] } + suspend fun streamById(id: Long): QuicStream? = streamsLock.withLock { streams[id] } - suspend fun queueDatagram(payload: ByteArray) = lock.withLock { pendingDatagrams.addLast(payload) } + suspend fun queueDatagram(payload: ByteArray) = streamsLock.withLock { pendingDatagrams.addLast(payload) } - suspend fun pollIncomingDatagram(): ByteArray? = lock.withLock { incomingDatagrams.removeFirstOrNull() } + suspend fun pollIncomingDatagram(): ByteArray? = streamsLock.withLock { incomingDatagrams.removeFirstOrNull() } /** * Suspending counterpart of [pollIncomingDatagram]. Returns null only when @@ -612,7 +1037,7 @@ class QuicConnection( */ suspend fun awaitIncomingDatagram(): ByteArray? { while (true) { - lock.withLock { incomingDatagrams.removeFirstOrNull() }?.let { return it } + streamsLock.withLock { incomingDatagrams.removeFirstOrNull() }?.let { return it } if (status == Status.CLOSED) return null val keepWaiting = select { @@ -620,7 +1045,7 @@ class QuicConnection( closedSignal.onReceiveCatching { false } } if (!keepWaiting) { - lock.withLock { incomingDatagrams.removeFirstOrNull() }?.let { return it } + streamsLock.withLock { incomingDatagrams.removeFirstOrNull() }?.let { return it } return null } } @@ -631,12 +1056,15 @@ class QuicConnection( errorCode: Long, reason: String, ) { - lock.withLock { + var firedQlog = false + lifecycleLock.withLock { if (status == Status.CLOSED || status == Status.CLOSING) return@withLock closeErrorCode = errorCode closeReason = reason status = Status.CLOSING + firedQlog = true } + if (firedQlog) qlogObserver.onConnectionClosed("local", errorCode, reason) // If a caller is suspended on awaitHandshake() and we're tearing down // before completion, fail the deferred so the caller throws instead // of hanging forever. @@ -648,7 +1076,16 @@ class QuicConnection( /** Called by the parser on inbound CONNECTION_CLOSE or by the driver on read-loop death. */ internal fun markClosedExternally(reason: String) { + val wasClosed = status == Status.CLOSED if (status != Status.CLOSED) status = Status.CLOSED + if (!wasClosed) { + // "remote" covers both peer-initiated CONNECTION_CLOSE and + // local invariant violations (CID mismatch, frame decode + // failure) that the parser surfaces as markClosedExternally. + // The reason string is the discriminator the trace consumer + // reads. + qlogObserver.onConnectionClosed("remote", closeErrorCode, reason) + } if (!handshakeComplete) { signalHandshakeFailed(QuicConnectionClosedException("connection closed externally: $reason")) } @@ -662,16 +1099,34 @@ class QuicConnection( * still `trySend(Unit)` into a never-consumed channel. All three channels * close idempotently, so calling this from both `close()` and * `markClosedExternally` is safe. + * + * Also closes every per-stream `incomingChannel` so application + * coroutines suspended on `stream.incoming.collect { … }` unblock with + * a clean Flow termination instead of hanging forever waiting for a + * FIN that will never come. Without this an interop run that drops + * the connection mid-response (e.g. quic-interop-runner's + * `multiplexing` case where 677 collectors were waiting for replies + * when the parser tripped INTERNAL_ERROR) leaves every per-stream + * collector pinned indefinitely. Closing the channel after the + * channel already has buffered chunks is safe — `consumeAsFlow` + * drains the buffer before terminating, so any bytes already + * delivered are surfaced to the collector before the Flow completes. */ private fun closeAllSignals() { closedSignal.close() peerStreamSignal.close() incomingDatagramSignal.close() + // Iterate the snapshot list (safe: we never remove from it). + // closeIncoming is idempotent on the underlying Channel.close(). + for (stream in streamsList) { + stream.closeIncoming() + } } /** - * Caller must hold [lock]. Used by [QuicConnectionParser] inside the - * driver's read loop, which already holds the connection lock. + * Caller must hold [streamsLock]. Used by [QuicConnectionParser] inside + * the driver's read loop, which already holds [streamsLock] around the + * stream-domain section of frame dispatch. */ internal fun getOrCreatePeerStreamLocked(id: Long): QuicStream { streams[id]?.let { return it } @@ -741,6 +1196,38 @@ class QuicConnection( EncryptionLevel.APPLICATION -> application } + /** + * RFC 9002 §6.2.4 PTO probe — spec-correct retransmit path. Move + * every byte currently sent-but-not-yet-ACK'd in the [level]'s + * CRYPTO send buffer back to its retransmit queue, so the next + * [com.vitorpamplona.quic.connection.drainOutbound] re-emits the + * same bytes (at the same offsets) inside a fresh CRYPTO frame on + * a new packet number. + * + * The driver calls this from its PTO branch when 1-RTT keys + * aren't yet installed — i.e. the handshake hasn't finished, so + * the only thing the peer could be missing is our ClientHello / + * ClientFinished. A bare PING is insufficient because if the + * server never saw our original Initial it has no DCID state to + * correlate a PING against (it'll be dropped). Retransmitting the + * CRYPTO actually advances the handshake. + * + * Idempotent: a second consecutive call is a no-op because the + * first call moved everything out of inFlight. Old `RecoveryToken.Crypto` + * entries in [LevelState.sentPackets] for the still-tracked + * original PNs remain harmless — when loss detection eventually + * declares them lost, [onTokensLost] re-runs `markLost` on the + * same offset/length range, which is itself idempotent (the bytes + * are already in retransmit or already ACK'd by then). + * + * Caller must hold [lock] (or call from inside an existing locked + * region — typically the driver's PTO branch under + * [QuicConnectionDriver.sendLoop]). + */ + internal fun requeueAllInflightCrypto(level: EncryptionLevel) { + levelState(level).cryptoSend.requeueAllInflight() + } + /** Caller must hold [lock]. Snapshot of streams for the driver's send loop. */ internal fun streamsLocked(): Map = streams @@ -954,6 +1441,17 @@ class QuicStreamLimitException( message: String, ) : RuntimeException(message) +/** + * RFC 9000 §6: the server replied with a Version Negotiation packet but + * the supported_versions list does not contain any version the client can + * speak (today: only [com.vitorpamplona.quic.packet.QuicVersion.V1]). + * The handshake is unrecoverable — caller must treat the connection as + * permanently failed. + */ +class QuicVersionNegotiationException( + message: String, +) : RuntimeException(message) + /** * Diagnostic snapshot of [QuicConnection]'s flow-control accounting at * a single moment. Returned by [QuicConnection.flowControlSnapshot]. diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionDriver.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionDriver.kt index 48f1986078..f0be55c393 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionDriver.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionDriver.kt @@ -35,10 +35,13 @@ import kotlinx.coroutines.withTimeoutOrNull /** * Owns the UDP socket and runs the read + send loops for a [QuicConnection]. * - * Synchronization: every public mutator on [QuicConnection] takes - * `connection.lock`; the driver acquires the same lock around feed + drain. - * That guarantees the read loop, send loop, and app coroutines never see a - * mid-mutation state of the streams map / datagram queues / counters. + * Synchronization (post lock-split refactor 2026-05-08): the driver no + * longer takes a single connection-wide lock around feed/drain. Instead + * [feedDatagram] and [drainOutbound] internally acquire `streamsLock` + * for the precise critical sections they touch — leaving app + * coroutines (`openBidiStream`, etc.) free to run in parallel with the + * I/O loops. Per-stream and per-level buffers serialize through their + * leaf `synchronized(this)` blocks. * * The send loop is woken by a `Channel(CONFLATED)` rather than a * polling timer — no idle CPU. App writes ([QuicConnection.queueDatagram] @@ -99,7 +102,9 @@ class QuicConnectionDriver( try { while (connection.status != QuicConnection.Status.CLOSED) { val datagram = socket.receive() ?: break - connection.lock.withLock { + // Phase 1 of the lock-split refactor: parser holds + // streamsLock for a single datagram-feed pass. + connection.streamsLock.withLock { feedDatagram(connection, datagram, nowMillis()) } // Inbound data may have produced new outbound (acks, crypto, etc.). @@ -123,11 +128,16 @@ class QuicConnectionDriver( // floor (the same prior-shipping behavior, kept for // handshake-timeout safety on lossy paths). while (connection.status != QuicConnection.Status.CLOSED) { - connection.lock.withLock { - while (true) { - val out = drainOutbound(connection, nowMillis()) ?: break - socket.send(out) - } + // Phase 1 of the lock-split refactor: the writer holds + // streamsLock for the build, releases it for the actual + // socket.send() so a slow socket doesn't stall app + // coroutines (open/close streams, queue datagrams). + while (true) { + val out = + connection.streamsLock.withLock { + drainOutbound(connection, nowMillis()) + } ?: break + socket.send(out) } val ptoBaseMs = if (connection.lossDetection.hasFirstRttSample) { @@ -145,15 +155,7 @@ class QuicConnectionDriver( Unit } if (woke == null) { - // PTO fired. Set pendingPing so the writer emits a - // PING on the next drain (RFC 9002 §6.2.4 probe - // packet). The peer's ACK feeds loss detection + - // retransmit (steps 5–6). - connection.lock.withLock { - connection.pendingPing = true - connection.consecutivePtoCount = - (connection.consecutivePtoCount + 1).coerceAtMost(6) - } + handlePtoFired(connection) } } } @@ -223,3 +225,67 @@ class QuicConnectionDriver( private const val CLOSE_FLUSH_TIMEOUT_MILLIS = 250L } } + +/** + * Spec-correct response to a PTO timer firing (RFC 9002 §6.2.4). Pre-1-RTT + * the probe packet MUST be ack-eliciting at the encryption level with + * unacknowledged data, and SHOULD retransmit the lost data rather than + * emit a bare PING — so we requeue ALL inflight CRYPTO bytes at the + * highest active pre-application level (Initial or Handshake), and the + * next [drainOutbound] emits a CRYPTO frame at the original offset. + * + * `pendingPing` stays set as a fallback. `collectHandshakeLevelFrames` + * suppresses the PING when CRYPTO is in the same frame list, so we + * don't waste a frame on top of the retransmit. Post-1-RTT we keep + * the bare-PING behavior — STREAM loss detection drives retransmit + * from the ACK that the PING elicits. + * + * Why aioquic interop demands this: aioquic strictly rejects pre- + * handshake Initials that contain no CRYPTO frame + * (`CONNECTION_CLOSE 0x0 "Packet contains no CRYPTO frame"`). A + * bare-PING probe before the ClientHello is acknowledged is fatal. + * + * Extracted from [QuicConnectionDriver.sendLoop]'s PTO branch into a + * top-level helper so the unit test in + * [com.vitorpamplona.quic.connection.PtoCryptoRetransmitTest] + * can invoke the EXACT logic the live driver does, without standing + * up a UDP socket. Earlier shapes simulated the steps inline in the + * test, which let the driver-side wiring regress twice + * (commits c0d7b6031, then again in the lock-split refactor) without + * any test breaking. + * + * Concurrency: `pendingPing` and `consecutivePtoCount` are `@Volatile`. + * [QuicConnection.requeueAllInflightCrypto] delegates to + * [com.vitorpamplona.quic.stream.SendBuffer.requeueAllInflight] which + * is `synchronized(this)` internally, so it's safe to call without + * an external lock — even concurrent with the writer's `takeChunk`. + * If the parser concurrently runs `discardKeys` on the same level, + * `requeueAllInflight` operates on the buffer reference we captured + * (or the fresh one — both are valid) and is at worst a no-op. + */ +internal fun handlePtoFired(conn: QuicConnection) { + conn.pendingPing = true + if (conn.application.sendProtection == null) { + val level = highestPreApplicationLevel(conn) + if (level != null) { + conn.requeueAllInflightCrypto(level) + } + } + conn.consecutivePtoCount = (conn.consecutivePtoCount + 1).coerceAtMost(6) +} + +/** + * Highest encryption level for which `conn` currently holds send keys + * AND hasn't yet discarded them, given that 1-RTT keys are NOT + * installed. Returns null when the level state has been completely + * cleared (e.g. CLOSED after a CONNECTION_CLOSE was sent). Mirrors the + * private helper in [com.vitorpamplona.quic.connection.QuicConnectionWriter] + * — kept in lockstep so the driver's PTO branch and the writer's PING + * placement target the same level. + */ +private fun highestPreApplicationLevel(conn: QuicConnection): EncryptionLevel? = + when { + conn.handshake.sendProtection != null -> EncryptionLevel.HANDSHAKE + conn.initial.sendProtection != null && !conn.initial.keysDiscarded -> EncryptionLevel.INITIAL + else -> null + } diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionParser.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionParser.kt index 284b05f4f4..9efcbafead 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionParser.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionParser.kt @@ -37,8 +37,11 @@ import com.vitorpamplona.quic.frame.ResetStreamFrame import com.vitorpamplona.quic.frame.StopSendingFrame import com.vitorpamplona.quic.frame.StreamFrame import com.vitorpamplona.quic.frame.decodeFrames +import com.vitorpamplona.quic.observability.qlogFrameName import com.vitorpamplona.quic.packet.LongHeaderPacket import com.vitorpamplona.quic.packet.LongHeaderType +import com.vitorpamplona.quic.packet.QuicVersion +import com.vitorpamplona.quic.packet.RetryPacket import com.vitorpamplona.quic.packet.ShortHeaderPacket import com.vitorpamplona.quic.stream.StreamId import com.vitorpamplona.quic.tls.TlsClient @@ -51,6 +54,15 @@ import com.vitorpamplona.quic.tls.TlsClient * — typically Initial + Handshake from the server in the same datagram during * the handshake. We loop until the datagram is fully consumed or a packet * fails to parse (which we drop silently per RFC 9001 §5.5). + * + * Lock-split refactor (2026-05-08): caller must hold + * [QuicConnection.streamsLock]. The driver wraps its read loop in + * `streamsLock.withLock { feedDatagram(...) }`. Test harnesses that drive + * single-threaded packet flow (no concurrent app code) may invoke this + * directly without lock acquisition; the runtime invariants still hold + * because there's no contending thread. Phase 1 wraps the whole feed + * under streamsLock so frame-dispatch / stream creation / level state + * remains a single critical section. */ fun feedDatagram( conn: QuicConnection, @@ -62,6 +74,23 @@ fun feedDatagram( val first = datagram[offset].toInt() and 0xFF val isLong = (first and 0x80) != 0 if (isLong) { + // RFC 9000 §17.2.1: a Version Negotiation packet has the form + // bit set but version=0. Detect it BEFORE peekHeader, which + // assumes a v1-shaped layout (token, length fields). + if (offset + 5 <= datagram.size) { + val version = + ((datagram[offset + 1].toInt() and 0xFF) shl 24) or + ((datagram[offset + 2].toInt() and 0xFF) shl 16) or + ((datagram[offset + 3].toInt() and 0xFF) shl 8) or + (datagram[offset + 4].toInt() and 0xFF) + if (version == QuicVersion.VERSION_NEGOTIATION) { + feedVersionNegotiationPacket(conn, datagram, offset) + // VN packets MUST be the only packet in their datagram + // (RFC 9000 §17.2.1: no length field, body is rest of + // datagram). Stop walking. + return + } + } // Per RFC 9001 §5.5, drop ONLY the failing packet, not subsequent // coalesced ones. Use peekHeader to advance over a packet whose // payload we couldn't decrypt; only break the loop on a header @@ -78,6 +107,66 @@ fun feedDatagram( } } +/** + * RFC 9000 §17.2.1 / §6: parse a Version Negotiation packet and dispatch + * to [QuicConnection.applyVersionNegotiation]. + * + * Wire layout: + * + * first byte (form=1, unused 4 bits — server fills with random) + * version (4 bytes, fixed at 0x00000000) + * dcid_len (1 byte) + dcid (dcid_len bytes) + * scid_len (1 byte) + scid (scid_len bytes) + * supported_versions: sequence of 32-bit big-endian version numbers, + * consuming the rest of the UDP datagram. + * + * VN packets are NOT AEAD-protected — there's nothing to decrypt. We do + * a minimal sanity check (DCID matches our SCID) and then hand the + * version list off. Malformed packets are dropped silently per RFC 9000 + * §17.2.1 ("an endpoint MUST NOT send … in response to a Version + * Negotiation packet"). + */ +private fun feedVersionNegotiationPacket( + conn: QuicConnection, + datagram: ByteArray, + offset: Int, +) { + // Layout fields above; bail early if any read would run past the + // end of the datagram (truncated VN — drop silently). + var pos = offset + 5 + if (pos >= datagram.size) return + val dcidLen = datagram[pos].toInt() and 0xFF + pos += 1 + if (dcidLen > 20 || pos + dcidLen > datagram.size) return + val dcid = datagram.copyOfRange(pos, pos + dcidLen) + pos += dcidLen + if (pos >= datagram.size) return + val scidLen = datagram[pos].toInt() and 0xFF + pos += 1 + if (scidLen > 20 || pos + scidLen > datagram.size) return + pos += scidLen // SCID body — not validated; servers may pick anything. + + // RFC 9000 §6.1: the VN packet's destination CID MUST equal the SCID + // the client put in its first Initial. Mismatch ⇒ probable spoof + // from an off-path attacker; drop without state change. + if (!dcid.contentEquals(conn.sourceConnectionId.bytes)) return + + val versionsRegion = datagram.size - pos + if (versionsRegion <= 0 || versionsRegion % 4 != 0) return // malformed + + val supportedVersions = ArrayList(versionsRegion / 4) + while (pos + 4 <= datagram.size) { + val v = + ((datagram[pos].toInt() and 0xFF) shl 24) or + ((datagram[pos + 1].toInt() and 0xFF) shl 16) or + ((datagram[pos + 2].toInt() and 0xFF) shl 8) or + (datagram[pos + 3].toInt() and 0xFF) + supportedVersions += v + pos += 4 + } + conn.applyVersionNegotiation(supportedVersions) +} + private fun feedLongHeaderPacket( conn: QuicConnection, datagram: ByteArray, @@ -85,14 +174,49 @@ private fun feedLongHeaderPacket( nowMillis: Long, ): Int? { val peeked = LongHeaderPacket.peekHeader(datagram, offset) ?: return null + // RFC 9000 §17.2.5 + RFC 9001 §5.8: Retry has no PN space, no AEAD + // payload protection, and cannot be coalesced with anything else + // (it consumes the rest of the datagram via peekHeader.totalLength). + // Branch out before the standard parse-and-decrypt path. + if (peeked.type == LongHeaderType.RETRY) { + val retryBytes = datagram.copyOfRange(offset, offset + peeked.totalLength) + val retryPacket = RetryPacket.parse(retryBytes) + if (retryPacket != null) { + // applyRetry returns false on bad-tag / second-Retry / pre-start — + // in all of those cases we silently drop without advancing state. + conn.applyRetry(retryPacket, retryBytes) + } + return peeked.totalLength + } val level = when (peeked.type) { - LongHeaderType.INITIAL -> EncryptionLevel.INITIAL - LongHeaderType.HANDSHAKE -> EncryptionLevel.HANDSHAKE - LongHeaderType.ZERO_RTT, LongHeaderType.RETRY -> return null // unsupported in client + LongHeaderType.INITIAL -> { + EncryptionLevel.INITIAL + } + + LongHeaderType.HANDSHAKE -> { + EncryptionLevel.HANDSHAKE + } + + LongHeaderType.ZERO_RTT, LongHeaderType.RETRY -> { + // Not supported by client; surface as a drop so qvis can + // see we ignored a packet rather than silently moving on. + conn.qlogObserver.onPacketDropped( + "unsupported long-header type ${peeked.type}", + peeked.totalLength, + ) + return null + } } val state = conn.levelState(level) - val proto = state.receiveProtection ?: return null + val proto = state.receiveProtection + if (proto == null) { + conn.qlogObserver.onPacketDropped( + "no receive keys at level $level", + peeked.totalLength, + ) + return null + } val parsed = LongHeaderPacket.parseAndDecrypt( bytes = datagram, @@ -103,7 +227,14 @@ private fun feedLongHeaderPacket( hp = proto.hp, hpKey = proto.hpKey, largestReceivedInSpace = state.pnSpace.largestReceived, - ) ?: return null + ) + if (parsed == null) { + conn.qlogObserver.onPacketDropped( + "AEAD auth failed or header parse failed at level $level", + peeked.totalLength, + ) + return null + } state.pnSpace.observeInbound(parsed.packet.packetNumber, nowMillis) @@ -112,6 +243,14 @@ private fun feedLongHeaderPacket( conn.destinationConnectionId = parsed.packet.scid } + if (conn.qlogObserver !== com.vitorpamplona.quic.observability.QlogObserver.NoOp) { + conn.qlogObserver.onPacketReceived( + level = level, + packetNumber = parsed.packet.packetNumber, + sizeBytes = parsed.consumed, + frames = peekFrameNames(parsed.packet.payload), + ) + } dispatchFrames(conn, level, parsed.packet.payload, parsed.packet.packetNumber, nowMillis) return parsed.consumed } @@ -123,7 +262,14 @@ private fun feedShortHeaderPacket( nowMillis: Long, ) { val state = conn.levelState(EncryptionLevel.APPLICATION) - val proto = state.receiveProtection ?: return + val proto = state.receiveProtection + if (proto == null) { + conn.qlogObserver.onPacketDropped( + "no application receive keys", + datagram.size - offset, + ) + return + } val parsed = ShortHeaderPacket.parseAndDecrypt( bytes = datagram, @@ -135,11 +281,42 @@ private fun feedShortHeaderPacket( hp = proto.hp, hpKey = proto.hpKey, largestReceivedInSpace = state.pnSpace.largestReceived, - ) ?: return + ) + if (parsed == null) { + conn.qlogObserver.onPacketDropped( + "AEAD auth failed or header parse failed at level APPLICATION", + datagram.size - offset, + ) + return + } state.pnSpace.observeInbound(parsed.packet.packetNumber, nowMillis) + if (conn.qlogObserver !== com.vitorpamplona.quic.observability.QlogObserver.NoOp) { + conn.qlogObserver.onPacketReceived( + level = EncryptionLevel.APPLICATION, + packetNumber = parsed.packet.packetNumber, + sizeBytes = datagram.size - offset, + frames = peekFrameNames(parsed.packet.payload), + ) + } dispatchFrames(conn, EncryptionLevel.APPLICATION, parsed.packet.payload, parsed.packet.packetNumber, nowMillis) } +/** + * Decode the payload's frames just to surface their qlog names. Reuses + * the same [com.vitorpamplona.quic.frame.decodeFrames] path as + * [dispatchFrames]; if it throws (malformed peer payload), we return + * an empty list — the dispatch path will catch the same exception + * and surface the close via `markClosedExternally`. + */ +private fun peekFrameNames(payload: ByteArray): List = + try { + com.vitorpamplona.quic.frame + .decodeFrames(payload) + .map { qlogFrameName(it::class.simpleName ?: "frame") } + } catch (_: QuicCodecException) { + emptyList() + } + private fun dispatchFrames( conn: QuicConnection, level: EncryptionLevel, @@ -225,6 +402,9 @@ private fun dispatchFrames( for (lostPacket in lost) { conn.onTokensLost(lostPacket.tokens) } + if (lost.isNotEmpty()) { + conn.qlogObserver.onLossDetected(level, lost.map { it.packetNumber }) + } } } diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionWriter.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionWriter.kt index 7b55987385..88792a4e3a 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionWriter.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/QuicConnectionWriter.kt @@ -37,6 +37,8 @@ import com.vitorpamplona.quic.frame.ResetStreamFrame import com.vitorpamplona.quic.frame.StopSendingFrame import com.vitorpamplona.quic.frame.StreamFrame import com.vitorpamplona.quic.frame.encodeFrames +import com.vitorpamplona.quic.observability.QlogObserver +import com.vitorpamplona.quic.observability.qlogFrameName import com.vitorpamplona.quic.packet.LongHeaderPacket import com.vitorpamplona.quic.packet.LongHeaderPlaintextPacket import com.vitorpamplona.quic.packet.LongHeaderType @@ -55,6 +57,15 @@ import com.vitorpamplona.quic.packet.ShortHeaderPlaintextPacket * * RFC 9000 §14: any datagram containing an Initial packet from the client * MUST be padded to at least 1200 bytes total. + * + * Lock-split refactor (2026-05-08): caller must hold + * [QuicConnection.streamsLock]. Phase 1 keeps level-state mutation + * inline under the same critical section as the streams-domain work + * the writer needs — the win comes from `lifecycleLock`-only callers + * (close(), status reads, PTO bookkeeping) no longer fighting this lock. + * The driver wraps `streamsLock.withLock { drainOutbound(...) }`; tests + * that drive single-threaded send paths can call this without holding + * the lock — there's no contending thread. */ fun drainOutbound( conn: QuicConnection, @@ -62,12 +73,19 @@ fun drainOutbound( ): ByteArray? { val parts = mutableListOf() - // Closing — emit a CONNECTION_CLOSE at the highest available level. + // Closing — emit a CONNECTION_CLOSE at the highest available level. The + // datagram-build paths below MUST satisfy two RFC 9000 constraints we + // got wrong before: + // - §10.2.3: at Initial / Handshake levels, only CONNECTION_CLOSE + // (Transport, 0x1c) is allowed; the application-level close (0x1d) is + // forbidden because app state would leak before the handshake is + // encrypted with the application key. + // - §14.1: a client datagram containing an Initial MUST be ≥ 1200 bytes + // in UDP-payload terms, even when carrying only a CONNECTION_CLOSE. if (conn.status == QuicConnection.Status.CLOSING) { - val frame = ConnectionCloseFrame(conn.closeErrorCode, null, conn.closeReason ?: "") - val packet = buildBestLevelPacket(conn, listOf(frame)) ?: return null + val datagram = buildClosingDatagram(conn, nowMillis) conn.status = QuicConnection.Status.CLOSED - return packet + return datagram } // Drain destructive frame sources into local lists, ONCE. @@ -118,21 +136,42 @@ fun drainOutbound( var natural = 0 for (p in firstPass) natural += p.size if (natural < 1200) { - val deficit = 1200 - natural - // Rewind the Initial PN — we'll reissue with the same PN and the - // same captured frames plus padding. The SentPacket entry recorded - // by the natural-size build will be overwritten by the rebuild - // below since both use the same PN. - initialState.pnSpace.rewindOutboundForRebuild() - val paddedInitial = - buildLongHeaderFromFrames( - conn = conn, - level = EncryptionLevel.INITIAL, - frames = initialContents!!.frames, // null-safe: gated by `initialNatural != null` above - tokens = initialContents.tokens, - nowMillis = nowMillis, - padBytes = deficit, - ) + // Padding rebuild: re-issue the Initial with PADDING frames inside + // the AEAD envelope so the on-wire datagram clears the §14.1 floor. + // + // Off-by-one trap: the QUIC long-header Length field is a varint + // (RFC 9000 §16). When the natural-size payload is tiny enough + // for Length to fit in 1 byte (body ≤ 63 bytes), the rebuild's + // larger body crosses the 64-byte threshold and Length grows to + // 2 bytes — adding 1 wire byte that wasn't in `natural`. Same + // shape at 16384 bytes (2 → 4) and 2^30 (4 → 8). A naive + // `padBytes = 1200 - natural` then produces a 1199-byte + // datagram for PING-only Initials. + // + // Solution: rebuild with the initial deficit, measure, and if we + // still fall short, bump by the residual and rebuild once more. + // Each iteration adds PADDING bytes 1:1 to the wire size; the + // varint grows monotonically so this terminates in ≤ 2 rounds + // for any reachable payload size. + var padBytes = 1200 - natural + var paddedInitial: ByteArray + while (true) { + initialState.pnSpace.rewindOutboundForRebuild() + paddedInitial = + buildLongHeaderFromFrames( + conn = conn, + level = EncryptionLevel.INITIAL, + frames = initialContents!!.frames, // null-safe: gated by `initialNatural != null` above + tokens = initialContents.tokens, + nowMillis = nowMillis, + padBytes = padBytes, + ) + var totalAfterRebuild = paddedInitial.size + if (handshakeNatural != null) totalAfterRebuild += handshakeNatural.size + if (applicationPkt != null) totalAfterRebuild += applicationPkt.size + if (totalAfterRebuild >= 1200) break + padBytes += 1200 - totalAfterRebuild + } concat(listOfNotNull(paddedInitial, handshakeNatural, applicationPkt)) } else { concat(firstPass) @@ -166,6 +205,93 @@ private fun concat(parts: List): ByteArray { return out } +/** + * Build a CONNECTION_CLOSE-only datagram at the highest encryption level we + * have keys for. Two RFC 9000 constraints make this trickier than a normal + * packet build: + * + * - §10.2.3 — at Initial / Handshake levels, only CONNECTION_CLOSE + * (Transport, 0x1c) is allowed. An application-level close is replaced + * with `APPLICATION_ERROR (0x0c)` + frameType=0 + empty reason so we don't + * leak app state pre-handshake. + * - §14.1 — a client datagram containing an Initial MUST be ≥ 1200 bytes, + * even a close-only one. We do this by building once at natural size and, + * if short, rewinding the PN and rebuilding with a PADDING-frame deficit + * inside the AEAD envelope. The rebuild loops because the long-header + * Length varint (RFC 9000 §16) can grow by 1 byte once the body crosses + * the 64-byte threshold, so a single-shot deficit can land 1 byte short + * of 1200. + */ +private fun buildClosingDatagram( + conn: QuicConnection, + nowMillis: Long, +): ByteArray? { + val app = conn.application + if (app.sendProtection != null) { + // 1-RTT level reached: app close (0x1d) is allowed and carries the + // original error code + reason. + val frame = ConnectionCloseFrame(conn.closeErrorCode, null, conn.closeReason ?: "") + return buildBestLevelPacket(conn, listOf(frame)) + } + + // Pre-1-RTT: must use transport-encoded close. RFC 9000 §20.1 + // APPLICATION_ERROR = 0x0c — "the application or application protocol + // caused the connection to be closed during the handshake". + val transportFrame = + ConnectionCloseFrame( + errorCode = APPLICATION_ERROR, + frameType = 0L, + reason = "", + ) + + val hs = conn.handshake + if (hs.sendProtection != null) { + return buildLongHeaderFromFrames( + conn = conn, + level = EncryptionLevel.HANDSHAKE, + frames = listOf(transportFrame), + tokens = emptyList(), + nowMillis = nowMillis, + padBytes = 0, + ) + } + + val init = conn.initial + if (init.sendProtection != null && !init.keysDiscarded) { + val natural = + buildLongHeaderFromFrames( + conn = conn, + level = EncryptionLevel.INITIAL, + frames = listOf(transportFrame), + tokens = emptyList(), + nowMillis = nowMillis, + padBytes = 0, + ) + if (natural.size >= 1200) return natural + var padBytes = 1200 - natural.size + var padded: ByteArray + do { + init.pnSpace.rewindOutboundForRebuild() + padded = + buildLongHeaderFromFrames( + conn = conn, + level = EncryptionLevel.INITIAL, + frames = listOf(transportFrame), + tokens = emptyList(), + nowMillis = nowMillis, + padBytes = padBytes, + ) + if (padded.size >= 1200) break + padBytes += 1200 - padded.size + } while (true) + return padded + } + return null +} + +/** RFC 9000 §20.1 — application-or-protocol caused close during handshake. */ +private const val APPLICATION_ERROR: Long = 0x0cL + private fun buildBestLevelPacket( conn: QuicConnection, frames: List, @@ -176,23 +302,26 @@ private fun buildBestLevelPacket( if (app.sendProtection != null) { val proto = app.sendProtection!! val pn = app.pnSpace.allocateOutbound() - return ShortHeaderPacket.build( - ShortHeaderPlaintextPacket(conn.destinationConnectionId, pn, payload), - proto.aead, - proto.key, - proto.iv, - proto.hp, - proto.hpKey, - largestAckedInSpace = -1L, - ) + val built = + ShortHeaderPacket.build( + ShortHeaderPlaintextPacket(conn.destinationConnectionId, pn, payload), + proto.aead, + proto.key, + proto.iv, + proto.hp, + proto.hpKey, + largestAckedInSpace = -1L, + ) + emitQlogSent(conn, EncryptionLevel.APPLICATION, pn, built.size, frames) + return built } val hs = conn.handshake if (hs.sendProtection != null) { - return buildLongHeaderPacket(conn, EncryptionLevel.HANDSHAKE, payload) + return buildLongHeaderPacket(conn, EncryptionLevel.HANDSHAKE, payload, frames) } val init = conn.initial if (init.sendProtection != null) { - return buildLongHeaderPacket(conn, EncryptionLevel.INITIAL, payload) + return buildLongHeaderPacket(conn, EncryptionLevel.INITIAL, payload, frames) } return null } @@ -201,6 +330,7 @@ private fun buildLongHeaderPacket( conn: QuicConnection, level: EncryptionLevel, payload: ByteArray, + frames: List, ): ByteArray { val state = conn.levelState(level) val proto = state.sendProtection!! @@ -211,22 +341,25 @@ private fun buildLongHeaderPacket( EncryptionLevel.HANDSHAKE -> LongHeaderType.HANDSHAKE EncryptionLevel.APPLICATION -> error("APPLICATION uses short-header packets") } - return LongHeaderPacket.build( - LongHeaderPlaintextPacket( - type = type, - version = QuicVersion.V1, - dcid = conn.destinationConnectionId, - scid = conn.sourceConnectionId, - packetNumber = pn, - payload = payload, - ), - proto.aead, - proto.key, - proto.iv, - proto.hp, - proto.hpKey, - largestAckedInSpace = -1L, - ) + val built = + LongHeaderPacket.build( + LongHeaderPlaintextPacket( + type = type, + version = QuicVersion.V1, + dcid = conn.destinationConnectionId, + scid = conn.sourceConnectionId, + packetNumber = pn, + payload = payload, + ), + proto.aead, + proto.key, + proto.iv, + proto.hp, + proto.hpKey, + largestAckedInSpace = -1L, + ) + emitQlogSent(conn, level, pn, built.size, frames) + return built } /** @@ -280,10 +413,51 @@ private fun collectHandshakeLevelFrames( length = cryptoChunk.data.size.toLong(), ) } + // RFC 9002 §6.2.4: PTO probe MUST be ack-eliciting at the + // encryption level with unacknowledged data. `buildApplicationPacket` + // consumes [pendingPing] when 1-RTT keys exist; pre-1-RTT we + // honor it here at the highest active level (Handshake > Initial). + // + // The driver's PTO branch also calls + // [QuicConnection.requeueAllInflightCrypto], which moves any + // unacknowledged ClientHello / ClientFinished bytes back onto + // the cryptoSend retransmit queue — so the takeChunk above + // already produced a CRYPTO frame in that case, and the PING + // would be redundant. We only emit a bare PING when there's no + // CRYPTO retransmit available (e.g. the unacknowledged data was + // already sitting in cryptoSend's retransmit queue from a + // previous PTO and got drained, or the original Initial was + // never sent at all). This preserves the "send something + // ack-eliciting on every PTO" contract without wasting a frame. + if (conn.pendingPing && + conn.application.sendProtection == null && + level == highestPreApplicationLevel(conn) + ) { + if (frames.none { it is CryptoFrame }) { + frames += PingFrame + } + // Either the CRYPTO frame already covers the ack-eliciting + // requirement at this level, or we just appended a PING. + // Clear the flag so the next drain doesn't double-fire. + conn.pendingPing = false + } if (frames.isEmpty()) return null return HandshakeLevelContents(frames = frames, tokens = tokens) } +/** + * Highest encryption level for which we currently hold send keys, given + * 1-RTT keys are NOT installed. Used by [collectHandshakeLevelFrames] + * to place a PTO PING (RFC 9002 §6.2.4) at the right level when the + * application path can't carry it. + */ +private fun highestPreApplicationLevel(conn: QuicConnection): EncryptionLevel = + when { + conn.handshake.sendProtection != null -> EncryptionLevel.HANDSHAKE + conn.initial.sendProtection != null && !conn.initial.keysDiscarded -> EncryptionLevel.INITIAL + else -> EncryptionLevel.INITIAL // collectHandshakeLevelFrames already returned null on no keys + } + /** * Build a long-header packet from already-collected frames, with optional * trailing PADDING (0x00) bytes inside the encryption envelope. RFC 9000 @@ -310,13 +484,24 @@ private fun buildLongHeaderFromFrames( } val pn = state.pnSpace.allocateOutbound() val type = if (level == EncryptionLevel.INITIAL) LongHeaderType.INITIAL else LongHeaderType.HANDSHAKE + // RFC 9000 §17.2.5.1: after a Retry, every Initial we send MUST carry + // the server-issued Retry token verbatim in the Initial header's Token + // field. Handshake packets have no Token field, so this only affects + // Initial-level builds. + val token = + if (type == LongHeaderType.INITIAL) { + conn.retryToken ?: ByteArray(0) + } else { + ByteArray(0) + } val packet = LongHeaderPacket.build( LongHeaderPlaintextPacket( type = type, - version = QuicVersion.V1, + version = conn.currentVersion, dcid = conn.destinationConnectionId, scid = conn.sourceConnectionId, + token = token, packetNumber = pn, payload = payload, ), @@ -345,9 +530,34 @@ private fun buildLongHeaderFromFrames( sizeBytes = packet.size, tokens = tokens, ) + emitQlogSent(conn, level, pn, packet.size, frames) return packet } +/** + * Fire [QlogObserver.onPacketSent] for one outbound packet. Skipped + * fast for the [QlogObserver.NoOp] default so production callers pay + * only one identity comparison + one virtual call. + */ +private fun emitQlogSent( + conn: QuicConnection, + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, +) { + val observer = conn.qlogObserver + if (observer === QlogObserver.NoOp) return + val frameNames = ArrayList(frames.size) + for (f in frames) frameNames += qlogFrameName(f::class.simpleName ?: "frame") + observer.onPacketSent( + level = level, + packetNumber = packetNumber, + sizeBytes = sizeBytes, + frames = frameNames, + ) +} + private fun buildApplicationPacket( conn: QuicConnection, nowMillis: Long, @@ -423,11 +633,25 @@ private fun buildApplicationPacket( // pre-priority round-robin behaviour exactly. // // Cost: O(N log N) per drain pass plus one transient sorted - // list. N is small (1–10 in the moq-lite audio path); if it - // ever grows enough to matter, switch to an indirect index sort - // or maintain an incrementally-sorted view on setPriority. + // list. N is small (1–10 in the moq-lite audio path) but the + // multiplexing interop testcase pushes N to ~2000, so we + // optimize: + // 1. filter out streams that are fully closed (no data + no + // retransmits coming) — drops "done" streams from the + // walk, which is most of them under bursty interop loads. + // 2. skip the sort entirely when every stream has the + // default priority (the common case) — preserving the + // insertion-ordered round-robin shape. + // The combination drops drainOutbound's per-call cost from + // O(N log N) where N=total-streams to roughly O(active) under + // realistic loads. + val active = streamsView.filter { !it.isClosed } val sorted = - if (streamsView.size > 1) streamsView.sortedByDescending { it.priority } else streamsView + when { + active.size <= 1 -> active + active.all { it.priority == 0 } -> active + else -> active.sortedByDescending { it.priority } + } val rotation = conn.streamRoundRobinStart var tierStart = 0 outer@ while (tierStart < sorted.size) { @@ -481,6 +705,25 @@ private fun buildApplicationPacket( } if (frames.isEmpty()) return null + + // Diagnostic: gated by QUIC_INTEROP_DEBUG=1 so prod is silent. Tells + // us exactly what state the writer was in when it built this packet. + // Hunting the multiplexing-on-the-wire bug where MultiplexingCoalescing + // Test passes (~9 streams/packet) but the live driver emits 1 STREAM + // per packet against aioquic. + if (com.vitorpamplona.quic.connection.writerDebugEnabled) { + val streamFrameCount = frames.count { it is StreamFrame } + val totalFrames = frames.size + val streamsViewSize = conn.streamsListLocked().size + val activeSize = conn.streamsListLocked().count { !it.isClosed } + System.err.println( + "[writer.app] frames=$totalFrames stream_frames=$streamFrameCount " + + "streamsView=$streamsViewSize active=$activeSize " + + "packetBudget_remaining=${1100 - (frames.filterIsInstance().sumOf { it.data.size + 32 })} " + + "connBudget_initial=${(conn.sendConnectionFlowCredit - conn.sendConnectionFlowConsumed).coerceAtLeast(0L)}", + ) + } + val payload = encodeFrames(frames) val pn = state.pnSpace.allocateOutbound() // Retain the packet for RFC 9002 loss detection BEFORE the encrypt @@ -510,6 +753,9 @@ private fun buildApplicationPacket( sizeBytes = sizeBytes.getOrNull()?.size ?: 0, tokens = tokens.toList(), ) + sizeBytes.getOrNull()?.let { built -> + emitQlogSent(conn, EncryptionLevel.APPLICATION, pn, built.size, frames) + } // Re-throw on encrypt failure so callers (driver loop) see the // same exception they did before this change. The bookkeeping // entry is still in place; if the throw was transient, retransmit diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/WriterDebug.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/WriterDebug.kt new file mode 100644 index 0000000000..7b5d0facf8 --- /dev/null +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/connection/WriterDebug.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +/** + * Opt-in writer-side debug logging. Toggled by external code (the + * interop endpoint sets it from the QUIC_INTEROP_DEBUG env var at + * startup). Off by default — production must be silent. + * + * Cost when off: a single volatile read in the writer hot path, + * negligible against the encode + AEAD seal cost. Worth keeping + * inert in shipped code so the next interop investigation can flip + * it on without code changes. + */ +@Volatile +var writerDebugEnabled: Boolean = false + +/** + * Build identifier injected into the boot log so we can verify the + * deployed image actually has the latest debug code (i.e. that the + * docker layer cache didn't serve a stale jar). Bump when adding new + * trace lines to make them traceable from the wire run. + */ +const val WRITER_DEBUG_BUILD_ID: String = "2026-05-07-fix-parallel-detection-v1" diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/observability/QlogObserver.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/observability/QlogObserver.kt new file mode 100644 index 0000000000..e977c4f114 --- /dev/null +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/observability/QlogObserver.kt @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.observability + +import com.vitorpamplona.quic.connection.EncryptionLevel + +/** + * qlog (draft-marx-qlog) observer interface for QUIC connection events. + * + * Tools like qvis (https://qvis.quictools.info/) and Wireshark consume + * qlog files to render sequence diagrams + RTT graphs + recovery + * timelines. The on-the-wire format is JSON-NDJSON (one JSON object + * per line); this interface decouples event emission from format — + * production code can pass [NoOp] (zero overhead), and the + * `:quic` interop runner attaches a JSON-writing implementation that + * produces a `client.sqlog` consumable by qvis. + * + * Goal: every interop-runner test failure produces a qlog file the + * caller can drop into qvis to see exactly what we did differently + * from the spec. + * + * Performance: hooks are on the hot packet-send/receive path, so + * [NoOp] must be a JIT-able single virtual call with no allocation. + * Default-method bodies in Kotlin compile to single bytecode `RETURN`, + * which HotSpot inlines reliably. + */ +interface QlogObserver { + /** Called once after [com.vitorpamplona.quic.connection.QuicConnection.start] runs. */ + fun onConnectionStarted( + serverName: String, + dcid: ByteArray, + scid: ByteArray, + ) + + /** + * Called when the connection transitions to CLOSED, either locally + * (close()) or due to an inbound CONNECTION_CLOSE / read-loop + * termination ([com.vitorpamplona.quic.connection.QuicConnection.markClosedExternally]). + * + * @param initiator `"local"` if we initiated, `"remote"` otherwise. + */ + fun onConnectionClosed( + initiator: String, + errorCode: Long, + reason: String, + ) + + /** + * One outbound packet at [level] just hit the wire. Called once per + * coalesced packet inside a UDP datagram, so a single + * datagram carrying Initial + Handshake fires twice. + */ + fun onPacketSent( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) + + /** One inbound packet at [level] was successfully decrypted + dispatched. */ + fun onPacketReceived( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) + + /** + * An inbound packet (or whole datagram) was dropped on the floor — + * AEAD AUTH FAIL, unknown DCID, missing receive keys, version + * mismatch, frame decode failure, etc. + */ + fun onPacketDropped( + reason: String, + sizeBytes: Int, + ) + + /** + * TLS produced new keys at the given encryption level. + * + * @param keyType `"server"` or `"client"` (which direction the + * key applies to). + */ + fun onKeyUpdated( + keyType: String, + level: EncryptionLevel, + ) + + /** + * RFC 9002 §6.1 loss detection declared one or more outbound + * packets at [level] lost. + */ + fun onLossDetected( + level: EncryptionLevel, + lostPacketNumbers: List, + ) + + /** + * RFC 9002 §6.2 PTO timer expired; the writer will emit a PING on + * the next drain to elicit an ACK from the peer. + * + * @param consecutivePtoCount the new PTO count (post-increment). + * @param ptoMillis the PTO duration that just expired. + */ + fun onPtoFired( + consecutivePtoCount: Int, + ptoMillis: Long, + ) + + /** + * Congestion-controller state transition (slow-start ↔ + * recovery ↔ congestion-avoidance). qvis renders this as + * background bands on the RTT timeline. + */ + fun onCongestionStateUpdated(newState: String) + + /** + * QUIC transport parameters were set by [initiator] (`"local"` at + * connection-open, `"remote"` once the peer's params arrive in + * EncryptedExtensions). [params] is a flat label→value map of + * the parameters that the implementation chose to surface; the + * exact key set is not part of the qlog 0.3 contract. + */ + fun onTransportParametersSet( + initiator: String, + params: Map, + ) + + /** + * The TLS handshake completed and an ALPN was selected (or `null` + * was chosen — [alpn] is the human-readable name, e.g. `"h3"`). + */ + fun onAlpnNegotiated(alpn: String) + + /** + * RFC 9000 §6 Version Information. Single-fire — emitted once + * after Version Negotiation resolves. + */ + fun onVersionInformation( + chosenVersion: String, + otherVersionsOffered: List, + ) + + /** + * No-op observer. Default for production callers — every method + * is an empty body that the JIT inlines. No allocation, no I/O. + */ + object NoOp : QlogObserver { + override fun onConnectionStarted( + serverName: String, + dcid: ByteArray, + scid: ByteArray, + ) = Unit + + override fun onConnectionClosed( + initiator: String, + errorCode: Long, + reason: String, + ) = Unit + + override fun onPacketSent( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) = Unit + + override fun onPacketReceived( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) = Unit + + override fun onPacketDropped( + reason: String, + sizeBytes: Int, + ) = Unit + + override fun onKeyUpdated( + keyType: String, + level: EncryptionLevel, + ) = Unit + + override fun onLossDetected( + level: EncryptionLevel, + lostPacketNumbers: List, + ) = Unit + + override fun onPtoFired( + consecutivePtoCount: Int, + ptoMillis: Long, + ) = Unit + + override fun onCongestionStateUpdated(newState: String) = Unit + + override fun onTransportParametersSet( + initiator: String, + params: Map, + ) = Unit + + override fun onAlpnNegotiated(alpn: String) = Unit + + override fun onVersionInformation( + chosenVersion: String, + otherVersionsOffered: List, + ) = Unit + } +} + +/** + * Map a [com.vitorpamplona.quic.frame.Frame] subclass simple-name to + * the qlog frame_type label. qlog 0.3 specifies snake_case with + * `_frame` suffix stripped — e.g. `MaxStreamsFrame` → `max_streams`. + * + * Lives next to [QlogObserver] so writer + parser hot paths share the + * same conversion (used to fill [QlogObserver.onPacketSent.frames] and + * [QlogObserver.onPacketReceived.frames] without round-tripping through + * Jackson). + */ +fun qlogFrameName(simpleClassName: String): String { + // Strip trailing "Frame" then convert CamelCase to snake_case. + val noSuffix = + if (simpleClassName.endsWith("Frame")) { + simpleClassName.dropLast("Frame".length) + } else { + simpleClassName + } + if (noSuffix.isEmpty()) return simpleClassName.lowercase() + val sb = StringBuilder(noSuffix.length + 4) + for (i in noSuffix.indices) { + val c = noSuffix[i] + if (i > 0 && c.isUpperCase()) sb.append('_') + sb.append(c.lowercaseChar()) + } + return sb.toString() +} diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/packet/PacketTypes.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/packet/PacketTypes.kt index 0f1ff00da5..9eb1515297 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/packet/PacketTypes.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/packet/PacketTypes.kt @@ -42,4 +42,17 @@ enum class LongHeaderType( object QuicVersion { const val V1: Int = 0x00000001 const val VERSION_NEGOTIATION: Int = 0x00000000 + + /** + * RFC 9000 §6 / interop runner convention: a "force VN" version + * number that no QUIC server is allowed to support. Sending this + * in the first Initial guarantees the server replies with a + * Version Negotiation packet listing the versions it actually + * supports. The interop runner's `versionnegotiation` testcase + * uses this to drive the client through the VN code path. + * + * Value 0x1a2a3a4a is conventional for this purpose; any + * non-spec-assigned 32-bit value would work equally well. + */ + const val FORCE_VERSION_NEGOTIATION: Int = 0x1a2a3a4a } diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/stream/SendBuffer.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/stream/SendBuffer.kt index 2b9d7807d3..ec56ee9ada 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/stream/SendBuffer.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/stream/SendBuffer.kt @@ -315,6 +315,54 @@ class SendBuffer( } } + /** + * Re-queue every byte range currently sent-but-not-yet-ACK'd for + * retransmission. The writer's next [takeChunk] drains the + * retransmit queue before any fresh bytes, at the original + * offsets — so the peer sees the same CRYPTO/STREAM bytes again + * with the same offset, only at a new packet number. + * + * Used by the RFC 9002 §6.2.4 PTO probe path: when the timer + * fires before the handshake completes, the client MUST + * retransmit the unacknowledged ClientHello (the CRYPTO bytes + * sitting in [inFlight] for the Initial level). A PING alone + * isn't enough — the peer may never have seen the original + * datagram and therefore has no state to correlate the PING + * against. + * + * Idempotent: ranges already in [retransmit] are not affected + * (only [inFlight] is walked); calling twice in a row is a no-op + * the second time because the first call moved everything out of + * [inFlight]. FIN bit is preserved per range — a lost FIN-bearing + * range will re-emit FIN. + * + * Best-effort buffers ([bestEffort] = true) drop the inflight + * ranges instead of retransmitting them, matching [markLost]'s + * semantics. The data buffer can compact as if those ranges had + * been ACK'd. + */ + fun requeueAllInflight() { + synchronized(this) { + if (inFlight.isEmpty()) return + if (bestEffort) { + inFlight.clear() + advanceFlushedFloorIfPossible() + return + } + // Move every inflight range to the retransmit queue, + // preserving offset order (inFlight is sorted by offset + // ascending, so addLast preserves sort within retransmit + // for these new entries — though retransmit is a FIFO + // and doesn't strictly require sorted order, takeChunk + // pops front-first regardless). + for (r in inFlight) { + if (r.fin && !_finAcked) _finSent = false + retransmit.addLast(r) + } + inFlight.clear() + } + } + /** * Disposition for a range that overlapped a [markAcked] / [markLost] * range. ACK drops the range and latches `_finAcked` if the FIN diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/tls/TlsClient.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/tls/TlsClient.kt index 330912a53a..38f37b1592 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/tls/TlsClient.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/tls/TlsClient.kt @@ -70,6 +70,17 @@ class TlsClient( val fixedKeyPair: X25519KeyPair? = null, /** When non-null, used as the ClientHello random (for deterministic tests). */ val fixedRandom: ByteArray? = null, + /** + * Cipher suites offered in the ClientHello, in preference order. The + * default offers AES-128-GCM first then ChaCha20-Poly1305. Override to + * force a specific negotiation (e.g. `chacha20`-only for the matching + * quic-interop-runner testcase). + */ + val cipherSuites: IntArray = + intArrayOf( + TlsConstants.CIPHER_TLS_AES_128_GCM_SHA256, + TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256, + ), ) { enum class State { INITIAL, @@ -120,6 +131,12 @@ class TlsClient( private var sharedSecret: ByteArray? = null private var negotiatedCipherSuite: Int = -1 + /** The 32-byte ClientHello random, available after [start]. Exposed so + * observers (e.g. SSLKEYLOGFILE writer) can correlate secrets with + * this connection. */ + var clientRandom: ByteArray? = null + private set + /** Begin the handshake by emitting a ClientHello at Initial level. */ fun start() { check(state == State.INITIAL) { "TlsClient already started" } @@ -127,14 +144,18 @@ class TlsClient( keySchedule.deriveEarly() + val random = + fixedRandom ?: com.vitorpamplona.quartz.utils.RandomInstance + .bytes(32) + clientRandom = random + val ch = buildQuicClientHello( serverName = serverName, x25519PublicKey = keyPair!!.publicKey, quicTransportParams = transportParameters, - random = - fixedRandom ?: com.vitorpamplona.quartz.utils.RandomInstance - .bytes(32), + random = random, + cipherSuites = cipherSuites, ) val chBytes = ch.encode() diff --git a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/tls/TlsClientHello.kt b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/tls/TlsClientHello.kt index d238ed0866..3971a9b730 100644 --- a/quic/src/commonMain/kotlin/com/vitorpamplona/quic/tls/TlsClientHello.kt +++ b/quic/src/commonMain/kotlin/com/vitorpamplona/quic/tls/TlsClientHello.kt @@ -92,6 +92,11 @@ fun buildQuicClientHello( quicTransportParams: ByteArray, additionalAlpn: List = emptyList(), random: ByteArray = RandomInstance.bytes(32), + cipherSuites: IntArray = + intArrayOf( + TlsConstants.CIPHER_TLS_AES_128_GCM_SHA256, + TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256, + ), ): TlsClientHello { val alpn = mutableListOf() alpn += TlsConstants.ALPN_H3 @@ -107,5 +112,5 @@ fun buildQuicClientHello( TlsExtension(TlsConstants.EXT_ALPN, encodeAlpn(alpn)), TlsExtension(TlsConstants.EXT_QUIC_TRANSPORT_PARAMETERS, quicTransportParams), ) - return TlsClientHello(random = random, extensions = exts) + return TlsClientHello(random = random, cipherSuites = cipherSuites, extensions = exts) } diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/AckTrackerPurgeOnAckOfAckTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/AckTrackerPurgeOnAckOfAckTest.kt index 37e304ed06..594ed5421c 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/AckTrackerPurgeOnAckOfAckTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/AckTrackerPurgeOnAckOfAckTest.kt @@ -64,7 +64,7 @@ class AckTrackerPurgeOnAckOfAckTest { // Peer ACKs the packet that carried our outbound ACK // covering up to PN 4. - conn.lock.lock() + conn.streamsLock.lock() try { conn.onTokensAcked( listOf( @@ -75,7 +75,7 @@ class AckTrackerPurgeOnAckOfAckTest { ), ) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } // Tracker is now empty: peer has confirmed receipt of our // ACK that covered everything up to PN 4. Re-advertising @@ -96,7 +96,7 @@ class AckTrackerPurgeOnAckOfAckTest { } // Peer ACKs our Initial-level outbound ACK. - conn.lock.lock() + conn.streamsLock.lock() try { conn.onTokensAcked( listOf( @@ -104,7 +104,7 @@ class AckTrackerPurgeOnAckOfAckTest { ), ) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } // Initial tracker drained; Application tracker untouched. assertTrue(conn.initial.ackTracker.isEmpty()) @@ -120,7 +120,7 @@ class AckTrackerPurgeOnAckOfAckTest { } // Peer ACKs our outbound ACK that covered up to PN 4 only; // the tracker's higher-PN ranges (5..9) must survive. - conn.lock.lock() + conn.streamsLock.lock() try { conn.onTokensAcked( listOf( @@ -128,7 +128,7 @@ class AckTrackerPurgeOnAckOfAckTest { ), ) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } assertFalse(conn.application.ackTracker.isEmpty()) assertEquals(9L, conn.application.ackTracker.largestReceived()) @@ -145,7 +145,7 @@ class AckTrackerPurgeOnAckOfAckTest { for (pn in 0L..9L) { conn.application.ackTracker.receivedPacket(pn, ackEliciting = true, receivedAtMillis = 1L) } - conn.lock.lock() + conn.streamsLock.lock() try { conn.onTokensAcked( listOf( @@ -160,7 +160,7 @@ class AckTrackerPurgeOnAckOfAckTest { ) assertTrue(conn.application.ackTracker.isEmpty()) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } } } diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/BatchedOpenLockContractTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/BatchedOpenLockContractTest.kt new file mode 100644 index 0000000000..65c7bfb8e7 --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/BatchedOpenLockContractTest.kt @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.tls.InProcessTlsServer +import com.vitorpamplona.quic.tls.PermissiveCertificateValidator +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.withLock +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * Lock contract for batched stream opens — the production pattern used by + * `Http3GetClient.prepareRequests` / `HqInteropGetClient.prepareRequests` + * and any future caller that wants to open N streams atomically without + * the send loop interjecting between opens. + * + * Background. The interop runner's `multiplexing` testcase against aioquic + * timed out on 2026-05-06, downloading 1421/2000 files in 60s (~23 + * streams/sec, exactly 1 RTT per stream — the wire was emitting one STREAM + * frame per datagram, not 64). The qlog showed 2898 packets sent in 60s, + * each carrying ONE STREAM frame. Cause: `prepareRequests` had been written + * to hold `conn.lock` (the deprecated alias for `lifecycleLock`) while + * calling [QuicConnection.openBidiStreamLocked]. `lifecycleLock` doesn't + * gate the writer — `drainOutbound` takes [QuicConnection.streamsLock] — + * so the writer's send loop could (and did) interleave between every two + * `openBidiStreamLocked` invocations, draining one stream per pass. + * + * Two-layer guard: + * 1. [QuicConnection.openBidiStreamLocked] now `check`s that + * [QuicConnection.streamsLock] is held when called. This catches + * callers that pass NO lock or the WRONG lock at the source. + * 2. The "batched open under streamsLock yields a small fixed packet + * count" contract is pinned by [MultiplexingCoalescingTest] — + * already in the suite. Together they fail loudly if the prod + * callers regress. + * + * If you're adding a new caller for `openBidiStreamLocked`, the right + * pattern is: + * + * conn.streamsLock.withLock { + * repeat(n) { conn.openBidiStreamLocked() ... } + * } + */ +class BatchedOpenLockContractTest { + @Test + fun `openBidiStreamLocked throws when called without streamsLock held`() { + runBlocking { + val client = handshakedClient() + // No `streamsLock.withLock { ... }` wrapper. This is the + // shape the regressed `prepareRequests` had — except it + // held `lifecycleLock` instead. Either way streamsLock is + // not held, the assertion in openBidiStreamLocked fires. + val ex = + assertFailsWith { + client.openBidiStreamLocked() + } + assertNotNull(ex.message) + // The error should mention streamsLock so the caller knows + // what to fix. + kotlin.test.assertTrue( + ex.message!!.contains("streamsLock"), + "error message should name the lock to acquire; got: ${ex.message}", + ) + } + } + + @Test + fun `openBidiStreamLocked throws when caller holds the wrong lock (lifecycleLock)`() { + runBlocking { + val client = handshakedClient() + // Exact regression shape: caller holds lifecycleLock and + // calls openBidiStreamLocked. The check should fire because + // streamsLock is not held — even though SOME lock is. + assertFailsWith { + client.lifecycleLock.withLock { + client.openBidiStreamLocked() + } + } + } + } + + @Test + fun `openBidiStreamLocked succeeds when streamsLock is held`() { + runBlocking { + val client = handshakedClient() + // Happy path — caller holds streamsLock, batched open works. + // Sanity-check that the assertion in openBidiStreamLocked + // doesn't fire on the correct call shape. + val streams = + client.streamsLock.withLock { + List(64) { client.openBidiStreamLocked() } + } + assertEquals(64, streams.size) + assertEquals(64, streams.map { it.streamId }.toSet().size, "stream ids must be unique") + } + } + + @Test + fun `openBidiStreamsBatch holds streamsLock for the entire batch`() { + runBlocking { + val client = handshakedClient() + // Verify the API actually holds streamsLock during init — + // the whole point of the function. A regression that + // released the lock between opens (the 2026-05-06 bug + // shape) would let isLocked drop to false inside init. + val payloads = (0 until 64).map { "req-$it".encodeToByteArray() } + var lockedAtSomePoint = false + val streams = + client.openBidiStreamsBatch(payloads) { stream, payload -> + if (client.streamsLock.isLocked) lockedAtSomePoint = true + stream.send.enqueue(payload) + stream.send.finish() + stream + } + assertEquals(64, streams.size) + assertEquals(64, streams.map { it.streamId }.toSet().size) + assertTrue( + lockedAtSomePoint, + "streamsLock must be held while init runs — without that, " + + "the send loop interleaves between opens", + ) + } + } + + @Test + fun `openBidiStreamsBatch with empty list does not throw`() { + runBlocking { + val client = handshakedClient() + // Empty-batch short-circuit: no items, no lock acquisition, + // no per-item init call. Returns empty list cleanly. + val result: List = + client.openBidiStreamsBatch(emptyList()) { _, _ -> + error("init must not run for empty batch") + } + assertEquals(0, result.size) + } + } + + @Test + fun `openUniStreamLocked throws when streamsLock is not held`() { + runBlocking { + val client = handshakedClient() + assertFailsWith { + client.openUniStreamLocked() + } + } + } + + @Test + fun `openUniStreamsBatch holds streamsLock for the entire batch`() { + runBlocking { + val client = handshakedClient() + // moq audio-rooms shape: many uni streams in burst. Without + // the batched API, each open releases the lock and the + // send loop interjects (the same shape that broke bidi + // multiplexing on 2026-05-06). + val items = List(16) { it } + var lockedAtSomePoint = false + val streams = + client.openUniStreamsBatch(items) { stream, _ -> + if (client.streamsLock.isLocked) lockedAtSomePoint = true + stream + } + assertEquals(16, streams.size) + assertEquals(16, streams.map { it.streamId }.toSet().size) + assertTrue(lockedAtSomePoint, "streamsLock must be held while init runs") + } + } + + private fun handshakedClient(): QuicConnection = + runBlocking { + val client = + QuicConnection( + serverName = "example.test", + config = + QuicConnectionConfig( + initialMaxStreamsBidi = 256, + ), + tlsCertificateValidator = PermissiveCertificateValidator(), + ) + val serverScid = ConnectionId.random(8) + val tlsServer = + InProcessTlsServer( + transportParameters = + TransportParameters( + initialMaxData = 1_000_000, + initialMaxStreamDataBidiLocal = 100_000, + initialMaxStreamDataBidiRemote = 100_000, + initialMaxStreamDataUni = 100_000, + initialMaxStreamsBidi = 256, + initialMaxStreamsUni = 16, + initialSourceConnectionId = serverScid.bytes, + originalDestinationConnectionId = client.destinationConnectionId.bytes, + ).encode(), + ) + val pipe = + InMemoryQuicPipe( + client = client, + initialDcid = client.destinationConnectionId.bytes, + serverScid = serverScid, + tlsServer = tlsServer, + ) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + client + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/CloseDatagramRfcComplianceTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/CloseDatagramRfcComplianceTest.kt new file mode 100644 index 0000000000..6f78c43c0a --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/CloseDatagramRfcComplianceTest.kt @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.QuicWriter +import com.vitorpamplona.quic.frame.ConnectionCloseFrame +import com.vitorpamplona.quic.tls.PermissiveCertificateValidator +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * Regression tests for two RFC 9000 violations found via the + * `quic-interop-runner` against aioquic on 2026-05-06: + * + * - §10.2.3 — `CONNECTION_CLOSE (Application)` (0x1d) MUST NOT appear in + * Initial / Handshake packets. Application-error closes that fire before + * 1-RTT keys exist must be encoded as `CONNECTION_CLOSE (Transport)` + * (0x1c) with `errorCode = APPLICATION_ERROR (0x0c)`. + * - §14.1 — any client datagram containing an Initial MUST be ≥ 1200 + * bytes, even when carrying only a `CONNECTION_CLOSE`. + * + * Pre-fix, the writer's CLOSING branch built a tiny ~45-byte Initial with + * frame type 0x1d. The runner's aioquic server silently dropped it (correct + * per spec) and our handshake hung until our 10s timeout. + */ +class CloseDatagramRfcComplianceTest { + @Test + fun `pre-handshake close datagram is padded to at least 1200 bytes per RFC 9000 sec 14_1`() = + runBlocking { + val conn = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = PermissiveCertificateValidator(), + ) + // Initial sendProtection is wired in QuicConnection's init block; + // no handshake required to exercise the close path. + conn.status = QuicConnection.Status.CLOSING + + val datagram = drainOutbound(conn, nowMillis = 0L) + requireNotNull(datagram) { "drainOutbound must produce a close datagram when CLOSING with Initial keys" } + + assertTrue( + datagram.size >= 1200, + "client Initial datagram MUST be ≥ 1200 bytes per RFC 9000 §14.1, " + + "got ${datagram.size} (this was bug A from the aioquic interop run)", + ) + // First byte: 1100????b — long-header form + Initial type. + val firstByte = datagram[0].toInt() and 0xff + assertEquals(0xc0, firstByte and 0xf0, "must be a long-header packet") + assertEquals(0x00, firstByte and 0x30, "must be type=Initial (00)") + } + + @Test + fun `PTO probe emits a PING at Initial level pre-handshake (RFC 9002 sec 6_2_4)`() = + runBlocking { + // Reproduces the third bug found via the aioquic interop run on + // 2026-05-06: when the first ClientHello is dropped (e.g. sim + // queues it before the server is ready), the driver's PTO timer + // sets `pendingPing = true`, but the writer only honored that + // flag in the 1-RTT path. Pre-handshake the flag was silently + // discarded, so the second drain produced no Initial datagram — + // the connection sat mute until our 10s handshake timeout. + // + // Pre-fix: drainOutbound returned null and the connection slept + // on the next PTO. Post-fix: a PING-bearing Initial datagram is + // emitted, eliciting an ACK from the peer that feeds loss + // detection and unblocks CRYPTO retransmit. + val conn = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = PermissiveCertificateValidator(), + ) + // Initial sendProtection is wired in QuicConnection's init {} + // block; no handshake required to exercise the PTO probe path. + // Skip conn.start() so the cryptoSend buffer is empty — this is + // exactly the post-ClientHello-sent state when PTO fires. + conn.pendingPing = true + + val datagram = drainOutbound(conn, nowMillis = 0L) + assertNotNull( + datagram, + "PTO probe MUST produce an Initial datagram pre-handshake — RFC 9002 §6.2.4", + ) + // RFC 9000 §14.1: client datagrams containing an Initial MUST + // be ≥ 1200 bytes. The writer's padding rebuild now accounts + // for Length-varint growth (1 → 2 bytes) when the natural-size + // payload is small (PING-only) so the deficit calculation + // produces a final size that strictly meets the spec floor. + assertTrue( + datagram.size >= 1200, + "PTO Initial datagram MUST be ≥ 1200 bytes per RFC 9000 §14.1, got ${datagram.size}", + ) + assertEquals(false, conn.pendingPing, "pendingPing MUST be cleared after the probe") + } + + @Test + fun `single-byte PING-only Initial pads to exactly 1200 bytes (no overshoot beyond varint growth)`() = + runBlocking { + // Boundary case for the padding-rebuild deficit calculation. + // The smallest possible Initial-level frame payload is a single + // PING frame (1 byte, encoded as 0x01 — RFC 9000 §19.2). With + // no token, the 1-byte natural payload encodes a Length varint + // of 1 byte. Once the rebuild adds ~1170 bytes of PADDING the + // Length value crosses the 63-byte varint boundary and grows + // to 2 bytes. The deficit calculation MUST account for that + // growth, otherwise the rebuilt packet falls 1 byte short of + // the §14.1 1200-byte floor. + // + // Asserts the strict floor (≥ 1200) AND that we don't overshoot + // by more than the varint-growth delta plus a small slack — the + // padded packet should land in the 1200..1203 range, never + // 1199 (pre-fix) and never 1300+ (over-correction). + val conn = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = PermissiveCertificateValidator(), + ) + conn.pendingPing = true + + val datagram = drainOutbound(conn, nowMillis = 0L) + assertNotNull(datagram, "PING-only PTO probe must produce a datagram") + assertTrue( + datagram.size >= 1200, + "padded Initial MUST be ≥ 1200 bytes (RFC 9000 §14.1), got ${datagram.size}", + ) + assertTrue( + datagram.size <= 1203, + "padded Initial should not overshoot the 1200 floor by more than the " + + "Length-varint growth (≤ 3 bytes), got ${datagram.size}", + ) + } + + @Test + fun `ConnectionCloseFrame encodes type 0x1c when frameType is non-null (transport close)`() { + // Bug B fix relies on the writer passing frameType=0L (instead of + // null) when emitting a close at Initial / Handshake level. Encode + // path must produce 0x1c for that case, 0x1d only for app-level. + val transportClose = ConnectionCloseFrame(errorCode = 0x0c, frameType = 0L, reason = "") + val w1 = QuicWriter() + transportClose.encode(w1) + val transportBytes = w1.toByteArray() + assertEquals(0x1c.toByte(), transportBytes[0], "transport CONNECTION_CLOSE must serialize as 0x1c") + + val appClose = ConnectionCloseFrame(errorCode = 0, frameType = null, reason = "") + val w2 = QuicWriter() + appClose.encode(w2) + val appBytes = w2.toByteArray() + assertEquals(0x1d.toByte(), appBytes[0], "application CONNECTION_CLOSE must serialize as 0x1d") + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/CryptoRetransmitTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/CryptoRetransmitTest.kt index e3663062e8..d7f4a16353 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/CryptoRetransmitTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/CryptoRetransmitTest.kt @@ -81,12 +81,12 @@ class CryptoRetransmitTest { .single() // Simulate loss via direct dispatch. - client.lock.lock() + client.streamsLock.lock() try { client.onTokensLost(listOf(cryptoToken)) client.initial.sentPackets.remove(firstPn) } finally { - client.lock.unlock() + client.streamsLock.unlock() } // Initial-level cryptoSend should now have re-queued bytes @@ -126,11 +126,11 @@ class CryptoRetransmitTest { client.initial.sentPackets.entries .first { it.value.tokens.any { t -> t is RecoveryToken.Crypto } } // ACK via direct dispatch. - client.lock.lock() + client.streamsLock.lock() try { client.onTokensAcked(packet.value.tokens) } finally { - client.lock.unlock() + client.streamsLock.unlock() } // After ACK the Initial-level cryptoSend's flushedFloor should // have advanced — we check by observing that another takeChunk diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiStreamFinDeliveryTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiStreamFinDeliveryTest.kt new file mode 100644 index 0000000000..6e1150723b --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiStreamFinDeliveryTest.kt @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.frame.StreamFrame +import com.vitorpamplona.quic.tls.InProcessTlsServer +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Regression for the quic-interop-runner `multiplexing` failure observed + * 2026-05-06 against aioquic: 677 client-bidi streams opened, server FIN'd + * every one, zero responses surfaced to the application — every per-stream + * `incoming.collect` hung forever. + * + * The single-stream cases (`transfer`, `chacha20`, `http3`) all pass, so + * STREAM-frame routing for low concurrency works. The bug only fires when + * many streams' responses stream through together. + * + * Two regressions are pinned here: + * + * 1. Server replies to every client-opened bidi stream with `data + FIN` + * coalesced into a single datagram. Each stream's `incoming` Flow MUST + * complete (FIN delivery), and the bytes MUST surface (no truncation). + * + * 2. If the parser tears the connection down (e.g. STREAM channel + * saturated → INTERNAL_ERROR), every stream's `incoming` Flow MUST + * still terminate. Otherwise application-side `incoming.collect` + * callers leak as zombie coroutines stuck on a channel that nobody + * will ever close. This is the "FIN never arrives" symptom from the + * runner's perspective: the connection died mid-response and we + * forgot to release the per-stream channels. + */ +class MultiStreamFinDeliveryTest { + @Test + fun finIsDeliveredToAllParallelClientBidiStreamsUnderConcurrentLoad() = + runBlocking { + val (client, pipe) = newConnectedClient() + + // Open N streams. Match the multiplexing case shape — many streams + // each with a small response — without paying for full 1999-file + // overhead. + val n = 50 + val streams = (0 until n).map { client.openBidiStream() } + + // Build one server datagram per stream that responds with + // "resp-" + FIN. We keep them in separate datagrams (rather + // than coalescing all into one short-header packet which the + // packet codec doesn't support) so the parser sees a stream of + // back-to-back STREAM frames just as it would on the wire. + for ((i, stream) in streams.withIndex()) { + val payload = "resp-$i".encodeToByteArray() + val frame = + StreamFrame( + streamId = stream.streamId, + offset = 0L, + data = payload, + fin = true, + ) + val packet = pipe.buildServerApplicationDatagram(listOf(frame))!! + feedDatagram(client, packet, nowMillis = 0L) + } + + // Each stream's collector must terminate (FIN closed the + // channel) AND must have received the expected payload. + val collected = + coroutineScope { + streams + .mapIndexed { i, stream -> + async { + withTimeoutOrNull(5_000L) { + stream.incoming.toList() + } to i + } + }.awaitAll() + } + val hung = collected.firstOrNull { it.first == null } + assertTrue( + hung == null, + "stream index ${hung?.second} never received FIN — Flow.toList() timed out", + ) + for ((result, i) in collected) { + val chunks = result!! + val joined = ByteArray(chunks.sumOf { it.size }) + var p = 0 + for (c in chunks) { + c.copyInto(joined, p) + p += c.size + } + assertEquals( + "resp-$i", + joined.decodeToString(), + "stream $i: bytes mismatch", + ) + } + assertEquals( + QuicConnection.Status.CONNECTED, + client.status, + "connection must remain CONNECTED through the response burst", + ) + } + + @Test + fun finIsDeliveredEvenWhenChannelAlreadyHasBufferedChunks() = + runBlocking { + // Defence-in-depth: a stream that has bytes still queued in + // its incomingChannel when the connection tears down must still + // surface those bytes AND complete the Flow. consumeAsFlow drains + // the buffer before honouring the close, so closeIncoming after + // deliverIncoming is safe — we pin that contract here so a + // future "close the channel and reset the buffer" refactor can't + // silently regress the byte loss. + val (client, pipe) = newConnectedClient() + val stream = client.openBidiStream() + val payload = "buffered-then-torn-down".encodeToByteArray() + val packet = + pipe.buildServerApplicationDatagram( + listOf( + StreamFrame( + streamId = stream.streamId, + offset = 0L, + data = payload, + fin = false, // intentionally no FIN + ), + ), + )!! + feedDatagram(client, packet, nowMillis = 0L) + + // Tear down WITHOUT delivering FIN — bytes are now buffered in the + // incomingChannel and the connection-level close must still + // terminate the per-stream Flow. + client.markClosedExternally("test teardown after partial response") + + val chunks = + withTimeoutOrNull(2_000L) { stream.incoming.toList() } + assertTrue(chunks != null, "Flow leaked after teardown with buffered bytes") + val joined = ByteArray(chunks.sumOf { it.size }) + var p = 0 + for (c in chunks) { + c.copyInto(joined, p) + p += c.size + } + assertEquals( + "buffered-then-torn-down", + joined.decodeToString(), + "buffered bytes must surface before the Flow terminates", + ) + } + + @Test + fun connectionTeardownClosesEveryPerStreamIncomingChannel() = + runBlocking { + // Even when the parser kills the connection (e.g. channel + // overflow surfaces as INTERNAL_ERROR via markClosedExternally), + // every per-stream `incoming` Flow MUST terminate. Otherwise + // application coroutines that called `stream.incoming.toList()` + // leak forever — exactly the symptom seen in the multiplexing + // runner where 677 collectors hung. + val (client, _) = newConnectedClient() + val n = 20 + val streams = (0 until n).map { client.openBidiStream() } + + // Tear the connection down externally without delivering any FIN. + client.markClosedExternally("simulated parser-side teardown") + + // Every stream's collector must complete promptly. Pre-fix the + // per-stream incomingChannel stays open (closeIncoming is never + // called) and the collector hangs. + val results = + coroutineScope { + streams + .mapIndexed { i, stream -> + async { + withTimeoutOrNull(2_000L) { + stream.incoming.toList() + } to i + } + }.awaitAll() + } + val hung = results.firstOrNull { it.first == null } + assertTrue( + hung == null, + "stream index ${hung?.second} incoming Flow leaked after connection teardown", + ) + } + + private fun newConnectedClient(): Pair = + runBlocking { + val client = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = + com.vitorpamplona.quic.tls + .PermissiveCertificateValidator(), + ) + val serverScid = ConnectionId.random(8) + val tlsServer = + InProcessTlsServer( + transportParameters = + TransportParameters( + initialMaxData = 16L * 1024 * 1024, + initialMaxStreamDataBidiLocal = 1L * 1024 * 1024, + initialMaxStreamDataBidiRemote = 1L * 1024 * 1024, + initialMaxStreamDataUni = 1L * 1024 * 1024, + initialMaxStreamsBidi = 1000, + initialMaxStreamsUni = 1000, + initialSourceConnectionId = serverScid.bytes, + originalDestinationConnectionId = client.destinationConnectionId.bytes, + ).encode(), + ) + val pipe = + InMemoryQuicPipe( + client = client, + initialDcid = client.destinationConnectionId.bytes, + serverScid = serverScid, + tlsServer = tlsServer, + ) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + client to pipe + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingAioquicTpsTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingAioquicTpsTest.kt new file mode 100644 index 0000000000..cec71658fd --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingAioquicTpsTest.kt @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.tls.InProcessTlsServer +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Reproduce the aioquic-2026-05-07 multiplex-on-the-wire failure in + * a unit test. Server qlog showed: + * - 1406 packets with exactly 1 STREAM frame, 1 with 3, never more + * - first stream packets at 1665 / 1709ms (one RTT apart, NOT burst) + * - per-RTT cadence throughout + * + * Hypothesis: writer drains 1 stream per packet despite many being + * queued — under conditions specific to aioquic's TPs: + * - initial_max_data = 1MB (matches qlog) + * - initial_max_stream_data_bidi_remote = 1MB (matches qlog) + * - initial_max_streams_bidi = 128 (matches qlog) + * + * MultiplexingCoalescingTest passes with `initial_max_data = 100MB` + * — which is 100x more credit than aioquic gives. This test mirrors + * the exact aioquic TPs to see if the smaller credit triggers the + * 1-stream-per-packet shape. + * + * If this test asserts ≤6 packets and PASSES, the bug is NOT in the + * synchronous drain path — it's in the live driver flow (which + * MultiplexingCoalescingTest doesn't exercise). + * + * If it FAILS with one stream per packet, we have a deterministic + * reproduction and can fix the writer. + */ +class MultiplexingAioquicTpsTest { + @Test + fun multiplex_64_streams_with_aioquic_TPs_should_still_coalesce() = + runBlocking { + val client = handshakedClientMatchingAioquic() + + // ~80-byte HTTP/3 HEADERS frame payload — what + // Http3GetClient actually emits per stream after QPACK + // encoding. Smaller payloads (50 bytes) under-stress the + // coalescing path. + val n = 64 + val payloadPerStream = 80 + val streams = + client.openBidiStreamsBatch((0 until n).toList()) { stream, i -> + stream.send.enqueue(ByteArray(payloadPerStream) { (i and 0xff).toByte() }) + stream.send.finish() + stream + } + assertEquals(n, streams.size) + + val packets = mutableListOf() + while (true) { + val pkt = drainOutbound(client, nowMillis = 1L) ?: break + packets += pkt + } + + val totalBytes = packets.sumOf { it.size } + val streamsPerPacket = n.toDouble() / packets.size.coerceAtLeast(1) + println( + "[MultiplexingAioquicTpsTest] 64 streams of $payloadPerStream bytes " + + "drained into ${packets.size} packets " + + "(${(streamsPerPacket * 10).toInt() / 10.0} streams/pkt, " + + "totalBytes=$totalBytes)", + ) + + // Same threshold as MultiplexingCoalescingTest. 64 streams + // × ~110 bytes (80 payload + ~30 overhead) ≈ 7 KB; at + // ~1100-byte payload budget per packet that's 6-7 packets. + assertTrue( + packets.size <= 8, + "expected ≤8 packets, got ${packets.size} — " + + "writer regressed to one-stream-per-packet under aioquic TPs", + ) + assertTrue( + streamsPerPacket >= 8.0, + "expected ≥8 streams/packet, got $streamsPerPacket", + ) + } + + private fun handshakedClientMatchingAioquic(): QuicConnection = + runBlocking { + val client = + QuicConnection( + serverName = "example.test", + config = + QuicConnectionConfig( + // Mirror our actual local TPs from the qlog. + initialMaxData = 16L * 1024 * 1024, + initialMaxStreamsBidi = 100, + initialMaxStreamsUni = 10000, + initialMaxStreamDataBidiLocal = 1L * 1024 * 1024, + initialMaxStreamDataBidiRemote = 1L * 1024 * 1024, + initialMaxStreamDataUni = 1L * 1024 * 1024, + ), + tlsCertificateValidator = + com.vitorpamplona.quic.tls + .PermissiveCertificateValidator(), + ) + val serverScid = ConnectionId.random(8) + val tlsServer = + InProcessTlsServer( + transportParameters = + TransportParameters( + // Mirror aioquic-qns 2026-05-07 from the qlog. + initialMaxData = 1L * 1024 * 1024, + initialMaxStreamDataBidiLocal = 1L * 1024 * 1024, + initialMaxStreamDataBidiRemote = 1L * 1024 * 1024, + initialMaxStreamDataUni = 1L * 1024 * 1024, + initialMaxStreamsBidi = 128, + initialMaxStreamsUni = 128, + initialSourceConnectionId = serverScid.bytes, + originalDestinationConnectionId = client.destinationConnectionId.bytes, + ).encode(), + ) + val pipe = + InMemoryQuicPipe( + client = client, + initialDcid = client.destinationConnectionId.bytes, + serverScid = serverScid, + tlsServer = tlsServer, + ) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + client + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingCoalescingTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingCoalescingTest.kt new file mode 100644 index 0000000000..1b499fe05d --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingCoalescingTest.kt @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.tls.InProcessTlsServer +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Pin the multiplexing throughput contract that the runner's + * `multiplexing` testcase exercises. + * + * This is the fast in-memory version of what the Docker-based runner + * does (open N bidi streams, send a GET on each, collect responses): + * we drive a real handshake via [InMemoryQuicPipe] then enqueue request + * bytes on N bidi streams in two batching shapes, drive the writer, and + * count the resulting datagrams. + * + * The bug this test catches: pre-fix, the interop runner's multiplexing + * test ran ~25 streams/sec because each application-side enqueue woke + * the send loop while the OTHER coroutines hadn't yet queued bytes. + * Result: one stream per packet, ~80-byte packets, useless coalescing. + * + * Post-fix: enqueue bytes on N streams synchronously, then drain ONCE + * — the writer should pack many streams' data into each datagram. With + * 64 streams × 50 bytes = 3200 bytes of payload, plus ~50 bytes/stream + * STREAM-frame framing, total wire load ≈ 6.4 KB. At a 1452-byte UDP + * cap, that's ~5 datagrams. The pre-fix shape would emit 64. + */ +class MultiplexingCoalescingTest { + @Test + fun `64 streams enqueued before drain coalesce into a small fixed number of packets`() = + runBlocking { + val client = handshakedClient() + + // Phase 1: serial enqueue. NO drainOutbound between streams. + // This is exactly the shape InteropClient.runTransferTest's + // chunked-multiplex path uses: prepareRequest is synchronous + // for every URL in the chunk, then a single wakeup, then + // parallel awaits. + val n = 64 + val payloadPerStream = 50 + val streams = + (0 until n).map { i -> + val s = client.openBidiStream() + s.send.enqueue(ByteArray(payloadPerStream) { (i and 0xff).toByte() }) + s.send.finish() + s + } + assertEquals(n, streams.size) + + // Phase 2: drain everything at once. Count packets emitted. + val packets = mutableListOf() + while (true) { + val pkt = drainOutbound(client, nowMillis = 1L) ?: break + packets += pkt + } + + val totalBytes = packets.sumOf { it.size } + val payloadTotal = n * payloadPerStream + val avgBytesPerPacket = totalBytes / packets.size.coerceAtLeast(1) + val streamsPerPacket = n.toDouble() / packets.size.coerceAtLeast(1) + + // Threshold derivation: each stream's wire load is ~50 bytes + // payload + ~10 bytes STREAM-frame framing + per-packet AEAD + // overhead. 64 streams ≈ 4 KB of frame data; at 1452 bytes + // per UDP packet that's ≤ 4 packets of frames + 1 for any + // pending ACK / control frames. Pre-fix produced 64+ packets. + assertTrue( + packets.size <= 6, + "64 streams should coalesce into ≤6 packets, got ${packets.size}; " + + "totalBytes=$totalBytes avgBytesPerPacket=$avgBytesPerPacket " + + "streamsPerPacket=$streamsPerPacket payloadTotal=$payloadTotal", + ) + // Sanity: average packet should carry at least ~10 streams' frames. + // If it's ~1, we regressed back to one-stream-per-packet. + assertTrue( + streamsPerPacket >= 10.0, + "expected ≥10 streams/packet, got $streamsPerPacket — coalescing broke", + ) + } + + @Test + fun `1000 streams enqueued in batches of 64 produce a tractable packet count`() = + runBlocking { + // Stress version. 1000 streams in chunks of 64 = ~16 chunks. + // Each chunk should produce ≤6 packets per the test above, + // so total ≤ 100 packets. Pre-fix this test would have + // produced ~1000 packets. + val client = handshakedClient(maxStreamsBidi = 2000) + val chunkSize = 64 + val totalStreams = 1000 + val payloadPerStream = 50 + + val packets = mutableListOf() + for (chunk in (0 until totalStreams).chunked(chunkSize)) { + for (i in chunk) { + val s = client.openBidiStream() + s.send.enqueue(ByteArray(payloadPerStream) { (i and 0xff).toByte() }) + s.send.finish() + } + while (true) { + val pkt = drainOutbound(client, nowMillis = 1L) ?: break + packets += pkt + } + } + + val streamsPerPacket = totalStreams.toDouble() / packets.size.coerceAtLeast(1) + assertTrue( + packets.size <= 150, + "1000 streams should produce ≤150 packets, got ${packets.size} (streamsPerPacket=$streamsPerPacket)", + ) + } + + private fun handshakedClient(maxStreamsBidi: Long = 100L): QuicConnection = + runBlocking { + val client = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(initialMaxStreamsBidi = maxStreamsBidi), + tlsCertificateValidator = + com.vitorpamplona.quic.tls + .PermissiveCertificateValidator(), + ) + val serverScid = ConnectionId.random(8) + val tlsServer = + InProcessTlsServer( + transportParameters = + TransportParameters( + initialMaxData = 100_000_000, + initialMaxStreamDataBidiLocal = 1_000_000, + initialMaxStreamDataBidiRemote = 1_000_000, + initialMaxStreamDataUni = 1_000_000, + initialMaxStreamsBidi = maxStreamsBidi, + initialMaxStreamsUni = maxStreamsBidi, + initialSourceConnectionId = serverScid.bytes, + originalDestinationConnectionId = client.destinationConnectionId.bytes, + ).encode(), + ) + val pipe = + InMemoryQuicPipe( + client = client, + initialDcid = client.destinationConnectionId.bytes, + serverScid = serverScid, + tlsServer = tlsServer, + ) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + client + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingRoundTripTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingRoundTripTest.kt new file mode 100644 index 0000000000..016872499f --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingRoundTripTest.kt @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.frame.StreamFrame +import com.vitorpamplona.quic.tls.InProcessTlsServer +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * In-process mirror of the [quic-interop-runner](https://github.com/quic-interop/quic-interop-runner) + * `multiplexing` testcase. The runner generates ~50–2000 small files and + * has the client open one bidi stream per file in parallel, then asserts + * every file is downloaded with correct content. This test runs the same + * shape against the [InMemoryQuicPipe] harness — no UDP, no real server — + * so the regression that drove this test (aioquic CONNECTION_CLOSE on + * bare-PING PTO probes, plus the parallel-open / FIN-delivery bugs) can + * be caught in a unit-test loop instead of a Docker matrix run. + * + * Why this shape and not just `MultiStreamFinDeliveryTest`: + * + * - `MultiStreamFinDeliveryTest` only exercises server-pushed STREAM + * frames. It hand-builds N response datagrams and feeds them + * directly. The CLIENT never sends a STREAM frame, so any regression + * in the writer's multiplex-stream-coalesce path or in the parser's + * server-side request-decode logic is invisible to it. + * + * - This test drives the full request → response loop: + * + * 1. Client opens N bidi streams in parallel (matches the matrix) + * 2. Each stream enqueues a tiny request + FIN + * 3. drainOutbound pulls a coalesced 1-RTT datagram off the writer + * 4. The pipe decrypts, extracts STREAM frames, builds server + * responses (`response-` + FIN) and feeds them back + * 5. Per-stream `incoming.toList()` must yield the expected bytes + * + * If any step in that loop regresses — the writer drops streams, the + * parser routes STREAM frames wrong under load, the server-side state + * forgets which stream sent which request — the assertion fails with a + * specific stream id, not a vague timeout. + * + * Stream-count notes: the matrix uses up to 1999 files; we use 64 to keep + * the test under a second on CI. The *bug class* the test guards against + * is invariant to the count — any "loses a stream's response" regression + * will fire at 64 too because it's per-stream, not aggregate. + */ +class MultiplexingRoundTripTest { + @Test + fun parallel_streams_each_get_their_own_response_back() = + runBlocking { + val (client, pipe) = newConnectedClient() + + val n = 64 + val streams = + (0 until n).map { i -> + val s = client.openBidiStream() + // Tiny request body. Real H3/HQ would be a `GET /file-i\r\n`; + // the matrix doesn't care WHAT the bytes are, only that the + // client emitted them on a distinct stream and the server's + // response surfaces back. + s.send.enqueue("req-$i".encodeToByteArray()) + s.send.finish() + s + } + + // Drain everything the client wants to send. drainOutbound returns + // one datagram per call (up to ~1200 bytes); on each, decrypt the + // 1-RTT short-header packet and collect STREAM frames. The writer + // is supposed to coalesce many streams' frames into one datagram + // (MultiplexingCoalescingTest pins that contract); here we just + // walk every emitted datagram until the client has nothing left. + val seenRequests = mutableMapOf>() + val seenFin = mutableSetOf() + var totalDatagrams = 0 + while (true) { + val out = drainOutbound(client, nowMillis = 0L) ?: break + if (out.isEmpty()) break + totalDatagrams += 1 + val frames = pipe.decryptClientApplicationFrames(out) ?: break + for (frame in frames) { + if (frame is StreamFrame) { + // Append to per-stream chunk list — joined once at the + // end. Earlier shape merged eagerly per frame which is + // O(N²) over byte count. + seenRequests.getOrPut(frame.streamId) { mutableListOf() } += frame.data + if (frame.fin) seenFin += frame.streamId + } + } + if (seenFin.size == n) break + } + assertEquals( + n, + seenRequests.size, + "server saw STREAM frames from only ${seenRequests.size}/$n streams " + + "after $totalDatagrams datagrams — writer dropped streams under multiplex load", + ) + assertEquals( + n, + seenFin.size, + "server saw FINs from only ${seenFin.size}/$n streams — client failed " + + "to deliver request FIN on every stream", + ) + // End-to-end coalescing contract: 64 streams × ~10-byte + // requests fit in a handful of datagrams. The aioquic + // 2026-05-06 regression hit when the writer emitted ONE + // STREAM per datagram (so 64 datagrams for this batch). + // Threshold of 12 leaves headroom for the writer's choice + // of when to coalesce + any per-drain ACK frames. + assertTrue( + totalDatagrams <= 12, + "expected ≤12 datagrams for 64 streams, got $totalDatagrams — " + + "writer regressed to one-stream-per-datagram (the aioquic " + + "interop 2026-05-06 multiplexing failure mode)", + ) + // Sanity-check the request payloads match what each stream sent. + for ((i, stream) in streams.withIndex()) { + val expected = "req-$i" + val chunks = seenRequests[stream.streamId] + assertNotNull(chunks, "stream ${stream.streamId} (i=$i): no STREAM frames received") + val joined = ByteArray(chunks.sumOf { it.size }) + var p = 0 + for (c in chunks) { + c.copyInto(joined, p) + p += c.size + } + val actual = joined.decodeToString() + assertEquals( + expected, + actual, + "stream ${stream.streamId} (i=$i): server received '$actual', expected '$expected'", + ) + } + + // Server responds: one STREAM frame per stream with a known body + // + FIN, packed into one datagram each (the in-memory pipe doesn't + // coalesce frames into a single packet across stream-ids by + // design, since the writer side of the pipe is intentionally + // minimal). Client must surface every body via stream.incoming + // and terminate each Flow on FIN. + for (stream in streams) { + val body = "response-${stream.streamId}".encodeToByteArray() + val frame = + StreamFrame( + streamId = stream.streamId, + offset = 0L, + data = body, + fin = true, + ) + val datagram = pipe.buildServerApplicationDatagram(listOf(frame))!! + feedDatagram(client, datagram, nowMillis = 0L) + } + + val collected = + coroutineScope { + streams + .map { stream -> + async { + val chunks = + withTimeoutOrNull(5_000L) { stream.incoming.toList() } + stream.streamId to chunks + } + }.awaitAll() + } + val hung = collected.firstOrNull { it.second == null } + assertTrue( + hung == null, + "stream ${hung?.first} never received FIN — Flow.toList() timed out " + + "(this is the matrix's 'incomplete transfer' symptom)", + ) + for ((streamId, chunks) in collected) { + val joined = ByteArray(chunks!!.sumOf { it.size }) + var p = 0 + for (c in chunks) { + c.copyInto(joined, p) + p += c.size + } + assertEquals( + "response-$streamId", + joined.decodeToString(), + "stream $streamId: response bytes mismatch — wrong content delivered", + ) + } + assertEquals( + QuicConnection.Status.CONNECTED, + client.status, + "connection must remain CONNECTED through the full round-trip", + ) + } + + private fun newConnectedClient(): Pair = + runBlocking { + val client = + QuicConnection( + serverName = "example.test", + config = + QuicConnectionConfig( + initialMaxStreamsBidi = 1024, + initialMaxStreamsUni = 1024, + initialMaxData = 16L * 1024 * 1024, + initialMaxStreamDataBidiLocal = 64L * 1024, + initialMaxStreamDataBidiRemote = 64L * 1024, + initialMaxStreamDataUni = 64L * 1024, + ), + tlsCertificateValidator = + com.vitorpamplona.quic.tls + .PermissiveCertificateValidator(), + ) + val serverScid = ConnectionId.random(8) + val tlsServer = + InProcessTlsServer( + transportParameters = + TransportParameters( + initialMaxData = 16L * 1024 * 1024, + initialMaxStreamDataBidiLocal = 64L * 1024, + initialMaxStreamDataBidiRemote = 64L * 1024, + initialMaxStreamDataUni = 64L * 1024, + initialMaxStreamsBidi = 1024, + initialMaxStreamsUni = 1024, + initialSourceConnectionId = serverScid.bytes, + originalDestinationConnectionId = client.destinationConnectionId.bytes, + ).encode(), + ) + val pipe = + InMemoryQuicPipe( + client = client, + initialDcid = client.destinationConnectionId.bytes, + serverScid = serverScid, + tlsServer = tlsServer, + ) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + client to pipe + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingThroughputTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingThroughputTest.kt new file mode 100644 index 0000000000..8c90bed415 --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/MultiplexingThroughputTest.kt @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.tls.InProcessTlsServer +import com.vitorpamplona.quic.tls.PermissiveCertificateValidator +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Throughput contract for the lock-split refactor (2026-05-08): opening + * many parallel bidi streams + queueing requests must not be serialised + * by a single connection-wide mutex. Phase 1 of the split (separate + * `streamsLock` / `lifecycleLock` / per-level `levelLock`) targets the + * multiplexing testcase that drove this refactor — see + * `quic/plans/2026-05-08-lock-split-design.md`. + * + * The test stands up an in-memory client (no socket I/O), opens 1000 + * client-bidi streams concurrently, enqueues a small request body + FIN + * on each, and asserts the operation completes within a generous wall- + * clock budget. The number is deliberately loose: this is a contract + * for "lock contention isn't pathological", not a microbenchmark. + * + * NOTE: the in-memory pipe doesn't drive a concurrent send loop, so + * this test exercises the lock-acquisition cost of `openBidiStream` + * itself rather than full multiplexing throughput. The interop runner + * provides the end-to-end measurement. + */ +class MultiplexingThroughputTest { + @Test + fun open_1000_bidi_streams_completes_quickly() { + runBlocking { + val client = + QuicConnection( + serverName = "example.test", + config = + QuicConnectionConfig( + initialMaxStreamsBidi = 2_000, + initialMaxStreamsUni = 2_000, + initialMaxData = 100_000_000, + initialMaxStreamDataBidiLocal = 100_000, + initialMaxStreamDataBidiRemote = 100_000, + initialMaxStreamDataUni = 100_000, + ), + tlsCertificateValidator = PermissiveCertificateValidator(), + ) + val serverScid = ConnectionId.random(8) + val tlsServer = + InProcessTlsServer( + transportParameters = + TransportParameters( + initialMaxData = 100_000_000, + initialMaxStreamDataBidiLocal = 100_000, + initialMaxStreamDataBidiRemote = 100_000, + initialMaxStreamDataUni = 100_000, + initialMaxStreamsBidi = 2_000, + initialMaxStreamsUni = 2_000, + initialSourceConnectionId = serverScid.bytes, + originalDestinationConnectionId = client.destinationConnectionId.bytes, + ).encode(), + ) + val pipe = + InMemoryQuicPipe( + client = client, + initialDcid = client.destinationConnectionId.bytes, + serverScid = serverScid, + tlsServer = tlsServer, + ) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + + val request = ByteArray(50) { it.toByte() } + val streamCount = 1_000 + + // Open all streams in parallel — each launch contends for + // streamsLock briefly. Pre-refactor this serialised against + // any in-flight drainOutbound call; phase 1 keeps openBidi + // contention scoped to streamsLock-only. + val started = + kotlin.time.TimeSource.Monotonic + .markNow() + val opens = + (0 until streamCount).map { + async { + val stream = client.openBidiStream() + stream.send.enqueue(request) + stream.send.finish() + stream.streamId + } + } + val ids = opens.awaitAll() + val elapsed = started.elapsedNow() + + // Useful diagnostic for measuring future regressions: stdout + // shows up in the test report so phase-1 vs phase-2 can be + // compared against the same test. + println( + "[MultiplexingThroughputTest] opened $streamCount bidi streams in " + + "${elapsed.inWholeMilliseconds}ms " + + "(${(streamCount * 1000.0 / elapsed.inWholeMilliseconds.coerceAtLeast(1L)).toLong()} streams/sec)", + ) + assertEquals(streamCount, ids.size) + assertEquals(streamCount, ids.toSet().size, "stream ids must be unique") + // Generous bound; in-process opens of 1000 streams should + // complete in well under half a second on any developer + // machine — pre-refactor this was minutes due to lock + // contention against the (idle) send-loop drain. The looser + // 2-second bound is still 100x what's expected on actual + // hardware while accounting for slow CI workers. + assertTrue( + elapsed.inWholeMilliseconds < 2_000L, + "1000 parallel openBidiStream calls took ${elapsed.inWholeMilliseconds}ms; expected <2000ms", + ) + } + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/OnTokensLostTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/OnTokensLostTest.kt index c14da3ff49..9f3cf3a787 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/OnTokensLostTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/OnTokensLostTest.kt @@ -47,11 +47,11 @@ class OnTokensLostTest { fun ackToken_doesNotPopulateAnyPending() = runBlocking { val conn = newConn() - conn.lock.lock() + conn.streamsLock.lock() try { conn.onTokensLost(listOf(RecoveryToken.Ack(level = EncryptionLevel.APPLICATION, largestAcked = 0L))) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } assertNull(conn.pendingMaxStreamsUni) assertNull(conn.pendingMaxStreamsBidi) @@ -64,12 +64,12 @@ class OnTokensLostTest { runBlocking { val conn = newConn() // Simulate the writer having advertised a higher cap. - conn.lock.lock() + conn.streamsLock.lock() try { conn.advertisedMaxStreamsUni = 150L conn.onTokensLost(listOf(RecoveryToken.MaxStreamsUni(maxStreams = 150L))) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } assertEquals(150L, conn.pendingMaxStreamsUni) } @@ -82,12 +82,12 @@ class OnTokensLostTest { // the value carried by the lost token (150). The lost // frame is irrelevant — re-emitting 150 would not extend // the cap. neqo's fc.rs line 322 supersede check. - conn.lock.lock() + conn.streamsLock.lock() try { conn.advertisedMaxStreamsUni = 200L conn.onTokensLost(listOf(RecoveryToken.MaxStreamsUni(maxStreams = 150L))) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } assertNull(conn.pendingMaxStreamsUni, "stale lost extension must not be re-emitted") } @@ -96,12 +96,12 @@ class OnTokensLostTest { fun lostMaxStreamsBidi_matchingAdvertised_setsPending() = runBlocking { val conn = newConn() - conn.lock.lock() + conn.streamsLock.lock() try { conn.advertisedMaxStreamsBidi = 200L conn.onTokensLost(listOf(RecoveryToken.MaxStreamsBidi(maxStreams = 200L))) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } assertEquals(200L, conn.pendingMaxStreamsBidi) } @@ -110,12 +110,12 @@ class OnTokensLostTest { fun lostMaxData_matchingAdvertised_setsPending() = runBlocking { val conn = newConn() - conn.lock.lock() + conn.streamsLock.lock() try { conn.advertisedMaxData = 1_000_000L conn.onTokensLost(listOf(RecoveryToken.MaxData(maxData = 1_000_000L))) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } assertEquals(1_000_000L, conn.pendingMaxData) } @@ -124,12 +124,12 @@ class OnTokensLostTest { fun lostMaxData_supersededIsDropped() = runBlocking { val conn = newConn() - conn.lock.lock() + conn.streamsLock.lock() try { conn.advertisedMaxData = 2_000_000L conn.onTokensLost(listOf(RecoveryToken.MaxData(maxData = 1_000_000L))) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } assertNull(conn.pendingMaxData) } @@ -138,13 +138,13 @@ class OnTokensLostTest { fun lostMaxStreamData_unknownStream_dropped() = runBlocking { val conn = newConn() - conn.lock.lock() + conn.streamsLock.lock() try { conn.onTokensLost( listOf(RecoveryToken.MaxStreamData(streamId = 999L, maxData = 1024L)), ) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } // No stream with id 999 exists ⇒ token is dropped silently. assertEquals(emptyMap(), conn.pendingMaxStreamData) @@ -154,7 +154,7 @@ class OnTokensLostTest { fun multipleLostTokens_dispatchAll() = runBlocking { val conn = newConn() - conn.lock.lock() + conn.streamsLock.lock() try { conn.advertisedMaxStreamsUni = 150L conn.advertisedMaxStreamsBidi = 200L @@ -168,7 +168,7 @@ class OnTokensLostTest { ), ) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } assertEquals(150L, conn.pendingMaxStreamsUni) assertEquals(200L, conn.pendingMaxStreamsBidi) @@ -183,7 +183,7 @@ class OnTokensLostTest { // most one value (the last setter wins; the supersede // check filters older losses). val conn = newConn() - conn.lock.lock() + conn.streamsLock.lock() try { conn.advertisedMaxStreamsUni = 200L // First lost packet had MaxStreamsUni(150) — stale, dropped. @@ -193,7 +193,7 @@ class OnTokensLostTest { conn.onTokensLost(listOf(RecoveryToken.MaxStreamsUni(maxStreams = 200L))) assertEquals(200L, conn.pendingMaxStreamsUni) } finally { - conn.lock.unlock() + conn.streamsLock.unlock() } } } diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PeerStreamCreditExtensionTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PeerStreamCreditExtensionTest.kt index e89c51da09..54a0a97be3 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PeerStreamCreditExtensionTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PeerStreamCreditExtensionTest.kt @@ -94,7 +94,7 @@ class PeerStreamCreditExtensionTest { // Simulate the relay opening uni streams to us. SERVER_UNI // stream IDs use the encoding `index << 2 | 0x3`. Two streams // (cap=4, half-window=2) is the threshold for a refresh. - client.lock + client.streamsLock .let { // Acquire under lock since getOrCreatePeerStreamLocked requires it. it @@ -103,7 +103,7 @@ class PeerStreamCreditExtensionTest { kotlinx.coroutines.sync .Mutex() .let { /* noop: silence unused-import linter */ } - client.lock.let { l -> + client.streamsLock.let { l -> kotlinx.coroutines.runBlocking { l.lock() try { @@ -179,7 +179,7 @@ class PeerStreamCreditExtensionTest { // Open 10 peer streams — half-window for cap=100 is 50, so // we're well below the threshold. - client.lock.let { l -> + client.streamsLock.let { l -> kotlinx.coroutines.runBlocking { l.lock() try { diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PeerUniStreamDrainTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PeerUniStreamDrainTest.kt new file mode 100644 index 0000000000..9647217a4b --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PeerUniStreamDrainTest.kt @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.frame.StreamFrame +import com.vitorpamplona.quic.stream.StreamId +import com.vitorpamplona.quic.tls.InProcessTlsServer +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals + +/** + * Regression coverage for the multiplexing-interop tear-down (audit-4 #3 + * "slow consumer" overflow on peer-initiated uni streams). + * + * Reproduction of the production symptom: + * + * The interop runner's `multiplexing` testcase opens many parallel + * client-bidi GET streams via an H3 client. The H3 client opens its + * three local uni streams (control + QPACK encoder + QPACK decoder) + * per RFC 9114 §6.2.1 but **does not consume the server's three + * counterpart peer-initiated uni streams**. Their bytes + * (SETTINGS, dynamic-table inserts, ack signals) are routed by + * [QuicConnectionParser] into each stream's bounded + * `incomingChannel` (capacity 64). Once the QPACK encoder + * stream's burst of dynamic-table inserts saturates it, the next + * delivery trips the audit-4 #3 escape hatch: + * + * conn.markClosedExternally("INTERNAL_ERROR: stream … consumer + * overflowed incoming channel + * (slow consumer)") + * + * and the entire connection dies after ~4.5 s with zero requests + * completed. + * + * The test pair below pins both halves of the contract: + * + * - [pre_fix_no_consumer_overflows_and_tears_down_connection] — without + * a consumer the connection MUST close with the documented reason. + * - [drainPeerInitiatedUniStreamsIntoBlackHole_keeps_connection_alive] — + * with [drainPeerInitiatedUniStreamsIntoBlackHole] running the + * connection MUST stay CONNECTED and absorb the same byte volume. + * + * The fix philosophy (variant B from the prompt's three-way menu): the + * `:quic` library stays strict about backpressure — silently dropping + * app-data bytes is worse than failing fast. The new public helper is + * the explicit "I do not care about these particular peer streams" + * opt-in for an H3 GET client that runs with the QPACK dynamic table + * off. Default behaviour is unchanged. + */ +class PeerUniStreamDrainTest { + @Test + fun pre_fix_no_consumer_overflows_and_tears_down_connection() { + runBlocking { + val client = buildClient() + val pipe = buildPipe(client) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + + // Push 65 chunks on a server-initiated uni stream — one more + // than the per-stream incomingChannel capacity (64). The + // first 64 land; the 65th overflows trySend, sets + // QuicStream.overflowed, and the parser maps that to + // markClosedExternally with the documented reason. + // + // Each chunk fits comfortably under the per-stream receive + // limit (initialMaxStreamDataUni = 1 MiB) so the prior + // receive-limit guard does NOT fire — this test is about + // the *channel* overflow specifically, not flow control. + val streamId = StreamId.build(StreamId.Kind.SERVER_UNI, 0) + var offset = 0L + for (i in 0 until 65) { + val chunk = ByteArray(8) { (i + it).toByte() } + val frame = StreamFrame(streamId = streamId, offset = offset, data = chunk, fin = false) + offset += chunk.size.toLong() + val packet = pipe.buildServerApplicationDatagram(listOf(frame)) + assertNotEquals(null, packet, "server has app keys after handshake") + feedDatagram(client, packet!!, nowMillis = 0L) + } + + // The 65th chunk must have torn the connection down. + assertEquals( + QuicConnection.Status.CLOSED, + client.status, + "without a peer-uni-stream consumer the audit-4 #3 escape " + + "hatch must fire on the 65th chunk", + ) + } + } + + @Test + fun drainPeerInitiatedUniStreamsIntoBlackHole_keeps_connection_alive() { + runBlocking { + val client = buildClient() + val pipe = buildPipe(client) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + + // Wire the explicit drainer BEFORE pushing the bytes. + val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + try { + client.drainPeerInitiatedUniStreamsIntoBlackHole(scope) + + // Same volume as the pre-fix test, except this time we go + // a long way past the channel capacity to prove sustained + // operation (4× the bound). If the drainer were absent the + // connection would have died at chunk 65; with the + // drainer reading them as fast as the parser delivers, no + // backpressure builds up. + val streamId = StreamId.build(StreamId.Kind.SERVER_UNI, 0) + var offset = 0L + for (i in 0 until 256) { + val chunk = ByteArray(8) { (i + it).toByte() } + val frame = StreamFrame(streamId = streamId, offset = offset, data = chunk, fin = false) + offset += chunk.size.toLong() + val packet = pipe.buildServerApplicationDatagram(listOf(frame))!! + feedDatagram(client, packet, nowMillis = 0L) + // Yield occasionally so the drainer coroutine actually + // gets a chance to consume — feedDatagram is synchronous + // and the drainer launched with Dispatchers.Default needs + // a scheduling tick. + if (i % 16 == 15) { + withTimeout(2_000) { delay(1) } + } + } + // Ensure the drainer has caught up before we sample status. + withTimeout(2_000) { delay(50) } + + assertEquals( + QuicConnection.Status.CONNECTED, + client.status, + "with drainPeerInitiatedUniStreamsIntoBlackHole the " + + "connection must absorb arbitrary peer-uni traffic " + + "without overflowing the channel; saw closeReason=" + + "${client.closeReason}", + ) + } finally { + scope.cancel() + } + } + } + + private fun buildClient(): QuicConnection = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = + com.vitorpamplona.quic.tls + .PermissiveCertificateValidator(), + ) + + private fun buildPipe(client: QuicConnection): InMemoryQuicPipe { + val serverScid = ConnectionId.random(8) + val tlsServer = + InProcessTlsServer( + transportParameters = + TransportParameters( + initialMaxData = 10_000_000, + initialMaxStreamDataBidiLocal = 1_000_000, + initialMaxStreamDataBidiRemote = 1_000_000, + initialMaxStreamDataUni = 1_000_000, + initialMaxStreamsBidi = 16, + initialMaxStreamsUni = 16, + initialSourceConnectionId = serverScid.bytes, + originalDestinationConnectionId = client.destinationConnectionId.bytes, + ).encode(), + ) + return InMemoryQuicPipe( + client = client, + initialDcid = client.destinationConnectionId.bytes, + serverScid = serverScid, + tlsServer = tlsServer, + ) + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PendingFlowControlEmitTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PendingFlowControlEmitTest.kt index 1c42c9aebc..1971459fcc 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PendingFlowControlEmitTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PendingFlowControlEmitTest.kt @@ -46,11 +46,11 @@ class PendingFlowControlEmitTest { fun pendingMaxStreamsUni_drainEmitsFrameAndToken() = runBlocking { val client = handshakedClient() - client.lock.lock() + client.streamsLock.lock() try { client.pendingMaxStreamsUni = 150L } finally { - client.lock.unlock() + client.streamsLock.unlock() } val sizeBefore = client.application.sentPackets.size @@ -79,11 +79,11 @@ class PendingFlowControlEmitTest { fun pendingMaxStreamsBidi_drainEmitsFrameAndToken(): Unit = runBlocking { val client = handshakedClient() - client.lock.lock() + client.streamsLock.lock() try { client.pendingMaxStreamsBidi = 200L } finally { - client.lock.unlock() + client.streamsLock.unlock() } val sizeBefore = client.application.sentPackets.size runCatching { drainOutbound(client, nowMillis = 1L) } @@ -103,11 +103,11 @@ class PendingFlowControlEmitTest { fun pendingMaxData_drainEmitsFrameAndToken(): Unit = runBlocking { val client = handshakedClient() - client.lock.lock() + client.streamsLock.lock() try { client.pendingMaxData = 5_000_000L } finally { - client.lock.unlock() + client.streamsLock.unlock() } val sizeBefore = client.application.sentPackets.size runCatching { drainOutbound(client, nowMillis = 1L) } @@ -127,12 +127,12 @@ class PendingFlowControlEmitTest { fun pendingMaxStreamData_perStreamDrain() = runBlocking { val client = handshakedClient() - client.lock.lock() + client.streamsLock.lock() try { client.pendingMaxStreamData[3L] = 1_024L client.pendingMaxStreamData[7L] = 2_048L } finally { - client.lock.unlock() + client.streamsLock.unlock() } val sizeBefore = client.application.sentPackets.size runCatching { drainOutbound(client, nowMillis = 1L) } @@ -159,13 +159,13 @@ class PendingFlowControlEmitTest { fun multiplePending_drainEmitsAllInOnePacket(): Unit = runBlocking { val client = handshakedClient() - client.lock.lock() + client.streamsLock.lock() try { client.pendingMaxStreamsUni = 150L client.pendingMaxStreamsBidi = 200L client.pendingMaxData = 1_000_000L } finally { - client.lock.unlock() + client.streamsLock.unlock() } val sizeBefore = client.application.sentPackets.size runCatching { drainOutbound(client, nowMillis = 1L) } @@ -216,11 +216,11 @@ class PendingFlowControlEmitTest { // advertised cap. The writer drains it as-is — supersede check // is in step 6 (the setter side), not here. val client = handshakedClient() - client.lock.lock() + client.streamsLock.lock() try { client.pendingMaxStreamsUni = 50L } finally { - client.lock.unlock() + client.streamsLock.unlock() } val sizeBefore = client.application.sentPackets.size runCatching { drainOutbound(client, nowMillis = 1L) } @@ -242,11 +242,11 @@ class PendingFlowControlEmitTest { fun pendingClearedAcrossDrains() = runBlocking { val client = handshakedClient() - client.lock.lock() + client.streamsLock.lock() try { client.pendingMaxStreamsUni = 150L } finally { - client.lock.unlock() + client.streamsLock.unlock() } // Drain once: pending consumed. runCatching { drainOutbound(client, nowMillis = 1L) } diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PtoCryptoRetransmitTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PtoCryptoRetransmitTest.kt new file mode 100644 index 0000000000..ad610c1e3c --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/PtoCryptoRetransmitTest.kt @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.connection.recovery.RecoveryToken +import com.vitorpamplona.quic.frame.CryptoFrame +import com.vitorpamplona.quic.frame.Frame +import com.vitorpamplona.quic.packet.LongHeaderPacket +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * RFC 9002 §6.2.4 spec-correct PTO probe behavior: when the timer fires + * pre-handshake the client MUST send an ack-eliciting packet at the + * encryption level with unacknowledged data, and SHOULD retransmit the + * lost data rather than emit a bare PING. + * + * Regression scenario from the quic-interop-runner ns-3 transfer test + * against aioquic: the first ClientHello datagram is dropped because + * the simulated server isn't fully started at t≈0.5s. Our PTO at + * t≈1.5s previously sent only a PING with the same DCID; the server + * had no state for that DCID (never saw the original Initial), so + * the PING was silently ignored and the connection died. + * + * The fix has two cooperating pieces: + * 1. [com.vitorpamplona.quic.stream.SendBuffer.requeueAllInflight] + * moves every sent-but-not-ACK'd byte range from the inflight + * list back onto the retransmit queue. + * 2. [QuicConnection.requeueAllInflightCrypto] exposes that to the + * driver, which calls it from the PTO branch when 1-RTT keys + * aren't installed yet (i.e. handshake not done). + * + * After the call, the next [drainOutbound] naturally emits a CRYPTO + * frame at the original offset — the server actually sees a fresh + * ClientHello attempt and can advance the handshake. + */ +class PtoCryptoRetransmitTest { + @Test + fun ptoPreHandshake_retransmitsClientHelloCryptoBytes() = + runBlocking { + val client = newClientWithStartedHandshake() + + // First drain — emits the ClientHello inside an Initial datagram. + val firstDrain = drainOutbound(client, nowMillis = 1L) + assertNotNull(firstDrain, "first drain must produce the ClientHello datagram") + assertTrue( + firstDrain.size >= 1200, + "RFC 9000 §14.1 — Initial-bearing datagram pads to ≥ 1200 bytes (got ${firstDrain.size})", + ) + + // Capture the original ClientHello bytes and offset by + // pulling them out of the SentPacket bookkeeping. + val firstSent = + client.initial.sentPackets.entries.firstOrNull { entry -> + entry.value.tokens.any { it is RecoveryToken.Crypto } + } + assertNotNull(firstSent, "Initial-level SentPacket must carry a Crypto token after first drain") + val firstCrypto = + firstSent.value.tokens + .filterIsInstance() + .single() + assertEquals(0L, firstCrypto.offset, "ClientHello starts at offset 0") + assertTrue(firstCrypto.length > 0L, "ClientHello has non-zero length") + + // Sanity: the second drain (immediately, no PTO yet) must + // produce nothing — bytes are inflight, not unsent. + val emptyDrain = drainOutbound(client, nowMillis = 2L) + assertTrue( + emptyDrain == null || emptyDrain.isEmpty(), + "no PTO yet, no fresh data → second drain must be empty (got ${emptyDrain?.size} bytes)", + ) + + // Drive the EXACT helper QuicConnectionDriver.sendLoop + // calls when its PTO timer fires. Earlier versions of + // this test inlined the simulation (set pendingPing, + // call requeueAllInflightCrypto manually) — but that + // hid the regression where the driver itself stopped + // calling requeueAllInflightCrypto. Calling + // [handlePtoFired] keeps the test in lockstep with the + // production code path: if anyone unwires the requeue + // again, this test breaks. + handlePtoFired(client) + + // Next drain must emit a fresh Initial packet carrying + // the ClientHello CRYPTO at the original offset. + val ptoDrain = drainOutbound(client, nowMillis = 3L) + assertNotNull(ptoDrain, "PTO drain must produce a retransmit datagram") + assertTrue( + ptoDrain.size >= 1200, + "RFC 9000 §14.1 — Initial-bearing datagram still pads to ≥ 1200 bytes on PTO retransmit (got ${ptoDrain.size})", + ) + + // The retransmit packet must carry a Crypto token at the + // original offset and length — that's how we know the + // CRYPTO frame went out (not a PING-only probe). + val replayEntries = + client.initial.sentPackets.entries + .filter { it.key != firstSent.key } + val replaySent = + replayEntries.firstOrNull { entry -> + entry.value.tokens.any { it is RecoveryToken.Crypto } + } + assertNotNull( + replaySent, + "PTO drain must produce a fresh Initial SentPacket carrying Crypto " + + "(saw ${replayEntries.map { it.value.tokens.map { t -> t::class.simpleName } }})", + ) + val replayCrypto = + replaySent.value.tokens + .filterIsInstance() + .single() + assertEquals(EncryptionLevel.INITIAL, replayCrypto.level) + assertEquals(firstCrypto.offset, replayCrypto.offset, "PTO retransmit replays original offset") + assertEquals(firstCrypto.length, replayCrypto.length, "PTO retransmit replays original length") + + // Decode the retransmit packet and assert the CRYPTO + // frame's payload bytes match the original. + val firstFrames = decodeInitialFrames(firstDrain, client) + val firstHello = + firstFrames.filterIsInstance().firstOrNull { + it.offset == firstCrypto.offset && it.data.size.toLong() == firstCrypto.length + } + assertNotNull(firstHello, "first drain's Initial must contain the ClientHello CRYPTO") + + val ptoFrames = decodeInitialFrames(ptoDrain, client) + val replayHello = + ptoFrames.filterIsInstance().firstOrNull { + it.offset == firstCrypto.offset + } + assertNotNull( + replayHello, + "PTO drain's Initial must contain a CRYPTO frame at offset 0 — bare PING is not enough " + + "(saw frames ${ptoFrames.map { it::class.simpleName }})", + ) + assertTrue( + replayHello.data.contentEquals(firstHello.data), + "PTO retransmit must carry the same ClientHello bytes (size first=${firstHello.data.size} replay=${replayHello.data.size})", + ) + + // pendingPing should be cleared since the CRYPTO frame + // satisfied the ack-eliciting requirement at the level. + assertEquals( + false, + client.pendingPing, + "pendingPing must be consumed once the PTO emit went out (CRYPTO covered the probe)", + ) + } + + /** + * Decode an Initial-level long-header packet from a freshly-drained + * datagram and return its frames. We open with the client's own + * SEND protection (client-side keys) — symmetric AEAD opens with + * the same key/iv that sealed it. + */ + private fun decodeInitialFrames( + datagram: ByteArray, + client: QuicConnection, + ): List { + val send = client.initial.sendProtection!! + val parsed = + LongHeaderPacket.parseAndDecrypt( + bytes = datagram, + offset = 0, + aead = send.aead, + key = send.key, + iv = send.iv, + hp = send.hp, + hpKey = send.hpKey, + largestReceivedInSpace = -1L, + ) + assertNotNull(parsed, "must parse the Initial packet header (datagram size=${datagram.size})") + return com.vitorpamplona.quic.frame + .decodeFrames(parsed.packet.payload) + } + + private fun newClientWithStartedHandshake(): QuicConnection = + runBlocking { + val client = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = + com.vitorpamplona.quic.tls + .PermissiveCertificateValidator(), + ) + client.start() + client + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/ResetStopSendingEmitTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/ResetStopSendingEmitTest.kt index d71ca978cc..43af5e5b06 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/ResetStopSendingEmitTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/ResetStopSendingEmitTest.kt @@ -93,12 +93,12 @@ class ResetStopSendingEmitTest { .single() // Simulate loss. - client.lock.lock() + client.streamsLock.lock() try { client.onTokensLost(listOf(token)) client.application.sentPackets.remove(firstEntry.key) } finally { - client.lock.unlock() + client.streamsLock.unlock() } // Per-stream emit-pending should be re-flagged. assertTrue(stream.resetEmitPending, "loss must re-flag resetEmitPending") @@ -137,21 +137,21 @@ class ResetStopSendingEmitTest { .single() // ACK first. - client.lock.lock() + client.streamsLock.lock() try { client.onTokensAcked(listOf(token)) } finally { - client.lock.unlock() + client.streamsLock.unlock() } assertEquals(true, stream.resetAcked) assertEquals(false, stream.resetEmitPending) // Now a stale loss notification arrives. Defensive: drop. - client.lock.lock() + client.streamsLock.lock() try { client.onTokensLost(listOf(token)) } finally { - client.lock.unlock() + client.streamsLock.unlock() } assertEquals(false, stream.resetEmitPending, "stale loss after ACK must not re-flag emit-pending") } @@ -248,11 +248,11 @@ class ResetStopSendingEmitTest { connectionId = byteArrayOf(1, 2, 3, 4), statelessResetToken = ByteArray(16) { it.toByte() }, ) - client.lock.lock() + client.streamsLock.lock() try { client.onTokensLost(listOf(token)) } finally { - client.lock.unlock() + client.streamsLock.unlock() } assertEquals(token, client.pendingNewConnectionId[1L]) diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/RetransmitIntegrationTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/RetransmitIntegrationTest.kt index 73a05ce1c8..dc35823f57 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/RetransmitIntegrationTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/RetransmitIntegrationTest.kt @@ -69,7 +69,7 @@ class RetransmitIntegrationTest { // ACK'd by reordering — its PN < largestAckedPn - // PACKET_THRESHOLD ⇒ declared lost. val futurePn = msuPn + 4L - client.lock.lock() + client.streamsLock.lock() try { // Inject a phantom SentPacket at futurePn so the loss // detector has a credible "newly acked" reference, then @@ -102,7 +102,7 @@ class RetransmitIntegrationTest { // 5. Dispatch lost tokens — pendingMaxStreamsUni gets set. client.onTokensLost(lostMsuPacket.tokens) } finally { - client.lock.unlock() + client.streamsLock.unlock() } assertEquals( capAfterFirstDrain, @@ -148,24 +148,24 @@ class RetransmitIntegrationTest { // Second bump: open more peer-uni streams to cross the // (already extended) threshold again. - client.lock.lock() + client.streamsLock.lock() try { client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 2)) client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 3)) client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 4)) } finally { - client.lock.unlock() + client.streamsLock.unlock() } runCatching { drainOutbound(client, nowMillis = 2L) } val secondCap = client.advertisedMaxStreamsUni assertTrue(secondCap > firstCap, "second drain must advertise a still-higher cap; saw $firstCap → $secondCap") // Now declare the FIRST emit lost via direct dispatch. - client.lock.lock() + client.streamsLock.lock() try { client.onTokensLost(listOf(RecoveryToken.MaxStreamsUni(maxStreams = firstCap))) } finally { - client.lock.unlock() + client.streamsLock.unlock() } // Supersede check: firstCap != advertisedMaxStreamsUni (now == secondCap), // so pending must remain null. @@ -216,12 +216,12 @@ class RetransmitIntegrationTest { private fun crossPeerUniHalfWindow(client: QuicConnection) = runBlocking { - client.lock.lock() + client.streamsLock.lock() try { client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 0)) client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 1)) } finally { - client.lock.unlock() + client.streamsLock.unlock() } } } diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/RetryHandlingTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/RetryHandlingTest.kt new file mode 100644 index 0000000000..d5e79d790f --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/RetryHandlingTest.kt @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.QuicReader +import com.vitorpamplona.quic.QuicWriter +import com.vitorpamplona.quic.packet.LongHeaderPacket +import com.vitorpamplona.quic.packet.LongHeaderType +import com.vitorpamplona.quic.packet.QuicVersion +import com.vitorpamplona.quic.packet.RetryPacket +import com.vitorpamplona.quic.tls.PermissiveCertificateValidator +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Retry packet handling end-to-end through [QuicConnection], per RFC 9000 + * §17.2.5 (semantics) + RFC 9001 §5.8 (integrity tag). + * + * Synthesizes valid-and-invalid Retry packets, feeds them through + * [feedDatagram], and asserts the resulting connection state: + * + * 1. Happy path: DCID swaps, retryToken stored, Initial PN reset to 0, + * next outbound Initial carries the token in its header, contains the + * ClientHello CRYPTO, and the datagram is padded to ≥ 1200 bytes. + * 2. Bad-tag path: corrupting the integrity tag must be silently dropped; + * no state advances. + * 3. Second-Retry path: a second valid Retry after a first one is dropped + * (RFC 9000 §17.2.5.2 — at most one Retry per connection). + */ +class RetryHandlingTest { + private fun newClient(): QuicConnection = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = PermissiveCertificateValidator(), + ) + + /** + * Build the on-wire bytes of a valid Retry packet for [client], with the + * given [retryScid] and [retryToken]. Computes the integrity tag using + * the client's [QuicConnection.originalDestinationConnectionId] so the + * client's [RetryPacket.verifyIntegrityTag] check passes. + * + * The Retry packet's DCID is the client's source CID (servers echo it + * even though it's unused — RFC 9000 §17.2.5.1). The high 4 bits of + * the first byte are 1100 (long header + RETRY type); the low 4 bits + * are unused — we set them to 0. + */ + private fun buildRetry( + client: QuicConnection, + retryScid: ConnectionId, + retryToken: ByteArray, + ): ByteArray { + val w = QuicWriter() + // Header form (1) | fixed bit (1) | long packet type RETRY (11) | unused (0000) + w.writeByte(0xC0 or (LongHeaderType.RETRY.code shl 4)) + w.writeUint32(QuicVersion.V1) + w.writeByte(client.sourceConnectionId.length) + w.writeBytes(client.sourceConnectionId.bytes) + w.writeByte(retryScid.length) + w.writeBytes(retryScid.bytes) + w.writeBytes(retryToken) + val withoutTag = w.toByteArray() + val tag = + RetryPacket.computeIntegrityTag( + retryPacketWithoutTag = withoutTag, + originalDestinationConnectionId = client.originalDestinationConnectionId.bytes, + ) + return withoutTag + tag + } + + /** + * Pull the Initial packet's Token field out of an on-wire datagram so + * we can assert on it. [LongHeaderPacket.parseAndDecrypt] decrypts the + * payload but doesn't surface the unprotected Token; we re-walk the + * header here to extract it without crypto. + */ + private fun extractInitialToken(datagram: ByteArray): ByteArray { + val r = QuicReader(datagram, 0) + val first = r.readByte() + require((first and 0x80) != 0) { "expected long header" } + val type = (first ushr 4) and 0x03 + require(type == LongHeaderType.INITIAL.code) { "expected INITIAL, got type=$type" } + r.readUint32() // version + val dcidLen = r.readByte() + r.readBytes(dcidLen) + val scidLen = r.readByte() + r.readBytes(scidLen) + val tokenLen = r.readVarint().toInt() + return r.readBytes(tokenLen) + } + + @Test + fun valid_retry_swaps_dcid_resets_pn_and_threads_token_into_next_initial() { + val client = newClient() + val originalDcid = client.originalDestinationConnectionId.bytes.copyOf() + client.start() + + // Drain the initial datagram (carries ClientHello at PN=0 with empty + // token field) so we can assert the pre-Retry state. + val firstDatagram = drainOutbound(client, nowMillis = 0L) + assertNotNull(firstDatagram, "client.start() should produce an Initial datagram") + assertEquals(0, extractInitialToken(firstDatagram).size, "pre-Retry Initial must have empty token") + + // Server picks a fresh source connection id and a token of its choice. + val retryScid = ConnectionId(byteArrayOf(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x78)) + val retryToken = "server-issued-retry-token".encodeToByteArray() + val retryDatagram = buildRetry(client, retryScid, retryToken) + + feedDatagram(client, retryDatagram, nowMillis = 1L) + + // DCID is now the Retry's SCID, originalDcid is unchanged. + assertContentEquals(retryScid.bytes, client.destinationConnectionId.bytes) + assertContentEquals(originalDcid, client.originalDestinationConnectionId.bytes) + assertNotEquals(originalDcid.toList(), retryScid.bytes.toList()) + + // Retry token captured. + assertContentEquals(retryToken, client.retryToken) + assertTrue(client.retryConsumed) + + // RFC 9001 §5.7 + RFC 9000 §17.2.5: the Initial PN namespace + // CONTINUES across the Retry boundary. The first ClientHello at + // PN=0 already consumed PN=0 (it was sent on the wire even though + // the server discarded it in favor of replying with Retry). The + // post-Retry Initial uses PN=1, not PN=0. The runner's retry + // testcase explicitly checks this — a client that resets PN to 0 + // gets "Client reset the packet number. Check failed for PN 0". + assertEquals(1L, client.initial.pnSpace.nextPacketNumber) + assertEquals(-1L, client.initial.pnSpace.largestReceived) + + // Next drain produces the retried Initial: token in header, ClientHello + // CRYPTO inside, datagram padded to ≥ 1200 (RFC 9000 §14.1). + val secondDatagram = drainOutbound(client, nowMillis = 2L) + assertNotNull(secondDatagram, "post-Retry drain must produce another Initial") + assertContentEquals(retryToken, extractInitialToken(secondDatagram)) + assertTrue( + secondDatagram.size >= 1200, + "retried Initial datagram must be padded to >= 1200 bytes (was ${secondDatagram.size})", + ) + + // The Initial is encrypted under the new keys derived from the retryScid + // DCID. Decrypt + verify it carries CRYPTO with the captured ClientHello + // bytes (== the prefix of the original ClientHello — drained ALL the + // bytes from cryptoSend on Retry replay). + val newSecrets = + com.vitorpamplona.quic.crypto.InitialSecrets + .derive(retryScid.bytes) + val proto = client.initial.sendProtection!! + val parsed = + LongHeaderPacket.parseAndDecrypt( + bytes = secondDatagram, + offset = 0, + aead = proto.aead, + key = newSecrets.clientKey, + iv = newSecrets.clientIv, + hp = + com.vitorpamplona.quic.crypto.AesEcbHeaderProtection( + com.vitorpamplona.quic.crypto.PlatformAesOneBlock, + ), + hpKey = newSecrets.clientHp, + largestReceivedInSpace = -1L, + ) + assertNotNull(parsed, "retried Initial must decrypt under keys derived from new DCID") + // RFC 9001 §5.7: Initial PN namespace continues across Retry. The + // pre-Retry Initial in this test already consumed PN=0, so the + // retried Initial uses PN=1. + assertEquals(1L, parsed.packet.packetNumber, "retried Initial PN continues from pre-Retry counter (RFC 9001 §5.7)") + // Decoded payload starts with at least one CRYPTO frame (frame type 0x06). + val frames = + com.vitorpamplona.quic.frame + .decodeFrames(parsed.packet.payload) + val cryptoFrames = frames.filterIsInstance() + assertTrue(cryptoFrames.isNotEmpty(), "retried Initial payload must contain CRYPTO frames (the ClientHello)") + assertEquals(0L, cryptoFrames.first().offset, "CRYPTO must restart at offset 0 on the new keys") + } + + @Test + fun retry_with_corrupted_integrity_tag_is_silently_dropped() { + val client = newClient() + val originalDcid = client.destinationConnectionId.bytes.copyOf() + client.start() + // Drain pre-Retry datagram so the test mirrors a realistic ordering. + drainOutbound(client, nowMillis = 0L) + + val retryScid = ConnectionId(byteArrayOf(0xAA.toByte(), 0xBB.toByte(), 0xCC.toByte(), 0xDD.toByte())) + val good = buildRetry(client, retryScid, "tk".encodeToByteArray()) + // Flip a bit in the last byte — the integrity tag. + val corrupted = good.copyOf() + corrupted[corrupted.size - 1] = (corrupted[corrupted.size - 1].toInt() xor 0x01).toByte() + + feedDatagram(client, corrupted, nowMillis = 1L) + + // No state advanced. + assertNull(client.retryToken) + assertFalse(client.retryConsumed) + assertContentEquals(originalDcid, client.destinationConnectionId.bytes) + } + + @Test + fun second_valid_retry_after_one_is_consumed_is_dropped() { + val client = newClient() + client.start() + drainOutbound(client, nowMillis = 0L) + + val firstScid = ConnectionId(byteArrayOf(0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08)) + val firstToken = "first".encodeToByteArray() + feedDatagram(client, buildRetry(client, firstScid, firstToken), nowMillis = 1L) + + // Sanity: first applied. + assertTrue(client.retryConsumed) + assertContentEquals(firstToken, client.retryToken) + assertContentEquals(firstScid.bytes, client.destinationConnectionId.bytes) + + // Build a second VALID Retry. The integrity tag is computed against + // [originalDestinationConnectionId] (still the very first random one, + // unchanged), so this packet's tag genuinely verifies. + val secondScid = ConnectionId(byteArrayOf(0x99.toByte(), 0x88.toByte(), 0x77.toByte(), 0x66.toByte())) + val secondToken = "second-should-be-ignored".encodeToByteArray() + val secondRetry = buildRetry(client, secondScid, secondToken) + + // Confirm the integrity tag really would verify in isolation — + // otherwise this test would conflate "bad tag" with "second retry". + val parsedSecond = RetryPacket.parse(secondRetry) + assertNotNull(parsedSecond) + assertTrue( + parsedSecond.verifyIntegrityTag(secondRetry, client.originalDestinationConnectionId.bytes), + "second retry's tag must be valid in isolation; otherwise this test is meaningless", + ) + + feedDatagram(client, secondRetry, nowMillis = 2L) + + // State unchanged from after the first retry. + assertContentEquals(firstToken, client.retryToken) + assertContentEquals(firstScid.bytes, client.destinationConnectionId.bytes) + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/SentPacketTrackingTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/SentPacketTrackingTest.kt index b11e72a554..81032e72fb 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/SentPacketTrackingTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/SentPacketTrackingTest.kt @@ -200,12 +200,12 @@ class SentPacketTrackingTest { /** Cross the half-window threshold (cap=4, two peer-uni streams ⇒ count >= cap-half=2). */ private fun crossPeerUniHalfWindow(client: QuicConnection) = runBlocking { - client.lock.lock() + client.streamsLock.lock() try { client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 0)) client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 1)) } finally { - client.lock.unlock() + client.streamsLock.unlock() } } } diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/StreamRetransmitTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/StreamRetransmitTest.kt index b6cb893d84..25e039b4c4 100644 --- a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/StreamRetransmitTest.kt +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/StreamRetransmitTest.kt @@ -82,7 +82,7 @@ class StreamRetransmitTest { val firstPn = firstPacketEntry.key // Simulate loss via direct dispatch. - client.lock.lock() + client.streamsLock.lock() try { val streamToken = firstPacketEntry.value.tokens @@ -93,7 +93,7 @@ class StreamRetransmitTest { // detector would have done this). client.application.sentPackets.remove(firstPn) } finally { - client.lock.unlock() + client.streamsLock.unlock() } // SendBuffer should have re-queued the bytes for retransmit. @@ -133,11 +133,11 @@ class StreamRetransmitTest { val packet = client.application.sentPackets.entries .first { it.value.tokens.any { t -> t is RecoveryToken.Stream } } - client.lock.lock() + client.streamsLock.lock() try { client.onTokensAcked(packet.value.tokens) } finally { - client.lock.unlock() + client.streamsLock.unlock() } // After ACK: enqueue more, observe that the buffer diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/VersionNegotiationTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/VersionNegotiationTest.kt new file mode 100644 index 0000000000..657aedb7ce --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/connection/VersionNegotiationTest.kt @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.connection + +import com.vitorpamplona.quic.packet.QuicVersion +import com.vitorpamplona.quic.tls.PermissiveCertificateValidator +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Version Negotiation flow per RFC 9000 §6. + * + * The runner's `versionnegotiation` testcase has the server reply to the + * client's first Initial with a VN packet whose supported_versions list + * contains v1 (and not the version we offered). The client must: + * + * 1. Validate the VN packet (DCID echoed = our SCID, list does NOT + * include the version we offered). + * 2. Pick a version it can speak from the list (we only support v1). + * 3. Generate a fresh DCID, re-derive Initial keys, reset Initial PN + * space, re-emit the cached ClientHello, and switch the writer's + * stamped version to v1. + * 4. Latch `vnConsumed` so a second VN is dropped. + * + * Failure modes also covered: + * + * - Downgrade defense: VN that lists the version we offered is dropped + * (RFC 9000 §6.2 — anti-replay). + * - Unsupported list: VN whose supported_versions doesn't include any + * version we can speak fails the handshake with + * [QuicVersionNegotiationException]. + * - Second-VN: after one consumed VN, any subsequent VN is dropped + * even if otherwise valid. + */ +class VersionNegotiationTest { + /** + * Synthesize a VN packet on the wire (RFC 9000 §17.2.1): + * first byte : 0x80 | + * version : 0x00000000 + * dcid_len + dcid (echoes the client's source CID per §6.1) + * scid_len + scid (server picks) + * supported_versions: 32-bit big-endian numbers, one per offered version + */ + private fun encodeVnPacket( + echoedDcid: ConnectionId, + serverScid: ConnectionId, + supportedVersions: List, + ): ByteArray { + val out = ArrayList() + out += 0x80.toByte() // form bit set; remaining bits unused / random + // version=0 + out += 0x00.toByte() + out += 0x00.toByte() + out += 0x00.toByte() + out += 0x00.toByte() + out += echoedDcid.length.toByte() + for (b in echoedDcid.bytes) out += b + out += serverScid.length.toByte() + for (b in serverScid.bytes) out += b + for (v in supportedVersions) { + out += ((v ushr 24) and 0xFF).toByte() + out += ((v ushr 16) and 0xFF).toByte() + out += ((v ushr 8) and 0xFF).toByte() + out += (v and 0xFF).toByte() + } + return out.toByteArray() + } + + private fun newClient(initialVersion: Int) = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = PermissiveCertificateValidator(), + initialVersion = initialVersion, + ) + + @Test + fun happy_path_vn_switches_to_v1_and_resets_dcid_pn_keys() { + val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION) + client.start() + // The first Initial would be stamped with the forced version. + assertEquals(QuicVersion.FORCE_VERSION_NEGOTIATION, client.currentVersion) + val originalDcid = client.destinationConnectionId + + val serverScid = ConnectionId.random(8) + val vn = + encodeVnPacket( + echoedDcid = client.sourceConnectionId, + serverScid = serverScid, + supportedVersions = listOf(QuicVersion.V1), + ) + + feedDatagram(client, vn, nowMillis = 0L) + + assertTrue(client.vnConsumed, "valid VN must latch vnConsumed") + assertEquals(QuicVersion.V1, client.currentVersion, "currentVersion must switch to v1") + assertNotEquals( + originalDcid, + client.destinationConnectionId, + "DCID must be regenerated (RFC 9000 §6 fresh handshake)", + ) + // Initial PN space is fresh — next outbound allocate returns 0. + assertEquals( + 0L, + client.initial.pnSpace.nextPacketNumber, + "Initial PN space must reset to 0 after VN", + ) + // The cached ClientHello must be re-queued so the next drain emits a v1 + // Initial with the same handshake bytes. + val drained = drainOutbound(client, nowMillis = 1L) + assertTrue(drained != null && drained.isNotEmpty(), "post-VN drain must emit a fresh Initial") + // Wire-level check: bytes 1..4 of the long-header packet are the version. + val versionOnWire = + ((drained[1].toInt() and 0xFF) shl 24) or + ((drained[2].toInt() and 0xFF) shl 16) or + ((drained[3].toInt() and 0xFF) shl 8) or + (drained[4].toInt() and 0xFF) + assertEquals( + QuicVersion.V1, + versionOnWire, + "post-VN Initial datagram must carry v1 in the long-header version field", + ) + } + + @Test + fun downgrade_defense_vn_listing_offered_version_is_dropped() { + val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION) + client.start() + val originalDcid = client.destinationConnectionId + val originalVersion = client.currentVersion + + val vn = + encodeVnPacket( + echoedDcid = client.sourceConnectionId, + serverScid = ConnectionId.random(8), + // The list MUST NOT contain the version we offered. If it does, + // it's a probable replay/spoof — RFC 9000 §6.2 says drop. + supportedVersions = listOf(QuicVersion.V1, QuicVersion.FORCE_VERSION_NEGOTIATION), + ) + + feedDatagram(client, vn, nowMillis = 0L) + + assertFalse(client.vnConsumed, "anti-replay: VN containing offered version must be dropped") + assertEquals(originalVersion, client.currentVersion, "currentVersion unchanged") + assertEquals(originalDcid, client.destinationConnectionId, "DCID unchanged") + } + + @Test + fun unsupported_list_fails_handshake() { + val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION) + client.start() + + // quic-go's force-VN test version. We don't support it, so the + // handshake must fail. + val vn = + encodeVnPacket( + echoedDcid = client.sourceConnectionId, + serverScid = ConnectionId.random(8), + supportedVersions = listOf(0x6b3343cf), + ) + + feedDatagram(client, vn, nowMillis = 0L) + + // Unsupported list ⇒ handshake fails BEFORE the latch is set. + // vnConsumed therefore stays false; the connection is forced + // closed via signalHandshakeFailed → markClosedExternally. + assertFalse(client.vnConsumed, "vnConsumed only latches on successful version pick") + assertEquals( + QuicConnection.Status.CLOSED, + client.status, + "no mutually-supported version must close the connection", + ) + } + + @Test + fun second_vn_is_ignored_after_first() { + val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION) + client.start() + + // First VN: valid, switches to v1. + val serverScid1 = ConnectionId.random(8) + feedDatagram( + client, + encodeVnPacket( + echoedDcid = client.sourceConnectionId, + serverScid = serverScid1, + supportedVersions = listOf(QuicVersion.V1), + ), + nowMillis = 0L, + ) + assertTrue(client.vnConsumed) + assertEquals(QuicVersion.V1, client.currentVersion) + val dcidAfterFirstVn = client.destinationConnectionId + + // Second VN: even if structurally fine, must be dropped. We craft + // one whose supported list does NOT include the version we + // ORIGINALLY offered — so it would otherwise look valid. + feedDatagram( + client, + encodeVnPacket( + echoedDcid = client.sourceConnectionId, + serverScid = ConnectionId.random(8), + supportedVersions = listOf(QuicVersion.V1), + ), + nowMillis = 1L, + ) + + assertEquals( + dcidAfterFirstVn, + client.destinationConnectionId, + "second VN must NOT regenerate the DCID", + ) + assertEquals(QuicVersion.V1, client.currentVersion, "current version stays at v1") + } + + @Test + fun vn_with_dcid_mismatch_is_dropped() { + // Defensive: a VN whose echoed DCID doesn't equal our SCID is + // probably an off-path attacker's spoof — drop without state change. + val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION) + client.start() + + val vn = + encodeVnPacket( + echoedDcid = ConnectionId.random(8), // wrong + serverScid = ConnectionId.random(8), + supportedVersions = listOf(QuicVersion.V1), + ) + + feedDatagram(client, vn, nowMillis = 0L) + + assertFalse(client.vnConsumed, "DCID mismatch ⇒ no state change") + assertEquals(QuicVersion.FORCE_VERSION_NEGOTIATION, client.currentVersion) + } + + @Test + fun default_initial_version_is_v1_for_existing_callers() { + // Backward-compat: not passing initialVersion must keep the writer + // emitting v1 (no behavior change for existing tests). + val client = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = PermissiveCertificateValidator(), + ) + assertEquals(QuicVersion.V1, client.currentVersion) + assertFalse(client.vnConsumed) + assertNull(client.peerTransportParameters) + } +} diff --git a/quic/src/commonTest/kotlin/com/vitorpamplona/quic/observability/QlogObserverTest.kt b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/observability/QlogObserverTest.kt new file mode 100644 index 0000000000..eaec428080 --- /dev/null +++ b/quic/src/commonTest/kotlin/com/vitorpamplona/quic/observability/QlogObserverTest.kt @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.observability + +import com.vitorpamplona.quic.connection.ConnectionId +import com.vitorpamplona.quic.connection.EncryptionLevel +import com.vitorpamplona.quic.connection.InMemoryQuicPipe +import com.vitorpamplona.quic.connection.QuicConnection +import com.vitorpamplona.quic.connection.QuicConnectionConfig +import com.vitorpamplona.quic.connection.TransportParameters +import com.vitorpamplona.quic.connection.feedDatagram +import com.vitorpamplona.quic.tls.InProcessTlsServer +import com.vitorpamplona.quic.tls.PermissiveCertificateValidator +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.withLock +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * End-to-end smoke for [QlogObserver]: + * + * 1. NoOp safety — a connection driven through start → handshake → + * close with [QlogObserver.NoOp] must complete without throwing. + * 2. RecordingQlogObserver captures the expected events (start, + * local transport_params, packet_sent for ClientHello, close). + * 3. Malformed inbound datagram surfaces a `packet_dropped` event. + * + * The recorded-event list documented in the implementation report + * comes from the second test below. + */ +class QlogObserverTest { + @Test + fun noOpObserver_handshakeAndCloseDoNotThrow(): Unit = + runBlocking { + val client = handshakedClient(QlogObserver.NoOp) + // Drive a tiny operation post-handshake. + client.lock.withLock { + // No-op — just confirm we can still take the lock. + assertEquals(QuicConnection.Status.CONNECTED, client.status) + } + client.close(0L, "done") + // close() flips status to CLOSING; the writer's next + // drainOutbound emits CONNECTION_CLOSE and transitions to + // CLOSED. We don't run the driver here, so stop at CLOSING + // and assert that — the point of this test is just that + // the no-op observer doesn't throw on any call site. + assertTrue( + client.status == QuicConnection.Status.CLOSING || + client.status == QuicConnection.Status.CLOSED, + "expected CLOSING or CLOSED, was ${client.status}", + ) + } + + @Test + fun recordingObserver_capturesStartParametersSentAndClose(): Unit = + runBlocking { + val recorder = RecordingQlogObserver() + val client = handshakedClient(recorder) + client.close(0L, "shutdown") + + val names = recorder.events.map { it.name } + assertTrue( + names.contains("connectionStarted"), + "expected connectionStarted in $names", + ) + // Local transport parameters set at start. + val localTps = + recorder.events + .filter { it.name == "transportParametersSet" } + .map { it.payload["initiator"] } + assertTrue( + localTps.contains("local"), + "expected local transportParametersSet in $localTps", + ) + // At least one packet_sent for the ClientHello at INITIAL level. + val initialSends = + recorder.events.filter { + it.name == "packetSent" && it.payload["level"] == EncryptionLevel.INITIAL + } + assertTrue( + initialSends.isNotEmpty(), + "expected at least one INITIAL-level packetSent (ClientHello) " + + "but saw ${recorder.events.map { it.name to it.payload["level"] }}", + ) + assertTrue( + names.contains("connectionClosed"), + "expected connectionClosed in $names", + ) + } + + @Test + fun malformedDatagram_recordsPacketDropped(): Unit = + runBlocking { + val recorder = RecordingQlogObserver() + val client = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = PermissiveCertificateValidator(), + qlogObserver = recorder, + ) + client.start() + // Long-header packet bytes that look parseable enough to peek + // but won't decrypt — the receive keys for HANDSHAKE/APPLICATION + // aren't installed yet, so feedDatagram drops with "no receive + // keys at level …". 0xC0 = long header, INITIAL with the + // smallest valid layout: 0xC0 | version(0x00000001) | dcil=0 + // | scil=0 | token_len=0 | length=2 | pn=0x00 | one byte of + // garbage. AEAD-decrypt will fail on this Initial too, since + // the keys are derived from a different DCID. + val garbage = + byteArrayOf( + 0xC0.toByte(), // long header initial + 0x00, + 0x00, + 0x00, + 0x01, // version v1 + 0x00, // dcil = 0 + 0x00, // scil = 0 + 0x00, // token length varint = 0 + 0x02, // length = 2 + 0x00, // pn byte + 0x00, // payload byte (will fail AEAD) + ) + client.lock.withLock { + feedDatagram(client, garbage, nowMillis = 1L) + } + val drops = recorder.events.filter { it.name == "packetDropped" } + assertTrue( + drops.isNotEmpty(), + "expected at least one packetDropped event but saw ${recorder.events.map { it.name }}", + ) + } + + private fun handshakedClient(observer: QlogObserver): QuicConnection = + runBlocking { + val client = + QuicConnection( + serverName = "example.test", + config = QuicConnectionConfig(), + tlsCertificateValidator = PermissiveCertificateValidator(), + qlogObserver = observer, + ) + val serverScid = ConnectionId.random(8) + val tlsServer = + InProcessTlsServer( + transportParameters = + TransportParameters( + initialMaxData = 1_000_000, + initialMaxStreamDataBidiLocal = 100_000, + initialMaxStreamDataBidiRemote = 100_000, + initialMaxStreamDataUni = 100_000, + initialMaxStreamsBidi = 100, + initialMaxStreamsUni = 100, + initialSourceConnectionId = serverScid.bytes, + originalDestinationConnectionId = client.destinationConnectionId.bytes, + ).encode(), + ) + val pipe = + InMemoryQuicPipe( + client = client, + initialDcid = client.destinationConnectionId.bytes, + serverScid = serverScid, + tlsServer = tlsServer, + ) + client.start() + pipe.drive(maxRounds = 16) + assertEquals(QuicConnection.Status.CONNECTED, client.status) + client + } +} + +/** + * Test-only [QlogObserver] that appends every callback into a list, + * with a tiny serialized-payload shape so assertions read like + * structured pattern-matches rather than a soup of positional args. + */ +internal class RecordingQlogObserver : QlogObserver { + data class Event( + val name: String, + val payload: Map, + ) + + val events: MutableList = mutableListOf() + + private fun add( + name: String, + payload: Map, + ) { + events += Event(name, payload) + } + + override fun onConnectionStarted( + serverName: String, + dcid: ByteArray, + scid: ByteArray, + ) = add( + "connectionStarted", + mapOf("serverName" to serverName, "dcid_size" to dcid.size, "scid_size" to scid.size), + ) + + override fun onConnectionClosed( + initiator: String, + errorCode: Long, + reason: String, + ) = add( + "connectionClosed", + mapOf("initiator" to initiator, "errorCode" to errorCode, "reason" to reason), + ) + + override fun onPacketSent( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) = add( + "packetSent", + mapOf("level" to level, "pn" to packetNumber, "size" to sizeBytes, "frames" to frames), + ) + + override fun onPacketReceived( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) = add( + "packetReceived", + mapOf("level" to level, "pn" to packetNumber, "size" to sizeBytes, "frames" to frames), + ) + + override fun onPacketDropped( + reason: String, + sizeBytes: Int, + ) = add("packetDropped", mapOf("reason" to reason, "size" to sizeBytes)) + + override fun onKeyUpdated( + keyType: String, + level: EncryptionLevel, + ) = add("keyUpdated", mapOf("keyType" to keyType, "level" to level)) + + override fun onLossDetected( + level: EncryptionLevel, + lostPacketNumbers: List, + ) = add("lossDetected", mapOf("level" to level, "lost" to lostPacketNumbers)) + + override fun onPtoFired( + consecutivePtoCount: Int, + ptoMillis: Long, + ) = add("ptoFired", mapOf("count" to consecutivePtoCount, "ptoMillis" to ptoMillis)) + + override fun onCongestionStateUpdated(newState: String) = add("congestionStateUpdated", mapOf("newState" to newState)) + + override fun onTransportParametersSet( + initiator: String, + params: Map, + ) = add("transportParametersSet", mapOf("initiator" to initiator, "params" to params)) + + override fun onAlpnNegotiated(alpn: String) = add("alpnNegotiated", mapOf("alpn" to alpn)) + + override fun onVersionInformation( + chosenVersion: String, + otherVersionsOffered: List, + ) = add( + "versionInformation", + mapOf("chosen" to chosenVersion, "offered" to otherVersionsOffered), + ) +} diff --git a/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/InteropRunner.kt b/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/InteropRunner.kt index 514df4d5d9..33f3084714 100644 --- a/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/InteropRunner.kt +++ b/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/InteropRunner.kt @@ -23,6 +23,8 @@ package com.vitorpamplona.quic.interop import com.vitorpamplona.quic.connection.QuicConnection import com.vitorpamplona.quic.connection.QuicConnectionConfig import com.vitorpamplona.quic.connection.QuicConnectionDriver +import com.vitorpamplona.quic.observability.QlogObserver +import com.vitorpamplona.quic.packet.QuicVersion import com.vitorpamplona.quic.tls.PermissiveCertificateValidator import com.vitorpamplona.quic.transport.UdpSocket import kotlinx.coroutines.CoroutineScope @@ -31,6 +33,7 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull +import java.io.File /** * Standalone interop runner. Drives [QuicConnection] against a real QUIC @@ -52,12 +55,41 @@ fun main(args: Array) { val host = System.getProperty("interopHost") ?: args.getOrNull(0) ?: "127.0.0.1" val port = (System.getProperty("interopPort") ?: args.getOrNull(1) ?: "4433").toInt() val timeoutSec = (System.getProperty("interopTimeoutSec") ?: "10").toLong() + // Public quic-interop-runner contract: TESTCASE env names the scenario. + // We currently only special-case `versionnegotiation`; everything else + // falls through to the default v1-handshake path, which is enough for + // the `transfer` / `handshake` / `multiconnect` testcases. + val testcase = + System.getProperty("interopTestcase") + ?: System.getenv("TESTCASE") + ?: args.getOrNull(2) + val initialVersion = + when (testcase) { + "versionnegotiation" -> QuicVersion.FORCE_VERSION_NEGOTIATION + else -> QuicVersion.V1 + } println("== :quic interop runner ==") - println("target: $host:$port") - println("timeout: ${timeoutSec}s") + println("target: $host:$port") + println("testcase: ${testcase ?: "(default)"}") + println("init ver: 0x${initialVersion.toUInt().toString(16)}") + println("timeout: ${timeoutSec}s") println() + // qlog: if QLOGDIR is set, drop a `client.sqlog` file at + // /client.sqlog so the operator can hand the failed run + // to qvis. The interop-runner contract from the IETF QUIC team's + // Docker harness (quic-interop-runner) sets this env var on every + // test invocation. + val qlogDir = + (System.getenv("QLOGDIR") ?: System.getProperty("QLOGDIR"))?.takeIf { it.isNotBlank() } + val qlogWriter: QlogWriter? = + qlogDir?.let { dir -> + val parent = File(dir).also { it.mkdirs() } + QlogWriter(File(parent, "client.sqlog"), odcidHex = "00") + } + val qlogObserver: QlogObserver = qlogWriter ?: QlogObserver.NoOp + val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) val outcome = runBlocking { @@ -73,6 +105,7 @@ fun main(args: Array) { serverName = host, config = QuicConnectionConfig(), tlsCertificateValidator = PermissiveCertificateValidator(), + qlogObserver = qlogObserver, ) val driver = QuicConnectionDriver(conn, socket, scope) driver.start() @@ -110,6 +143,10 @@ fun main(args: Array) { // are torn down before main() exits. Without this the JVM hangs on stray // IO-dispatcher threads. scope.cancel() + // Flush + close the qlog file before exiting. Without this, an + // exitProcess() call below could leave the trailing events + // unflushed. + runCatching { qlogWriter?.close() } when (outcome) { is InteropOutcome.Connected -> { diff --git a/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/QlogWriter.kt b/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/QlogWriter.kt new file mode 100644 index 0000000000..585d17c3e6 --- /dev/null +++ b/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/QlogWriter.kt @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.vitorpamplona.quic.connection.EncryptionLevel +import com.vitorpamplona.quic.observability.QlogObserver +import java.io.BufferedWriter +import java.io.Closeable +import java.io.File +import java.io.FileWriter +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +/** + * JSON-NDJSON qlog writer (qlog 0.3 / JSON-SEQ format) used by the + * `:quic` interop runner. One JSON object per line; first line is the + * qlog header, subsequent lines are events. + * + * Tools like qvis (https://qvis.quictools.info/) consume the resulting + * `.sqlog` file to render sequence diagrams + RTT graphs + recovery + * timelines. + * + * **Goal: every interop-runner test failure produces a qlog file the + * caller can drop into qvis to see exactly what we did differently + * from the spec.** + * + * Threading: [java.io.BufferedWriter] is not safe for concurrent + * writers; we hold a [ReentrantLock] around each line emit so the + * read + send loops can fire events concurrently without interleaving + * partial JSON. + */ +class QlogWriter( + file: File, + private val odcidHex: String, + private val mapper: ObjectMapper = DEFAULT_MAPPER, + /** Wall-clock provider; tests inject a deterministic source. */ + private val nowMillis: () -> Long = { System.currentTimeMillis() }, +) : QlogObserver, + Closeable { + // append=false → truncate any prior trace at this path so a reused + // QLOGDIR doesn't accumulate stale events from a previous run. + private val writer: BufferedWriter = BufferedWriter(FileWriter(file, false)) + private val lock = ReentrantLock() + private val startMillis: Long = nowMillis() + + init { + // qlog 0.3 JSON-SEQ header. qvis tolerates both `qlog_format` + // values "JSON-SEQ" and "NDJSON"; we use JSON-SEQ to match the + // most-common production qlog files (Chromium, mvfst). + val header = + mapOf( + "qlog_version" to "0.3", + "qlog_format" to "JSON-SEQ", + "title" to "amethyst :quic client trace", + "trace" to + mapOf( + "vantage_point" to mapOf("type" to "client", "name" to "amethyst-quic"), + "common_fields" to + mapOf( + "ODCID" to odcidHex, + "reference_time" to startMillis, + "time_format" to "relative", + ), + ), + ) + writeLineLocked(mapper.writeValueAsString(header)) + } + + override fun onConnectionStarted( + serverName: String, + dcid: ByteArray, + scid: ByteArray, + ) { + emit( + "transport:connection_started", + mapOf( + "ip_version" to "v4_or_v6", + "server_name" to serverName, + "dst_cid" to hex(dcid), + "src_cid" to hex(scid), + ), + ) + } + + override fun onConnectionClosed( + initiator: String, + errorCode: Long, + reason: String, + ) { + emit( + "transport:connection_closed", + mapOf( + "owner" to initiator, + "application_code" to errorCode, + "reason" to reason, + ), + ) + } + + override fun onPacketSent( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) { + emit( + "transport:packet_sent", + mapOf( + "header" to + mapOf( + "packet_type" to packetTypeFor(level), + "packet_number" to packetNumber, + ), + "raw" to mapOf("length" to sizeBytes), + "frames" to frames.map { mapOf("frame_type" to it) }, + ), + ) + } + + override fun onPacketReceived( + level: EncryptionLevel, + packetNumber: Long, + sizeBytes: Int, + frames: List, + ) { + emit( + "transport:packet_received", + mapOf( + "header" to + mapOf( + "packet_type" to packetTypeFor(level), + "packet_number" to packetNumber, + ), + "raw" to mapOf("length" to sizeBytes), + "frames" to frames.map { mapOf("frame_type" to it) }, + ), + ) + } + + override fun onPacketDropped( + reason: String, + sizeBytes: Int, + ) { + emit( + "transport:packet_dropped", + mapOf( + "trigger" to reason, + "raw" to mapOf("length" to sizeBytes), + ), + ) + } + + override fun onKeyUpdated( + keyType: String, + level: EncryptionLevel, + ) { + emit( + "security:key_updated", + mapOf( + "key_type" to "${keyType}_${packetTypeFor(level)}_secret", + "trigger" to "tls", + ), + ) + } + + override fun onLossDetected( + level: EncryptionLevel, + lostPacketNumbers: List, + ) { + for (pn in lostPacketNumbers) { + emit( + "recovery:packet_lost", + mapOf( + "header" to + mapOf( + "packet_type" to packetTypeFor(level), + "packet_number" to pn, + ), + "trigger" to "reordering_threshold_or_time_threshold", + ), + ) + } + } + + override fun onPtoFired( + consecutivePtoCount: Int, + ptoMillis: Long, + ) { + emit( + "recovery:loss_timer_updated", + mapOf( + "event_type" to "expired", + "timer_type" to "pto", + "pto_count" to consecutivePtoCount, + "delta" to ptoMillis, + ), + ) + } + + override fun onCongestionStateUpdated(newState: String) { + emit( + "recovery:congestion_state_updated", + mapOf("new" to newState), + ) + } + + override fun onTransportParametersSet( + initiator: String, + params: Map, + ) { + emit( + "transport:parameters_set", + mapOf( + "owner" to initiator, + "params" to params, + ), + ) + } + + override fun onAlpnNegotiated(alpn: String) { + emit( + "transport:alpn_information", + mapOf("chosen_alpn" to alpn), + ) + } + + override fun onVersionInformation( + chosenVersion: String, + otherVersionsOffered: List, + ) { + emit( + "transport:version_information", + mapOf( + "chosen_version" to chosenVersion, + "client_versions" to otherVersionsOffered, + ), + ) + } + + override fun close() { + lock.withLock { + try { + writer.flush() + } finally { + writer.close() + } + } + } + + private fun emit( + name: String, + data: Map, + ) { + val event = + linkedMapOf( + "time" to (nowMillis() - startMillis), + "name" to name, + "data" to data, + ) + // Serialize OUTSIDE the lock so concurrent emitters don't + // serialize their JSON serially. The lock is only held while + // appending the line to the file. + val line = mapper.writeValueAsString(event) + writeLineLocked(line) + } + + private fun writeLineLocked(line: String) { + lock.withLock { + writer.write(line) + writer.write("\n") + // Flush every line so a hard-killed process still leaves a + // partial-but-parseable trace. + writer.flush() + } + } + + companion object { + private val DEFAULT_MAPPER: ObjectMapper = jacksonObjectMapper() + + private fun packetTypeFor(level: EncryptionLevel): String = + when (level) { + EncryptionLevel.INITIAL -> "initial" + EncryptionLevel.HANDSHAKE -> "handshake" + EncryptionLevel.APPLICATION -> "1RTT" + } + + private val HEX_CHARS = "0123456789abcdef".toCharArray() + + fun hex(bytes: ByteArray): String { + val sb = StringBuilder(bytes.size * 2) + for (b in bytes) { + val v = b.toInt() and 0xFF + sb.append(HEX_CHARS[v ushr 4]) + sb.append(HEX_CHARS[v and 0x0F]) + } + return sb.toString() + } + } +} diff --git a/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/QlogWriterTest.kt b/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/QlogWriterTest.kt new file mode 100644 index 0000000000..72f5d13845 --- /dev/null +++ b/quic/src/jvmTest/kotlin/com/vitorpamplona/quic/interop/QlogWriterTest.kt @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quic.interop + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.vitorpamplona.quic.connection.EncryptionLevel +import org.junit.Test +import java.io.File +import java.nio.file.Files +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * Validates the JSON-NDJSON shape qvis (https://qvis.quictools.info/) + * expects: header line + one event per line, every line independently + * parseable as JSON. + * + * Drives [QlogWriter] through one event of each type to confirm we + * don't emit anything that breaks the format. + */ +class QlogWriterTest { + @Test + fun headerThenOneEventPerLine_allParseable() { + val tmp = Files.createTempFile("amethyst-qlog-test", ".sqlog").toFile() + tmp.deleteOnExit() + var clock = 0L + QlogWriter(tmp, odcidHex = "deadbeef", nowMillis = { clock }).use { w -> + clock = 5L + w.onConnectionStarted("example.test", byteArrayOf(1, 2, 3), byteArrayOf(4, 5, 6)) + clock = 7L + w.onTransportParametersSet("local", mapOf("initial_max_data" to "1000000")) + clock = 10L + w.onPacketSent(EncryptionLevel.INITIAL, packetNumber = 0, sizeBytes = 1200, frames = listOf("crypto")) + clock = 15L + w.onPacketReceived(EncryptionLevel.INITIAL, packetNumber = 0, sizeBytes = 800, frames = listOf("crypto", "ack")) + clock = 20L + w.onPacketDropped("AEAD auth failed", sizeBytes = 80) + clock = 25L + w.onKeyUpdated("server", EncryptionLevel.HANDSHAKE) + clock = 30L + w.onLossDetected(EncryptionLevel.APPLICATION, lostPacketNumbers = listOf(3L, 5L)) + clock = 35L + w.onPtoFired(consecutivePtoCount = 1, ptoMillis = 333) + clock = 40L + w.onCongestionStateUpdated("recovery") + clock = 45L + w.onAlpnNegotiated("h3") + clock = 50L + w.onVersionInformation("v1", emptyList()) + clock = 55L + w.onConnectionClosed("local", errorCode = 0, reason = "done") + } + + val lines = tmp.readLines().filter { it.isNotBlank() } + assertTrue(lines.size >= 12, "expected >= 12 lines (header + at least 11 events) but got ${lines.size}") + + val mapper = jacksonObjectMapper() + + // Line 1: qlog header. + val header = mapper.readTree(lines[0]) + assertEquals("0.3", header.get("qlog_version").asText(), "qlog_version must be 0.3") + assertEquals("JSON-SEQ", header.get("qlog_format").asText(), "qlog_format must be JSON-SEQ") + val vp = header.get("trace").get("vantage_point") + assertEquals("client", vp.get("type").asText()) + assertEquals( + "deadbeef", + header + .get("trace") + .get("common_fields") + .get("ODCID") + .asText(), + ) + + // Lines 2..N: event objects with `time`, `name`, `data`. + for (i in 1 until lines.size) { + val node = mapper.readTree(lines[i]) + assertNotNull(node.get("time"), "line $i missing 'time': ${lines[i]}") + val name = node.get("name") + assertNotNull(name, "line $i missing 'name': ${lines[i]}") + assertTrue( + name.asText().contains(":"), + "name '${name.asText()}' must be in ':' form", + ) + assertNotNull(node.get("data"), "line $i missing 'data': ${lines[i]}") + } + + // Spot-check specific events made it through. + val names = lines.drop(1).map { mapper.readTree(it).get("name").asText() } + assertTrue(names.contains("transport:connection_started"), names.toString()) + assertTrue(names.contains("transport:packet_sent"), names.toString()) + assertTrue(names.contains("transport:packet_received"), names.toString()) + assertTrue(names.contains("transport:packet_dropped"), names.toString()) + assertTrue(names.contains("security:key_updated"), names.toString()) + assertTrue(names.contains("recovery:packet_lost"), names.toString()) + assertTrue(names.contains("recovery:loss_timer_updated"), names.toString()) + assertTrue(names.contains("transport:parameters_set"), names.toString()) + assertTrue(names.contains("transport:alpn_information"), names.toString()) + assertTrue(names.contains("transport:version_information"), names.toString()) + assertTrue(names.contains("transport:connection_closed"), names.toString()) + } + + @Test + fun timesAreRelativeToConstructorTime() { + val tmp = Files.createTempFile("amethyst-qlog-rel", ".sqlog").toFile() + tmp.deleteOnExit() + var clock = 1_000L + QlogWriter(tmp, odcidHex = "00", nowMillis = { clock }).use { w -> + clock = 1_050L + w.onAlpnNegotiated("h3") + } + val lines = tmp.readLines().filter { it.isNotBlank() } + val mapper = jacksonObjectMapper() + val event = mapper.readTree(lines[1]) + assertEquals(50L, event.get("time").asLong(), "time must be relative to constructor (1050 - 1000)") + } + + @Test + fun fileEndsWithNewline_qvisCompatible() { + val tmp = Files.createTempFile("amethyst-qlog-nl", ".sqlog").toFile() + tmp.deleteOnExit() + QlogWriter(tmp, odcidHex = "00").use { w -> + w.onAlpnNegotiated("h3") + } + val bytes = tmp.readBytes() + assertTrue(bytes.isNotEmpty(), "file must not be empty") + assertEquals('\n'.code.toByte(), bytes.last(), "file must end with '\\n' so trailing event parses") + } + + @Test + fun handlesEmptyFramesList(): Unit = + File.createTempFile("amethyst-qlog-empty", ".sqlog").let { tmp -> + tmp.deleteOnExit() + QlogWriter(tmp, odcidHex = "00").use { w -> + w.onPacketSent(EncryptionLevel.INITIAL, 0, 1200, emptyList()) + } + val mapper = jacksonObjectMapper() + val lines = tmp.readLines().filter { it.isNotBlank() } + val frames = mapper.readTree(lines[1]).get("data").get("frames") + assertTrue(frames.isArray, "frames must be an array even when empty") + assertEquals(0, frames.size()) + } +} diff --git a/settings.gradle b/settings.gradle index 6d88537ff4..b6eb94ee6a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -41,3 +41,5 @@ include ':quic' include ':nestsClient' include ':desktopApp' include ':cli' +include ':quic-interop' +project(':quic-interop').projectDir = file('quic/interop')