Merge pull request #2764 from vitorpamplona/claude/research-quic-libraries-hH1Dc

Add QUIC interop runner support and observability infrastructure
This commit is contained in:
Vitor Pamplona
2026-05-07 11:48:06 -04:00
committed by GitHub
56 changed files with 7645 additions and 201 deletions
+23
View File
@@ -0,0 +1,23 @@
# quic-interop-runner endpoint image for :quic.
#
# Build (from repo root):
# ./gradlew :quic-interop:installDist
# docker build -t amethyst-quic-interop -f quic/interop/Dockerfile .
#
# Or run `make build` inside quic/interop/.
FROM martenseemann/quic-network-simulator-endpoint:latest
RUN apt-get update \
&& apt-get install -y --no-install-recommends openjdk-21-jre-headless \
&& rm -rf /var/lib/apt/lists/*
COPY quic/interop/build/install/quic-interop /opt/quic-interop
COPY quic/interop/run_endpoint.sh /run_endpoint.sh
RUN chmod +x /run_endpoint.sh /opt/quic-interop/bin/quic-interop
# Build-time toggle for verbose tracing. Pass --build-arg DEBUG=1 (or
# `make build DEBUG=1` via the Makefile) to bake QUIC_INTEROP_DEBUG into
# the image so the InteropClient + writer emit per-drain stats to stderr.
# Off by default — production matrix runs stay quiet.
ARG DEBUG=0
ENV QUIC_INTEROP_DEBUG=${DEBUG}
+33
View File
@@ -0,0 +1,33 @@
# Local iteration loop for the quic-interop-runner endpoint image.
#
# make build # compile JVM dist + build Docker image
# make image-name # print the image tag (for use in implementations_quic.json)
# make clean # nuke the dist + image
#
# Real testing happens via the runner: clone quic-interop-runner separately
# and drive it with `quic/interop/run-matrix.sh -s <peer> -t <tests>`.
IMAGE ?= amethyst-quic-interop:latest
REPO_ROOT := $(shell git rev-parse --show-toplevel)
DIST_DIR := build/install/quic-interop
# Build-time toggle: `make build DEBUG=1` bakes QUIC_INTEROP_DEBUG=1 into
# the image so the InteropClient + writer emit per-drain diagnostics to
# stderr. Off by default — production matrix runs stay silent.
DEBUG ?= 0
.PHONY: build dist docker image-name clean
build: docker
dist:
cd $(REPO_ROOT) && ./gradlew :quic-interop:installDist
docker: dist
cd $(REPO_ROOT) && docker build -t $(IMAGE) --build-arg DEBUG=$(DEBUG) -f quic/interop/Dockerfile .
image-name:
@echo $(IMAGE)
clean:
cd $(REPO_ROOT) && ./gradlew :quic-interop:clean
docker image rm -f $(IMAGE) 2>/dev/null || true
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
plugins {
alias(libs.plugins.jetbrainsKotlinJvm)
application
}
kotlin {
jvmToolchain(21)
compilerOptions {
jvmTarget.set(JvmTarget.JVM_21)
}
}
sourceSets {
main {
kotlin.srcDir("src/main/kotlin")
}
test {
kotlin.srcDir("src/test/kotlin")
}
}
dependencies {
implementation(project(":quic"))
implementation(libs.kotlinx.coroutines.core)
implementation(libs.jackson.module.kotlin)
testImplementation(libs.kotlin.test)
}
application {
mainClass.set("com.vitorpamplona.quic.interop.runner.InteropClientKt")
applicationName = "quic-interop"
}
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
# Pull the post-mortem diagnostics for the most recent multiplexing run.
# Runs from anywhere; resolves logs relative to the runner clone path
# (../quic-interop-runner from this repo).
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
RUNNER_LOGS="${REPO_ROOT}/../quic-interop-runner/logs"
if [[ ! -d "$RUNNER_LOGS" ]]; then
echo "no runner logs at $RUNNER_LOGS" >&2
exit 1
fi
RUN_DIR="$(ls -1dt "$RUNNER_LOGS"/run-* 2>/dev/null | head -n 1 || true)"
if [[ -z "$RUN_DIR" ]]; then
echo "no run-* dirs under $RUNNER_LOGS" >&2
exit 1
fi
echo "==> run dir: $RUN_DIR"
# Layout: <run>/<server>_<client>/<testcase>/{client,server,sim}/
CASE_DIR="$(ls -1d "$RUN_DIR"/*amethyst*/multiplexing 2>/dev/null | head -n 1 || true)"
if [[ -z "$CASE_DIR" ]]; then
echo "no <pair>/multiplexing dir; tree under run:" >&2
find "$RUN_DIR" -maxdepth 3 -type d >&2
exit 1
fi
echo "==> case dir: $CASE_DIR"
echo
echo "=============== writer-side debug traces (DEBUG=1 build only) ==============="
# Per-drain frame/budget stats from buildApplicationPacket. The
# smoking-gun section for "writer is iterating N active streams but
# emits only 1 STREAM frame per packet". Empty unless the image was
# built with `make build DEBUG=1`.
#
# `grep -c` exits 1 on no matches AND prints "0", so a naive
# `grep -c ... || echo 0` doubles up. Suppress the exit code
# instead.
echo
echo "=============== boot line (verifies image has latest debug build) ==============="
grep '\[boot\]' "$CASE_DIR/output.txt" 2>/dev/null | head -n 5 \
|| echo "(no [boot] lines — image is older than the boot log; rebuild with: DEBUG=1 ./quic/interop/run-matrix.sh ...)"
WRITER_LINES=$(grep -cE '\[(writer|batch|interop|boot)' "$CASE_DIR/output.txt" 2>/dev/null) || WRITER_LINES=0
if [[ "$WRITER_LINES" -gt 0 ]]; then
echo "($WRITER_LINES diagnostic lines; [batch] / [interop] entries first then first 30 [writer.app]:)"
# Use `|| true` because grep returns 1 on no matches and the
# script runs under `set -euo pipefail` — otherwise an empty
# [batch]/[interop] subset (run from a build without those logs)
# would abort the whole script.
grep -E '\[(batch|interop)' "$CASE_DIR/output.txt" | head -n 20 || true
echo "..."
grep '\[writer' "$CASE_DIR/output.txt" | head -n 30 || true
echo
echo "stream_frames histogram (writer-reported):"
grep -oE 'stream_frames=[0-9]+' "$CASE_DIR/output.txt" | sort | uniq -c | sort -rn || true
echo
echo "active histogram (active stream count at drain time):"
grep -oE 'active=[0-9]+' "$CASE_DIR/output.txt" | sort | uniq -c | sort -rn || true
else
echo "(no diagnostic lines yet — to enable, REBUILD the image with DEBUG=1:"
echo " DEBUG=1 ./quic/interop/run-matrix.sh -s aioquic -t multiplexing"
echo " then re-run this script)"
fi
echo
echo "=============== server stderr (last 50 lines) ==============="
# Peer's CONNECTION_CLOSE reason if any; processing pace via stream
# create/discard cadence.
tail -n 50 "$CASE_DIR/server/stderr.log" 2>/dev/null \
|| echo "(no server/stderr.log)"
QLOG="$(ls -1 "$CASE_DIR"/client/qlog/*.sqlog \
"$CASE_DIR"/client/qlog/*.qlog \
2>/dev/null | head -n 1 || true)"
if [[ -z "$QLOG" ]]; then
echo
echo "(no qlog under $CASE_DIR/client/qlog — skipping qlog sections)"
exit 0
fi
echo
echo "=============== qlog ($QLOG, $(wc -l <"$QLOG" | tr -d ' ') lines) ==============="
echo
echo "stream-frames-per-sent-packet histogram:"
# Many `0` = ack-only; many `1` = the bug; many `>=4` = writer
# coalescing as intended.
grep '"name":"transport:packet_sent"' "$QLOG" \
| awk -F'"frame_type":"stream"' '{print NF - 1}' \
| sort -n | uniq -c
echo
echo "transport_parameters (local + remote):"
grep '"name":"transport:parameters_set"' "$QLOG"
echo
echo "first 10 packet_sent (full) — burst shape after handshake:"
grep '"name":"transport:packet_sent"' "$QLOG" | head -n 10
echo
echo "connection_closed events (peer CCs are the smoking gun for spec violations):"
grep '"name":"transport:connection_closed"' "$QLOG" || echo "(none)"
echo
echo "packet_dropped events (last 10):"
grep '"name":"transport:packet_dropped"' "$QLOG" | tail -n 10 \
|| echo "(none)"
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env bash
# Per-testcase deep-dive diagnostic. Use after running any single
# testcase that failed:
# ./quic/interop/inspect-testcase.sh longrtt
#
# Auto-finds the most recent run dir that has the named testcase and
# reports the runner status line + qlog summary + frame histograms
# (sent/received) + connection-close events. Designed to fit in one
# screen of output.
set -o pipefail
TC="${1:-}"
if [[ -z "$TC" ]]; then
echo "usage: $0 <testcase> (e.g. longrtt, retry, http3)" >&2
exit 2
fi
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
RUNNER_LOGS="${REPO_ROOT}/../quic-interop-runner/logs"
# Most recent run dir that has the named testcase.
RUN_DIR=""
# Filter to actual directories — zsh's glob also matches the
# sibling .stdout.log files that run-matrix.sh tees, which start
# with the same 'run-' prefix and would resolve as not-a-dir.
for d in $(ls -1dt "$RUNNER_LOGS"/run-* 2>/dev/null); do
[[ -d "$d" ]] || continue
if ls -d "$d"/*amethyst*/"$TC" >/dev/null 2>&1; then
RUN_DIR="$d"
break
fi
done
if [[ -z "$RUN_DIR" ]]; then
echo "no run dir with testcase '$TC' under $RUNNER_LOGS" >&2
exit 1
fi
echo "==> run dir: $RUN_DIR"
TC_DIR="$(ls -1d "$RUN_DIR"/*amethyst*/"$TC" 2>/dev/null | head -n 1)"
echo "==> testcase dir: $TC_DIR"
echo
echo "=============== runner status ==============="
if [[ -f "${RUN_DIR}.stdout.log" ]]; then
grep -E "Test: $TC took" "${RUN_DIR}.stdout.log" | tail -n 5
else
echo "(no .stdout.log — run-matrix.sh predates the tee, status not saved)"
fi
echo
echo "=============== client diagnostic traces (DEBUG=1) ==============="
# Three places to look, in priority order:
# 1. Per-testcase client output (testcases that write to file)
# 2. Per-testcase output.txt (some runner versions)
# 3. The runner's tee'd .stdout.log narrowed to this testcase's
# window — but only works for short tests, since matrix runs
# restart containers and longer tests' stderr can be lost
# when the container is killed mid-test.
CLIENT_TRACES=()
[[ -f "$TC_DIR/client/output.txt" ]] && CLIENT_TRACES+=("$TC_DIR/client/output.txt")
[[ -f "$TC_DIR/output.txt" ]] && CLIENT_TRACES+=("$TC_DIR/output.txt")
[[ -f "${RUN_DIR}.stdout.log" ]] && CLIENT_TRACES+=("${RUN_DIR}.stdout.log")
found=0
for f in "${CLIENT_TRACES[@]}"; do
n=$(grep -cE '\[(boot|interop|batch|writer\.)' "$f" 2>/dev/null || echo 0)
if [[ "$n" -gt 0 ]]; then
echo "(found $n diagnostic lines in $f)"
if [[ "$f" == *.stdout.log ]]; then
# Narrow to this testcase's window.
awk -v tc="$TC" '
$0 ~ "Running test case: " tc { in_tc = 1; next }
in_tc && /Running test case:/ { exit }
in_tc && /\[(boot|interop|batch|writer\.)/ { print }
' "$f" | head -n 50 || true
else
grep -E '\[(boot|interop|batch|writer\.)' "$f" | head -n 50 || true
fi
found=1
break
fi
done
if [[ "$found" -eq 0 ]]; then
echo "(no diagnostic lines for this testcase)"
echo
echo "Possible causes:"
echo " - The image was built without DEBUG=1: use 'DEBUG=1 ./run-matrix.sh ...'"
echo " - The container was killed mid-test before flushing stderr"
echo " - This testcase ran in a matrix and only the FIRST testcase's"
echo " traces were captured. Run this single testcase in isolation:"
echo " DEBUG=1 ./quic/interop/run-matrix.sh -s <peer> -t $TC"
fi
echo
echo "=============== file sizes generated for this testcase ==============="
if [[ -f "${RUN_DIR}.stdout.log" ]]; then
# 'Generated random file: NAME of size: N' lines printed before
# each test run. Extract the ones that came RIGHT BEFORE this
# testcase's "Running test case" line.
awk -v tc="$TC" '
/^Running test case: / { current = $0 }
/^Generated random file:/ { last_files = last_files "\n" $0 }
$0 ~ ("Running test case: " tc) {
print last_files
last_files = ""
exit
}
' "${RUN_DIR}.stdout.log" | tail -n 10
else
echo "(no .stdout.log)"
fi
echo
echo "=============== qlog event-type histogram ==============="
QLOG="$(ls -1 "$TC_DIR"/client/qlog/*.sqlog "$TC_DIR"/client/qlog/*.qlog 2>/dev/null | head -n 1 || true)"
if [[ -z "$QLOG" ]]; then
echo "(no qlog — connection probably never made it past TLS)"
else
echo "(qlog: $QLOG, $(wc -l <"$QLOG" | tr -d ' ') lines)"
grep -oE '"name":"[^"]+"' "$QLOG" | sort | uniq -c | sort -rn | head
fi
if [[ -n "$QLOG" ]]; then
echo
echo "=============== transport_parameters (peer's flow-control budget) ==============="
grep '"name":"transport:parameters_set"' "$QLOG"
echo
echo "=============== connection_closed (smoking gun for spec violations) ==============="
grep '"name":"transport:connection_closed"' "$QLOG" | head -n 5 \
|| echo "(none — connection didn't formally close; ran out of time?)"
echo
echo "=============== last 10 sent / received packets (steady-state shape) ==============="
echo "-- received --"
grep '"name":"transport:packet_received"' "$QLOG" | tail -n 10
echo "-- sent --"
grep '"name":"transport:packet_sent"' "$QLOG" | tail -n 10
echo
echo "=============== first/last received packet timestamps (transfer rate hint) ==============="
FIRST=$(grep '"name":"transport:packet_received"' "$QLOG" | head -n 1 | grep -oE '"time":[0-9]+' | head -n 1)
LAST=$(grep '"name":"transport:packet_received"' "$QLOG" | tail -n 1 | grep -oE '"time":[0-9]+' | head -n 1)
PKTS=$(grep -c '"name":"transport:packet_received"' "$QLOG")
echo "first=$FIRST last=$LAST pkts=$PKTS"
fi
echo
echo "=============== server stderr (last 30 lines) ==============="
tail -n 30 "$TC_DIR/server/stderr.log" 2>/dev/null \
|| echo "(no server/stderr.log)"
@@ -0,0 +1,257 @@
# quic-interop-runner endpoint — Phase 0 scaffolding
Date: 2026-05-06
## Why
We want the [`quic-interop-runner`](https://github.com/quic-interop/quic-interop-runner)
matrix as a **bug-finding harness**, not a vanity scoreboard. Each peer impl
(quiche, aioquic, picoquic, ngtcp2, msquic, mvfst, lsquic, neqo, kwik) enforces
different parts of RFC 9000/9001/9002/9114 strictly, so failures triangulate
to specific bugs in `:quic`. The runner's ns-3 sim also exposes loss /
reorder / migration scenarios that are awkward to reproduce in unit tests.
## What's in Phase 0
- `:quic-interop` Gradle module (JVM-only application, registered at
`quic/interop/` via `settings.gradle`).
- `InteropClient.kt` reads the runner's env-var contract (`ROLE`, `TESTCASE`,
`REQUESTS`, …) and dispatches by testcase. Phase 0 implements only
`handshake`; everything else returns `127` (runner-skip).
- `Dockerfile` based on `martenseemann/quic-network-simulator-endpoint` +
OpenJDK 21 runtime, copies the `installDist` output.
- `run_endpoint.sh` sources the base image's `/setup.sh` then execs our JVM
binary.
- `Makefile` wrappers: `make build`, `make clean`. (A `make smoke`
target previously stood up picoquic + our endpoint on a private Docker
bridge to bisect runner failures from impl failures; dropped once
the runner reliably exercised both paths.)
## Local iteration loop
The fast path: `quic/interop/run-matrix.sh` clones the runner alongside
this repo, sets up a venv, merges our `implementations_quic.json` snippet,
builds the endpoint image, and invokes `run.py`. All steps are
idempotent so repeated invocations just iterate.
```
# Single test against the most permissive peer:
quic/interop/run-matrix.sh -s aioquic -t handshake
# A focused triangulation:
quic/interop/run-matrix.sh -s aioquic -t handshake,chacha20
quic/interop/run-matrix.sh -s quic-go -t handshake,chacha20
quic/interop/run-matrix.sh -s picoquic -t handshake,chacha20
# Tight inner loop — skip the image rebuild between test selections:
SKIP_BUILD=1 quic/interop/run-matrix.sh -s aioquic -t transfer
```
Manual flow (if `run-matrix.sh` doesn't fit):
```
make -C quic/interop build
# Then in a sibling clone of quic-interop-runner, merge our snippet:
jq -s '.[0] * .[1]' implementations_quic.json \
../amethyst/quic/interop/quic-interop-runner-snippet.json \
> implementations_quic.json.new && mv implementations_quic.json.new implementations_quic.json
python run.py -d -i amethyst -s aioquic -t handshake,chacha20 --log-dir ./logs
```
Inspect `./logs/<run>/client_qlog/*.qlog` in qvis when something breaks.
## Phase ladder (excerpt — full plan in conversation)
| Phase | Goal | Tests | Exit criterion |
|---|---|---|---|
| 0 | Minimum harness | `handshake` | one test reproducible end-to-end ✅ |
| 1 | Triangulate handshake bugs | + `versionnegotiation`, `chacha20` | green vs aioquic + quiche + picoquic |
| 2 | Streams + loss + multiplexing | + `transfer`, `multiplexing`, `*loss`, `http3` | `transfer` / `multiplexing` / `http3` ✅ landed; loss tests pending |
| 3 | Edge cases | `retry`, `resumption`, `zerortt`, `keyupdate`, `rebinding-*`, `blackhole`, `amplificationlimit` | every test green or unsupported-127 with a written reason |
| 4 | CI gate | nightly Phases 1–2; PR-blocking subset on every push | qlogs uploaded as artifacts on red |
## Phase 2 — landed 2026-05-06
- Minimal `Http3GetClient` (in `:quic-interop`, NOT `:quic` — interop-test
surface, not a production HTTP/3 client). Opens the three required
client uni streams (control + QPACK encoder + QPACK decoder), sends
empty SETTINGS, then per request opens a bidi stream, encodes a HEADERS
frame with the four pseudo-headers using the existing literal-only
`QpackEncoder`, FINs, and reassembles HEADERS+DATA frames from the
response. Out-of-scope: GOAWAY, PUSH_PROMISE, dynamic QPACK table,
trailers, priority.
- `transfer` + `http3` testcases: GET each URL in `REQUESTS` sequentially,
write each body to `$DOWNLOADS/<basename>`. Status != 200 fails.
- `multiplexing` testcase: same as `transfer` but issues each GET in a
parallel `coroutineScope { async { … } }` so the request streams
genuinely overlap on the wire (what tshark verifies).
- Aliased sim-driven testcases — these reuse the same client code paths;
the runner injects the network condition via the ns-3 sim. Failures
here are exactly the bug-finding signal we want, since they exercise
loss recovery / RTT estimator / congestion behaviour against real peers:
- `transferloss` → transfer (random packet loss)
- `transfercorruption` → transfer (random bit-flip; AEAD AUTH FAIL → drop + retransmit)
- `longrtt` → transfer (emulated high-latency link)
- `goodput` → transfer (throughput floor)
- `crosstraffic` → transfer (competing UDP flows on the same link)
- `handshakeloss` → handshake (loss during handshake — tests CRYPTO retransmit)
## Phase 3 — landed 2026-05-07 (post-quic-go interop)
- ALPN per testcase (`:quic-interop`'s `Alpn` enum + per-test switch in
`InteropClient.main`). quic-go enforces strictly with TLS
`no_application_protocol` (CRYPTO_ERROR 0x178); aioquic / picoquic accept
either. Convention: `h3` for `http3`/`multiplexing`, `hq-interop`
everywhere else.
- `HqInteropGetClient` (HTTP/0.9 over QUIC) — open bidi, send `GET /path\r\n`,
FIN, read body until server FINs. No framing, no QPACK. ~30 lines.
- Multi-stream FIN delivery fix in `QuicConnection.closeAllSignals()`: pre-fix
iterated only the connection-wide signal channels, leaving every per-stream
`incomingChannel` open after teardown — coroutines suspended on
`stream.incoming.collect { ... }` hung forever. Fix iterates `streamsList`
on close. Three regression tests in `MultiStreamFinDeliveryTest`.
- `retry` + `ipv6` testcases enabled in dispatch. `retry` rides on agent 3's
RFC 9000 §17.2.5 + RFC 9001 §5.8 implementation. `ipv6` should "just work"
via JDK's `DatagramChannel` v6 support; runner-validated when run.
## Validated against (as of 2026-05-07 evening)
Latest run results before pushing the multi-ALPN fix `acfe815e1`:
| Peer | handshake | chacha20 | transfer | http3 | multiplexing |
|---|---|---|---|---|---|
| aioquic | ✓ | ✓ | ✓ | ✓ | ✕ (channel-saturation, agent C investigating) |
| picoquic | ✕ (alpn=hq-interop unsupported, fixed in `acfe815e1`) | ✕ | ✕ | ✓ | ✕ |
| quic-go | (untested post-fix) | | | | |
After `acfe815e1`'s multi-ALPN offer, predictions:
- picoquic returns to 4/4 (server picks `h3`, we run Http3GetClient).
- quic-go: handshake / chacha20 / transfer / http3 should green (server picks `hq-interop` for non-h3 tests).
- aioquic: still 4/5; multiplexing held back by channel-saturation bug.
## Phase 4 — landed 2026-05-07 (overnight agents)
All three agents merged onto the branch with three rounds of fixup
(each agent's worktree was based on `main`, not the branch HEAD, so
they clobbered each other's changes during merge):
- **Agent A — VN-handling defense in `:quic`**: configurable
`initialVersion` on `QuicConnection`, `applyVersionNegotiation`
state machine, `vnConsumed` latch, downgrade defenses. *NOTE*: the
runner does not have a `versionnegotiation` testcase (it has `v2`,
which tests QUIC v2 — we're v1-only). Code stays as defensive
support for any server that throws a VN at us, but unused by the
matrix.
- **Agent B — qlog observer**: `QlogObserver` interface in `:quic`
with NoOp default + 12 event hooks (packet-sent/received/dropped,
key-updated, conn-started/closed, loss-detected, PTO-fired,
transport-params, ALPN, version). `QlogWriter` (Jackson-backed
JSON-NDJSON) in `:quic-interop`. `InteropClient` reads `$QLOGDIR`
the runner already sets and writes `client.sqlog` per testcase.
Drag straight into qvis.quictools.info to see the trace.
- **Agent C — peer-uni-stream drainer**: `drainPeerInitiatedUniStreamsIntoBlackHole`
helper on `QuicConnection`, wired into `Http3GetClient.init(scope)`.
Fixes the multiplexing channel-saturation symptom (server's uni
streams accumulated bytes in 64-chunk per-stream channels until
parser tore down with INTERNAL_ERROR ~4.5s into a multi-stream run).
## Phase 5 — landed 2026-05-07 (post-overnight bug-hunt)
The full overnight session pulled hard on the bugs the matrix exposed.
qlog turned out to be the unblocking tool — every fix in this phase
came from staring at a `client.sqlog` and matching it against the RFC
pages it referenced.
| Commit | Diagnosis pattern |
|---|---|
| `bd9d717df` | `IOException: Stream closed` from QlogWriter mid-test → observer threw into the send loop, killed connections that had already completed transfer. |
| `99a1a91de` | qlog stops at t=400ms, no inbound packets → per-event `writer.flush()` stalled the connection lock on macOS Docker filesystem virtualization. |
| `c0d7b6031` | aioquic `CONNECTION_CLOSE: Packet contains no CRYPTO frame` after PTO probe → our PTO emitted bare PING, not CRYPTO retransmit. Restored agent 2's wiring lost in the qlog merge. |
| `17b80270d` | runner's retry verdict `Client reset the packet number. Check failed for PN 0` → `applyRetry` reset Initial PN to 0; RFC 9001 §5.7 says PN namespace continues across Retry. New `LevelState.resetForRetry` helper. |
| `9a74d1d5d` | multiplexing qlog showed STREAM frames still arriving at t=31s when local timeout fired → bumped `TRANSFER_TIMEOUT_SEC` 30→60. Doesn't fix throughput, just lets the test budget match the workload. |
| `32ccbd2b2` | `coroutineScope { urls.map { async }.map { it.await() } }` blocks on slowest hung stream → per-stream `withTimeoutOrNull` so a single bad stream surfaces as status=0 instead of hanging the matrix. |
After all of these, expectation is aioquic / picoquic ≥6/7 (M
might still be flaky). retry green via qlog-driven debugging is
the marquee result.
## Still open
- **`v2`** — server demands QUIC v2 (RFC 9369). We're v1-only.
Implementing v2 is its own project (different Initial-secret
derivation, different transport-parameter encoding, different
long-header type bits — not just a version-number swap).
- **Multiplexing throughput on Mac+Rosetta** — measured at ~23 streams/sec
(aioquic processed 1359 GETs in 58s). The runner's multiplexing test
generates 1999 files; even with a 60s budget we only get 70% of them
open before the test ends, with most coroutines failing to surface a
status=200 in time for the file-write loop. Throughput is hard-bottlenecked
by `:quic`'s single `conn.lock` serializing the send loop, the read
loop, and `openBidiStream` across all 1999 awaiting coroutines. Under
this lock contention pattern, parallelism doesn't help — coroutines
queue on the lock anyway.
Possible fixes (non-trivial):
- Per-level / per-stream lock split (big refactor; touches almost
everything in `:quic/connection`).
- Batch concurrency via a `Semaphore` capping in-flight `client.get()`
calls to e.g. 64 at a time. Doesn't lift the throughput ceiling but
reduces lock thrash and probably lets MORE files complete in 60s.
- QPACK dynamic-table support so aioquic doesn't need to fall back
to literal encoding (would help on the response-decode side).
None are urgent; this is a stress-test scenario, not a normal-traffic
one. Filed for follow-up.
- **Server role** — entire stack is client-only. Implementing the
server side would unlock `handshakeloss` against aioquic
(currently `?` because aioquic-server doesn't support it), and
would make us a self-validating peer.
## Concurrency
`run-matrix.sh` is NOT safe to run in parallel. The runner's
docker-compose.yml hardcodes `container_name: sim/server/client` —
Docker enforces those globally. Use a sequential loop:
```
for peer in aioquic picoquic quic-go; do
quic/interop/run-matrix.sh -s $peer -t handshake,chacha20,...
done
```
## Explicitly unsupported testcases (return 127, runner skips)
| Testcase | Reason |
|---|---|
| `versionnegotiation` | `QuicConnectionWriter` hard-codes `QuicVersion.V1`; needs a configurable initial version + VN-response retry path. **In progress (background agent)**. |
| `resumption` | session ticket parsing + persistence not yet implemented |
| `zerortt` | depends on `resumption` + early-data path |
| `keyupdate` | KEY_PHASE bit handling not yet implemented in 1-RTT |
| `rebinding-port`, `rebinding-addr` | client-side connection migration (re-bind UDP socket, NEW_CONNECTION_ID rotation) not implemented |
| `amplificationlimit` | server-side test, N/A for client role |
| `blackhole` | inverse test (verifies we *fail* on dead network in bounded time); needs special handling |
| `ipv6` | `UdpSocket` IPv6 path not exercised; risky to claim without testing |
## Phase 1a — landed 2026-05-06
- `SSLKEYLOGFILE` writer in `InteropClient` (NSS Key Log Format) so Wireshark
decrypts the sim's pcap captures. Backed by:
- `TlsClient.clientRandom` (new public read-only property, captured in
`start()` before sending ClientHello).
- `QuicConnection.extraSecretsListener` (new optional constructor param,
chained after the connection's own key-installation listener; no-op
default, so production callers are unaffected).
- `chacha20` testcase: forces ChaCha20-Poly1305 only via new
`QuicConnection.cipherSuites` knob (threaded through `TlsClient` →
`buildQuicClientHello` → `TlsClientHello`).
## Phase 1b — open
- `QLOGDIR`: `:quic` has no qlog observer infrastructure yet. Wiring needs
hooks at packet send/recv, frame dispatch, recovery, and TLS state
transitions, plus the qlog JSON-NDJSON schema. Sized as its own design
doc before implementation.
- `versionnegotiation`: `QuicConnectionWriter` hard-codes `QuicVersion.V1`
in two call sites; threading a configurable initial-version through and
wiring the response-handling path is non-trivial. Defer.
- Server role: we are client-first. Reassess after Phase 3.
- WebTransport is **not** part of the standard interop matrix; it needs a
separate harness against `moq-rs` / chrome-headless.
@@ -0,0 +1,7 @@
{
"amethyst": {
"image": "amethyst-quic-interop:latest",
"url": "https://github.com/vitorpamplona/amethyst",
"role": "client"
}
}
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env bash
# Drive the quic-interop-runner against the :quic-interop endpoint image.
#
# Idempotent: clones the runner alongside this repo if missing, sets up its
# venv, registers `amethyst` in implementations_quic.json, builds our endpoint
# image, then invokes run.py with the user's args.
#
# Usage:
# quic/interop/run-matrix.sh # full matrix vs aioquic
# quic/interop/run-matrix.sh -s aioquic -t handshake # one test
# quic/interop/run-matrix.sh -s quic-go -t handshake,chacha20 # different peer
#
# Env overrides:
# RUNNER_DIR — where to clone / find the runner (default: ../quic-interop-runner)
# LOG_DIR — qlog / pcap output (default: $RUNNER_DIR/logs)
# SKIP_BUILD=1 — skip `make build` (image already current)
set -euo pipefail
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd)
RUNNER_DIR="${RUNNER_DIR:-$REPO_ROOT/../quic-interop-runner}"
LOG_DIR="${LOG_DIR:-$RUNNER_DIR/logs}"
is_macos=false
case "${OSTYPE:-}" in
darwin*) is_macos=true ;;
esac
install_hint() {
if $is_macos; then
echo " brew install $1"
else
echo " sudo apt-get install -y $1"
fi
}
need() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "error: '$1' not found in PATH. Install with:" >&2
install_hint "$2" >&2
exit 1
fi
}
need docker docker.io
need python3 python3
need jq jq
need git git
need make make
# Docker daemon must be reachable, not just installed. On macOS this is the
# usual gotcha — Docker Desktop installed but not running.
if ! docker info >/dev/null 2>&1; then
echo "error: docker daemon not reachable." >&2
if $is_macos; then
echo " → launch Docker Desktop and retry." >&2
else
echo " → 'sudo systemctl start docker' (and add yourself to the docker group)." >&2
fi
exit 1
fi
# 1. Clone the runner if missing.
if [ ! -d "$RUNNER_DIR" ]; then
echo "==> cloning quic-interop-runner into $RUNNER_DIR"
git clone --depth 1 https://github.com/quic-interop/quic-interop-runner.git "$RUNNER_DIR"
fi
# 2. venv + Python deps. Prefer 3.13 — pyshark (the runner's pcap parser)
# trips on 3.14's asyncio.get_event_loop() removal. Fall back to whatever
# `python3` resolves to if 3.13 isn't installed.
PYTHON_BIN=""
for candidate in python3.13 python3.12 python3.11 python3; do
if command -v "$candidate" >/dev/null 2>&1; then
if "$candidate" -c 'import sys; sys.exit(0 if sys.version_info < (3,14) else 1)' 2>/dev/null; then
PYTHON_BIN="$candidate"
break
fi
fi
done
if [ -z "$PYTHON_BIN" ]; then
echo "warning: no Python < 3.14 found; pyshark will likely crash on pcap validation." >&2
install_hint python3.13
PYTHON_BIN="python3"
fi
if [ ! -d "$RUNNER_DIR/.venv" ]; then
echo "==> creating venv at $RUNNER_DIR/.venv (using $PYTHON_BIN)"
"$PYTHON_BIN" -m venv "$RUNNER_DIR/.venv"
fi
"$RUNNER_DIR/.venv/bin/pip" install -q -r "$RUNNER_DIR/requirements.txt"
# 3. Register our endpoint in implementations_quic.json (idempotent).
if ! jq -e '.amethyst' "$RUNNER_DIR/implementations_quic.json" >/dev/null 2>&1; then
echo "==> registering 'amethyst' in implementations_quic.json"
tmp=$(mktemp)
jq -s '.[0] * .[1]' \
"$RUNNER_DIR/implementations_quic.json" \
"$SCRIPT_DIR/quic-interop-runner-snippet.json" \
> "$tmp"
mv "$tmp" "$RUNNER_DIR/implementations_quic.json"
fi
# 4. Build the endpoint image (skippable for tight loops). Pass DEBUG=1
# to bake QUIC_INTEROP_DEBUG=1 into the image so the InteropClient +
# writer emit per-drain stats to stderr (visible via
# inspect-multiplexing.sh after the run). Off by default.
if [ "${SKIP_BUILD:-0}" != "1" ]; then
if [ "${DEBUG:-0}" = "1" ]; then
echo "==> building amethyst-quic-interop image (DEBUG=1)"
make -C "$SCRIPT_DIR" build DEBUG=1
else
echo "==> building amethyst-quic-interop image"
make -C "$SCRIPT_DIR" build
fi
fi
# 5. Drive the runner.
#
# run.py hard-exits if --log-dir already exists, so we use a fresh
# per-invocation subdirectory under $LOG_DIR. The parent must exist; the
# child must not. Tail of `ls -t "$LOG_DIR" | head -1` finds the latest.
#
# NOTE: this script is NOT safe to run concurrently against itself.
# quic-interop-runner's docker-compose.yml hardcodes `container_name:
# sim/server/client`, which Docker enforces globally regardless of
# COMPOSE_PROJECT_NAME. Two simultaneous invocations collide on
# `docker create container "sim"`. Run sequentially:
# for peer in aioquic picoquic quic-go; do
# quic/interop/run-matrix.sh -s $peer -t handshake,chacha20,...
# done
#
# Output filter: by default we drop the runner / container boilerplate
# (interface checksum offload toggles, route setup, container lifecycle,
# the long Command: WAITFORSERVER=... line, the platform-mismatch warning
# that fires once per test on Apple Silicon). Set VERBOSE=1 to bypass.
mkdir -p "$LOG_DIR"
RUN_LOG_DIR="$LOG_DIR/run-$(date +%Y%m%d-%H%M%S)"
echo "==> running matrix (args: $* | logs: $RUN_LOG_DIR)"
cd "$RUNNER_DIR"
if [ "${VERBOSE:-0}" = "1" ]; then
exec "$RUNNER_DIR/.venv/bin/python" run.py \
-d -i amethyst --log-dir "$RUN_LOG_DIR" "$@"
else
# Tee unfiltered stdout to a sibling file so summarize-matrix.sh
# can find the 'Test: X took Y, status:' lines later — the runner
# only writes those to its own stdout, not into per-testcase
# output.txt. Sibling rather than inside RUN_LOG_DIR because
# run.py refuses to start if its --log-dir already exists.
RUNNER_STDOUT="${RUN_LOG_DIR}.stdout.log"
"$RUNNER_DIR/.venv/bin/python" run.py \
-d -i amethyst --log-dir "$RUN_LOG_DIR" "$@" 2>&1 \
| tee "$RUNNER_STDOUT" \
| grep -Ev \
-e '^(client|server|sim) +\| +(Setting up routes|Actual changes:|tx-[a-z0-9-]+:|Endpoint'\''s IPv[46] address is)' \
-e '^ Container [a-z]+ +(Recreate|Recreated|Stopping|Stopped|Starting|Started)( [0-9.]+s)?$' \
-e '^ server The requested image'\''s platform' \
-e '^Attaching to client, server, sim$' \
-e '^Aborting on container exit\.\.\.$' \
-e '^(client|server|sim) exited with code [0-9]+$' \
-e '^Packets captured: [0-9]+$' \
-e '^sim +\| +Packets received/dropped on interface ' \
-e '^sim +\| +(Received signal:|msg=|NS_FATAL)' \
-e '^Using the client'\''s key log file\.$' \
-e '^Command: WAITFORSERVER=' \
-e '^[0-9-]+ [0-9:,]+ Generated random file: ' \
-e '^[0-9-]+ [0-9:,]+ Requests: \[' \
-e '^==> ' \
-e '^$'
exit "${PIPESTATUS[0]}"
fi
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
# quic-interop-runner endpoint entry point.
#
# The base image (martenseemann/quic-network-simulator-endpoint) provides
# /setup.sh, which configures routing for the runner's ns-3 sim. Source it
# inside the runner; tolerate failure (e.g. missing NET_ADMIN cap) so the
# JVM client still launches if a caller invokes the image outside the
# runner — they get default container networking instead.
set -uo pipefail
# shellcheck disable=SC1091
source /setup.sh 2>/dev/null || echo "(setup.sh skipped — not under the runner sim)" >&2
set -e
case "${ROLE:-client}" in
client)
exec /opt/quic-interop/bin/quic-interop
;;
server)
# Phase 0: server role unimplemented. Exit 127 so the runner skips.
echo "server role not implemented" >&2
exit 127
;;
*)
echo "unknown ROLE=${ROLE}" >&2
exit 127
;;
esac
@@ -0,0 +1,88 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop.runner
import com.vitorpamplona.quic.connection.QuicConnection
import com.vitorpamplona.quic.connection.QuicConnectionDriver
import kotlinx.coroutines.flow.toList
/**
* HQ-interop (HTTP/0.9 over QUIC) GET client. quic-interop-runner convention
* for the non-`http3` testcases (handshake / chacha20 / transfer / loss
* variants): the client opens a bidi stream, sends `GET /path\r\n` (raw
* ASCII, no framing), FINs the send side, and reads the response body
* verbatim until the server FINs. There is no status code, no headers, no
* QPACK, no control stream — the body bytes ARE the response.
*
* Per the runner's testcase validator, an empty body means failure, a
* non-empty body that matches what the server staged at $WWW/<path> means
* success. We surface non-empty as status=200, empty as status=0, so the
* caller's `if (resp.status != 200) anyFailed = true` check keeps working.
*/
class HqInteropGetClient(
private val conn: QuicConnection,
private val driver: QuicConnectionDriver,
) : GetClient {
override suspend fun prepareRequest(
@Suppress("UNUSED_PARAMETER") authority: String,
path: String,
): RequestHandle {
val stream = conn.openBidiStream()
val request = "GET $path\r\n".encodeToByteArray()
stream.send.enqueue(request)
stream.send.finish()
// Wake the send loop — same reasoning as Http3GetClient.
driver.wakeup()
return HqRequestHandle(stream)
}
override suspend fun prepareRequests(
@Suppress("UNUSED_PARAMETER") authority: String,
paths: List<String>,
): List<RequestHandle> {
// Pre-format outside the lock — see Http3GetClient. Then
// openBidiStreamsBatch atomically opens all N streams under
// streamsLock so the writer's next drain coalesces them.
val encoded = paths.map { "GET $it\r\n".encodeToByteArray() }
return conn.openBidiStreamsBatch(encoded) { stream, request ->
stream.send.enqueue(request)
stream.send.finish()
HqRequestHandle(stream)
}
}
override suspend fun awaitResponse(handle: RequestHandle): GetResponse {
val stream = (handle as HqRequestHandle).stream
val chunks = stream.incoming.toList()
val total = chunks.sumOf { it.size }
val body = ByteArray(total)
var off = 0
for (c in chunks) {
c.copyInto(body, off)
off += c.size
}
return GetResponse(status = if (body.isEmpty()) 0 else 200, body = body)
}
}
private class HqRequestHandle(
val stream: com.vitorpamplona.quic.stream.QuicStream,
) : RequestHandle
@@ -0,0 +1,253 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop.runner
import com.vitorpamplona.quic.QuicWriter
import com.vitorpamplona.quic.connection.QuicConnection
import com.vitorpamplona.quic.connection.QuicConnectionDriver
import com.vitorpamplona.quic.connection.drainPeerInitiatedUniStreamsIntoBlackHole
import com.vitorpamplona.quic.http3.Http3Frame
import com.vitorpamplona.quic.http3.Http3FrameReader
import com.vitorpamplona.quic.http3.Http3FrameType
import com.vitorpamplona.quic.http3.Http3Settings
import com.vitorpamplona.quic.http3.Http3StreamType
import com.vitorpamplona.quic.qpack.QpackDecoder
import com.vitorpamplona.quic.qpack.QpackEncoder
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.flow.collect
/** Common shape for the two interop GET clients (HTTP/3 and HQ-interop).
*
* The interface is split into two phases so the parallel multiplexing
* path can BATCH enqueues (synchronous, serial) and SINGLE-wakeup the
* send loop, vs. waking on every individual request which produces
* one tiny packet per stream instead of coalesced packets per drain. */
interface GetClient {
/** Open a stream + enqueue the request bytes + FIN. Does NOT wake the
* send loop — caller is responsible for batching wakes. Returns an
* opaque handle the caller passes to [awaitResponse]. */
suspend fun prepareRequest(
authority: String,
path: String,
): RequestHandle
/**
* Atomically open + enqueue + FIN N streams under a single hold of
* the connection lock. The send loop cannot interject between
* opens, so when it next drains it sees ALL N streams' data ready
* and packs them into coalesced packets instead of emitting one
* tiny packet per stream.
*
* Without this, the equivalent serial loop of [prepareRequest]
* yields between calls (lock release → send loop wakes → drains
* one stream → next prepareRequest acquires...) and we send one
* stream per packet — what cratered the multiplexing testcase.
*/
suspend fun prepareRequests(
authority: String,
paths: List<String>,
): List<RequestHandle>
/** Suspend until the server FINs the response stream associated with
* [handle]. Returns the assembled response. */
suspend fun awaitResponse(handle: RequestHandle): GetResponse
/** Convenience shortcut for the sequential / single-request paths. */
suspend fun get(
authority: String,
path: String,
): GetResponse {
val h = prepareRequest(authority, path)
return awaitResponse(h)
}
}
/** Opaque handle returned by [GetClient.prepareRequest]. Implementations
* cast it back to their internal stream representation. */
interface RequestHandle
data class GetResponse(
val status: Int,
val body: ByteArray,
)
/**
* Minimal HTTP/3 GET client used by the interop endpoint to satisfy the
* `http3` and `multiplexing` testcases.
*
* Opens the three required client-side unidirectional streams (control,
* QPACK encoder, QPACK decoder) per RFC 9114 §6.2.1. Encodes requests with
* the literal-only [QpackEncoder] (no dynamic table — RFC 9204 Required
* Insert Count = 0) so we don't need to push QPACK encoder instructions.
*
* Not a production HTTP/3 client. Specifically: no GOAWAY handling, no
* priority, no push-promise, no trailers, no dynamic QPACK table.
*/
class Http3GetClient(
private val conn: QuicConnection,
private val driver: QuicConnectionDriver,
) : GetClient {
suspend fun init(scope: CoroutineScope) {
// Control stream: type-0x00 prefix followed by a SETTINGS frame
// (empty body is legal — RFC 9114 §7.2.4). Empty body = spec
// defaults: QPACK_MAX_TABLE_CAPACITY=0, QPACK_BLOCKED_STREAMS=0,
// MAX_FIELD_SECTION_SIZE=unlimited. We deliberately do NOT
// explicitly send QPACK_MAX_TABLE_CAPACITY=0 — empirically
// (aioquic 2026-05-07) that worsened the multiplexing case
// from 1 file to 0 files.
val control = conn.openUniStream()
val w = QuicWriter()
w.writeVarint(Http3StreamType.CONTROL)
w.writeBytes(Http3Settings(emptyMap()).encodeFrame())
control.send.enqueue(w.toByteArray())
// Control stream stays open for the lifetime of the H3 connection;
// do NOT call finish() — peers treat that as H3_CLOSED_CRITICAL_STREAM.
// Required: open QPACK encoder + decoder streams, even though we
// never insert into the dynamic table. Just send the type prefix.
val qpackEnc = conn.openUniStream()
val w2 = QuicWriter()
w2.writeVarint(Http3StreamType.QPACK_ENCODER)
qpackEnc.send.enqueue(w2.toByteArray())
val qpackDec = conn.openUniStream()
val w3 = QuicWriter()
w3.writeVarint(Http3StreamType.QPACK_DECODER)
qpackDec.send.enqueue(w3.toByteArray())
// RFC 9114 §6.2: the server opens its own three uni streams
// (control, qpack encoder, qpack decoder). Their bytes accumulate
// in per-stream `incomingChannel`s (capacity 64); without an
// active consumer the channel saturates and `:quic` tears down
// the connection with INTERNAL_ERROR. We don't actually use the
// dynamic table or care about the server's settings, so drain
// and discard. Without this, multiplexing testcase fails after
// ~4.5s with "consumer overflowed" tear-down.
conn.drainPeerInitiatedUniStreamsIntoBlackHole(scope)
}
override suspend fun prepareRequest(
authority: String,
path: String,
): RequestHandle {
val stream = conn.openBidiStream()
stream.send.enqueue(encodeRequest(authority, path))
stream.send.finish()
// Without this, the data sits in the queue until the PTO
// timer fires (~1 s later). On the longrtt scenario (750 ms
// one-way, 1.5 s RTT) that's a fatal delay — the runner's
// 8 s timeout doesn't leave room for the PTO + RTT + RTT
// dance. The parallel path wakes after the chunk; the
// serial path was missing the same nudge.
driver.wakeup()
return Http3RequestHandle(stream)
}
override suspend fun prepareRequests(
authority: String,
paths: List<String>,
): List<RequestHandle> {
// Pre-encode all requests OUTSIDE the lock so QPACK encoding
// (allocations, hashing, varint emit) doesn't extend the
// critical section. With 1999 paths the encoding cost is
// non-trivial; doing it under streamsLock would stall the
// send loop for the full encode time across every chunk.
val encoded = paths.map { encodeRequest(authority, it) }
// openBidiStreamsBatch holds streamsLock for the entire
// batch — the send loop can't interject between opens so
// the writer's next drain finds all N streams' frames ready
// and packs them into coalesced packets (vs. the regressed
// shape that emitted one STREAM per packet against aioquic
// on 2026-05-06).
return conn.openBidiStreamsBatch(encoded) { stream, request ->
stream.send.enqueue(request)
stream.send.finish()
Http3RequestHandle(stream)
}
}
override suspend fun awaitResponse(handle: RequestHandle): GetResponse {
val stream = (handle as Http3RequestHandle).stream
val reader = Http3FrameReader()
var status = 0
val body = mutableListOf<ByteArray>()
stream.incoming.collect { chunk ->
reader.push(chunk)
while (true) {
val frame = reader.next() ?: break
when (frame) {
is Http3Frame.Headers -> {
val fields = QpackDecoder().decodeFieldSection(frame.qpackPayload)
status = fields.firstOrNull { it.first == ":status" }?.second?.toIntOrNull() ?: 0
}
is Http3Frame.Data -> {
body += frame.body
}
else -> {
Unit
}
}
}
}
return GetResponse(status = status, body = concat(body))
}
}
private class Http3RequestHandle(
val stream: com.vitorpamplona.quic.stream.QuicStream,
) : RequestHandle
/**
* Serialize a GET request as a single HEADERS frame ready to be enqueued
* onto a fresh bidi stream. Exposed for unit-testing the wire format
* without spinning up a QUIC connection.
*/
internal fun encodeRequest(
authority: String,
path: String,
): ByteArray {
val headers =
listOf(
":method" to "GET",
":scheme" to "https",
":authority" to authority,
":path" to path,
)
val qpack = QpackEncoder().encodeFieldSection(headers)
val w = QuicWriter()
w.writeVarint(Http3FrameType.HEADERS)
w.writeVarint(qpack.size.toLong())
w.writeBytes(qpack)
return w.toByteArray()
}
private fun concat(parts: List<ByteArray>): ByteArray {
val total = parts.sumOf { it.size }
val out = ByteArray(total)
var off = 0
for (p in parts) {
p.copyInto(out, off)
off += p.size
}
return out
}
@@ -0,0 +1,553 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop.runner
import com.vitorpamplona.quic.connection.QuicConnection
import com.vitorpamplona.quic.connection.QuicConnectionConfig
import com.vitorpamplona.quic.connection.QuicConnectionDriver
import com.vitorpamplona.quic.tls.PermissiveCertificateValidator
import com.vitorpamplona.quic.tls.TlsConstants
import com.vitorpamplona.quic.tls.TlsSecretsListener
import com.vitorpamplona.quic.transport.UdpSocket
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.async
import kotlinx.coroutines.cancel
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeoutOrNull
import java.io.File
import java.net.URI
import kotlin.system.exitProcess
/**
* Endpoint that implements the [quic-interop-runner](https://github.com/quic-interop/quic-interop-runner)
* Docker contract. Reads the env-var protocol the runner pushes into each
* client container and dispatches by [TESTCASE].
*
* Exit codes:
* - `0` — testcase passed
* - `1` — testcase failed (bug in our impl, OR partner)
* - `127` — testcase not implemented (runner skips, doesn't fail)
*/
private fun nowMs(): Long = System.currentTimeMillis()
private const val EXIT_OK = 0
private const val EXIT_FAIL = 1
private const val EXIT_UNSUPPORTED = 127
private const val HANDSHAKE_TIMEOUT_SEC = 10L
// Multiplexing generates ~hundreds-to-thousands of small files; download
// throughput on Mac+Rosetta is dominated by Docker filesystem overhead
// per-write. 30s wasn't enough for the larger file counts; the qlog
// against aioquic showed us still actively receiving STREAM frames at
// t=31s when our local timeout fired. 60s gives more headroom without
// inflating turnaround for the cases that actually complete fast.
private const val TRANSFER_TIMEOUT_SEC = 60L
// Per-stream timeout in the parallel-multiplexing path. If any single
// GET hangs past this (e.g. its FIN was lost in the shuffle of
// hundreds of concurrent streams), the await on that future returns
// a status=0 response and the others continue. Without this, a single
// stuck stream would block the whole .map { it.await() } chain.
private const val PER_STREAM_TIMEOUT_SEC = 20L
// Concurrency cap for the parallel-multiplexing path. Each chunk of this
// many requests fires fully in parallel; chunks process sequentially.
// Sized so that conn.lock contention stays manageable while still
// satisfying the runner's "streams overlap on the wire" check (which
// only needs a handful of streams concurrent at any given moment, not
// all of them simultaneously). Empirically validated against aioquic +
// picoquic at this value.
private const val MULTIPLEX_PARALLELISM = 64
fun main() {
// Single env-var check, propagated to library code that opts into
// verbose tracing only when this is set.
val debugEnv = System.getenv("QUIC_INTEROP_DEBUG")
if (debugEnv == "1") {
com.vitorpamplona.quic.connection.writerDebugEnabled = true
System.err.println(
"[boot] DEBUG=1; writerDebugEnabled=true; build_id=" +
"${com.vitorpamplona.quic.connection.WRITER_DEBUG_BUILD_ID}; " +
"TESTCASE=${System.getenv("TESTCASE") ?: "(unset)"}; " +
"ROLE=${System.getenv("ROLE") ?: "(unset)"}",
)
} else {
System.err.println("[boot] DEBUG=${debugEnv ?: "(unset)"} writerDebugEnabled=false")
}
val role = System.getenv("ROLE") ?: "client"
if (role != "client") {
System.err.println("server role not implemented")
exitProcess(EXIT_UNSUPPORTED)
}
val testcase = System.getenv("TESTCASE")?.trim().orEmpty()
val requests = System.getenv("REQUESTS")?.trim().orEmpty()
// The runner mounts $CLIENT_DOWNLOADS to /downloads as a Docker volume
// (see quic-interop-runner's docker-compose.yml `client.volumes`). It
// does NOT export a DOWNLOADS env var. Hard-code the mount path.
val downloadsDir = File("/downloads")
val keyLogPath = System.getenv("SSLKEYLOGFILE")?.takeIf { it.isNotBlank() }
val qlogDir = System.getenv("QLOGDIR")?.takeIf { it.isNotBlank() }?.let { File(it) }
// One-line context header. Verbose per-field dump deferred to debug
// mode (env var QUIC_INTEROP_DEBUG=1) so the runner's aggregated output
// stays readable across a full matrix run.
if (System.getenv("QUIC_INTEROP_DEBUG") == "1") {
System.err.println("== quic-interop client ==")
System.err.println("testcase: $testcase")
System.err.println("requests: $requests")
System.err.println("downloads dir: ${downloadsDir.absolutePath} (exists=${downloadsDir.isDirectory})")
System.err.println("sslkeylogfile: ${keyLogPath ?: "(unset)"}")
System.err.println("qlogdir: ${qlogDir?.absolutePath ?: "(unset)"}")
}
val cipherSuites =
when (testcase) {
"chacha20" -> intArrayOf(TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256)
else -> null
}
// For the versionnegotiation testcase the runner expects us to send
// an Initial advertising a version the server doesn't support, then
// honor its VN response by retrying with v1. agent A's
// QuicVersion.FORCE_VERSION_NEGOTIATION drives that flow.
val initialVersion =
when (testcase) {
"versionnegotiation" -> com.vitorpamplona.quic.packet.QuicVersion.FORCE_VERSION_NEGOTIATION
else -> com.vitorpamplona.quic.packet.QuicVersion.V1
}
// ALPN selection. Different servers configure different ALPNs PER
// testcase, with no consistent convention:
// - quic-go-qns — strictly hq-interop for non-http3 tests
// - aioquic-qns — accepts either
// - picoquic-qns — strictly h3 for ALL testcases
//
// Solution: offer BOTH `h3` and `hq-interop` in the ClientHello. TLS
// ALPN negotiation lets the server pick whichever matches its config.
// For testcases that REQUIRE H3 framing (http3, multiplexing) we
// restrict to h3 so any server that picks hq-interop fails fast.
val offeredAlpns =
when (testcase) {
"http3", "multiplexing" -> listOf(Alpn.H3)
else -> listOf(Alpn.HQ_INTEROP, Alpn.H3)
}
val code =
when (testcase) {
// All these testcases require: successful handshake + file
// transferred to /downloads. Per-testcase notes:
// chacha20 — runner verifies the negotiated cipher
// was ChaCha20-Poly1305 via tshark on the
// pcap (decrypted with SSLKEYLOGFILE).
// handshakeloss/ — same client behaviour against the
// transferloss runner's sim with random packet loss.
// transfercorruption — random bit-flips (recovery via
// AEAD AUTH FAIL → drop + retransmit).
// longrtt — emulated high-latency link.
// goodput / crosstraffic — throughput-floor / competing-flow
// scenarios.
// multiplexing — H3 GETs issued in parallel; runner
// verifies overlap on the wire via tshark.
// retry — server sends a Retry packet first; our
// applyRetry path (RFC 9000 §17.2.5 +
// RFC 9001 §5.8) handles DCID swap +
// token threading + key re-derivation.
// ipv6 — same flow over an IPv6 socket;
// JDK DatagramChannel.connect handles
// the v6 address resolution natively.
"handshake", "chacha20", "handshakeloss",
"transfer", "http3", "multiplexing",
"transferloss", "transfercorruption", "longrtt", "goodput", "crosstraffic",
"retry", "ipv6",
// NOTE: the runner does NOT have a `versionnegotiation` testcase
// (its Available list excludes it). The :quic VN-handling code
// (applyVersionNegotiation, FORCE_VERSION_NEGOTIATION constant)
// stays as defensive support for any server that decides to
// send a VN packet at us, but no testcase exercises it directly.
-> {
runTransferTest(
requests = requests,
downloadsDir = downloadsDir,
cipherSuites = cipherSuites,
offeredAlpns = offeredAlpns,
initialVersion = initialVersion,
keyLogPath = keyLogPath,
qlogDir = qlogDir,
// The runner reuses TESTCASE_CLIENT=transfer for the
// multiplexing testcase — discrimination is by URL
// count, not testcase name. We were checking
// testcase == "multiplexing" which is NEVER true
// (we'd see TESTCASE=multiplexing only on a
// hypothetical client where the runner explicitly
// sets it). Symptom: 60s timeout with 1421/2000
// files at 1 stream / RTT — exactly the serial
// client.get(...) path. Confirmed via the boot log:
// [boot] TESTCASE=transfer; transfer mode:
// parallel=false urls=1999
//
// Cheap whitespace tokenization here just counts;
// runTransferTest re-parses into URI[] inside.
parallel = requests.split(Regex("\\s+")).count { it.isNotBlank() } > 1,
)
}
else -> {
EXIT_UNSUPPORTED
}
}
exitProcess(code)
}
internal enum class Alpn(
val wireBytes: ByteArray,
) {
/** RFC 9114 — full HTTP/3 + QPACK + H3 framing. */
H3("h3".encodeToByteArray()),
/** quic-interop-runner convention — HTTP/0.9 over QUIC. Just `GET /path\r\n`
* on a fresh bidi stream, server returns the body, FIN both sides. No
* control stream, no QPACK, no SETTINGS. Used for handshake / chacha20 /
* transfer / loss-variant testcases. */
HQ_INTEROP("hq-interop".encodeToByteArray()),
}
private fun runTransferTest(
requests: String,
downloadsDir: File,
cipherSuites: IntArray?,
offeredAlpns: List<Alpn>,
initialVersion: Int,
keyLogPath: String?,
qlogDir: File?,
parallel: Boolean,
): Int {
val urls =
requests
.split(Regex("\\s+"))
.filter { it.isNotBlank() }
.map { runCatching { URI(it) }.getOrNull() }
.filter { it != null && it.host != null }
.map { it!! }
if (urls.isEmpty()) {
System.err.println("no parseable URL in REQUESTS")
return EXIT_FAIL
}
val first = urls[0]
val host = first.host
val port = first.port.takeIf { it > 0 } ?: 443
downloadsDir.mkdirs()
val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val outcome =
runBlocking {
val socket =
try {
UdpSocket.connect(host, port)
} catch (t: Throwable) {
return@runBlocking "udp_failed: ${t.message ?: t::class.simpleName}"
}
val keyLogger = keyLogPath?.let { SslKeyLogger(File(it)) }
val qlogWriter =
qlogDir?.let { dir ->
dir.mkdirs()
// ODCID is unknown until the connection generates one in
// its init block; we'd need to plumb through, but for the
// header it's fine to start with a placeholder and the
// packet-sent events will carry SCID/DCID anyway.
QlogWriter(file = File(dir, "client.sqlog"), odcidHex = "client")
}
val conn =
QuicConnection(
serverName = host,
config =
QuicConnectionConfig(
// Interop stress sizes — push the receive
// window past the largest single-file transfer
// any testcase exercises (longrtt does 5 MB,
// transferloss / transfercorruption do up to
// a few MB). Without this, the peer sends
// up to initialMaxStreamDataBidiLocal then
// stalls until our parser sends a
// MAX_STREAM_DATA — at high RTT (longrtt
// is 750 ms one-way) each stall is ~1.5 s
// round-trip lost. Setting both connection-
// and stream-level windows to 32 MB lets
// the peer's CC alone determine throughput.
initialMaxData = 32L * 1024 * 1024,
initialMaxStreamDataBidiLocal = 32L * 1024 * 1024,
initialMaxStreamDataBidiRemote = 32L * 1024 * 1024,
initialMaxStreamDataUni = 32L * 1024 * 1024,
),
tlsCertificateValidator = PermissiveCertificateValidator(),
alpnList = offeredAlpns.map { it.wireBytes },
initialVersion = initialVersion,
cipherSuites =
cipherSuites
?: intArrayOf(
TlsConstants.CIPHER_TLS_AES_128_GCM_SHA256,
TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256,
),
extraSecretsListener = keyLogger?.listener,
qlogObserver = qlogWriter ?: com.vitorpamplona.quic.observability.QlogObserver.NoOp,
)
val driver = QuicConnectionDriver(conn, socket, scope)
driver.start()
val handshake =
withTimeoutOrNull(HANDSHAKE_TIMEOUT_SEC * 1_000L) {
runCatching { conn.awaitHandshake() }
}
if (handshake == null || handshake.isFailure) {
runCatching { driver.close() }
conn.tls.clientRandom?.let { keyLogger?.flush(it) }
runCatching { qlogWriter?.close() }
return@runBlocking "handshake_failed"
}
// Dispatch the GET client by what the server actually picked.
// If the server didn't pick or picked something unrecognized,
// fall back to HQ-interop (simpler, more permissive).
val negotiated =
conn.tls.negotiatedAlpn
?.decodeToString()
.orEmpty()
val client: GetClient =
when (negotiated) {
"h3" -> {
Http3GetClient(conn, driver).also { it.init(scope) }
}
"hq-interop" -> {
HqInteropGetClient(conn, driver)
}
else -> {
System.err.println("unrecognized negotiated ALPN '$negotiated'; defaulting to hq-interop")
HqInteropGetClient(conn, driver)
}
}
val authority = if (port == 443) host else "$host:$port"
// Unconditional one-shot log so we can confirm which branch
// runs even when DEBUG=0 — this is a control-flow boundary,
// not a hot-path trace.
System.err.println("[boot] transfer mode: parallel=$parallel urls=${urls.size}")
val outcome =
withTimeoutOrNull(TRANSFER_TIMEOUT_SEC * 1_000L) {
val responses =
if (parallel) {
// Multiplexing throughput note. Spawning 1999
// simultaneous coroutines all racing the same
// conn.lock cratered throughput (~23 streams/sec
// on Mac+Rosetta, qlog-measured against aioquic).
// Lock contention scales superlinearly with
// suspended coroutines: every drainOutbound
// walks streamsList O(N), every openBidiStream
// queues behind every other waiter, and the
// dispatcher thrashes context-switching.
//
// Bound concurrency: process in chunks of
// [MULTIPLEX_PARALLELISM]. Each chunk is
// batched in two phases:
// 1. SERIAL prepareRequest for every URL
// in the chunk — opens the bidi
// stream, encodes the request, FINs.
// Synchronous; no async / no per-call
// wakeup.
// 2. SINGLE driver.wakeup() so the send
// loop drains all 64 enqueued requests
// in coalesced packets (multi-stream
// framing per drain) instead of one
// tiny packet per stream.
// 3. PARALLEL await — one async per
// stream collects its response with
// a per-stream timeout so a hung
// stream surfaces as status=0 instead
// of blocking its peers.
//
// Earlier shape (per-call wakeup inside
// client.get()) produced ~23 streams/sec
// because each individual enqueue tripped
// the send loop, which then drained alone
// (the other 63 coroutines hadn't queued
// yet on the dispatcher). Result: one
// ~80-byte packet per stream instead of
// ~10 streams/packet. Coalescing recovered
// by batching enqueues + single wake.
val collected = mutableListOf<Pair<URI, GetResponse>>()
// Quiet by default. QUIC_INTEROP_DEBUG=1 emits one
// line per chunk to stderr — wall-clock split between
// "all enqueued" and "all responded" lets us see
// whether time is spent in the writer (lots of ms
// before responses start arriving) or the server
// (responses dribble in over a long stretch).
val debug = System.getenv("QUIC_INTEROP_DEBUG") == "1"
val transferStartMs = nowMs()
if (debug) {
System.err.println(
"[interop] multiplex start: total_urls=${urls.size} " +
"MULTIPLEX_PARALLELISM=$MULTIPLEX_PARALLELISM " +
"expected_chunks=${(urls.size + MULTIPLEX_PARALLELISM - 1) / MULTIPLEX_PARALLELISM}",
)
}
urls.chunked(MULTIPLEX_PARALLELISM).forEachIndexed { chunkIdx, chunk ->
val chunkStartMs = nowMs()
if (debug && chunkIdx < 3) {
System.err.println(
"[interop] chunk=$chunkIdx size=${chunk.size} starting prepareRequests",
)
}
// Single lock-held batch open + enqueue.
// Without this, openBidiStream's per-call
// lock acquire / release lets the send loop
// interject between opens and drain one
// stream per packet.
val handles = client.prepareRequests(authority, chunk.map { it.path })
driver.wakeup()
val enqueuedMs = nowMs()
coroutineScope {
val deferreds =
chunk.zip(handles).map { (url, handle) ->
async {
val resp =
withTimeoutOrNull(PER_STREAM_TIMEOUT_SEC * 1_000L) {
client.awaitResponse(handle)
}
url to (resp ?: GetResponse(status = 0, body = ByteArray(0)))
}
}
deferreds.forEach { collected += it.await() }
}
if (debug) {
val doneMs = nowMs()
System.err.println(
"[interop] chunk=$chunkIdx size=${chunk.size} " +
"enqueue=${enqueuedMs - chunkStartMs}ms " +
"responses=${doneMs - enqueuedMs}ms " +
"cumulative=${doneMs - transferStartMs}ms",
)
}
}
collected
} else {
urls.map { url -> url to client.get(authority, url.path) }
}
var anyFailed = false
for ((url, resp) in responses) {
if (resp.status != 200) {
System.err.println("GET ${url.path} → status ${resp.status}")
anyFailed = true
continue
}
val name = url.path.substringAfterLast('/').ifBlank { "index" }
File(downloadsDir, name).writeBytes(resp.body)
}
if (anyFailed) "request_failed" else "ok"
} ?: "transfer_timeout"
runCatching { driver.close() }
conn.tls.clientRandom?.let { keyLogger?.flush(it) }
runCatching { qlogWriter?.close() }
delay(50)
outcome
}
scope.cancel()
return if (outcome == "ok") {
EXIT_OK
} else {
System.err.println("transfer $outcome")
EXIT_FAIL
}
}
/**
* Writes [NSS Key Log Format](https://firefox-source-docs.mozilla.org/security/nss/legacy/key_log_format/index.html)
* lines so Wireshark can decrypt the sim's captured pcap.
*
* CLIENT_HANDSHAKE_TRAFFIC_SECRET <client_random_hex> <secret_hex>
* SERVER_HANDSHAKE_TRAFFIC_SECRET <client_random_hex> <secret_hex>
* CLIENT_TRAFFIC_SECRET_0 <client_random_hex> <secret_hex>
* SERVER_TRAFFIC_SECRET_0 <client_random_hex> <secret_hex>
*/
internal class SslKeyLogger(
private val file: File,
) {
private val pending = mutableListOf<Pair<String, ByteArray>>()
val listener: TlsSecretsListener =
object : TlsSecretsListener {
override fun onHandshakeKeysReady(
cipherSuite: Int,
clientSecret: ByteArray,
serverSecret: ByteArray,
) {
pending += "CLIENT_HANDSHAKE_TRAFFIC_SECRET" to clientSecret
pending += "SERVER_HANDSHAKE_TRAFFIC_SECRET" to serverSecret
}
override fun onApplicationKeysReady(
cipherSuite: Int,
clientSecret: ByteArray,
serverSecret: ByteArray,
) {
pending += "CLIENT_TRAFFIC_SECRET_0" to clientSecret
pending += "SERVER_TRAFFIC_SECRET_0" to serverSecret
}
override fun onHandshakeComplete() = Unit
}
fun flush(clientRandom: ByteArray) {
val randomHex = clientRandom.toHex()
file.parentFile?.mkdirs()
file.appendText(
buildString {
for ((label, secret) in pending) {
append(label)
.append(' ')
.append(randomHex)
.append(' ')
.append(secret.toHex())
.append('\n')
}
},
)
pending.clear()
}
}
internal fun ByteArray.toHex(): String =
buildString(size * 2) {
for (b in this@toHex) {
val v = b.toInt() and 0xff
append("0123456789abcdef"[v ushr 4])
append("0123456789abcdef"[v and 0xf])
}
}
@@ -0,0 +1,344 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop.runner
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.vitorpamplona.quic.connection.EncryptionLevel
import com.vitorpamplona.quic.observability.QlogObserver
import java.io.BufferedWriter
import java.io.Closeable
import java.io.File
import java.io.FileWriter
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
/**
* JSON-NDJSON qlog writer (qlog 0.3 / JSON-SEQ format) used by the
* `:quic` interop runner. One JSON object per line; first line is the
* qlog header, subsequent lines are events.
*
* Tools like qvis (https://qvis.quictools.info/) consume the resulting
* `.sqlog` file to render sequence diagrams + RTT graphs + recovery
* timelines.
*
* **Goal: every interop-runner test failure produces a qlog file the
* caller can drop into qvis to see exactly what we did differently
* from the spec.**
*
* Threading: [java.io.BufferedWriter] is not safe for concurrent
* writers; we hold a [ReentrantLock] around each line emit so the
* read + send loops can fire events concurrently without interleaving
* partial JSON.
*/
class QlogWriter(
file: File,
private val odcidHex: String,
private val mapper: ObjectMapper = DEFAULT_MAPPER,
/** Wall-clock provider; tests inject a deterministic source. */
private val nowMillis: () -> Long = { System.currentTimeMillis() },
) : QlogObserver,
Closeable {
// append=false → truncate any prior trace at this path so a reused
// QLOGDIR doesn't accumulate stale events from a previous run.
private val writer: BufferedWriter = BufferedWriter(FileWriter(file, false))
/** Latched true once close() runs OR once a write throws IOException.
* Subsequent emits short-circuit instead of throwing into the
* send loop and tearing down an otherwise-healthy connection. */
@Volatile
private var closed: Boolean = false
private val lock = ReentrantLock()
private val startMillis: Long = nowMillis()
init {
// qlog 0.3 JSON-SEQ header. qvis tolerates both `qlog_format`
// values "JSON-SEQ" and "NDJSON"; we use JSON-SEQ to match the
// most-common production qlog files (Chromium, mvfst).
val header =
mapOf(
"qlog_version" to "0.3",
"qlog_format" to "JSON-SEQ",
"title" to "amethyst :quic client trace",
"trace" to
mapOf(
"vantage_point" to mapOf("type" to "client", "name" to "amethyst-quic"),
"common_fields" to
mapOf(
"ODCID" to odcidHex,
"reference_time" to startMillis,
"time_format" to "relative",
),
),
)
writeLineLocked(mapper.writeValueAsString(header))
}
override fun onConnectionStarted(
serverName: String,
dcid: ByteArray,
scid: ByteArray,
) {
emit(
"transport:connection_started",
mapOf(
"ip_version" to "v4_or_v6",
"server_name" to serverName,
"dst_cid" to hex(dcid),
"src_cid" to hex(scid),
),
)
}
override fun onConnectionClosed(
initiator: String,
errorCode: Long,
reason: String,
) {
emit(
"transport:connection_closed",
mapOf(
"owner" to initiator,
"application_code" to errorCode,
"reason" to reason,
),
)
}
override fun onPacketSent(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
) {
emit(
"transport:packet_sent",
mapOf(
"header" to
mapOf(
"packet_type" to packetTypeFor(level),
"packet_number" to packetNumber,
),
"raw" to mapOf("length" to sizeBytes),
"frames" to frames.map { mapOf("frame_type" to it) },
),
)
}
override fun onPacketReceived(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
) {
emit(
"transport:packet_received",
mapOf(
"header" to
mapOf(
"packet_type" to packetTypeFor(level),
"packet_number" to packetNumber,
),
"raw" to mapOf("length" to sizeBytes),
"frames" to frames.map { mapOf("frame_type" to it) },
),
)
}
override fun onPacketDropped(
reason: String,
sizeBytes: Int,
) {
emit(
"transport:packet_dropped",
mapOf(
"trigger" to reason,
"raw" to mapOf("length" to sizeBytes),
),
)
}
override fun onKeyUpdated(
keyType: String,
level: EncryptionLevel,
) {
emit(
"security:key_updated",
mapOf(
"key_type" to "${keyType}_${packetTypeFor(level)}_secret",
"trigger" to "tls",
),
)
}
override fun onLossDetected(
level: EncryptionLevel,
lostPacketNumbers: List<Long>,
) {
for (pn in lostPacketNumbers) {
emit(
"recovery:packet_lost",
mapOf(
"header" to
mapOf(
"packet_type" to packetTypeFor(level),
"packet_number" to pn,
),
"trigger" to "reordering_threshold_or_time_threshold",
),
)
}
}
override fun onPtoFired(
consecutivePtoCount: Int,
ptoMillis: Long,
) {
emit(
"recovery:loss_timer_updated",
mapOf(
"event_type" to "expired",
"timer_type" to "pto",
"pto_count" to consecutivePtoCount,
"delta" to ptoMillis,
),
)
}
override fun onCongestionStateUpdated(newState: String) {
emit(
"recovery:congestion_state_updated",
mapOf("new" to newState),
)
}
override fun onTransportParametersSet(
initiator: String,
params: Map<String, String>,
) {
emit(
"transport:parameters_set",
mapOf(
"owner" to initiator,
"params" to params,
),
)
}
override fun onAlpnNegotiated(alpn: String) {
emit(
"transport:alpn_information",
mapOf("chosen_alpn" to alpn),
)
}
override fun onVersionInformation(
chosenVersion: String,
otherVersionsOffered: List<String>,
) {
emit(
"transport:version_information",
mapOf(
"chosen_version" to chosenVersion,
"client_versions" to otherVersionsOffered,
),
)
}
override fun close() {
lock.withLock {
closed = true
try {
writer.flush()
} finally {
writer.close()
}
}
}
private fun emit(
name: String,
data: Map<String, Any?>,
) {
val event =
linkedMapOf<String, Any?>(
"time" to (nowMillis() - startMillis),
"name" to name,
"data" to data,
)
// Serialize OUTSIDE the lock so concurrent emitters don't
// serialize their JSON serially. The lock is only held while
// appending the line to the file.
val line = mapper.writeValueAsString(event)
writeLineLocked(line)
}
private fun writeLineLocked(line: String) {
lock.withLock {
// The send loop may still emit packet_sent events for the
// CONNECTION_CLOSE packet after the application calls close()
// — observers MUST NOT break the connection. Silently swallow
// post-close IOExceptions; the trace is what we have.
if (closed) return
try {
writer.write(line)
writer.write("\n")
// Deliberately NOT flushing per event: this method runs
// inside conn.lock.withLock { drainOutbound(...) } on the
// hot send path. On macOS Docker Desktop the filesystem
// is virtualized and per-event flush is multi-millisecond.
// Every flush stalls the connection lock, blocks the
// receive loop, and the connection silently dies
// mid-handshake. close() does the only flush we need
// for normal completion. A hard-killed JVM loses recent
// events but that's an acceptable trade — the alternative
// is the connection never completing in the first place.
} catch (_: java.io.IOException) {
// Stream closed under us. Latch closed so subsequent emits
// skip the lock and return immediately.
closed = true
}
}
}
companion object {
private val DEFAULT_MAPPER: ObjectMapper = jacksonObjectMapper()
private fun packetTypeFor(level: EncryptionLevel): String =
when (level) {
EncryptionLevel.INITIAL -> "initial"
EncryptionLevel.HANDSHAKE -> "handshake"
EncryptionLevel.APPLICATION -> "1RTT"
}
private val HEX_CHARS = "0123456789abcdef".toCharArray()
fun hex(bytes: ByteArray): String {
val sb = StringBuilder(bytes.size * 2)
for (b in bytes) {
val v = b.toInt() and 0xFF
sb.append(HEX_CHARS[v ushr 4])
sb.append(HEX_CHARS[v and 0x0F])
}
return sb.toString()
}
}
}
@@ -0,0 +1,59 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop.runner
import com.vitorpamplona.quic.http3.Http3Frame
import com.vitorpamplona.quic.http3.Http3FrameReader
import com.vitorpamplona.quic.qpack.QpackDecoder
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class Http3GetClientTest {
@Test
fun `encodeRequest produces a single HEADERS frame with the four pseudo-headers`() {
val bytes = encodeRequest(authority = "example.com", path = "/file.bin")
val reader = Http3FrameReader().apply { push(bytes) }
val frame = reader.next()
assertTrue(frame is Http3Frame.Headers, "first frame must be HEADERS, got $frame")
assertEquals(null, reader.next(), "should be exactly one frame")
val fields = QpackDecoder().decodeFieldSection(frame.qpackPayload)
// QPACK emits headers in order; confirm pseudo-headers come first
// and carry the expected values.
val map = fields.associate { it.first to it.second }
assertEquals("GET", map[":method"])
assertEquals("https", map[":scheme"])
assertEquals("example.com", map[":authority"])
assertEquals("/file.bin", map[":path"])
}
@Test
fun `encodeRequest survives an authority with a non-default port`() {
val bytes = encodeRequest(authority = "example.com:8443", path = "/")
val frame = Http3FrameReader().apply { push(bytes) }.next()
require(frame is Http3Frame.Headers)
val map = QpackDecoder().decodeFieldSection(frame.qpackPayload).associate { it.first to it.second }
assertEquals("example.com:8443", map[":authority"])
assertEquals("/", map[":path"])
}
}
@@ -0,0 +1,161 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop.runner
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.vitorpamplona.quic.connection.EncryptionLevel
import org.junit.Test
import java.io.File
import java.nio.file.Files
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
/**
* Validates the JSON-NDJSON shape qvis (https://qvis.quictools.info/)
* expects: header line + one event per line, every line independently
* parseable as JSON.
*
* Drives [QlogWriter] through one event of each type to confirm we
* don't emit anything that breaks the format.
*/
class QlogWriterTest {
@Test
fun headerThenOneEventPerLine_allParseable() {
val tmp = Files.createTempFile("amethyst-qlog-test", ".sqlog").toFile()
tmp.deleteOnExit()
var clock = 0L
QlogWriter(tmp, odcidHex = "deadbeef", nowMillis = { clock }).use { w ->
clock = 5L
w.onConnectionStarted("example.test", byteArrayOf(1, 2, 3), byteArrayOf(4, 5, 6))
clock = 7L
w.onTransportParametersSet("local", mapOf("initial_max_data" to "1000000"))
clock = 10L
w.onPacketSent(EncryptionLevel.INITIAL, packetNumber = 0, sizeBytes = 1200, frames = listOf("crypto"))
clock = 15L
w.onPacketReceived(EncryptionLevel.INITIAL, packetNumber = 0, sizeBytes = 800, frames = listOf("crypto", "ack"))
clock = 20L
w.onPacketDropped("AEAD auth failed", sizeBytes = 80)
clock = 25L
w.onKeyUpdated("server", EncryptionLevel.HANDSHAKE)
clock = 30L
w.onLossDetected(EncryptionLevel.APPLICATION, lostPacketNumbers = listOf(3L, 5L))
clock = 35L
w.onPtoFired(consecutivePtoCount = 1, ptoMillis = 333)
clock = 40L
w.onCongestionStateUpdated("recovery")
clock = 45L
w.onAlpnNegotiated("h3")
clock = 50L
w.onVersionInformation("v1", emptyList())
clock = 55L
w.onConnectionClosed("local", errorCode = 0, reason = "done")
}
val lines = tmp.readLines().filter { it.isNotBlank() }
assertTrue(lines.size >= 12, "expected >= 12 lines (header + at least 11 events) but got ${lines.size}")
val mapper = jacksonObjectMapper()
// Line 1: qlog header.
val header = mapper.readTree(lines[0])
assertEquals("0.3", header.get("qlog_version").asText(), "qlog_version must be 0.3")
assertEquals("JSON-SEQ", header.get("qlog_format").asText(), "qlog_format must be JSON-SEQ")
val vp = header.get("trace").get("vantage_point")
assertEquals("client", vp.get("type").asText())
assertEquals(
"deadbeef",
header
.get("trace")
.get("common_fields")
.get("ODCID")
.asText(),
)
// Lines 2..N: event objects with `time`, `name`, `data`.
for (i in 1 until lines.size) {
val node = mapper.readTree(lines[i])
assertNotNull(node.get("time"), "line $i missing 'time': ${lines[i]}")
val name = node.get("name")
assertNotNull(name, "line $i missing 'name': ${lines[i]}")
assertTrue(
name.asText().contains(":"),
"name '${name.asText()}' must be in '<category>:<event>' form",
)
assertNotNull(node.get("data"), "line $i missing 'data': ${lines[i]}")
}
// Spot-check specific events made it through.
val names = lines.drop(1).map { mapper.readTree(it).get("name").asText() }
assertTrue(names.contains("transport:connection_started"), names.toString())
assertTrue(names.contains("transport:packet_sent"), names.toString())
assertTrue(names.contains("transport:packet_received"), names.toString())
assertTrue(names.contains("transport:packet_dropped"), names.toString())
assertTrue(names.contains("security:key_updated"), names.toString())
assertTrue(names.contains("recovery:packet_lost"), names.toString())
assertTrue(names.contains("recovery:loss_timer_updated"), names.toString())
assertTrue(names.contains("transport:parameters_set"), names.toString())
assertTrue(names.contains("transport:alpn_information"), names.toString())
assertTrue(names.contains("transport:version_information"), names.toString())
assertTrue(names.contains("transport:connection_closed"), names.toString())
}
@Test
fun timesAreRelativeToConstructorTime() {
val tmp = Files.createTempFile("amethyst-qlog-rel", ".sqlog").toFile()
tmp.deleteOnExit()
var clock = 1_000L
QlogWriter(tmp, odcidHex = "00", nowMillis = { clock }).use { w ->
clock = 1_050L
w.onAlpnNegotiated("h3")
}
val lines = tmp.readLines().filter { it.isNotBlank() }
val mapper = jacksonObjectMapper()
val event = mapper.readTree(lines[1])
assertEquals(50L, event.get("time").asLong(), "time must be relative to constructor (1050 - 1000)")
}
@Test
fun fileEndsWithNewline_qvisCompatible() {
val tmp = Files.createTempFile("amethyst-qlog-nl", ".sqlog").toFile()
tmp.deleteOnExit()
QlogWriter(tmp, odcidHex = "00").use { w ->
w.onAlpnNegotiated("h3")
}
val bytes = tmp.readBytes()
assertTrue(bytes.isNotEmpty(), "file must not be empty")
assertEquals('\n'.code.toByte(), bytes.last(), "file must end with '\\n' so trailing event parses")
}
@Test
fun handlesEmptyFramesList(): Unit =
File.createTempFile("amethyst-qlog-empty", ".sqlog").let { tmp ->
tmp.deleteOnExit()
QlogWriter(tmp, odcidHex = "00").use { w ->
w.onPacketSent(EncryptionLevel.INITIAL, 0, 1200, emptyList())
}
val mapper = jacksonObjectMapper()
val lines = tmp.readLines().filter { it.isNotBlank() }
val frames = mapper.readTree(lines[1]).get("data").get("frames")
assertTrue(frames.isArray, "frames must be an array even when empty")
assertEquals(0, frames.size())
}
}
@@ -0,0 +1,91 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop.runner
import java.io.File
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class SslKeyLoggerTest {
@Test
fun `emits NSS Key Log lines for handshake and application secrets`() {
val tmp = File.createTempFile("ssl-keylog-test", ".log").also { it.deleteOnExit() }
val logger = SslKeyLogger(tmp)
logger.listener.onHandshakeKeysReady(
cipherSuite = 0x1301,
clientSecret = ByteArray(32) { 0xAA.toByte() },
serverSecret = ByteArray(32) { 0xBB.toByte() },
)
logger.listener.onApplicationKeysReady(
cipherSuite = 0x1301,
clientSecret = ByteArray(32) { 0xCC.toByte() },
serverSecret = ByteArray(32) { 0xDD.toByte() },
)
val clientRandom = ByteArray(32) { it.toByte() }
logger.flush(clientRandom)
val lines =
tmp
.readText()
.lineSequence()
.filter { it.isNotEmpty() }
.toList()
assertEquals(4, lines.size, "one line per secret")
val randomHex = clientRandom.toHex()
val expectedClientHs = "CLIENT_HANDSHAKE_TRAFFIC_SECRET $randomHex ${ByteArray(32) { 0xAA.toByte() }.toHex()}"
val expectedServerHs = "SERVER_HANDSHAKE_TRAFFIC_SECRET $randomHex ${ByteArray(32) { 0xBB.toByte() }.toHex()}"
val expectedClientApp = "CLIENT_TRAFFIC_SECRET_0 $randomHex ${ByteArray(32) { 0xCC.toByte() }.toHex()}"
val expectedServerApp = "SERVER_TRAFFIC_SECRET_0 $randomHex ${ByteArray(32) { 0xDD.toByte() }.toHex()}"
assertEquals(expectedClientHs, lines[0])
assertEquals(expectedServerHs, lines[1])
assertEquals(expectedClientApp, lines[2])
assertEquals(expectedServerApp, lines[3])
}
@Test
fun `flush is idempotent — second flush emits nothing`() {
val tmp = File.createTempFile("ssl-keylog-test", ".log").also { it.deleteOnExit() }
val logger = SslKeyLogger(tmp)
logger.listener.onHandshakeKeysReady(
cipherSuite = 0x1301,
clientSecret = ByteArray(32) { 1 },
serverSecret = ByteArray(32) { 2 },
)
logger.flush(ByteArray(32))
val firstLen = tmp.length()
logger.flush(ByteArray(32))
assertEquals(firstLen, tmp.length(), "second flush must not append")
}
@Test
fun `toHex round-trips lowercase`() {
val bytes = byteArrayOf(0x00, 0x0f, 0x10.toByte(), 0xff.toByte())
assertEquals("000f10ff", bytes.toHex())
assertTrue(bytes.toHex().all { it.isDigit() || it in 'a'..'f' })
}
}
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/env bash
# Summarize per-testcase results for the most recent matrix run.
# Useful when run-matrix.sh terminated early — the per-testcase
# output.txt files have status lines we can extract.
set -o pipefail
# NOT set -u: bash 3.2 (macOS default) treats `${arr[@]}` on an
# empty array as an unbound-variable error, which we hit on runs
# that have no log files (the artifact-inspection path is still
# valid).
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
RUNNER_LOGS="${REPO_ROOT}/../quic-interop-runner/logs"
if [[ ! -d "$RUNNER_LOGS" ]]; then
echo "no runner logs at $RUNNER_LOGS" >&2
exit 1
fi
RUN_DIR="$(ls -1dt "$RUNNER_LOGS"/run-* 2>/dev/null | head -n 1 || true)"
if [[ -z "$RUN_DIR" ]]; then
echo "no run-* dirs under $RUNNER_LOGS" >&2
exit 1
fi
echo "==> run dir: $RUN_DIR"
echo
# Layout: <run>/<pair>/<testcase>/output.txt
# The runner writes a "Test: <name> took X, status: TestResult.{SUCCEEDED|FAILED|UNSUPPORTED}"
# line at the end of each test.
PAIR_DIR="$(ls -1d "$RUN_DIR"/*amethyst* 2>/dev/null | head -n 1 || true)"
if [[ -z "$PAIR_DIR" ]]; then
echo "no <pair> dir under $RUN_DIR" >&2
exit 1
fi
# Some runner versions don't write status to per-testcase output.txt.
# Fall back to inferring from artifacts (qlog connection_closed events,
# pcap presence, etc.). We tag these with [inf] in the result column.
infer_status_from_artifacts() {
local tc_dir="$1"
local tc="$2"
# Did the connection formally close with an error?
local qlog
qlog=$(ls -1 "$tc_dir"/client/qlog/*.sqlog "$tc_dir"/client/qlog/*.qlog 2>/dev/null | head -n 1 || true)
if [[ -n "$qlog" ]]; then
# Did the server close us with an error code?
local cc
cc=$(grep '"name":"transport:connection_closed"' "$qlog" 2>/dev/null | tail -n 1)
if [[ -n "$cc" ]]; then
local reason
reason=$(echo "$cc" | sed -nE 's/.*"reason":"([^"]+)".*/\1/p')
echo "FAILED: $reason"
return
fi
# No close → did we get >0 packets received? If so, infer
# something happened. Status is genuinely uncertain.
local rx
rx=$(grep -c '"name":"transport:packet_received"' "$qlog" 2>/dev/null || echo 0)
if [[ "$rx" -gt 0 ]]; then
echo "RAN ($rx pkts rx; no formal close)"
return
fi
fi
echo "UNKNOWN (no qlog)"
}
# The "Test: X took Y, status: TestResult.Z" line is written by run.py
# to its own stdout, not into the per-testcase output.txt. Search a few
# likely locations: the per-testcase output.txt (in case the runner
# version we're using writes there), the run dir, the runner-logs root,
# and the working dir we were invoked from.
# Build list of files to grep, in priority order. Bash 3.2 (macOS
# default) chokes on multiline process-substitution with comments,
# so we just push to an array imperatively.
SEARCH_FILES=()
# 1. run-matrix.sh tees the runner stdout to a sibling .stdout.log
# file — that has the authoritative "Test: X took Y, status:" lines.
if [[ -f "${RUN_DIR}.stdout.log" ]]; then
SEARCH_FILES+=("${RUN_DIR}.stdout.log")
fi
# 2. Per-testcase output.txt — older runner versions wrote status here.
while IFS= read -r f; do
SEARCH_FILES+=("$f")
done < <(find "$PAIR_DIR" -maxdepth 3 -name 'output.txt' -type f 2>/dev/null)
# 3. Run-dir / runner-logs *.log — catch-all.
while IFS= read -r f; do
SEARCH_FILES+=("$f")
done < <(find "$RUN_DIR" -maxdepth 1 -name '*.log' -type f 2>/dev/null)
while IFS= read -r f; do
SEARCH_FILES+=("$f")
done < <(find "$RUNNER_LOGS" -maxdepth 1 -name 'run-*.log' -type f 2>/dev/null)
printf "%-22s %-15s %-10s\n" "TESTCASE" "RESULT" "TIME"
printf "%-22s %-15s %-10s\n" "----------------------" "---------------" "----------"
for tc_dir in "$PAIR_DIR"/*/; do
tc=$(basename "$tc_dir")
# Search every candidate file for a status line matching this
# testcase. Allow optional leading timestamp from the runner's
# logging format (`2026-05-07 12:34:56,789 Test: ...`).
line=""
for f in "${SEARCH_FILES[@]}"; do
[[ -f "$f" ]] || continue
match=$(grep -E "Test: $tc took [0-9.]+s, status: TestResult\." "$f" 2>/dev/null | tail -n 1)
if [[ -n "$match" ]]; then
line="$match"
break
fi
done
if [[ -n "$line" ]]; then
status=$(echo "$line" | sed -nE 's/.*TestResult\.([A-Z_]+).*/\1/p')
time=$(echo "$line" | sed -nE 's/.*took ([0-9.]+s).*/\1/p')
case "$status" in
SUCCEEDED) marker="✓" ;;
UNSUPPORTED) marker="?" ;;
FAILED) marker="✕" ;;
*) marker="·" ;;
esac
printf "%-22s %s %-13s %-10s\n" "$tc" "$marker" "$status" "$time"
else
# Fall back to artifact inspection.
inferred=$(infer_status_from_artifacts "$tc_dir" "$tc")
printf "%-22s %s [inf] %s\n" "$tc" "·" "$inferred"
fi
done
+223
View File
@@ -0,0 +1,223 @@
# QuicConnection Lock Split — Design Note
Date: 2026-05-08
## Problem
`QuicConnection.lock: Mutex` serialises every meaningful operation:
- `drainOutbound` (send loop, holds lock during a full datagram build —
iterates every stream, allocates packet numbers, encrypts).
- `feedDatagram` (read loop, holds lock during decrypt + frame dispatch
+ per-stream insert).
- `openBidiStream` / `openUniStream` (app code, holds lock for stream
allocation + map insert).
- `getOrCreatePeerStreamLocked` (parser path on the read loop's
critical section, but app code can also call it from tests).
Multiplexing test against aioquic measures ~25 streams/sec — every
coroutine fights this single mutex.
## Goal
Split the mutex into per-domain mutexes so the read loop, send loop, and
app code can mostly progress concurrently. Per-stream `synchronized(this)`
inside `SendBuffer`/`ReceiveBuffer` already handles per-stream
serialisation; we don't touch those.
## Domain Map
### Domain A — `streamsLock: Mutex` (the streams registry)
Fields:
- `streams: MutableMap<Long, QuicStream>`
- `streamsList: MutableList<QuicStream>` (insertion-ordered list parallel
to `streams`, used by writer round-robin)
- `nextLocalBidiIndex`, `nextLocalUniIndex`
- `streamRoundRobinStart` — read+written by writer; used in the
same critical section it holds `streamsLock` for the iteration
- `peerInitiatedUniCount`, `peerInitiatedBidiCount`
- `advertisedMaxStreamsUni`, `advertisedMaxStreamsBidi`,
`advertisedMaxData`
- `pendingMaxStreamsUni`, `pendingMaxStreamsBidi`, `pendingMaxData`
- `pendingMaxStreamData: MutableMap<Long, Long>`
- `pendingNewConnectionId: MutableMap<Long, …>`
- `newPeerStreams: ArrayDeque<QuicStream>`
- `pendingDatagrams: ArrayDeque<ByteArray>` — outbound DATAGRAMs
- `incomingDatagrams: ArrayDeque<ByteArray>` — inbound DATAGRAMs
- `sendConnectionFlowCredit`, `sendConnectionFlowConsumed`
- `receiveConnectionFlowLimit`
Rationale: the writer needs an atomic snapshot of "all streams + all
pending control-frame retransmits + datagram queues + flow-control
counters" in one critical section to assemble a packet. The parser needs
the same coverage when delivering a STREAM frame (look up or create
the stream + queue receive bytes + bump pending* fields). Splitting
these into multiple sub-locks would force the writer/parser to acquire
several locks per pass — same contention, more deadlock risk.
`peerMaxStreamsBidi`, `peerMaxStreamsUni` stay `@Volatile` (already are):
the writer reads them once at the top of a stream open; the parser
writes once on inbound MAX_STREAMS. Atomic long write is sufficient on
all supported platforms.
### Domain B — `LevelState.levelLock: Mutex` (one per level: initial / handshake / application)
Fields per `LevelState`:
- `pnSpace: PacketNumberSpaceState`
- `sentPackets: MutableMap<Long, SentPacket>`
- `ackTracker`
- `cryptoSend: SendBuffer`, `cryptoReceive: ReceiveBuffer`
- `sendProtection: PacketProtection?`, `receiveProtection: PacketProtection?`
- `keysDiscarded`
- `largestAckedPn`, `largestAckedSentTimeMs`
The writer iterates through levels in order (initial → handshake →
application) when building a coalesced datagram. Each level's critical
section is independent, so the lock is held only for the duration of
build at that level (which doesn't touch the streams registry except
to read `streamsListLocked()` for stream frames inside the application
build — that read transitions through `streamsLock`).
### Domain C — `lifecycleLock: Mutex` (status + handshake metadata)
Fields:
- `status: Status`
- `closeReason: String?`, `closeErrorCode: Long`
- `peerTransportParameters: TransportParameters?` — read-mostly after
handshake; using `@Volatile` reference + write-once-after-handshake
is sufficient here. Promoted to `@Volatile` so writer/parser can
snapshot without a lock.
- `handshakeComplete: Boolean`
- `closeAllSignals` (the channels are themselves thread-safe; lock is
only required to serialise the status transition)
### Domain D — Atomic / `@Volatile` (no lock)
Fields:
- `pendingPing` — toggled by driver under PTO; observed by writer.
Promote to `@Volatile`.
- `consecutivePtoCount` — already `@Volatile`. Driver writes it under
its own logic; no further protection needed because it's only read
inside the same loop iteration that wrote it.
- `destinationConnectionId` — already has volatile semantics
(`internal set` on a `@Volatile var`). Stays as is.
- `udpStatsSupplier` — already `@Volatile`.
- `peerMaxStreamsBidi`, `peerMaxStreamsUni` — already `@Volatile`.
- `handshakeDoneSignal: CompletableDeferred<Unit>` — coroutines
primitive, thread-safe.
### Domain E — Per-stream (UNCHANGED)
`QuicStream` already protects its `SendBuffer` / `ReceiveBuffer` with
internal `synchronized(this)` blocks. Nothing changes here.
## Lock Acquisition Order
To prevent deadlock, document and enforce:
```
lifecycleLock < streamsLock < (any LevelState.levelLock)
```
Per-stream `synchronized(...)` blocks inside `SendBuffer`/`ReceiveBuffer`
remain at the leaf — never acquire any QuicConnection mutex while
holding a per-stream lock.
In practice the only nesting that happens is:
- `drainOutbound` acquires `streamsLock` (for the streams loop +
stream-frame creation) but the per-level builds happen *outside*
that block — each level acquires its own `levelLock` separately.
No nested `streamsLock` ⊃ `levelLock` chain.
- Actually re-checking the design: the writer needs to allocate a
PN at the chosen level *while* it has decided which streams to
flush. Two options:
(1) acquire streamsLock, snapshot streams + frames, release;
acquire each levelLock to encode + record.
(2) hold streamsLock during level encode for the application
packet (because stream-frame retransmit tokens get recorded
into level.sentPackets in the same operation).
We take option (2) — encode under both locks, with strict order
`streamsLock` → `levelLock`. The other levels (initial/handshake)
don't touch streamsLock at all, so they only acquire `levelLock`.
## Public API Compatibility
`QuicConnection.lock: Mutex` is `val`-public. External callers exist
(tests + InMemoryQuicPipe-driven harnesses). To avoid breaking those:
- Keep the `lock: Mutex` field as a deprecated forwarder. It now
*also* exists, but it is an alias for `lifecycleLock`. New code
must NOT use it. Existing tests that lock it before mutating
state used to cover all domains; we update them in place to use
the appropriate lock(s).
Actually simpler: keep `lock: Mutex` as a *no-op* lock (still a
`Mutex` so external code compiles), document that it no longer
guards anything, update the tests that lock it.
After review: tests use `conn.lock` to serialise their direct calls to
`onTokensAcked`/`onTokensLost`/`getOrCreatePeerStreamLocked`. We update
those tests to acquire `streamsLock` instead (since those routines
mutate stream-domain state). The `lock` field is kept as deprecated
for source compatibility but is functionally a leaf no-op.
## Migration Plan
1. Add `streamsLock`, `lifecycleLock` fields. Keep `lock` as alias of
`lifecycleLock`.
2. Add `levelLock` to `LevelState`.
3. Convert `getOrCreatePeerStreamLocked` → `getOrCreatePeerStream` doing
its own `streamsLock` acquisition. Keep the old name as a forwarder
for backwards compat.
4. Update `openBidiStream`, `openUniStream`, `streamById`, `pollIncomingPeerStream`,
`awaitIncomingPeerStream`, `pollIncomingDatagram`, `awaitIncomingDatagram`,
`queueDatagram`, `flowControlSnapshot` to acquire `streamsLock`.
5. Update `close`, `markClosedExternally` to use `lifecycleLock`.
6. Update driver's `readLoop`/`sendLoop`:
- `feedDatagram` no longer wraps in conn-wide lock. Instead the
parser acquires `streamsLock` around stream-touching code,
and `levelLock` around level-touching code.
- `drainOutbound` is restructured similarly.
7. Update tests that hold `conn.lock` to use the relevant new lock.
## Risk + Mitigation
- **Deadlock**: enforce order via code review + (where practical)
inline comments at each acquisition site. Keep nesting shallow.
- **Missed coverage**: enumerate every field in this doc; if a field
can be mutated from two domains we either move it to a single domain
or annotate it as @Volatile.
- **Performance regression**: more mutex acquisitions overall; but
the critical path (multiplexing test) sees parallel execution
instead of serial, which more than compensates.
## Implementation Phases
This commit implements **phase 1** — separate domain locks but
`drainOutbound` and `feedDatagram` still hold `streamsLock` for the
entire pass. The wins from phase 1 alone:
- App code (`openBidiStream`, `streamById`, `flowControlSnapshot`) no
longer contends with `lifecycleLock`-only operations.
- The PTO timer path stops touching any mutex (volatile fields).
- `markClosedExternally` no longer needs a lock.
- `close()` only takes lifecycleLock — opens the path for in-progress
drain to finish without status-write contention.
Phase 2 (deferred follow-up): split `buildApplicationPacket` into a
"collect frames under streamsLock" stage and an "encrypt + record
under levelLock" stage so app coroutines can intersperse during the
encrypt window. That requires more invasive surgery on the writer's
internals; phase 1 ships first to lock in the safer subset.
## Verification
- `:quic:jvmTest` — full suite must pass.
- `MultiplexingThroughputTest` (new): 1000 streams in <500 ms on
InMemoryQuicPipe.
@@ -24,7 +24,18 @@ import com.vitorpamplona.quic.connection.recovery.SentPacket
import com.vitorpamplona.quic.stream.ReceiveBuffer
import com.vitorpamplona.quic.stream.SendBuffer
/** Per-encryption-level state owned by [QuicConnection]. */
/**
* Per-encryption-level state owned by [QuicConnection].
*
* Concurrency: [cryptoSend] / [cryptoReceive] use their internal
* [SendBuffer] / [ReceiveBuffer] `synchronized(this)` blocks for
* thread safety — the writer's `takeChunk`, the parser's `markAcked`,
* and PTO-driven `requeueAllInflight` are all serialized through
* those leaf locks. [sentPackets] is currently mutated by the writer
* (under [QuicConnection.streamsLock]) and read by the parser without
* synchronization; that race is pre-existing audit-tracked and not
* fixed by [LevelState] today.
*/
class LevelState {
val pnSpace = PacketNumberSpaceState()
@@ -122,4 +133,79 @@ class LevelState {
largestAckedSentTimeMs = null
keysDiscarded = true
}
/**
* RFC 9000 §6: reset every per-level field to a constructor-fresh
* state, then install [sendProtection] / [receiveProtection] keyed
* to the post-VN destination CID.
*
* Differs from [discardKeys] in that this re-arms the level for
* a fresh handshake — caller (
* [QuicConnection.applyVersionNegotiation]) re-enqueues the cached
* ClientHello onto [cryptoSend] immediately afterwards, so the
* next outbound drain emits a v1 Initial with PN=0 and the same
* TLS bytes the original Initial carried.
*/
internal fun resetForVersionNegotiation(
sendProtection: PacketProtection,
receiveProtection: PacketProtection,
) {
// pnSpace is val (lock-split refactor); reset its fields in place.
// The PacketNumberSpaceState.resetForRetry() name is historical but
// the semantics are right for VN too: zero out the PN counter +
// received-side state.
pnSpace.resetForRetry()
ackTracker =
com.vitorpamplona.quic.recovery
.AckTracker()
cryptoSend = SendBuffer()
cryptoReceive = ReceiveBuffer()
sentPackets.clear()
largestAckedPn = null
largestAckedSentTimeMs = null
keysDiscarded = false
this.sendProtection = sendProtection
this.receiveProtection = receiveProtection
}
/**
* Reset the Initial-level state for a successful Retry.
*
* Differs from [resetForVersionNegotiation] in that the packet number
* namespace is **preserved**: per RFC 9001 §5.7 + RFC 9000 §17.2.5,
* the Initial PN space spans the original AND the post-Retry Initials.
* The client MUST NOT reuse a packet number across the boundary —
* the next Initial sent after Retry uses PN = (last-used) + 1.
*
* The runner's retry test specifically checks for this: a client that
* resets the PN gets "Client reset the packet number. Check failed
* for PN 0".
*
* Everything else resets to mirror VN semantics:
* - cryptoSend cleared so the caller can re-enqueue the cached
* ClientHello on top of fresh empty buffer state.
* - sentPackets cleared because the pre-Retry Initial's loss-recovery
* state references PNs the server will never ACK (the server
* discarded that packet when it sent the Retry).
* - keys re-derived from the new DCID and reinstalled.
* - keysDiscarded latch reset (the pre-Retry keys are gone, but the
* level itself is still alive with the new keys).
*/
internal fun resetForRetry(
sendProtection: PacketProtection,
receiveProtection: PacketProtection,
) {
// pnSpace is INTENTIONALLY preserved — see kdoc above.
ackTracker =
com.vitorpamplona.quic.recovery
.AckTracker()
cryptoSend = SendBuffer()
cryptoReceive = ReceiveBuffer()
sentPackets.clear()
largestAckedPn = null
largestAckedSentTimeMs = null
keysDiscarded = false
this.sendProtection = sendProtection
this.receiveProtection = receiveProtection
}
}
@@ -64,6 +64,20 @@ class PacketNumberSpaceState {
if (nextPacketNumber > 0) nextPacketNumber--
}
/**
* RFC 9000 §17.2.5.2: reset the Initial-level packet-number space
* after a Retry packet rolls our DCID. The new Initial keys are
* derived from a different secret, so the (PN, key) pair the AEAD
* nonce relies on is unique even with the PN going back to 0.
* Inbound state is reset because no Initial packet has been
* received yet on the new keys.
*/
internal fun resetForRetry() {
nextPacketNumber = 0L
largestReceived = -1L
largestReceivedTime = 0L
}
/** Note that an inbound packet was successfully decrypted. */
fun observeInbound(
packetNumber: Long,
@@ -0,0 +1,114 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.stream.StreamId
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
/**
* Spawn a long-lived coroutine that accepts every peer-initiated
* unidirectional stream this connection surfaces and drains it to
* `/dev/null`. Returns the [Job] of the launched dispatcher so the
* caller can join / cancel it.
*
* Why this exists: RFC 9114 §6.2.1 mandates that an HTTP/3 server
* opens at least three peer-initiated uni streams (CONTROL +
* QPACK_ENCODER + QPACK_DECODER) immediately after the handshake.
* [QuicConnectionParser] routes their bytes into each stream's
* bounded `incomingChannel` (capacity 64 chunks) — if no consumer
* reads them, the next chunk delivery overflows and the connection
* tears down with `INTERNAL_ERROR: stream … consumer overflowed`
* (audit-4 #3). The symptom for the multiplexing interop test was
* a zero-request connection that died after ~5 seconds the moment
* the server's QPACK encoder pushed dynamic-table inserts.
*
* This helper is the explicit "I do not care about these particular
* peer streams" knob for callers (e.g. an H3 GET client that runs
* with QPACK dynamic-table off) that don't need to interpret the
* SETTINGS / QPACK bytes. The :quic library does NOT default to
* silently dropping app bytes — apps that DO care about peer-uni
* streams (WebTransport, MoQ-over-WT) call
* [QuicConnection.awaitIncomingPeerStream] directly and route each
* stream by inspecting its leading varint.
*
* Bidi peer streams are deliberately re-queued back to
* [QuicConnection.newPeerStreams] (well — left untouched on the
* head when surfaced; we just don't consume them here) so that an
* application that opts in to draining uni streams doesn't
* accidentally swallow peer-initiated bidi requests. In the
* H3-client multiplexing case we never expect the server to open
* a bidi stream against us, but if it does the connection-level
* handling stays correct.
*
* Usage from an integrator (sketch — `Http3GetClient` is on a
* different branch on this repo today):
*
* ```
* suspend fun init(scope: CoroutineScope) {
* // Open our own H3 control + QPACK uni streams.
* openH3ControlStream()
* openQpackEncoderStream()
* openQpackDecoderStream()
*
* // Accept the server's three counterparts and discard their bytes.
* conn.drainPeerInitiatedUniStreamsIntoBlackHole(scope)
* }
* ```
*
* Lifecycle: the launched coroutine exits cleanly when
* [QuicConnection.awaitIncomingPeerStream] returns null (the
* connection has reached `CLOSED`). Cancelling the [scope] also
* tears it down.
*/
fun QuicConnection.drainPeerInitiatedUniStreamsIntoBlackHole(scope: CoroutineScope): Job =
scope.launch {
while (true) {
val stream = awaitIncomingPeerStream() ?: return@launch
// Only drain peer-initiated UNI streams. Peer bidi streams are
// returned to whatever else the application wants to do with
// them — but we have to put them somewhere because
// awaitIncomingPeerStream removed them from the queue. The
// pragmatic choice on a connection that uses this helper:
// log + ignore. If the application ALSO cares about peer
// bidi streams, it should NOT use this helper and instead
// implement its own routing dispatcher.
if (StreamId.kindOf(stream.streamId) != StreamId.Kind.SERVER_UNI) {
// Drain the bidi too — silently dropping bytes is bad
// policy, but tearing down the connection because the
// server opened an unexpected bidi is worse. The uni
// case is the documented one.
launch { drainStreamSilently(stream) }
continue
}
launch { drainStreamSilently(stream) }
}
}
private suspend fun drainStreamSilently(stream: com.vitorpamplona.quic.stream.QuicStream) {
@Suppress("UNUSED_VARIABLE")
stream.incoming.collect { _ ->
// intentionally discarded; this stream is one the caller has
// declared it does not care about (typically the server's H3
// CONTROL / QPACK_ENCODER / QPACK_DECODER streams).
}
}
@@ -24,6 +24,8 @@ import com.vitorpamplona.quic.crypto.AesEcbHeaderProtection
import com.vitorpamplona.quic.crypto.InitialSecrets
import com.vitorpamplona.quic.crypto.PlatformAesOneBlock
import com.vitorpamplona.quic.crypto.bestAes128GcmAead
import com.vitorpamplona.quic.observability.QlogObserver
import com.vitorpamplona.quic.packet.QuicVersion
import com.vitorpamplona.quic.stream.QuicStream
import com.vitorpamplona.quic.stream.StreamId
import com.vitorpamplona.quic.tls.TlsClient
@@ -73,24 +75,119 @@ class QuicConnection(
.toEpochMilliseconds()
},
val alpnList: List<ByteArray> = listOf(TlsConstants.ALPN_H3),
/**
* Optional second listener invoked after the connection's own
* key-installation listener. Used by the interop runner endpoint to
* dump SSLKEYLOG lines so Wireshark can decrypt captured pcaps.
* Default `null` keeps production callers unaffected.
*/
val extraSecretsListener: TlsSecretsListener? = null,
/**
* TLS cipher suites to offer in the ClientHello. Override to e.g.
* `intArrayOf(TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256)` for the
* `chacha20` interop testcase. Default matches [TlsClient]'s default.
*/
val cipherSuites: IntArray =
intArrayOf(
TlsConstants.CIPHER_TLS_AES_128_GCM_SHA256,
TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256,
),
/**
* Version this connection puts in the FIRST Initial it sends. Defaults
* to [QuicVersion.V1]; the interop runner sets it to
* [QuicVersion.FORCE_VERSION_NEGOTIATION] for the `versionnegotiation`
* testcase, which drives the client through the RFC 9000 §6 VN flow.
*/
val initialVersion: Int = QuicVersion.V1,
/**
* Optional qlog observer (draft-marx-qlog). Production callers
* leave this at [QlogObserver.NoOp] (zero overhead). Interop /
* test runners attach a JSON-NDJSON writer so a failed run
* produces a `client.sqlog` consumable by qvis.
*/
val qlogObserver: QlogObserver = QlogObserver.NoOp,
) {
val sourceConnectionId: ConnectionId = ConnectionId.random(8)
var destinationConnectionId: ConnectionId = ConnectionId.random(8)
internal set
val originalDestinationConnectionId: ConnectionId = destinationConnectionId
/**
* Version the writer stamps into the long-header version field on the
* NEXT outbound Initial / Handshake packet. Initialised to
* [initialVersion]; switched to [QuicVersion.V1] by
* [applyVersionNegotiation] after a successful VN exchange.
*/
@Volatile
var currentVersion: Int = initialVersion
internal set
/**
* RFC 9000 §6.2: a client MUST consume at most one VN response per
* connection. After [applyVersionNegotiation] runs once, any further
* inbound VN packet is dropped silently — the latch defends against a
* mid-handshake attacker who replays an old VN datagram to wedge us
* into an endless re-negotiation loop.
*/
@Volatile
var vnConsumed: Boolean = false
internal set
/**
* RFC 9000 §17.2.5.1: the Retry token the server handed us in a Retry
* packet, which we must echo verbatim in the Token field of every
* subsequent Initial we send. Null until [applyRetry] runs.
*/
@Volatile
var retryToken: ByteArray? = null
internal set
/**
* RFC 9000 §17.2.5.2: a client MUST NOT process more than one Retry
* packet per connection. Any subsequent Retry is silently dropped.
* Latched true by [applyRetry] on a successfully-verified Retry.
*/
@Volatile
var retryConsumed: Boolean = false
internal set
/**
* Cached ClientHello bytes captured by [start]. Re-enqueued onto the
* fresh Initial-level [LevelState.cryptoSend] when
* [applyVersionNegotiation] or [applyRetry] resets the encryption
* level so the new Initial datagram still carries a valid TLS handshake.
* Without this the reset wipes the bytes that [TlsClient] already
* enqueued and the post-VN/post-Retry Initial would carry an empty
* CRYPTO frame.
*/
private var originalClientHello: ByteArray? = null
val initial = LevelState()
val handshake = LevelState()
val application = LevelState()
@Volatile
var handshakeComplete: Boolean = false
private set
/**
* Lock-split refactor (2026-05-08): @Volatile because the writer/parser
* read this without acquiring [lifecycleLock] (the field is written
* once at handshake completion, then immutable).
*/
@Volatile
var peerTransportParameters: TransportParameters? = null
private set
enum class Status { HANDSHAKING, CONNECTED, CLOSING, CLOSED }
/**
* Lock-split refactor (2026-05-08): @Volatile so concurrent loops can
* read the status without a lock — coarse "are we still alive?" checks.
* Mutating transitions still go through [lifecycleLock] for atomicity
* with [closeReason]/[closeErrorCode] updates.
*/
@Volatile
var status: Status = Status.HANDSHAKING
internal set
@@ -234,7 +331,11 @@ class QuicConnection(
* emits a PING frame on the next drain. The PING elicits an
* ACK from the peer; that ACK runs through loss detection and
* declares any in-flight packets lost, triggering retransmit.
*
* Lock-split refactor (2026-05-08): @Volatile so the driver
* sets it without acquiring any mutex.
*/
@Volatile
internal var pendingPing: Boolean = false
/**
@@ -317,6 +418,8 @@ class QuicConnection(
) {
handshake.sendProtection = packetProtectionFromSecret(cipherSuite, clientSecret)
handshake.receiveProtection = packetProtectionFromSecret(cipherSuite, serverSecret)
qlogObserver.onKeyUpdated("client", EncryptionLevel.HANDSHAKE)
qlogObserver.onKeyUpdated("server", EncryptionLevel.HANDSHAKE)
}
override fun onApplicationKeysReady(
@@ -326,13 +429,17 @@ class QuicConnection(
) {
application.sendProtection = packetProtectionFromSecret(cipherSuite, clientSecret)
application.receiveProtection = packetProtectionFromSecret(cipherSuite, serverSecret)
qlogObserver.onKeyUpdated("client", EncryptionLevel.APPLICATION)
qlogObserver.onKeyUpdated("server", EncryptionLevel.APPLICATION)
}
override fun onHandshakeComplete() {
handshakeComplete = true
if (status == Status.HANDSHAKING) status = Status.CONNECTED
applyPeerTransportParameters()
tls.negotiatedAlpn?.let { qlogObserver.onAlpnNegotiated(it.decodeToString()) }
handshakeDoneSignal.complete(Unit)
extraSecretsListener?.onHandshakeComplete()
}
}
@@ -358,6 +465,7 @@ class QuicConnection(
secretsListener = tlsListener,
certificateValidator = tlsCertificateValidator,
offeredAlpns = alpnList,
cipherSuites = cipherSuites,
)
init {
@@ -375,9 +483,165 @@ class QuicConnection(
/** Begin the handshake — emits ClientHello into Initial CRYPTO. */
fun start() {
// qlog: emit connection_started + initial transport_parameters_set
// before any wire traffic so the trace makes chronological sense
// when handed to qvis.
qlogObserver.onConnectionStarted(
serverName = serverName,
dcid = destinationConnectionId.bytes,
scid = sourceConnectionId.bytes,
)
qlogObserver.onTransportParametersSet("local", localTransportParametersSummary())
// RFC 9000 §6: we're not doing version negotiation, so the
// chosen version is unconditional.
qlogObserver.onVersionInformation("v1", emptyList())
tls.start()
// Drain ClientHello bytes into the Initial-level CRYPTO send buffer.
tls.pollOutbound(TlsClient.Level.INITIAL)?.let { initial.cryptoSend.enqueue(it) }
// Cache the bytes so [applyVersionNegotiation] can re-enqueue them
// onto a fresh cryptoSend after resetting Initial-level state.
// Cannot re-pollOutbound — the queue is destructive.
tls.pollOutbound(TlsClient.Level.INITIAL)?.let {
originalClientHello = it
initial.cryptoSend.enqueue(it)
}
}
/**
* Apply a Version Negotiation packet (RFC 9000 §6) received from the
* server. The client offered [initialVersion]; the server replies with
* a list of versions it supports. We pick [QuicVersion.V1] from the
* list, regenerate the destination CID + Initial keys, reset the
* Initial encryption level, and re-emit the cached ClientHello so the
* next drain produces a valid v1 Initial packet.
*
* RFC 9000 §6.2 invariants enforced here:
* - The supported_versions list MUST NOT contain
* [initialVersion] — including it would mean the server received
* our offer and STILL replied with VN, which is a downgrade signal.
* Drop the packet (treat as no-op).
* - At most one VN per connection (latched via [vnConsumed]).
* - If we cannot speak any of the offered versions, fail the
* handshake.
*
* Caller MUST hold [lock] (the parser already does).
*/
internal fun applyVersionNegotiation(supportedVersions: List<Int>) {
// RFC 9000 §6.2: a second VN must be ignored.
if (vnConsumed) return
// Anti-downgrade: server-claimed support for the version we
// already offered indicates VN replay / spoof. Drop silently.
if (supportedVersions.contains(initialVersion)) return
// Pick a version we can speak. Today that's only v1.
if (!supportedVersions.contains(QuicVersion.V1)) {
signalHandshakeFailed(
QuicVersionNegotiationException(
"VERSION_NEGOTIATION: server offered ${supportedVersions.map { v -> "0x" + v.toUInt().toString(16) }}, " +
"client only supports 0x" + QuicVersion.V1.toUInt().toString(16),
),
)
markClosedExternally("VERSION_NEGOTIATION: no mutually supported version")
return
}
// Latch BEFORE reset so a re-entrant inbound VN during the reset
// window is rejected by the early-return at the top.
vnConsumed = true
// Generate a fresh destination CID. RFC 9000 §6.2 doesn't strictly
// require this (the server hasn't indexed our CID with any state
// since its only response was VN), but it matches what reference
// implementations do and keeps the post-VN connection
// cryptographically isolated from the pre-VN exchange.
val newDcid = ConnectionId.random(8)
destinationConnectionId = newDcid
// Reset Initial-level state in place: fresh PN space (next
// allocateOutbound returns 0), fresh ackTracker, fresh
// cryptoSend / cryptoReceive, fresh sentPackets retention.
// Writer + parser only ever reach the level via [conn.initial],
// so we mutate the fields rather than swap the instance.
val proto = InitialSecrets.derive(newDcid.bytes)
val hp = AesEcbHeaderProtection(PlatformAesOneBlock)
val newSend =
PacketProtection(bestAes128GcmAead(proto.clientKey), proto.clientKey, proto.clientIv, hp, proto.clientHp)
val newReceive =
PacketProtection(bestAes128GcmAead(proto.serverKey), proto.serverKey, proto.serverIv, hp, proto.serverHp)
initial.resetForVersionNegotiation(sendProtection = newSend, receiveProtection = newReceive)
// Re-enqueue the ClientHello so the next drainOutbound emits a v1
// Initial datagram with the same TLS handshake the original carried.
originalClientHello?.let { initial.cryptoSend.enqueue(it) }
// Switch the writer's stamp to v1 so the next Initial / Handshake
// long-header carries the right version.
currentVersion = QuicVersion.V1
}
/**
* Apply a verified Retry packet per RFC 9000 §17.2.5 + RFC 9001 §5.8.
* Validates the retry-integrity tag, swaps DCID, re-derives Initial keys,
* resets the Initial PN space, and re-enqueues the cached ClientHello so
* the next outbound Initial carries `Token = retryPacket.retryToken`.
*
* Returns false on bad tag, second Retry (RFC 9000 §17.2.5.2), or
* pre-start (no original ClientHello captured) — all silently dropped.
*/
internal fun applyRetry(
retryPacket: com.vitorpamplona.quic.packet.RetryPacket,
originalPacketBytes: ByteArray,
): Boolean {
if (retryConsumed) return false
if (!retryPacket.verifyIntegrityTag(originalPacketBytes, originalDestinationConnectionId.bytes)) {
return false
}
val savedClientHello = originalClientHello ?: return false
destinationConnectionId = retryPacket.scid
val proto = InitialSecrets.derive(destinationConnectionId.bytes)
val hp = AesEcbHeaderProtection(PlatformAesOneBlock)
// Use resetForRetry, NOT resetForVersionNegotiation: RFC 9001 §5.7
// requires the Initial PN namespace to continue across the Retry
// boundary. Resetting PN to 0 caused the runner to flag
// "Client reset the packet number. Check failed for PN 0".
initial.resetForRetry(
sendProtection =
PacketProtection(bestAes128GcmAead(proto.clientKey), proto.clientKey, proto.clientIv, hp, proto.clientHp),
receiveProtection =
PacketProtection(bestAes128GcmAead(proto.serverKey), proto.serverKey, proto.serverIv, hp, proto.serverHp),
)
initial.cryptoSend.enqueue(savedClientHello)
retryToken = retryPacket.retryToken
retryConsumed = true
return true
}
private fun localTransportParametersSummary(): Map<String, String> {
val out = LinkedHashMap<String, String>(8)
out["initial_max_data"] = config.initialMaxData.toString()
out["initial_max_stream_data_bidi_local"] = config.initialMaxStreamDataBidiLocal.toString()
out["initial_max_stream_data_bidi_remote"] = config.initialMaxStreamDataBidiRemote.toString()
out["initial_max_stream_data_uni"] = config.initialMaxStreamDataUni.toString()
out["initial_max_streams_bidi"] = config.initialMaxStreamsBidi.toString()
out["initial_max_streams_uni"] = config.initialMaxStreamsUni.toString()
out["max_idle_timeout"] = config.maxIdleTimeoutMillis.toString()
out["max_udp_payload_size"] = config.maxUdpPayloadSize.toString()
out["max_datagram_frame_size"] = config.maxDatagramFrameSize.toString()
return out
}
private fun peerTransportParametersSummary(tp: TransportParameters): Map<String, String> {
val out = LinkedHashMap<String, String>(10)
tp.initialMaxData?.let { out["initial_max_data"] = it.toString() }
tp.initialMaxStreamDataBidiLocal?.let { out["initial_max_stream_data_bidi_local"] = it.toString() }
tp.initialMaxStreamDataBidiRemote?.let { out["initial_max_stream_data_bidi_remote"] = it.toString() }
tp.initialMaxStreamDataUni?.let { out["initial_max_stream_data_uni"] = it.toString() }
tp.initialMaxStreamsBidi?.let { out["initial_max_streams_bidi"] = it.toString() }
tp.initialMaxStreamsUni?.let { out["initial_max_streams_uni"] = it.toString() }
tp.maxIdleTimeoutMillis?.let { out["max_idle_timeout"] = it.toString() }
tp.maxUdpPayloadSize?.let { out["max_udp_payload_size"] = it.toString() }
tp.maxDatagramFrameSize?.let { out["max_datagram_frame_size"] = it.toString() }
tp.maxAckDelay?.let { out["max_ack_delay"] = it.toString() }
return out
}
private fun buildLocalTransportParameters(): TransportParameters =
@@ -397,6 +661,13 @@ class QuicConnection(
maxDatagramFrameSize = config.maxDatagramFrameSize,
)
/**
* Lock-split refactor (2026-05-08): caller must hold [streamsLock]
* because we mutate [streams], [peerMaxStreamsBidi]/Uni, and
* [sendConnectionFlowCredit]. Invoked from the TLS listener inside
* [QuicConnectionParser.feedDatagram] which acquires [streamsLock]
* around CRYPTO-frame handling.
*/
private fun applyPeerTransportParameters() {
val raw = tls.peerTransportParameters ?: return
val tp = TransportParameters.decode(raw)
@@ -425,6 +696,7 @@ class QuicConnection(
return
}
peerTransportParameters = tp
qlogObserver.onTransportParametersSet("remote", peerTransportParametersSummary(tp))
sendConnectionFlowCredit = tp.initialMaxData ?: 0L
peerMaxStreamsBidi = tp.initialMaxStreamsBidi ?: 0L
peerMaxStreamsUni = tp.initialMaxStreamsUni ?: 0L
@@ -444,13 +716,41 @@ class QuicConnection(
}
/**
* Single mutex protecting connection-wide mutable state: streams map,
* datagram queues, stream-id counters, status. The driver acquires this
* around its read/send loops; public API methods listed below acquire it
* before mutating. Internal-only methods (used only from inside the
* driver loops) do NOT lock — caller must hold the lock.
* Lock-split refactor (2026-05-08): split the previous single
* `lock` into two independent mutexes so the read loop, send
* loop, and app coroutines can mostly progress in parallel.
*
* - [streamsLock] guards the streams registry, datagram queues,
* stream-id counters, connection-level flow-control bookkeeping,
* packet-number space + sentPackets retention + CRYPTO buffer
* mutations at every encryption level. The writer's drain and
* the parser's feed both take it.
* - [lifecycleLock] guards [status] / [closeReason] /
* [closeErrorCode] transitions.
*
* Per-stream and per-level buffer mutations serialize through
* `synchronized(this)` inside `SendBuffer` / `ReceiveBuffer` /
* `AckTracker` — those leaf locks are safe to take with or
* without an outer mutex held.
*
* Acquisition order to prevent deadlock:
* `lifecycleLock` → `streamsLock`. Never go the other way.
*
* The historical `lock` field is retained as an alias of
* [lifecycleLock] for source-compatibility with external callers
* (tests, harnesses, in-process bridges). New code MUST NOT use it
* — it no longer protects streams or level state.
*/
val lock: Mutex = Mutex()
val streamsLock: Mutex = Mutex()
val lifecycleLock: Mutex = Mutex()
@Deprecated(
"Use streamsLock or lifecycleLock as appropriate. Lock-split refactor 2026-05-08.",
replaceWith = ReplaceWith("streamsLock"),
)
val lock: Mutex
get() = lifecycleLock
/**
* Allocate a new client-initiated bidirectional stream. Locked.
@@ -460,22 +760,97 @@ class QuicConnection(
* check capacity proactively if the caller wants to back-pressure rather
* than throw.
*/
suspend fun openBidiStream(): QuicStream =
lock.withLock {
if (nextLocalBidiIndex >= peerMaxStreamsBidi) {
throw QuicStreamLimitException(
"peer-granted bidi stream cap reached " +
"(used=$nextLocalBidiIndex limit=$peerMaxStreamsBidi)",
)
suspend fun openBidiStream(): QuicStream = streamsLock.withLock { openBidiStreamLocked() }
/**
* Atomically open one bidi stream per [items] entry under a single
* [streamsLock] hold and run [init] for each (stream, item) inside
* the lock. The send loop cannot interject between opens — when it
* next drains it sees ALL N streams' frames ready and packs them
* into coalesced packets instead of emitting one tiny packet per
* stream.
*
* **`init` runs under `streamsLock`.** It must not suspend
* (the type signature enforces this) and SHOULD be fast — any
* expensive work (encoding, allocation-heavy formatting) belongs
* outside the call so it doesn't extend the lock-hold time. The
* intended shape per caller:
*
* val encoded = items.map { encode(it) } // outside
* conn.openBidiStreamsBatch(encoded) { stream, payload -> // under lock
* stream.send.enqueue(payload)
* stream.send.finish()
* Handle(stream)
* }
*
* This is the bug-resistant API for the prepareRequests pattern.
* The previous shape (caller manually wraps `streamsLock.withLock`
* around a loop of [openBidiStreamLocked]) regressed twice: once
* by holding the wrong lock, and once by skipping the wrapper
* entirely. Both shapes failed silently as "one STREAM per packet"
* under multiplex load, while the unit tests passed.
*
* Callers that just need a single stream should still use
* [openBidiStream]. [openBidiStreamLocked] remains public for the
* rare custom-batching scenarios that need finer control, but
* those callers should generally migrate to this API.
*/
suspend fun <I, R> openBidiStreamsBatch(
items: List<I>,
init: (QuicStream, I) -> R,
): List<R> {
if (items.isEmpty()) return emptyList()
val streamsBefore = if (writerDebugEnabled) streams.size else 0
val result =
streamsLock.withLock {
items.map { init(openBidiStreamLocked(), it) }
}
val id = StreamId.build(StreamId.Kind.CLIENT_BIDI, nextLocalBidiIndex++)
val stream = QuicStream(id, QuicStream.Direction.BIDIRECTIONAL)
stream.sendCredit = peerTransportParameters?.initialMaxStreamDataBidiRemote ?: config.initialMaxStreamDataBidiRemote
stream.receiveLimit = config.initialMaxStreamDataBidiLocal
streams[id] = stream
streamsList += stream
stream
if (writerDebugEnabled) {
System.err.println(
"[batch] openBidiStreamsBatch items=${items.size} returned=${result.size} " +
"streamsList_before=$streamsBefore streamsList_after=${streams.size}",
)
}
return result
}
/**
* The streamsLock-holding primitive used by [openBidiStream] and
* [openBidiStreamsBatch]. Public so callers that need a custom
* batching shape (e.g. mixed bidi+uni opens) can compose it under
* a manual [streamsLock] hold. Caller MUST hold [streamsLock].
*/
fun openBidiStreamLocked(): QuicStream {
// Mutex.isLocked is the only check we have — kotlinx.coroutines
// Mutex doesn't expose ownership without an `owner` argument,
// and we don't pass one in production. So this catches the
// common bug — caller used the wrong lock or no lock — but
// not the rarer case of "caller held a DIFFERENT lock that
// happens to be locked too." The interop runner's multiplexing
// failure on 2026-05-06 was precisely this: prepareRequests
// held lifecycleLock (`conn.lock`) and called this fn, the
// send loop's drainOutbound interleaved between opens, and
// we emitted one STREAM per packet (1421/2000 files in 60s).
check(streamsLock.isLocked) {
"openBidiStreamLocked requires streamsLock to be held — caller " +
"must wrap with streamsLock.withLock { ... }. Without that, " +
"drainOutbound can race the streams mutation and emit one " +
"STREAM per packet under multiplex load."
}
if (nextLocalBidiIndex >= peerMaxStreamsBidi) {
throw QuicStreamLimitException(
"peer-granted bidi stream cap reached " +
"(used=$nextLocalBidiIndex limit=$peerMaxStreamsBidi)",
)
}
val id = StreamId.build(StreamId.Kind.CLIENT_BIDI, nextLocalBidiIndex++)
val stream = QuicStream(id, QuicStream.Direction.BIDIRECTIONAL)
stream.sendCredit = peerTransportParameters?.initialMaxStreamDataBidiRemote ?: config.initialMaxStreamDataBidiRemote
stream.receiveLimit = config.initialMaxStreamDataBidiLocal
streams[id] = stream
streamsList += stream
return stream
}
/**
* Allocate a new client-initiated unidirectional (write-only) stream.
@@ -486,22 +861,57 @@ class QuicConnection(
* [QuicStream.bestEffort]). Used for moq-lite group streams
* carrying real-time Opus audio.
*/
suspend fun openUniStream(bestEffort: Boolean = false): QuicStream =
lock.withLock {
if (nextLocalUniIndex >= peerMaxStreamsUni) {
throw QuicStreamLimitException(
"peer-granted uni stream cap reached " +
"(used=$nextLocalUniIndex limit=$peerMaxStreamsUni)",
)
}
val id = StreamId.build(StreamId.Kind.CLIENT_UNI, nextLocalUniIndex++)
val stream = QuicStream(id, QuicStream.Direction.UNIDIRECTIONAL_LOCAL_TO_REMOTE, bestEffort = bestEffort)
stream.sendCredit = peerTransportParameters?.initialMaxStreamDataUni ?: config.initialMaxStreamDataUni
stream.receiveLimit = 0L // can't receive
streams[id] = stream
streamsList += stream
stream
suspend fun openUniStream(bestEffort: Boolean = false): QuicStream = streamsLock.withLock { openUniStreamLocked(bestEffort) }
/**
* The streamsLock-holding primitive used by [openUniStream] and
* [openUniStreamsBatch]. Caller MUST hold [streamsLock].
*/
fun openUniStreamLocked(bestEffort: Boolean = false): QuicStream {
check(streamsLock.isLocked) {
"openUniStreamLocked requires streamsLock to be held"
}
if (nextLocalUniIndex >= peerMaxStreamsUni) {
throw QuicStreamLimitException(
"peer-granted uni stream cap reached " +
"(used=$nextLocalUniIndex limit=$peerMaxStreamsUni)",
)
}
val id = StreamId.build(StreamId.Kind.CLIENT_UNI, nextLocalUniIndex++)
val stream = QuicStream(id, QuicStream.Direction.UNIDIRECTIONAL_LOCAL_TO_REMOTE, bestEffort = bestEffort)
stream.sendCredit = peerTransportParameters?.initialMaxStreamDataUni ?: config.initialMaxStreamDataUni
stream.receiveLimit = 0L // can't receive
streams[id] = stream
streamsList += stream
return stream
}
/**
* Bug-resistant counterpart to [openBidiStreamsBatch] for uni
* streams. Atomically open one client-uni stream per [items]
* entry under a single [streamsLock] hold and run [init] for
* each (stream, item).
*
* **`init` runs under `streamsLock`** — same caveat as
* [openBidiStreamsBatch]: keep it fast, encode outside the call.
*
* Use this for moq audio-rooms and any other path that opens many
* uni streams in burst — without batching, each open releases the
* lock and the send loop can interject, emitting one stream per
* packet (the same shape that broke bidi multiplexing on
* 2026-05-06). [bestEffort] applies uniformly to every stream
* in the batch; mixed-mode batches need separate calls.
*/
suspend fun <I, R> openUniStreamsBatch(
items: List<I>,
bestEffort: Boolean = false,
init: (QuicStream, I) -> R,
): List<R> {
if (items.isEmpty()) return emptyList()
return streamsLock.withLock {
items.map { init(openUniStreamLocked(bestEffort), it) }
}
}
/** Snapshot of peer-granted bidi cap. Reads do not need the lock — long writes are atomic on every supported platform. */
fun peerMaxStreamsBidiSnapshot(): Long = peerMaxStreamsBidi
@@ -529,7 +939,7 @@ class QuicConnection(
* See `nestsClient/plans/2026-05-01-quic-stream-cliff-investigation.md`.
*/
suspend fun flowControlSnapshot(): QuicFlowControlSnapshot =
lock.withLock {
streamsLock.withLock {
val tp = peerTransportParameters
// Sum bytes the application has enqueued but the writer
// hasn't yet handed to a STREAM frame. A non-zero value
@@ -568,17 +978,32 @@ class QuicConnection(
)
}
suspend fun pollIncomingPeerStream(): QuicStream? = lock.withLock { newPeerStreams.removeFirstOrNull() }
suspend fun pollIncomingPeerStream(): QuicStream? = streamsLock.withLock { newPeerStreams.removeFirstOrNull() }
/**
* Suspends until a peer-initiated stream is queued OR the connection
* closes. Returns null on close. Replaces the older `pollIncomingPeerStream
* + delay(5)` busy-loop — this version wakes within microseconds of the
* parser appending a stream and parks the coroutine the rest of the time.
*
* **An H3 application MUST consume peer-initiated streams.** RFC 9114
* §6.2.1 mandates that the server opens at least three peer-initiated
* uni streams (CONTROL + QPACK_ENCODER + QPACK_DECODER). The parser
* routes their bytes into the per-[QuicStream] `incomingChannel`
* (capacity 64 chunks); if nothing accepts and reads them, the channel
* fills and the next inbound chunk trips the audit-4 #3 "slow consumer"
* tear-down at [QuicConnectionParser] (`INTERNAL_ERROR: stream …
* consumer overflowed`). Symptoms: under H3 multiplexing of many bidi
* request streams, the server's QPACK encoder issues a burst of
* dynamic-table inserts on its uni stream and the connection dies
* after ~5 s with zero requests completed. See
* [drainPeerInitiatedUniStreamsIntoBlackHole] for a one-line opt-in
* drainer that satisfies the contract when the H3 layer doesn't
* actually need the SETTINGS / QPACK bytes.
*/
suspend fun awaitIncomingPeerStream(): QuicStream? {
while (true) {
lock.withLock { newPeerStreams.removeFirstOrNull() }?.let { return it }
streamsLock.withLock { newPeerStreams.removeFirstOrNull() }?.let { return it }
if (status == Status.CLOSED) return null
// select between "wakeup" and "closed" so neither path can hang.
val keepWaiting =
@@ -593,17 +1018,17 @@ class QuicConnection(
if (!keepWaiting) {
// After a close-wake, drain one more time to surface any
// streams added between the last drain and the close.
lock.withLock { newPeerStreams.removeFirstOrNull() }?.let { return it }
streamsLock.withLock { newPeerStreams.removeFirstOrNull() }?.let { return it }
return null
}
}
}
suspend fun streamById(id: Long): QuicStream? = lock.withLock { streams[id] }
suspend fun streamById(id: Long): QuicStream? = streamsLock.withLock { streams[id] }
suspend fun queueDatagram(payload: ByteArray) = lock.withLock { pendingDatagrams.addLast(payload) }
suspend fun queueDatagram(payload: ByteArray) = streamsLock.withLock { pendingDatagrams.addLast(payload) }
suspend fun pollIncomingDatagram(): ByteArray? = lock.withLock { incomingDatagrams.removeFirstOrNull() }
suspend fun pollIncomingDatagram(): ByteArray? = streamsLock.withLock { incomingDatagrams.removeFirstOrNull() }
/**
* Suspending counterpart of [pollIncomingDatagram]. Returns null only when
@@ -612,7 +1037,7 @@ class QuicConnection(
*/
suspend fun awaitIncomingDatagram(): ByteArray? {
while (true) {
lock.withLock { incomingDatagrams.removeFirstOrNull() }?.let { return it }
streamsLock.withLock { incomingDatagrams.removeFirstOrNull() }?.let { return it }
if (status == Status.CLOSED) return null
val keepWaiting =
select<Boolean> {
@@ -620,7 +1045,7 @@ class QuicConnection(
closedSignal.onReceiveCatching { false }
}
if (!keepWaiting) {
lock.withLock { incomingDatagrams.removeFirstOrNull() }?.let { return it }
streamsLock.withLock { incomingDatagrams.removeFirstOrNull() }?.let { return it }
return null
}
}
@@ -631,12 +1056,15 @@ class QuicConnection(
errorCode: Long,
reason: String,
) {
lock.withLock {
var firedQlog = false
lifecycleLock.withLock {
if (status == Status.CLOSED || status == Status.CLOSING) return@withLock
closeErrorCode = errorCode
closeReason = reason
status = Status.CLOSING
firedQlog = true
}
if (firedQlog) qlogObserver.onConnectionClosed("local", errorCode, reason)
// If a caller is suspended on awaitHandshake() and we're tearing down
// before completion, fail the deferred so the caller throws instead
// of hanging forever.
@@ -648,7 +1076,16 @@ class QuicConnection(
/** Called by the parser on inbound CONNECTION_CLOSE or by the driver on read-loop death. */
internal fun markClosedExternally(reason: String) {
val wasClosed = status == Status.CLOSED
if (status != Status.CLOSED) status = Status.CLOSED
if (!wasClosed) {
// "remote" covers both peer-initiated CONNECTION_CLOSE and
// local invariant violations (CID mismatch, frame decode
// failure) that the parser surfaces as markClosedExternally.
// The reason string is the discriminator the trace consumer
// reads.
qlogObserver.onConnectionClosed("remote", closeErrorCode, reason)
}
if (!handshakeComplete) {
signalHandshakeFailed(QuicConnectionClosedException("connection closed externally: $reason"))
}
@@ -662,16 +1099,34 @@ class QuicConnection(
* still `trySend(Unit)` into a never-consumed channel. All three channels
* close idempotently, so calling this from both `close()` and
* `markClosedExternally` is safe.
*
* Also closes every per-stream `incomingChannel` so application
* coroutines suspended on `stream.incoming.collect { … }` unblock with
* a clean Flow termination instead of hanging forever waiting for a
* FIN that will never come. Without this an interop run that drops
* the connection mid-response (e.g. quic-interop-runner's
* `multiplexing` case where 677 collectors were waiting for replies
* when the parser tripped INTERNAL_ERROR) leaves every per-stream
* collector pinned indefinitely. Closing the channel after the
* channel already has buffered chunks is safe — `consumeAsFlow`
* drains the buffer before terminating, so any bytes already
* delivered are surfaced to the collector before the Flow completes.
*/
private fun closeAllSignals() {
closedSignal.close()
peerStreamSignal.close()
incomingDatagramSignal.close()
// Iterate the snapshot list (safe: we never remove from it).
// closeIncoming is idempotent on the underlying Channel.close().
for (stream in streamsList) {
stream.closeIncoming()
}
}
/**
* Caller must hold [lock]. Used by [QuicConnectionParser] inside the
* driver's read loop, which already holds the connection lock.
* Caller must hold [streamsLock]. Used by [QuicConnectionParser] inside
* the driver's read loop, which already holds [streamsLock] around the
* stream-domain section of frame dispatch.
*/
internal fun getOrCreatePeerStreamLocked(id: Long): QuicStream {
streams[id]?.let { return it }
@@ -741,6 +1196,38 @@ class QuicConnection(
EncryptionLevel.APPLICATION -> application
}
/**
* RFC 9002 §6.2.4 PTO probe — spec-correct retransmit path. Move
* every byte currently sent-but-not-yet-ACK'd in the [level]'s
* CRYPTO send buffer back to its retransmit queue, so the next
* [com.vitorpamplona.quic.connection.drainOutbound] re-emits the
* same bytes (at the same offsets) inside a fresh CRYPTO frame on
* a new packet number.
*
* The driver calls this from its PTO branch when 1-RTT keys
* aren't yet installed — i.e. the handshake hasn't finished, so
* the only thing the peer could be missing is our ClientHello /
* ClientFinished. A bare PING is insufficient because if the
* server never saw our original Initial it has no DCID state to
* correlate a PING against (it'll be dropped). Retransmitting the
* CRYPTO actually advances the handshake.
*
* Idempotent: a second consecutive call is a no-op because the
* first call moved everything out of inFlight. Old `RecoveryToken.Crypto`
* entries in [LevelState.sentPackets] for the still-tracked
* original PNs remain harmless — when loss detection eventually
* declares them lost, [onTokensLost] re-runs `markLost` on the
* same offset/length range, which is itself idempotent (the bytes
* are already in retransmit or already ACK'd by then).
*
* Caller must hold [lock] (or call from inside an existing locked
* region — typically the driver's PTO branch under
* [QuicConnectionDriver.sendLoop]).
*/
internal fun requeueAllInflightCrypto(level: EncryptionLevel) {
levelState(level).cryptoSend.requeueAllInflight()
}
/** Caller must hold [lock]. Snapshot of streams for the driver's send loop. */
internal fun streamsLocked(): Map<Long, QuicStream> = streams
@@ -954,6 +1441,17 @@ class QuicStreamLimitException(
message: String,
) : RuntimeException(message)
/**
* RFC 9000 §6: the server replied with a Version Negotiation packet but
* the supported_versions list does not contain any version the client can
* speak (today: only [com.vitorpamplona.quic.packet.QuicVersion.V1]).
* The handshake is unrecoverable — caller must treat the connection as
* permanently failed.
*/
class QuicVersionNegotiationException(
message: String,
) : RuntimeException(message)
/**
* Diagnostic snapshot of [QuicConnection]'s flow-control accounting at
* a single moment. Returned by [QuicConnection.flowControlSnapshot].
@@ -35,10 +35,13 @@ import kotlinx.coroutines.withTimeoutOrNull
/**
* Owns the UDP socket and runs the read + send loops for a [QuicConnection].
*
* Synchronization: every public mutator on [QuicConnection] takes
* `connection.lock`; the driver acquires the same lock around feed + drain.
* That guarantees the read loop, send loop, and app coroutines never see a
* mid-mutation state of the streams map / datagram queues / counters.
* Synchronization (post lock-split refactor 2026-05-08): the driver no
* longer takes a single connection-wide lock around feed/drain. Instead
* [feedDatagram] and [drainOutbound] internally acquire `streamsLock`
* for the precise critical sections they touch — leaving app
* coroutines (`openBidiStream`, etc.) free to run in parallel with the
* I/O loops. Per-stream and per-level buffers serialize through their
* leaf `synchronized(this)` blocks.
*
* The send loop is woken by a `Channel<Unit>(CONFLATED)` rather than a
* polling timer — no idle CPU. App writes ([QuicConnection.queueDatagram]
@@ -99,7 +102,9 @@ class QuicConnectionDriver(
try {
while (connection.status != QuicConnection.Status.CLOSED) {
val datagram = socket.receive() ?: break
connection.lock.withLock {
// Phase 1 of the lock-split refactor: parser holds
// streamsLock for a single datagram-feed pass.
connection.streamsLock.withLock {
feedDatagram(connection, datagram, nowMillis())
}
// Inbound data may have produced new outbound (acks, crypto, etc.).
@@ -123,11 +128,16 @@ class QuicConnectionDriver(
// floor (the same prior-shipping behavior, kept for
// handshake-timeout safety on lossy paths).
while (connection.status != QuicConnection.Status.CLOSED) {
connection.lock.withLock {
while (true) {
val out = drainOutbound(connection, nowMillis()) ?: break
socket.send(out)
}
// Phase 1 of the lock-split refactor: the writer holds
// streamsLock for the build, releases it for the actual
// socket.send() so a slow socket doesn't stall app
// coroutines (open/close streams, queue datagrams).
while (true) {
val out =
connection.streamsLock.withLock {
drainOutbound(connection, nowMillis())
} ?: break
socket.send(out)
}
val ptoBaseMs =
if (connection.lossDetection.hasFirstRttSample) {
@@ -145,15 +155,7 @@ class QuicConnectionDriver(
Unit
}
if (woke == null) {
// PTO fired. Set pendingPing so the writer emits a
// PING on the next drain (RFC 9002 §6.2.4 probe
// packet). The peer's ACK feeds loss detection +
// retransmit (steps 5–6).
connection.lock.withLock {
connection.pendingPing = true
connection.consecutivePtoCount =
(connection.consecutivePtoCount + 1).coerceAtMost(6)
}
handlePtoFired(connection)
}
}
}
@@ -223,3 +225,67 @@ class QuicConnectionDriver(
private const val CLOSE_FLUSH_TIMEOUT_MILLIS = 250L
}
}
/**
* Spec-correct response to a PTO timer firing (RFC 9002 §6.2.4). Pre-1-RTT
* the probe packet MUST be ack-eliciting at the encryption level with
* unacknowledged data, and SHOULD retransmit the lost data rather than
* emit a bare PING — so we requeue ALL inflight CRYPTO bytes at the
* highest active pre-application level (Initial or Handshake), and the
* next [drainOutbound] emits a CRYPTO frame at the original offset.
*
* `pendingPing` stays set as a fallback. `collectHandshakeLevelFrames`
* suppresses the PING when CRYPTO is in the same frame list, so we
* don't waste a frame on top of the retransmit. Post-1-RTT we keep
* the bare-PING behavior — STREAM loss detection drives retransmit
* from the ACK that the PING elicits.
*
* Why aioquic interop demands this: aioquic strictly rejects pre-
* handshake Initials that contain no CRYPTO frame
* (`CONNECTION_CLOSE 0x0 "Packet contains no CRYPTO frame"`). A
* bare-PING probe before the ClientHello is acknowledged is fatal.
*
* Extracted from [QuicConnectionDriver.sendLoop]'s PTO branch into a
* top-level helper so the unit test in
* [com.vitorpamplona.quic.connection.PtoCryptoRetransmitTest]
* can invoke the EXACT logic the live driver does, without standing
* up a UDP socket. Earlier shapes simulated the steps inline in the
* test, which let the driver-side wiring regress twice
* (commits c0d7b6031, then again in the lock-split refactor) without
* any test breaking.
*
* Concurrency: `pendingPing` and `consecutivePtoCount` are `@Volatile`.
* [QuicConnection.requeueAllInflightCrypto] delegates to
* [com.vitorpamplona.quic.stream.SendBuffer.requeueAllInflight] which
* is `synchronized(this)` internally, so it's safe to call without
* an external lock — even concurrent with the writer's `takeChunk`.
* If the parser concurrently runs `discardKeys` on the same level,
* `requeueAllInflight` operates on the buffer reference we captured
* (or the fresh one — both are valid) and is at worst a no-op.
*/
internal fun handlePtoFired(conn: QuicConnection) {
conn.pendingPing = true
if (conn.application.sendProtection == null) {
val level = highestPreApplicationLevel(conn)
if (level != null) {
conn.requeueAllInflightCrypto(level)
}
}
conn.consecutivePtoCount = (conn.consecutivePtoCount + 1).coerceAtMost(6)
}
/**
* Highest encryption level for which `conn` currently holds send keys
* AND hasn't yet discarded them, given that 1-RTT keys are NOT
* installed. Returns null when the level state has been completely
* cleared (e.g. CLOSED after a CONNECTION_CLOSE was sent). Mirrors the
* private helper in [com.vitorpamplona.quic.connection.QuicConnectionWriter]
* — kept in lockstep so the driver's PTO branch and the writer's PING
* placement target the same level.
*/
private fun highestPreApplicationLevel(conn: QuicConnection): EncryptionLevel? =
when {
conn.handshake.sendProtection != null -> EncryptionLevel.HANDSHAKE
conn.initial.sendProtection != null && !conn.initial.keysDiscarded -> EncryptionLevel.INITIAL
else -> null
}
@@ -37,8 +37,11 @@ import com.vitorpamplona.quic.frame.ResetStreamFrame
import com.vitorpamplona.quic.frame.StopSendingFrame
import com.vitorpamplona.quic.frame.StreamFrame
import com.vitorpamplona.quic.frame.decodeFrames
import com.vitorpamplona.quic.observability.qlogFrameName
import com.vitorpamplona.quic.packet.LongHeaderPacket
import com.vitorpamplona.quic.packet.LongHeaderType
import com.vitorpamplona.quic.packet.QuicVersion
import com.vitorpamplona.quic.packet.RetryPacket
import com.vitorpamplona.quic.packet.ShortHeaderPacket
import com.vitorpamplona.quic.stream.StreamId
import com.vitorpamplona.quic.tls.TlsClient
@@ -51,6 +54,15 @@ import com.vitorpamplona.quic.tls.TlsClient
* — typically Initial + Handshake from the server in the same datagram during
* the handshake. We loop until the datagram is fully consumed or a packet
* fails to parse (which we drop silently per RFC 9001 §5.5).
*
* Lock-split refactor (2026-05-08): caller must hold
* [QuicConnection.streamsLock]. The driver wraps its read loop in
* `streamsLock.withLock { feedDatagram(...) }`. Test harnesses that drive
* single-threaded packet flow (no concurrent app code) may invoke this
* directly without lock acquisition; the runtime invariants still hold
* because there's no contending thread. Phase 1 wraps the whole feed
* under streamsLock so frame-dispatch / stream creation / level state
* remains a single critical section.
*/
fun feedDatagram(
conn: QuicConnection,
@@ -62,6 +74,23 @@ fun feedDatagram(
val first = datagram[offset].toInt() and 0xFF
val isLong = (first and 0x80) != 0
if (isLong) {
// RFC 9000 §17.2.1: a Version Negotiation packet has the form
// bit set but version=0. Detect it BEFORE peekHeader, which
// assumes a v1-shaped layout (token, length fields).
if (offset + 5 <= datagram.size) {
val version =
((datagram[offset + 1].toInt() and 0xFF) shl 24) or
((datagram[offset + 2].toInt() and 0xFF) shl 16) or
((datagram[offset + 3].toInt() and 0xFF) shl 8) or
(datagram[offset + 4].toInt() and 0xFF)
if (version == QuicVersion.VERSION_NEGOTIATION) {
feedVersionNegotiationPacket(conn, datagram, offset)
// VN packets MUST be the only packet in their datagram
// (RFC 9000 §17.2.1: no length field, body is rest of
// datagram). Stop walking.
return
}
}
// Per RFC 9001 §5.5, drop ONLY the failing packet, not subsequent
// coalesced ones. Use peekHeader to advance over a packet whose
// payload we couldn't decrypt; only break the loop on a header
@@ -78,6 +107,66 @@ fun feedDatagram(
}
}
/**
* RFC 9000 §17.2.1 / §6: parse a Version Negotiation packet and dispatch
* to [QuicConnection.applyVersionNegotiation].
*
* Wire layout:
*
* first byte (form=1, unused 4 bits — server fills with random)
* version (4 bytes, fixed at 0x00000000)
* dcid_len (1 byte) + dcid (dcid_len bytes)
* scid_len (1 byte) + scid (scid_len bytes)
* supported_versions: sequence of 32-bit big-endian version numbers,
* consuming the rest of the UDP datagram.
*
* VN packets are NOT AEAD-protected — there's nothing to decrypt. We do
* a minimal sanity check (DCID matches our SCID) and then hand the
* version list off. Malformed packets are dropped silently per RFC 9000
* §17.2.1 ("an endpoint MUST NOT send … in response to a Version
* Negotiation packet").
*/
private fun feedVersionNegotiationPacket(
conn: QuicConnection,
datagram: ByteArray,
offset: Int,
) {
// Layout fields above; bail early if any read would run past the
// end of the datagram (truncated VN — drop silently).
var pos = offset + 5
if (pos >= datagram.size) return
val dcidLen = datagram[pos].toInt() and 0xFF
pos += 1
if (dcidLen > 20 || pos + dcidLen > datagram.size) return
val dcid = datagram.copyOfRange(pos, pos + dcidLen)
pos += dcidLen
if (pos >= datagram.size) return
val scidLen = datagram[pos].toInt() and 0xFF
pos += 1
if (scidLen > 20 || pos + scidLen > datagram.size) return
pos += scidLen // SCID body — not validated; servers may pick anything.
// RFC 9000 §6.1: the VN packet's destination CID MUST equal the SCID
// the client put in its first Initial. Mismatch ⇒ probable spoof
// from an off-path attacker; drop without state change.
if (!dcid.contentEquals(conn.sourceConnectionId.bytes)) return
val versionsRegion = datagram.size - pos
if (versionsRegion <= 0 || versionsRegion % 4 != 0) return // malformed
val supportedVersions = ArrayList<Int>(versionsRegion / 4)
while (pos + 4 <= datagram.size) {
val v =
((datagram[pos].toInt() and 0xFF) shl 24) or
((datagram[pos + 1].toInt() and 0xFF) shl 16) or
((datagram[pos + 2].toInt() and 0xFF) shl 8) or
(datagram[pos + 3].toInt() and 0xFF)
supportedVersions += v
pos += 4
}
conn.applyVersionNegotiation(supportedVersions)
}
private fun feedLongHeaderPacket(
conn: QuicConnection,
datagram: ByteArray,
@@ -85,14 +174,49 @@ private fun feedLongHeaderPacket(
nowMillis: Long,
): Int? {
val peeked = LongHeaderPacket.peekHeader(datagram, offset) ?: return null
// RFC 9000 §17.2.5 + RFC 9001 §5.8: Retry has no PN space, no AEAD
// payload protection, and cannot be coalesced with anything else
// (it consumes the rest of the datagram via peekHeader.totalLength).
// Branch out before the standard parse-and-decrypt path.
if (peeked.type == LongHeaderType.RETRY) {
val retryBytes = datagram.copyOfRange(offset, offset + peeked.totalLength)
val retryPacket = RetryPacket.parse(retryBytes)
if (retryPacket != null) {
// applyRetry returns false on bad-tag / second-Retry / pre-start —
// in all of those cases we silently drop without advancing state.
conn.applyRetry(retryPacket, retryBytes)
}
return peeked.totalLength
}
val level =
when (peeked.type) {
LongHeaderType.INITIAL -> EncryptionLevel.INITIAL
LongHeaderType.HANDSHAKE -> EncryptionLevel.HANDSHAKE
LongHeaderType.ZERO_RTT, LongHeaderType.RETRY -> return null // unsupported in client
LongHeaderType.INITIAL -> {
EncryptionLevel.INITIAL
}
LongHeaderType.HANDSHAKE -> {
EncryptionLevel.HANDSHAKE
}
LongHeaderType.ZERO_RTT, LongHeaderType.RETRY -> {
// Not supported by client; surface as a drop so qvis can
// see we ignored a packet rather than silently moving on.
conn.qlogObserver.onPacketDropped(
"unsupported long-header type ${peeked.type}",
peeked.totalLength,
)
return null
}
}
val state = conn.levelState(level)
val proto = state.receiveProtection ?: return null
val proto = state.receiveProtection
if (proto == null) {
conn.qlogObserver.onPacketDropped(
"no receive keys at level $level",
peeked.totalLength,
)
return null
}
val parsed =
LongHeaderPacket.parseAndDecrypt(
bytes = datagram,
@@ -103,7 +227,14 @@ private fun feedLongHeaderPacket(
hp = proto.hp,
hpKey = proto.hpKey,
largestReceivedInSpace = state.pnSpace.largestReceived,
) ?: return null
)
if (parsed == null) {
conn.qlogObserver.onPacketDropped(
"AEAD auth failed or header parse failed at level $level",
peeked.totalLength,
)
return null
}
state.pnSpace.observeInbound(parsed.packet.packetNumber, nowMillis)
@@ -112,6 +243,14 @@ private fun feedLongHeaderPacket(
conn.destinationConnectionId = parsed.packet.scid
}
if (conn.qlogObserver !== com.vitorpamplona.quic.observability.QlogObserver.NoOp) {
conn.qlogObserver.onPacketReceived(
level = level,
packetNumber = parsed.packet.packetNumber,
sizeBytes = parsed.consumed,
frames = peekFrameNames(parsed.packet.payload),
)
}
dispatchFrames(conn, level, parsed.packet.payload, parsed.packet.packetNumber, nowMillis)
return parsed.consumed
}
@@ -123,7 +262,14 @@ private fun feedShortHeaderPacket(
nowMillis: Long,
) {
val state = conn.levelState(EncryptionLevel.APPLICATION)
val proto = state.receiveProtection ?: return
val proto = state.receiveProtection
if (proto == null) {
conn.qlogObserver.onPacketDropped(
"no application receive keys",
datagram.size - offset,
)
return
}
val parsed =
ShortHeaderPacket.parseAndDecrypt(
bytes = datagram,
@@ -135,11 +281,42 @@ private fun feedShortHeaderPacket(
hp = proto.hp,
hpKey = proto.hpKey,
largestReceivedInSpace = state.pnSpace.largestReceived,
) ?: return
)
if (parsed == null) {
conn.qlogObserver.onPacketDropped(
"AEAD auth failed or header parse failed at level APPLICATION",
datagram.size - offset,
)
return
}
state.pnSpace.observeInbound(parsed.packet.packetNumber, nowMillis)
if (conn.qlogObserver !== com.vitorpamplona.quic.observability.QlogObserver.NoOp) {
conn.qlogObserver.onPacketReceived(
level = EncryptionLevel.APPLICATION,
packetNumber = parsed.packet.packetNumber,
sizeBytes = datagram.size - offset,
frames = peekFrameNames(parsed.packet.payload),
)
}
dispatchFrames(conn, EncryptionLevel.APPLICATION, parsed.packet.payload, parsed.packet.packetNumber, nowMillis)
}
/**
* Decode the payload's frames just to surface their qlog names. Reuses
* the same [com.vitorpamplona.quic.frame.decodeFrames] path as
* [dispatchFrames]; if it throws (malformed peer payload), we return
* an empty list — the dispatch path will catch the same exception
* and surface the close via `markClosedExternally`.
*/
private fun peekFrameNames(payload: ByteArray): List<String> =
try {
com.vitorpamplona.quic.frame
.decodeFrames(payload)
.map { qlogFrameName(it::class.simpleName ?: "frame") }
} catch (_: QuicCodecException) {
emptyList()
}
private fun dispatchFrames(
conn: QuicConnection,
level: EncryptionLevel,
@@ -225,6 +402,9 @@ private fun dispatchFrames(
for (lostPacket in lost) {
conn.onTokensLost(lostPacket.tokens)
}
if (lost.isNotEmpty()) {
conn.qlogObserver.onLossDetected(level, lost.map { it.packetNumber })
}
}
}
@@ -37,6 +37,8 @@ import com.vitorpamplona.quic.frame.ResetStreamFrame
import com.vitorpamplona.quic.frame.StopSendingFrame
import com.vitorpamplona.quic.frame.StreamFrame
import com.vitorpamplona.quic.frame.encodeFrames
import com.vitorpamplona.quic.observability.QlogObserver
import com.vitorpamplona.quic.observability.qlogFrameName
import com.vitorpamplona.quic.packet.LongHeaderPacket
import com.vitorpamplona.quic.packet.LongHeaderPlaintextPacket
import com.vitorpamplona.quic.packet.LongHeaderType
@@ -55,6 +57,15 @@ import com.vitorpamplona.quic.packet.ShortHeaderPlaintextPacket
*
* RFC 9000 §14: any datagram containing an Initial packet from the client
* MUST be padded to at least 1200 bytes total.
*
* Lock-split refactor (2026-05-08): caller must hold
* [QuicConnection.streamsLock]. Phase 1 keeps level-state mutation
* inline under the same critical section as the streams-domain work
* the writer needs — the win comes from `lifecycleLock`-only callers
* (close(), status reads, PTO bookkeeping) no longer fighting this lock.
* The driver wraps `streamsLock.withLock { drainOutbound(...) }`; tests
* that drive single-threaded send paths can call this without holding
* the lock — there's no contending thread.
*/
fun drainOutbound(
conn: QuicConnection,
@@ -62,12 +73,19 @@ fun drainOutbound(
): ByteArray? {
val parts = mutableListOf<ByteArray>()
// Closing — emit a CONNECTION_CLOSE at the highest available level.
// Closing — emit a CONNECTION_CLOSE at the highest available level. The
// datagram-build paths below MUST satisfy two RFC 9000 constraints we
// got wrong before:
// - §10.2.3: at Initial / Handshake levels, only CONNECTION_CLOSE
// (Transport, 0x1c) is allowed; the application-level close (0x1d) is
// forbidden because app state would leak before the handshake is
// encrypted with the application key.
// - §14.1: a client datagram containing an Initial MUST be ≥ 1200 bytes
// in UDP-payload terms, even when carrying only a CONNECTION_CLOSE.
if (conn.status == QuicConnection.Status.CLOSING) {
val frame = ConnectionCloseFrame(conn.closeErrorCode, null, conn.closeReason ?: "")
val packet = buildBestLevelPacket(conn, listOf(frame)) ?: return null
val datagram = buildClosingDatagram(conn, nowMillis)
conn.status = QuicConnection.Status.CLOSED
return packet
return datagram
}
// Drain destructive frame sources into local lists, ONCE.
@@ -118,21 +136,42 @@ fun drainOutbound(
var natural = 0
for (p in firstPass) natural += p.size
if (natural < 1200) {
val deficit = 1200 - natural
// Rewind the Initial PN — we'll reissue with the same PN and the
// same captured frames plus padding. The SentPacket entry recorded
// by the natural-size build will be overwritten by the rebuild
// below since both use the same PN.
initialState.pnSpace.rewindOutboundForRebuild()
val paddedInitial =
buildLongHeaderFromFrames(
conn = conn,
level = EncryptionLevel.INITIAL,
frames = initialContents!!.frames, // null-safe: gated by `initialNatural != null` above
tokens = initialContents.tokens,
nowMillis = nowMillis,
padBytes = deficit,
)
// Padding rebuild: re-issue the Initial with PADDING frames inside
// the AEAD envelope so the on-wire datagram clears the §14.1 floor.
//
// Off-by-one trap: the QUIC long-header Length field is a varint
// (RFC 9000 §16). When the natural-size payload is tiny enough
// for Length to fit in 1 byte (body ≤ 63 bytes), the rebuild's
// larger body crosses the 64-byte threshold and Length grows to
// 2 bytes — adding 1 wire byte that wasn't in `natural`. Same
// shape at 16384 bytes (2 → 4) and 2^30 (4 → 8). A naive
// `padBytes = 1200 - natural` then produces a 1199-byte
// datagram for PING-only Initials.
//
// Solution: rebuild with the initial deficit, measure, and if we
// still fall short, bump by the residual and rebuild once more.
// Each iteration adds PADDING bytes 1:1 to the wire size; the
// varint grows monotonically so this terminates in ≤ 2 rounds
// for any reachable payload size.
var padBytes = 1200 - natural
var paddedInitial: ByteArray
while (true) {
initialState.pnSpace.rewindOutboundForRebuild()
paddedInitial =
buildLongHeaderFromFrames(
conn = conn,
level = EncryptionLevel.INITIAL,
frames = initialContents!!.frames, // null-safe: gated by `initialNatural != null` above
tokens = initialContents.tokens,
nowMillis = nowMillis,
padBytes = padBytes,
)
var totalAfterRebuild = paddedInitial.size
if (handshakeNatural != null) totalAfterRebuild += handshakeNatural.size
if (applicationPkt != null) totalAfterRebuild += applicationPkt.size
if (totalAfterRebuild >= 1200) break
padBytes += 1200 - totalAfterRebuild
}
concat(listOfNotNull(paddedInitial, handshakeNatural, applicationPkt))
} else {
concat(firstPass)
@@ -166,6 +205,93 @@ private fun concat(parts: List<ByteArray>): ByteArray {
return out
}
/**
* Build a CONNECTION_CLOSE-only datagram at the highest encryption level we
* have keys for. Two RFC 9000 constraints make this trickier than a normal
* packet build:
*
* - §10.2.3 — at Initial / Handshake levels, only CONNECTION_CLOSE
* (Transport, 0x1c) is allowed. An application-level close is replaced
* with `APPLICATION_ERROR (0x0c)` + frameType=0 + empty reason so we don't
* leak app state pre-handshake.
* - §14.1 — a client datagram containing an Initial MUST be ≥ 1200 bytes,
* even a close-only one. We do this by building once at natural size and,
* if short, rewinding the PN and rebuilding with a PADDING-frame deficit
* inside the AEAD envelope. The rebuild loops because the long-header
* Length varint (RFC 9000 §16) can grow by 1 byte once the body crosses
* the 64-byte threshold, so a single-shot deficit can land 1 byte short
* of 1200.
*/
private fun buildClosingDatagram(
conn: QuicConnection,
nowMillis: Long,
): ByteArray? {
val app = conn.application
if (app.sendProtection != null) {
// 1-RTT level reached: app close (0x1d) is allowed and carries the
// original error code + reason.
val frame = ConnectionCloseFrame(conn.closeErrorCode, null, conn.closeReason ?: "")
return buildBestLevelPacket(conn, listOf(frame))
}
// Pre-1-RTT: must use transport-encoded close. RFC 9000 §20.1
// APPLICATION_ERROR = 0x0c — "the application or application protocol
// caused the connection to be closed during the handshake".
val transportFrame =
ConnectionCloseFrame(
errorCode = APPLICATION_ERROR,
frameType = 0L,
reason = "",
)
val hs = conn.handshake
if (hs.sendProtection != null) {
return buildLongHeaderFromFrames(
conn = conn,
level = EncryptionLevel.HANDSHAKE,
frames = listOf(transportFrame),
tokens = emptyList(),
nowMillis = nowMillis,
padBytes = 0,
)
}
val init = conn.initial
if (init.sendProtection != null && !init.keysDiscarded) {
val natural =
buildLongHeaderFromFrames(
conn = conn,
level = EncryptionLevel.INITIAL,
frames = listOf(transportFrame),
tokens = emptyList(),
nowMillis = nowMillis,
padBytes = 0,
)
if (natural.size >= 1200) return natural
var padBytes = 1200 - natural.size
var padded: ByteArray
do {
init.pnSpace.rewindOutboundForRebuild()
padded =
buildLongHeaderFromFrames(
conn = conn,
level = EncryptionLevel.INITIAL,
frames = listOf(transportFrame),
tokens = emptyList(),
nowMillis = nowMillis,
padBytes = padBytes,
)
if (padded.size >= 1200) break
padBytes += 1200 - padded.size
} while (true)
return padded
}
return null
}
/** RFC 9000 §20.1 — application-or-protocol caused close during handshake. */
private const val APPLICATION_ERROR: Long = 0x0cL
private fun buildBestLevelPacket(
conn: QuicConnection,
frames: List<Frame>,
@@ -176,23 +302,26 @@ private fun buildBestLevelPacket(
if (app.sendProtection != null) {
val proto = app.sendProtection!!
val pn = app.pnSpace.allocateOutbound()
return ShortHeaderPacket.build(
ShortHeaderPlaintextPacket(conn.destinationConnectionId, pn, payload),
proto.aead,
proto.key,
proto.iv,
proto.hp,
proto.hpKey,
largestAckedInSpace = -1L,
)
val built =
ShortHeaderPacket.build(
ShortHeaderPlaintextPacket(conn.destinationConnectionId, pn, payload),
proto.aead,
proto.key,
proto.iv,
proto.hp,
proto.hpKey,
largestAckedInSpace = -1L,
)
emitQlogSent(conn, EncryptionLevel.APPLICATION, pn, built.size, frames)
return built
}
val hs = conn.handshake
if (hs.sendProtection != null) {
return buildLongHeaderPacket(conn, EncryptionLevel.HANDSHAKE, payload)
return buildLongHeaderPacket(conn, EncryptionLevel.HANDSHAKE, payload, frames)
}
val init = conn.initial
if (init.sendProtection != null) {
return buildLongHeaderPacket(conn, EncryptionLevel.INITIAL, payload)
return buildLongHeaderPacket(conn, EncryptionLevel.INITIAL, payload, frames)
}
return null
}
@@ -201,6 +330,7 @@ private fun buildLongHeaderPacket(
conn: QuicConnection,
level: EncryptionLevel,
payload: ByteArray,
frames: List<Frame>,
): ByteArray {
val state = conn.levelState(level)
val proto = state.sendProtection!!
@@ -211,22 +341,25 @@ private fun buildLongHeaderPacket(
EncryptionLevel.HANDSHAKE -> LongHeaderType.HANDSHAKE
EncryptionLevel.APPLICATION -> error("APPLICATION uses short-header packets")
}
return LongHeaderPacket.build(
LongHeaderPlaintextPacket(
type = type,
version = QuicVersion.V1,
dcid = conn.destinationConnectionId,
scid = conn.sourceConnectionId,
packetNumber = pn,
payload = payload,
),
proto.aead,
proto.key,
proto.iv,
proto.hp,
proto.hpKey,
largestAckedInSpace = -1L,
)
val built =
LongHeaderPacket.build(
LongHeaderPlaintextPacket(
type = type,
version = QuicVersion.V1,
dcid = conn.destinationConnectionId,
scid = conn.sourceConnectionId,
packetNumber = pn,
payload = payload,
),
proto.aead,
proto.key,
proto.iv,
proto.hp,
proto.hpKey,
largestAckedInSpace = -1L,
)
emitQlogSent(conn, level, pn, built.size, frames)
return built
}
/**
@@ -280,10 +413,51 @@ private fun collectHandshakeLevelFrames(
length = cryptoChunk.data.size.toLong(),
)
}
// RFC 9002 §6.2.4: PTO probe MUST be ack-eliciting at the
// encryption level with unacknowledged data. `buildApplicationPacket`
// consumes [pendingPing] when 1-RTT keys exist; pre-1-RTT we
// honor it here at the highest active level (Handshake > Initial).
//
// The driver's PTO branch also calls
// [QuicConnection.requeueAllInflightCrypto], which moves any
// unacknowledged ClientHello / ClientFinished bytes back onto
// the cryptoSend retransmit queue — so the takeChunk above
// already produced a CRYPTO frame in that case, and the PING
// would be redundant. We only emit a bare PING when there's no
// CRYPTO retransmit available (e.g. the unacknowledged data was
// already sitting in cryptoSend's retransmit queue from a
// previous PTO and got drained, or the original Initial was
// never sent at all). This preserves the "send something
// ack-eliciting on every PTO" contract without wasting a frame.
if (conn.pendingPing &&
conn.application.sendProtection == null &&
level == highestPreApplicationLevel(conn)
) {
if (frames.none { it is CryptoFrame }) {
frames += PingFrame
}
// Either the CRYPTO frame already covers the ack-eliciting
// requirement at this level, or we just appended a PING.
// Clear the flag so the next drain doesn't double-fire.
conn.pendingPing = false
}
if (frames.isEmpty()) return null
return HandshakeLevelContents(frames = frames, tokens = tokens)
}
/**
* Highest encryption level for which we currently hold send keys, given
* 1-RTT keys are NOT installed. Used by [collectHandshakeLevelFrames]
* to place a PTO PING (RFC 9002 §6.2.4) at the right level when the
* application path can't carry it.
*/
private fun highestPreApplicationLevel(conn: QuicConnection): EncryptionLevel =
when {
conn.handshake.sendProtection != null -> EncryptionLevel.HANDSHAKE
conn.initial.sendProtection != null && !conn.initial.keysDiscarded -> EncryptionLevel.INITIAL
else -> EncryptionLevel.INITIAL // collectHandshakeLevelFrames already returned null on no keys
}
/**
* Build a long-header packet from already-collected frames, with optional
* trailing PADDING (0x00) bytes inside the encryption envelope. RFC 9000
@@ -310,13 +484,24 @@ private fun buildLongHeaderFromFrames(
}
val pn = state.pnSpace.allocateOutbound()
val type = if (level == EncryptionLevel.INITIAL) LongHeaderType.INITIAL else LongHeaderType.HANDSHAKE
// RFC 9000 §17.2.5.1: after a Retry, every Initial we send MUST carry
// the server-issued Retry token verbatim in the Initial header's Token
// field. Handshake packets have no Token field, so this only affects
// Initial-level builds.
val token =
if (type == LongHeaderType.INITIAL) {
conn.retryToken ?: ByteArray(0)
} else {
ByteArray(0)
}
val packet =
LongHeaderPacket.build(
LongHeaderPlaintextPacket(
type = type,
version = QuicVersion.V1,
version = conn.currentVersion,
dcid = conn.destinationConnectionId,
scid = conn.sourceConnectionId,
token = token,
packetNumber = pn,
payload = payload,
),
@@ -345,9 +530,34 @@ private fun buildLongHeaderFromFrames(
sizeBytes = packet.size,
tokens = tokens,
)
emitQlogSent(conn, level, pn, packet.size, frames)
return packet
}
/**
* Fire [QlogObserver.onPacketSent] for one outbound packet. Skipped
* fast for the [QlogObserver.NoOp] default so production callers pay
* only one identity comparison + one virtual call.
*/
private fun emitQlogSent(
conn: QuicConnection,
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<com.vitorpamplona.quic.frame.Frame>,
) {
val observer = conn.qlogObserver
if (observer === QlogObserver.NoOp) return
val frameNames = ArrayList<String>(frames.size)
for (f in frames) frameNames += qlogFrameName(f::class.simpleName ?: "frame")
observer.onPacketSent(
level = level,
packetNumber = packetNumber,
sizeBytes = sizeBytes,
frames = frameNames,
)
}
private fun buildApplicationPacket(
conn: QuicConnection,
nowMillis: Long,
@@ -423,11 +633,25 @@ private fun buildApplicationPacket(
// pre-priority round-robin behaviour exactly.
//
// Cost: O(N log N) per drain pass plus one transient sorted
// list. N is small (1–10 in the moq-lite audio path); if it
// ever grows enough to matter, switch to an indirect index sort
// or maintain an incrementally-sorted view on setPriority.
// list. N is small (1–10 in the moq-lite audio path) but the
// multiplexing interop testcase pushes N to ~2000, so we
// optimize:
// 1. filter out streams that are fully closed (no data + no
// retransmits coming) — drops "done" streams from the
// walk, which is most of them under bursty interop loads.
// 2. skip the sort entirely when every stream has the
// default priority (the common case) — preserving the
// insertion-ordered round-robin shape.
// The combination drops drainOutbound's per-call cost from
// O(N log N) where N=total-streams to roughly O(active) under
// realistic loads.
val active = streamsView.filter { !it.isClosed }
val sorted =
if (streamsView.size > 1) streamsView.sortedByDescending { it.priority } else streamsView
when {
active.size <= 1 -> active
active.all { it.priority == 0 } -> active
else -> active.sortedByDescending { it.priority }
}
val rotation = conn.streamRoundRobinStart
var tierStart = 0
outer@ while (tierStart < sorted.size) {
@@ -481,6 +705,25 @@ private fun buildApplicationPacket(
}
if (frames.isEmpty()) return null
// Diagnostic: gated by QUIC_INTEROP_DEBUG=1 so prod is silent. Tells
// us exactly what state the writer was in when it built this packet.
// Hunting the multiplexing-on-the-wire bug where MultiplexingCoalescing
// Test passes (~9 streams/packet) but the live driver emits 1 STREAM
// per packet against aioquic.
if (com.vitorpamplona.quic.connection.writerDebugEnabled) {
val streamFrameCount = frames.count { it is StreamFrame }
val totalFrames = frames.size
val streamsViewSize = conn.streamsListLocked().size
val activeSize = conn.streamsListLocked().count { !it.isClosed }
System.err.println(
"[writer.app] frames=$totalFrames stream_frames=$streamFrameCount " +
"streamsView=$streamsViewSize active=$activeSize " +
"packetBudget_remaining=${1100 - (frames.filterIsInstance<StreamFrame>().sumOf { it.data.size + 32 })} " +
"connBudget_initial=${(conn.sendConnectionFlowCredit - conn.sendConnectionFlowConsumed).coerceAtLeast(0L)}",
)
}
val payload = encodeFrames(frames)
val pn = state.pnSpace.allocateOutbound()
// Retain the packet for RFC 9002 loss detection BEFORE the encrypt
@@ -510,6 +753,9 @@ private fun buildApplicationPacket(
sizeBytes = sizeBytes.getOrNull()?.size ?: 0,
tokens = tokens.toList(),
)
sizeBytes.getOrNull()?.let { built ->
emitQlogSent(conn, EncryptionLevel.APPLICATION, pn, built.size, frames)
}
// Re-throw on encrypt failure so callers (driver loop) see the
// same exception they did before this change. The bookkeeping
// entry is still in place; if the throw was transient, retransmit
@@ -0,0 +1,42 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
/**
* Opt-in writer-side debug logging. Toggled by external code (the
* interop endpoint sets it from the QUIC_INTEROP_DEBUG env var at
* startup). Off by default — production must be silent.
*
* Cost when off: a single volatile read in the writer hot path,
* negligible against the encode + AEAD seal cost. Worth keeping
* inert in shipped code so the next interop investigation can flip
* it on without code changes.
*/
@Volatile
var writerDebugEnabled: Boolean = false
/**
* Build identifier injected into the boot log so we can verify the
* deployed image actually has the latest debug code (i.e. that the
* docker layer cache didn't serve a stale jar). Bump when adding new
* trace lines to make them traceable from the wire run.
*/
const val WRITER_DEBUG_BUILD_ID: String = "2026-05-07-fix-parallel-detection-v1"
@@ -0,0 +1,255 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.observability
import com.vitorpamplona.quic.connection.EncryptionLevel
/**
* qlog (draft-marx-qlog) observer interface for QUIC connection events.
*
* Tools like qvis (https://qvis.quictools.info/) and Wireshark consume
* qlog files to render sequence diagrams + RTT graphs + recovery
* timelines. The on-the-wire format is JSON-NDJSON (one JSON object
* per line); this interface decouples event emission from format —
* production code can pass [NoOp] (zero overhead), and the
* `:quic` interop runner attaches a JSON-writing implementation that
* produces a `client.sqlog` consumable by qvis.
*
* Goal: every interop-runner test failure produces a qlog file the
* caller can drop into qvis to see exactly what we did differently
* from the spec.
*
* Performance: hooks are on the hot packet-send/receive path, so
* [NoOp] must be a JIT-able single virtual call with no allocation.
* Default-method bodies in Kotlin compile to single bytecode `RETURN`,
* which HotSpot inlines reliably.
*/
interface QlogObserver {
/** Called once after [com.vitorpamplona.quic.connection.QuicConnection.start] runs. */
fun onConnectionStarted(
serverName: String,
dcid: ByteArray,
scid: ByteArray,
)
/**
* Called when the connection transitions to CLOSED, either locally
* (close()) or due to an inbound CONNECTION_CLOSE / read-loop
* termination ([com.vitorpamplona.quic.connection.QuicConnection.markClosedExternally]).
*
* @param initiator `"local"` if we initiated, `"remote"` otherwise.
*/
fun onConnectionClosed(
initiator: String,
errorCode: Long,
reason: String,
)
/**
* One outbound packet at [level] just hit the wire. Called once per
* coalesced packet inside a UDP datagram, so a single
* datagram carrying Initial + Handshake fires twice.
*/
fun onPacketSent(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
)
/** One inbound packet at [level] was successfully decrypted + dispatched. */
fun onPacketReceived(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
)
/**
* An inbound packet (or whole datagram) was dropped on the floor —
* AEAD AUTH FAIL, unknown DCID, missing receive keys, version
* mismatch, frame decode failure, etc.
*/
fun onPacketDropped(
reason: String,
sizeBytes: Int,
)
/**
* TLS produced new keys at the given encryption level.
*
* @param keyType `"server"` or `"client"` (which direction the
* key applies to).
*/
fun onKeyUpdated(
keyType: String,
level: EncryptionLevel,
)
/**
* RFC 9002 §6.1 loss detection declared one or more outbound
* packets at [level] lost.
*/
fun onLossDetected(
level: EncryptionLevel,
lostPacketNumbers: List<Long>,
)
/**
* RFC 9002 §6.2 PTO timer expired; the writer will emit a PING on
* the next drain to elicit an ACK from the peer.
*
* @param consecutivePtoCount the new PTO count (post-increment).
* @param ptoMillis the PTO duration that just expired.
*/
fun onPtoFired(
consecutivePtoCount: Int,
ptoMillis: Long,
)
/**
* Congestion-controller state transition (slow-start ↔
* recovery ↔ congestion-avoidance). qvis renders this as
* background bands on the RTT timeline.
*/
fun onCongestionStateUpdated(newState: String)
/**
* QUIC transport parameters were set by [initiator] (`"local"` at
* connection-open, `"remote"` once the peer's params arrive in
* EncryptedExtensions). [params] is a flat label→value map of
* the parameters that the implementation chose to surface; the
* exact key set is not part of the qlog 0.3 contract.
*/
fun onTransportParametersSet(
initiator: String,
params: Map<String, String>,
)
/**
* The TLS handshake completed and an ALPN was selected (or `null`
* was chosen — [alpn] is the human-readable name, e.g. `"h3"`).
*/
fun onAlpnNegotiated(alpn: String)
/**
* RFC 9000 §6 Version Information. Single-fire — emitted once
* after Version Negotiation resolves.
*/
fun onVersionInformation(
chosenVersion: String,
otherVersionsOffered: List<String>,
)
/**
* No-op observer. Default for production callers — every method
* is an empty body that the JIT inlines. No allocation, no I/O.
*/
object NoOp : QlogObserver {
override fun onConnectionStarted(
serverName: String,
dcid: ByteArray,
scid: ByteArray,
) = Unit
override fun onConnectionClosed(
initiator: String,
errorCode: Long,
reason: String,
) = Unit
override fun onPacketSent(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
) = Unit
override fun onPacketReceived(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
) = Unit
override fun onPacketDropped(
reason: String,
sizeBytes: Int,
) = Unit
override fun onKeyUpdated(
keyType: String,
level: EncryptionLevel,
) = Unit
override fun onLossDetected(
level: EncryptionLevel,
lostPacketNumbers: List<Long>,
) = Unit
override fun onPtoFired(
consecutivePtoCount: Int,
ptoMillis: Long,
) = Unit
override fun onCongestionStateUpdated(newState: String) = Unit
override fun onTransportParametersSet(
initiator: String,
params: Map<String, String>,
) = Unit
override fun onAlpnNegotiated(alpn: String) = Unit
override fun onVersionInformation(
chosenVersion: String,
otherVersionsOffered: List<String>,
) = Unit
}
}
/**
* Map a [com.vitorpamplona.quic.frame.Frame] subclass simple-name to
* the qlog frame_type label. qlog 0.3 specifies snake_case with
* `_frame` suffix stripped — e.g. `MaxStreamsFrame` → `max_streams`.
*
* Lives next to [QlogObserver] so writer + parser hot paths share the
* same conversion (used to fill [QlogObserver.onPacketSent.frames] and
* [QlogObserver.onPacketReceived.frames] without round-tripping through
* Jackson).
*/
fun qlogFrameName(simpleClassName: String): String {
// Strip trailing "Frame" then convert CamelCase to snake_case.
val noSuffix =
if (simpleClassName.endsWith("Frame")) {
simpleClassName.dropLast("Frame".length)
} else {
simpleClassName
}
if (noSuffix.isEmpty()) return simpleClassName.lowercase()
val sb = StringBuilder(noSuffix.length + 4)
for (i in noSuffix.indices) {
val c = noSuffix[i]
if (i > 0 && c.isUpperCase()) sb.append('_')
sb.append(c.lowercaseChar())
}
return sb.toString()
}
@@ -42,4 +42,17 @@ enum class LongHeaderType(
object QuicVersion {
const val V1: Int = 0x00000001
const val VERSION_NEGOTIATION: Int = 0x00000000
/**
* RFC 9000 §6 / interop runner convention: a "force VN" version
* number that no QUIC server is allowed to support. Sending this
* in the first Initial guarantees the server replies with a
* Version Negotiation packet listing the versions it actually
* supports. The interop runner's `versionnegotiation` testcase
* uses this to drive the client through the VN code path.
*
* Value 0x1a2a3a4a is conventional for this purpose; any
* non-spec-assigned 32-bit value would work equally well.
*/
const val FORCE_VERSION_NEGOTIATION: Int = 0x1a2a3a4a
}
@@ -315,6 +315,54 @@ class SendBuffer(
}
}
/**
* Re-queue every byte range currently sent-but-not-yet-ACK'd for
* retransmission. The writer's next [takeChunk] drains the
* retransmit queue before any fresh bytes, at the original
* offsets — so the peer sees the same CRYPTO/STREAM bytes again
* with the same offset, only at a new packet number.
*
* Used by the RFC 9002 §6.2.4 PTO probe path: when the timer
* fires before the handshake completes, the client MUST
* retransmit the unacknowledged ClientHello (the CRYPTO bytes
* sitting in [inFlight] for the Initial level). A PING alone
* isn't enough — the peer may never have seen the original
* datagram and therefore has no state to correlate the PING
* against.
*
* Idempotent: ranges already in [retransmit] are not affected
* (only [inFlight] is walked); calling twice in a row is a no-op
* the second time because the first call moved everything out of
* [inFlight]. FIN bit is preserved per range — a lost FIN-bearing
* range will re-emit FIN.
*
* Best-effort buffers ([bestEffort] = true) drop the inflight
* ranges instead of retransmitting them, matching [markLost]'s
* semantics. The data buffer can compact as if those ranges had
* been ACK'd.
*/
fun requeueAllInflight() {
synchronized(this) {
if (inFlight.isEmpty()) return
if (bestEffort) {
inFlight.clear()
advanceFlushedFloorIfPossible()
return
}
// Move every inflight range to the retransmit queue,
// preserving offset order (inFlight is sorted by offset
// ascending, so addLast preserves sort within retransmit
// for these new entries — though retransmit is a FIFO
// and doesn't strictly require sorted order, takeChunk
// pops front-first regardless).
for (r in inFlight) {
if (r.fin && !_finAcked) _finSent = false
retransmit.addLast(r)
}
inFlight.clear()
}
}
/**
* Disposition for a range that overlapped a [markAcked] / [markLost]
* range. ACK drops the range and latches `_finAcked` if the FIN
@@ -70,6 +70,17 @@ class TlsClient(
val fixedKeyPair: X25519KeyPair? = null,
/** When non-null, used as the ClientHello random (for deterministic tests). */
val fixedRandom: ByteArray? = null,
/**
* Cipher suites offered in the ClientHello, in preference order. The
* default offers AES-128-GCM first then ChaCha20-Poly1305. Override to
* force a specific negotiation (e.g. `chacha20`-only for the matching
* quic-interop-runner testcase).
*/
val cipherSuites: IntArray =
intArrayOf(
TlsConstants.CIPHER_TLS_AES_128_GCM_SHA256,
TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256,
),
) {
enum class State {
INITIAL,
@@ -120,6 +131,12 @@ class TlsClient(
private var sharedSecret: ByteArray? = null
private var negotiatedCipherSuite: Int = -1
/** The 32-byte ClientHello random, available after [start]. Exposed so
* observers (e.g. SSLKEYLOGFILE writer) can correlate secrets with
* this connection. */
var clientRandom: ByteArray? = null
private set
/** Begin the handshake by emitting a ClientHello at Initial level. */
fun start() {
check(state == State.INITIAL) { "TlsClient already started" }
@@ -127,14 +144,18 @@ class TlsClient(
keySchedule.deriveEarly()
val random =
fixedRandom ?: com.vitorpamplona.quartz.utils.RandomInstance
.bytes(32)
clientRandom = random
val ch =
buildQuicClientHello(
serverName = serverName,
x25519PublicKey = keyPair!!.publicKey,
quicTransportParams = transportParameters,
random =
fixedRandom ?: com.vitorpamplona.quartz.utils.RandomInstance
.bytes(32),
random = random,
cipherSuites = cipherSuites,
)
val chBytes = ch.encode()
@@ -92,6 +92,11 @@ fun buildQuicClientHello(
quicTransportParams: ByteArray,
additionalAlpn: List<ByteArray> = emptyList(),
random: ByteArray = RandomInstance.bytes(32),
cipherSuites: IntArray =
intArrayOf(
TlsConstants.CIPHER_TLS_AES_128_GCM_SHA256,
TlsConstants.CIPHER_TLS_CHACHA20_POLY1305_SHA256,
),
): TlsClientHello {
val alpn = mutableListOf<ByteArray>()
alpn += TlsConstants.ALPN_H3
@@ -107,5 +112,5 @@ fun buildQuicClientHello(
TlsExtension(TlsConstants.EXT_ALPN, encodeAlpn(alpn)),
TlsExtension(TlsConstants.EXT_QUIC_TRANSPORT_PARAMETERS, quicTransportParams),
)
return TlsClientHello(random = random, extensions = exts)
return TlsClientHello(random = random, cipherSuites = cipherSuites, extensions = exts)
}
@@ -64,7 +64,7 @@ class AckTrackerPurgeOnAckOfAckTest {
// Peer ACKs the packet that carried our outbound ACK
// covering up to PN 4.
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.onTokensAcked(
listOf(
@@ -75,7 +75,7 @@ class AckTrackerPurgeOnAckOfAckTest {
),
)
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
// Tracker is now empty: peer has confirmed receipt of our
// ACK that covered everything up to PN 4. Re-advertising
@@ -96,7 +96,7 @@ class AckTrackerPurgeOnAckOfAckTest {
}
// Peer ACKs our Initial-level outbound ACK.
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.onTokensAcked(
listOf(
@@ -104,7 +104,7 @@ class AckTrackerPurgeOnAckOfAckTest {
),
)
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
// Initial tracker drained; Application tracker untouched.
assertTrue(conn.initial.ackTracker.isEmpty())
@@ -120,7 +120,7 @@ class AckTrackerPurgeOnAckOfAckTest {
}
// Peer ACKs our outbound ACK that covered up to PN 4 only;
// the tracker's higher-PN ranges (5..9) must survive.
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.onTokensAcked(
listOf(
@@ -128,7 +128,7 @@ class AckTrackerPurgeOnAckOfAckTest {
),
)
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
assertFalse(conn.application.ackTracker.isEmpty())
assertEquals(9L, conn.application.ackTracker.largestReceived())
@@ -145,7 +145,7 @@ class AckTrackerPurgeOnAckOfAckTest {
for (pn in 0L..9L) {
conn.application.ackTracker.receivedPacket(pn, ackEliciting = true, receivedAtMillis = 1L)
}
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.onTokensAcked(
listOf(
@@ -160,7 +160,7 @@ class AckTrackerPurgeOnAckOfAckTest {
)
assertTrue(conn.application.ackTracker.isEmpty())
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
}
}
@@ -0,0 +1,230 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.tls.InProcessTlsServer
import com.vitorpamplona.quic.tls.PermissiveCertificateValidator
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withLock
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
/**
* Lock contract for batched stream opens — the production pattern used by
* `Http3GetClient.prepareRequests` / `HqInteropGetClient.prepareRequests`
* and any future caller that wants to open N streams atomically without
* the send loop interjecting between opens.
*
* Background. The interop runner's `multiplexing` testcase against aioquic
* timed out on 2026-05-06, downloading 1421/2000 files in 60s (~23
* streams/sec, exactly 1 RTT per stream — the wire was emitting one STREAM
* frame per datagram, not 64). The qlog showed 2898 packets sent in 60s,
* each carrying ONE STREAM frame. Cause: `prepareRequests` had been written
* to hold `conn.lock` (the deprecated alias for `lifecycleLock`) while
* calling [QuicConnection.openBidiStreamLocked]. `lifecycleLock` doesn't
* gate the writer — `drainOutbound` takes [QuicConnection.streamsLock] —
* so the writer's send loop could (and did) interleave between every two
* `openBidiStreamLocked` invocations, draining one stream per pass.
*
* Two-layer guard:
* 1. [QuicConnection.openBidiStreamLocked] now `check`s that
* [QuicConnection.streamsLock] is held when called. This catches
* callers that pass NO lock or the WRONG lock at the source.
* 2. The "batched open under streamsLock yields a small fixed packet
* count" contract is pinned by [MultiplexingCoalescingTest] —
* already in the suite. Together they fail loudly if the prod
* callers regress.
*
* If you're adding a new caller for `openBidiStreamLocked`, the right
* pattern is:
*
* conn.streamsLock.withLock {
* repeat(n) { conn.openBidiStreamLocked() ... }
* }
*/
class BatchedOpenLockContractTest {
@Test
fun `openBidiStreamLocked throws when called without streamsLock held`() {
runBlocking {
val client = handshakedClient()
// No `streamsLock.withLock { ... }` wrapper. This is the
// shape the regressed `prepareRequests` had — except it
// held `lifecycleLock` instead. Either way streamsLock is
// not held, the assertion in openBidiStreamLocked fires.
val ex =
assertFailsWith<IllegalStateException> {
client.openBidiStreamLocked()
}
assertNotNull(ex.message)
// The error should mention streamsLock so the caller knows
// what to fix.
kotlin.test.assertTrue(
ex.message!!.contains("streamsLock"),
"error message should name the lock to acquire; got: ${ex.message}",
)
}
}
@Test
fun `openBidiStreamLocked throws when caller holds the wrong lock (lifecycleLock)`() {
runBlocking {
val client = handshakedClient()
// Exact regression shape: caller holds lifecycleLock and
// calls openBidiStreamLocked. The check should fire because
// streamsLock is not held — even though SOME lock is.
assertFailsWith<IllegalStateException> {
client.lifecycleLock.withLock {
client.openBidiStreamLocked()
}
}
}
}
@Test
fun `openBidiStreamLocked succeeds when streamsLock is held`() {
runBlocking {
val client = handshakedClient()
// Happy path — caller holds streamsLock, batched open works.
// Sanity-check that the assertion in openBidiStreamLocked
// doesn't fire on the correct call shape.
val streams =
client.streamsLock.withLock {
List(64) { client.openBidiStreamLocked() }
}
assertEquals(64, streams.size)
assertEquals(64, streams.map { it.streamId }.toSet().size, "stream ids must be unique")
}
}
@Test
fun `openBidiStreamsBatch holds streamsLock for the entire batch`() {
runBlocking {
val client = handshakedClient()
// Verify the API actually holds streamsLock during init —
// the whole point of the function. A regression that
// released the lock between opens (the 2026-05-06 bug
// shape) would let isLocked drop to false inside init.
val payloads = (0 until 64).map { "req-$it".encodeToByteArray() }
var lockedAtSomePoint = false
val streams =
client.openBidiStreamsBatch(payloads) { stream, payload ->
if (client.streamsLock.isLocked) lockedAtSomePoint = true
stream.send.enqueue(payload)
stream.send.finish()
stream
}
assertEquals(64, streams.size)
assertEquals(64, streams.map { it.streamId }.toSet().size)
assertTrue(
lockedAtSomePoint,
"streamsLock must be held while init runs — without that, " +
"the send loop interleaves between opens",
)
}
}
@Test
fun `openBidiStreamsBatch with empty list does not throw`() {
runBlocking {
val client = handshakedClient()
// Empty-batch short-circuit: no items, no lock acquisition,
// no per-item init call. Returns empty list cleanly.
val result: List<Unit> =
client.openBidiStreamsBatch(emptyList<Int>()) { _, _ ->
error("init must not run for empty batch")
}
assertEquals(0, result.size)
}
}
@Test
fun `openUniStreamLocked throws when streamsLock is not held`() {
runBlocking {
val client = handshakedClient()
assertFailsWith<IllegalStateException> {
client.openUniStreamLocked()
}
}
}
@Test
fun `openUniStreamsBatch holds streamsLock for the entire batch`() {
runBlocking {
val client = handshakedClient()
// moq audio-rooms shape: many uni streams in burst. Without
// the batched API, each open releases the lock and the
// send loop interjects (the same shape that broke bidi
// multiplexing on 2026-05-06).
val items = List(16) { it }
var lockedAtSomePoint = false
val streams =
client.openUniStreamsBatch(items) { stream, _ ->
if (client.streamsLock.isLocked) lockedAtSomePoint = true
stream
}
assertEquals(16, streams.size)
assertEquals(16, streams.map { it.streamId }.toSet().size)
assertTrue(lockedAtSomePoint, "streamsLock must be held while init runs")
}
}
private fun handshakedClient(): QuicConnection =
runBlocking {
val client =
QuicConnection(
serverName = "example.test",
config =
QuicConnectionConfig(
initialMaxStreamsBidi = 256,
),
tlsCertificateValidator = PermissiveCertificateValidator(),
)
val serverScid = ConnectionId.random(8)
val tlsServer =
InProcessTlsServer(
transportParameters =
TransportParameters(
initialMaxData = 1_000_000,
initialMaxStreamDataBidiLocal = 100_000,
initialMaxStreamDataBidiRemote = 100_000,
initialMaxStreamDataUni = 100_000,
initialMaxStreamsBidi = 256,
initialMaxStreamsUni = 16,
initialSourceConnectionId = serverScid.bytes,
originalDestinationConnectionId = client.destinationConnectionId.bytes,
).encode(),
)
val pipe =
InMemoryQuicPipe(
client = client,
initialDcid = client.destinationConnectionId.bytes,
serverScid = serverScid,
tlsServer = tlsServer,
)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
client
}
}
@@ -0,0 +1,174 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.QuicWriter
import com.vitorpamplona.quic.frame.ConnectionCloseFrame
import com.vitorpamplona.quic.tls.PermissiveCertificateValidator
import kotlinx.coroutines.runBlocking
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
/**
* Regression tests for two RFC 9000 violations found via the
* `quic-interop-runner` against aioquic on 2026-05-06:
*
* - §10.2.3 — `CONNECTION_CLOSE (Application)` (0x1d) MUST NOT appear in
* Initial / Handshake packets. Application-error closes that fire before
* 1-RTT keys exist must be encoded as `CONNECTION_CLOSE (Transport)`
* (0x1c) with `errorCode = APPLICATION_ERROR (0x0c)`.
* - §14.1 — any client datagram containing an Initial MUST be ≥ 1200
* bytes, even when carrying only a `CONNECTION_CLOSE`.
*
* Pre-fix, the writer's CLOSING branch built a tiny ~45-byte Initial with
* frame type 0x1d. The runner's aioquic server silently dropped it (correct
* per spec) and our handshake hung until our 10s timeout.
*/
class CloseDatagramRfcComplianceTest {
@Test
fun `pre-handshake close datagram is padded to at least 1200 bytes per RFC 9000 sec 14_1`() =
runBlocking {
val conn =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
)
// Initial sendProtection is wired in QuicConnection's init block;
// no handshake required to exercise the close path.
conn.status = QuicConnection.Status.CLOSING
val datagram = drainOutbound(conn, nowMillis = 0L)
requireNotNull(datagram) { "drainOutbound must produce a close datagram when CLOSING with Initial keys" }
assertTrue(
datagram.size >= 1200,
"client Initial datagram MUST be ≥ 1200 bytes per RFC 9000 §14.1, " +
"got ${datagram.size} (this was bug A from the aioquic interop run)",
)
// First byte: 1100????b — long-header form + Initial type.
val firstByte = datagram[0].toInt() and 0xff
assertEquals(0xc0, firstByte and 0xf0, "must be a long-header packet")
assertEquals(0x00, firstByte and 0x30, "must be type=Initial (00)")
}
@Test
fun `PTO probe emits a PING at Initial level pre-handshake (RFC 9002 sec 6_2_4)`() =
runBlocking {
// Reproduces the third bug found via the aioquic interop run on
// 2026-05-06: when the first ClientHello is dropped (e.g. sim
// queues it before the server is ready), the driver's PTO timer
// sets `pendingPing = true`, but the writer only honored that
// flag in the 1-RTT path. Pre-handshake the flag was silently
// discarded, so the second drain produced no Initial datagram —
// the connection sat mute until our 10s handshake timeout.
//
// Pre-fix: drainOutbound returned null and the connection slept
// on the next PTO. Post-fix: a PING-bearing Initial datagram is
// emitted, eliciting an ACK from the peer that feeds loss
// detection and unblocks CRYPTO retransmit.
val conn =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
)
// Initial sendProtection is wired in QuicConnection's init {}
// block; no handshake required to exercise the PTO probe path.
// Skip conn.start() so the cryptoSend buffer is empty — this is
// exactly the post-ClientHello-sent state when PTO fires.
conn.pendingPing = true
val datagram = drainOutbound(conn, nowMillis = 0L)
assertNotNull(
datagram,
"PTO probe MUST produce an Initial datagram pre-handshake — RFC 9002 §6.2.4",
)
// RFC 9000 §14.1: client datagrams containing an Initial MUST
// be ≥ 1200 bytes. The writer's padding rebuild now accounts
// for Length-varint growth (1 → 2 bytes) when the natural-size
// payload is small (PING-only) so the deficit calculation
// produces a final size that strictly meets the spec floor.
assertTrue(
datagram.size >= 1200,
"PTO Initial datagram MUST be ≥ 1200 bytes per RFC 9000 §14.1, got ${datagram.size}",
)
assertEquals(false, conn.pendingPing, "pendingPing MUST be cleared after the probe")
}
@Test
fun `single-byte PING-only Initial pads to exactly 1200 bytes (no overshoot beyond varint growth)`() =
runBlocking {
// Boundary case for the padding-rebuild deficit calculation.
// The smallest possible Initial-level frame payload is a single
// PING frame (1 byte, encoded as 0x01 — RFC 9000 §19.2). With
// no token, the 1-byte natural payload encodes a Length varint
// of 1 byte. Once the rebuild adds ~1170 bytes of PADDING the
// Length value crosses the 63-byte varint boundary and grows
// to 2 bytes. The deficit calculation MUST account for that
// growth, otherwise the rebuilt packet falls 1 byte short of
// the §14.1 1200-byte floor.
//
// Asserts the strict floor (≥ 1200) AND that we don't overshoot
// by more than the varint-growth delta plus a small slack — the
// padded packet should land in the 1200..1203 range, never
// 1199 (pre-fix) and never 1300+ (over-correction).
val conn =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
)
conn.pendingPing = true
val datagram = drainOutbound(conn, nowMillis = 0L)
assertNotNull(datagram, "PING-only PTO probe must produce a datagram")
assertTrue(
datagram.size >= 1200,
"padded Initial MUST be ≥ 1200 bytes (RFC 9000 §14.1), got ${datagram.size}",
)
assertTrue(
datagram.size <= 1203,
"padded Initial should not overshoot the 1200 floor by more than the " +
"Length-varint growth (≤ 3 bytes), got ${datagram.size}",
)
}
@Test
fun `ConnectionCloseFrame encodes type 0x1c when frameType is non-null (transport close)`() {
// Bug B fix relies on the writer passing frameType=0L (instead of
// null) when emitting a close at Initial / Handshake level. Encode
// path must produce 0x1c for that case, 0x1d only for app-level.
val transportClose = ConnectionCloseFrame(errorCode = 0x0c, frameType = 0L, reason = "")
val w1 = QuicWriter()
transportClose.encode(w1)
val transportBytes = w1.toByteArray()
assertEquals(0x1c.toByte(), transportBytes[0], "transport CONNECTION_CLOSE must serialize as 0x1c")
val appClose = ConnectionCloseFrame(errorCode = 0, frameType = null, reason = "")
val w2 = QuicWriter()
appClose.encode(w2)
val appBytes = w2.toByteArray()
assertEquals(0x1d.toByte(), appBytes[0], "application CONNECTION_CLOSE must serialize as 0x1d")
}
}
@@ -81,12 +81,12 @@ class CryptoRetransmitTest {
.single()
// Simulate loss via direct dispatch.
client.lock.lock()
client.streamsLock.lock()
try {
client.onTokensLost(listOf(cryptoToken))
client.initial.sentPackets.remove(firstPn)
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
// Initial-level cryptoSend should now have re-queued bytes
@@ -126,11 +126,11 @@ class CryptoRetransmitTest {
client.initial.sentPackets.entries
.first { it.value.tokens.any { t -> t is RecoveryToken.Crypto } }
// ACK via direct dispatch.
client.lock.lock()
client.streamsLock.lock()
try {
client.onTokensAcked(packet.value.tokens)
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
// After ACK the Initial-level cryptoSend's flushedFloor should
// have advanced — we check by observing that another takeChunk
@@ -0,0 +1,249 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.frame.StreamFrame
import com.vitorpamplona.quic.tls.InProcessTlsServer
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeoutOrNull
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Regression for the quic-interop-runner `multiplexing` failure observed
* 2026-05-06 against aioquic: 677 client-bidi streams opened, server FIN'd
* every one, zero responses surfaced to the application — every per-stream
* `incoming.collect` hung forever.
*
* The single-stream cases (`transfer`, `chacha20`, `http3`) all pass, so
* STREAM-frame routing for low concurrency works. The bug only fires when
* many streams' responses stream through together.
*
* Two regressions are pinned here:
*
* 1. Server replies to every client-opened bidi stream with `data + FIN`
* coalesced into a single datagram. Each stream's `incoming` Flow MUST
* complete (FIN delivery), and the bytes MUST surface (no truncation).
*
* 2. If the parser tears the connection down (e.g. STREAM channel
* saturated → INTERNAL_ERROR), every stream's `incoming` Flow MUST
* still terminate. Otherwise application-side `incoming.collect`
* callers leak as zombie coroutines stuck on a channel that nobody
* will ever close. This is the "FIN never arrives" symptom from the
* runner's perspective: the connection died mid-response and we
* forgot to release the per-stream channels.
*/
class MultiStreamFinDeliveryTest {
@Test
fun finIsDeliveredToAllParallelClientBidiStreamsUnderConcurrentLoad() =
runBlocking {
val (client, pipe) = newConnectedClient()
// Open N streams. Match the multiplexing case shape — many streams
// each with a small response — without paying for full 1999-file
// overhead.
val n = 50
val streams = (0 until n).map { client.openBidiStream() }
// Build one server datagram per stream that responds with
// "resp-<i>" + FIN. We keep them in separate datagrams (rather
// than coalescing all into one short-header packet which the
// packet codec doesn't support) so the parser sees a stream of
// back-to-back STREAM frames just as it would on the wire.
for ((i, stream) in streams.withIndex()) {
val payload = "resp-$i".encodeToByteArray()
val frame =
StreamFrame(
streamId = stream.streamId,
offset = 0L,
data = payload,
fin = true,
)
val packet = pipe.buildServerApplicationDatagram(listOf(frame))!!
feedDatagram(client, packet, nowMillis = 0L)
}
// Each stream's collector must terminate (FIN closed the
// channel) AND must have received the expected payload.
val collected =
coroutineScope {
streams
.mapIndexed { i, stream ->
async {
withTimeoutOrNull(5_000L) {
stream.incoming.toList()
} to i
}
}.awaitAll()
}
val hung = collected.firstOrNull { it.first == null }
assertTrue(
hung == null,
"stream index ${hung?.second} never received FIN — Flow.toList() timed out",
)
for ((result, i) in collected) {
val chunks = result!!
val joined = ByteArray(chunks.sumOf { it.size })
var p = 0
for (c in chunks) {
c.copyInto(joined, p)
p += c.size
}
assertEquals(
"resp-$i",
joined.decodeToString(),
"stream $i: bytes mismatch",
)
}
assertEquals(
QuicConnection.Status.CONNECTED,
client.status,
"connection must remain CONNECTED through the response burst",
)
}
@Test
fun finIsDeliveredEvenWhenChannelAlreadyHasBufferedChunks() =
runBlocking {
// Defence-in-depth: a stream that has bytes still queued in
// its incomingChannel when the connection tears down must still
// surface those bytes AND complete the Flow. consumeAsFlow drains
// the buffer before honouring the close, so closeIncoming after
// deliverIncoming is safe — we pin that contract here so a
// future "close the channel and reset the buffer" refactor can't
// silently regress the byte loss.
val (client, pipe) = newConnectedClient()
val stream = client.openBidiStream()
val payload = "buffered-then-torn-down".encodeToByteArray()
val packet =
pipe.buildServerApplicationDatagram(
listOf(
StreamFrame(
streamId = stream.streamId,
offset = 0L,
data = payload,
fin = false, // intentionally no FIN
),
),
)!!
feedDatagram(client, packet, nowMillis = 0L)
// Tear down WITHOUT delivering FIN — bytes are now buffered in the
// incomingChannel and the connection-level close must still
// terminate the per-stream Flow.
client.markClosedExternally("test teardown after partial response")
val chunks =
withTimeoutOrNull(2_000L) { stream.incoming.toList() }
assertTrue(chunks != null, "Flow leaked after teardown with buffered bytes")
val joined = ByteArray(chunks.sumOf { it.size })
var p = 0
for (c in chunks) {
c.copyInto(joined, p)
p += c.size
}
assertEquals(
"buffered-then-torn-down",
joined.decodeToString(),
"buffered bytes must surface before the Flow terminates",
)
}
@Test
fun connectionTeardownClosesEveryPerStreamIncomingChannel() =
runBlocking {
// Even when the parser kills the connection (e.g. channel
// overflow surfaces as INTERNAL_ERROR via markClosedExternally),
// every per-stream `incoming` Flow MUST terminate. Otherwise
// application coroutines that called `stream.incoming.toList()`
// leak forever — exactly the symptom seen in the multiplexing
// runner where 677 collectors hung.
val (client, _) = newConnectedClient()
val n = 20
val streams = (0 until n).map { client.openBidiStream() }
// Tear the connection down externally without delivering any FIN.
client.markClosedExternally("simulated parser-side teardown")
// Every stream's collector must complete promptly. Pre-fix the
// per-stream incomingChannel stays open (closeIncoming is never
// called) and the collector hangs.
val results =
coroutineScope {
streams
.mapIndexed { i, stream ->
async {
withTimeoutOrNull(2_000L) {
stream.incoming.toList()
} to i
}
}.awaitAll()
}
val hung = results.firstOrNull { it.first == null }
assertTrue(
hung == null,
"stream index ${hung?.second} incoming Flow leaked after connection teardown",
)
}
private fun newConnectedClient(): Pair<QuicConnection, InMemoryQuicPipe> =
runBlocking {
val client =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator =
com.vitorpamplona.quic.tls
.PermissiveCertificateValidator(),
)
val serverScid = ConnectionId.random(8)
val tlsServer =
InProcessTlsServer(
transportParameters =
TransportParameters(
initialMaxData = 16L * 1024 * 1024,
initialMaxStreamDataBidiLocal = 1L * 1024 * 1024,
initialMaxStreamDataBidiRemote = 1L * 1024 * 1024,
initialMaxStreamDataUni = 1L * 1024 * 1024,
initialMaxStreamsBidi = 1000,
initialMaxStreamsUni = 1000,
initialSourceConnectionId = serverScid.bytes,
originalDestinationConnectionId = client.destinationConnectionId.bytes,
).encode(),
)
val pipe =
InMemoryQuicPipe(
client = client,
initialDcid = client.destinationConnectionId.bytes,
serverScid = serverScid,
tlsServer = tlsServer,
)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
client to pipe
}
}
@@ -0,0 +1,150 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.tls.InProcessTlsServer
import kotlinx.coroutines.runBlocking
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Reproduce the aioquic-2026-05-07 multiplex-on-the-wire failure in
* a unit test. Server qlog showed:
* - 1406 packets with exactly 1 STREAM frame, 1 with 3, never more
* - first stream packets at 1665 / 1709ms (one RTT apart, NOT burst)
* - per-RTT cadence throughout
*
* Hypothesis: writer drains 1 stream per packet despite many being
* queued — under conditions specific to aioquic's TPs:
* - initial_max_data = 1MB (matches qlog)
* - initial_max_stream_data_bidi_remote = 1MB (matches qlog)
* - initial_max_streams_bidi = 128 (matches qlog)
*
* MultiplexingCoalescingTest passes with `initial_max_data = 100MB`
* — which is 100x more credit than aioquic gives. This test mirrors
* the exact aioquic TPs to see if the smaller credit triggers the
* 1-stream-per-packet shape.
*
* If this test asserts ≤6 packets and PASSES, the bug is NOT in the
* synchronous drain path — it's in the live driver flow (which
* MultiplexingCoalescingTest doesn't exercise).
*
* If it FAILS with one stream per packet, we have a deterministic
* reproduction and can fix the writer.
*/
class MultiplexingAioquicTpsTest {
@Test
fun multiplex_64_streams_with_aioquic_TPs_should_still_coalesce() =
runBlocking {
val client = handshakedClientMatchingAioquic()
// ~80-byte HTTP/3 HEADERS frame payload — what
// Http3GetClient actually emits per stream after QPACK
// encoding. Smaller payloads (50 bytes) under-stress the
// coalescing path.
val n = 64
val payloadPerStream = 80
val streams =
client.openBidiStreamsBatch((0 until n).toList()) { stream, i ->
stream.send.enqueue(ByteArray(payloadPerStream) { (i and 0xff).toByte() })
stream.send.finish()
stream
}
assertEquals(n, streams.size)
val packets = mutableListOf<ByteArray>()
while (true) {
val pkt = drainOutbound(client, nowMillis = 1L) ?: break
packets += pkt
}
val totalBytes = packets.sumOf { it.size }
val streamsPerPacket = n.toDouble() / packets.size.coerceAtLeast(1)
println(
"[MultiplexingAioquicTpsTest] 64 streams of $payloadPerStream bytes " +
"drained into ${packets.size} packets " +
"(${(streamsPerPacket * 10).toInt() / 10.0} streams/pkt, " +
"totalBytes=$totalBytes)",
)
// Same threshold as MultiplexingCoalescingTest. 64 streams
// × ~110 bytes (80 payload + ~30 overhead) ≈ 7 KB; at
// ~1100-byte payload budget per packet that's 6-7 packets.
assertTrue(
packets.size <= 8,
"expected ≤8 packets, got ${packets.size} — " +
"writer regressed to one-stream-per-packet under aioquic TPs",
)
assertTrue(
streamsPerPacket >= 8.0,
"expected ≥8 streams/packet, got $streamsPerPacket",
)
}
private fun handshakedClientMatchingAioquic(): QuicConnection =
runBlocking {
val client =
QuicConnection(
serverName = "example.test",
config =
QuicConnectionConfig(
// Mirror our actual local TPs from the qlog.
initialMaxData = 16L * 1024 * 1024,
initialMaxStreamsBidi = 100,
initialMaxStreamsUni = 10000,
initialMaxStreamDataBidiLocal = 1L * 1024 * 1024,
initialMaxStreamDataBidiRemote = 1L * 1024 * 1024,
initialMaxStreamDataUni = 1L * 1024 * 1024,
),
tlsCertificateValidator =
com.vitorpamplona.quic.tls
.PermissiveCertificateValidator(),
)
val serverScid = ConnectionId.random(8)
val tlsServer =
InProcessTlsServer(
transportParameters =
TransportParameters(
// Mirror aioquic-qns 2026-05-07 from the qlog.
initialMaxData = 1L * 1024 * 1024,
initialMaxStreamDataBidiLocal = 1L * 1024 * 1024,
initialMaxStreamDataBidiRemote = 1L * 1024 * 1024,
initialMaxStreamDataUni = 1L * 1024 * 1024,
initialMaxStreamsBidi = 128,
initialMaxStreamsUni = 128,
initialSourceConnectionId = serverScid.bytes,
originalDestinationConnectionId = client.destinationConnectionId.bytes,
).encode(),
)
val pipe =
InMemoryQuicPipe(
client = client,
initialDcid = client.destinationConnectionId.bytes,
serverScid = serverScid,
tlsServer = tlsServer,
)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
client
}
}
@@ -0,0 +1,172 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.tls.InProcessTlsServer
import kotlinx.coroutines.runBlocking
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Pin the multiplexing throughput contract that the runner's
* `multiplexing` testcase exercises.
*
* This is the fast in-memory version of what the Docker-based runner
* does (open N bidi streams, send a GET on each, collect responses):
* we drive a real handshake via [InMemoryQuicPipe] then enqueue request
* bytes on N bidi streams in two batching shapes, drive the writer, and
* count the resulting datagrams.
*
* The bug this test catches: pre-fix, the interop runner's multiplexing
* test ran ~25 streams/sec because each application-side enqueue woke
* the send loop while the OTHER coroutines hadn't yet queued bytes.
* Result: one stream per packet, ~80-byte packets, useless coalescing.
*
* Post-fix: enqueue bytes on N streams synchronously, then drain ONCE
* — the writer should pack many streams' data into each datagram. With
* 64 streams × 50 bytes = 3200 bytes of payload, plus ~50 bytes/stream
* STREAM-frame framing, total wire load ≈ 6.4 KB. At a 1452-byte UDP
* cap, that's ~5 datagrams. The pre-fix shape would emit 64.
*/
class MultiplexingCoalescingTest {
@Test
fun `64 streams enqueued before drain coalesce into a small fixed number of packets`() =
runBlocking {
val client = handshakedClient()
// Phase 1: serial enqueue. NO drainOutbound between streams.
// This is exactly the shape InteropClient.runTransferTest's
// chunked-multiplex path uses: prepareRequest is synchronous
// for every URL in the chunk, then a single wakeup, then
// parallel awaits.
val n = 64
val payloadPerStream = 50
val streams =
(0 until n).map { i ->
val s = client.openBidiStream()
s.send.enqueue(ByteArray(payloadPerStream) { (i and 0xff).toByte() })
s.send.finish()
s
}
assertEquals(n, streams.size)
// Phase 2: drain everything at once. Count packets emitted.
val packets = mutableListOf<ByteArray>()
while (true) {
val pkt = drainOutbound(client, nowMillis = 1L) ?: break
packets += pkt
}
val totalBytes = packets.sumOf { it.size }
val payloadTotal = n * payloadPerStream
val avgBytesPerPacket = totalBytes / packets.size.coerceAtLeast(1)
val streamsPerPacket = n.toDouble() / packets.size.coerceAtLeast(1)
// Threshold derivation: each stream's wire load is ~50 bytes
// payload + ~10 bytes STREAM-frame framing + per-packet AEAD
// overhead. 64 streams ≈ 4 KB of frame data; at 1452 bytes
// per UDP packet that's ≤ 4 packets of frames + 1 for any
// pending ACK / control frames. Pre-fix produced 64+ packets.
assertTrue(
packets.size <= 6,
"64 streams should coalesce into ≤6 packets, got ${packets.size}; " +
"totalBytes=$totalBytes avgBytesPerPacket=$avgBytesPerPacket " +
"streamsPerPacket=$streamsPerPacket payloadTotal=$payloadTotal",
)
// Sanity: average packet should carry at least ~10 streams' frames.
// If it's ~1, we regressed back to one-stream-per-packet.
assertTrue(
streamsPerPacket >= 10.0,
"expected ≥10 streams/packet, got $streamsPerPacket — coalescing broke",
)
}
@Test
fun `1000 streams enqueued in batches of 64 produce a tractable packet count`() =
runBlocking {
// Stress version. 1000 streams in chunks of 64 = ~16 chunks.
// Each chunk should produce ≤6 packets per the test above,
// so total ≤ 100 packets. Pre-fix this test would have
// produced ~1000 packets.
val client = handshakedClient(maxStreamsBidi = 2000)
val chunkSize = 64
val totalStreams = 1000
val payloadPerStream = 50
val packets = mutableListOf<ByteArray>()
for (chunk in (0 until totalStreams).chunked(chunkSize)) {
for (i in chunk) {
val s = client.openBidiStream()
s.send.enqueue(ByteArray(payloadPerStream) { (i and 0xff).toByte() })
s.send.finish()
}
while (true) {
val pkt = drainOutbound(client, nowMillis = 1L) ?: break
packets += pkt
}
}
val streamsPerPacket = totalStreams.toDouble() / packets.size.coerceAtLeast(1)
assertTrue(
packets.size <= 150,
"1000 streams should produce ≤150 packets, got ${packets.size} (streamsPerPacket=$streamsPerPacket)",
)
}
private fun handshakedClient(maxStreamsBidi: Long = 100L): QuicConnection =
runBlocking {
val client =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(initialMaxStreamsBidi = maxStreamsBidi),
tlsCertificateValidator =
com.vitorpamplona.quic.tls
.PermissiveCertificateValidator(),
)
val serverScid = ConnectionId.random(8)
val tlsServer =
InProcessTlsServer(
transportParameters =
TransportParameters(
initialMaxData = 100_000_000,
initialMaxStreamDataBidiLocal = 1_000_000,
initialMaxStreamDataBidiRemote = 1_000_000,
initialMaxStreamDataUni = 1_000_000,
initialMaxStreamsBidi = maxStreamsBidi,
initialMaxStreamsUni = maxStreamsBidi,
initialSourceConnectionId = serverScid.bytes,
originalDestinationConnectionId = client.destinationConnectionId.bytes,
).encode(),
)
val pipe =
InMemoryQuicPipe(
client = client,
initialDcid = client.destinationConnectionId.bytes,
serverScid = serverScid,
tlsServer = tlsServer,
)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
client
}
}
@@ -0,0 +1,261 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.frame.StreamFrame
import com.vitorpamplona.quic.tls.InProcessTlsServer
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeoutOrNull
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
/**
* In-process mirror of the [quic-interop-runner](https://github.com/quic-interop/quic-interop-runner)
* `multiplexing` testcase. The runner generates ~50–2000 small files and
* has the client open one bidi stream per file in parallel, then asserts
* every file is downloaded with correct content. This test runs the same
* shape against the [InMemoryQuicPipe] harness — no UDP, no real server —
* so the regression that drove this test (aioquic CONNECTION_CLOSE on
* bare-PING PTO probes, plus the parallel-open / FIN-delivery bugs) can
* be caught in a unit-test loop instead of a Docker matrix run.
*
* Why this shape and not just `MultiStreamFinDeliveryTest`:
*
* - `MultiStreamFinDeliveryTest` only exercises server-pushed STREAM
* frames. It hand-builds N response datagrams and feeds them
* directly. The CLIENT never sends a STREAM frame, so any regression
* in the writer's multiplex-stream-coalesce path or in the parser's
* server-side request-decode logic is invisible to it.
*
* - This test drives the full request → response loop:
*
* 1. Client opens N bidi streams in parallel (matches the matrix)
* 2. Each stream enqueues a tiny request + FIN
* 3. drainOutbound pulls a coalesced 1-RTT datagram off the writer
* 4. The pipe decrypts, extracts STREAM frames, builds server
* responses (`response-<id>` + FIN) and feeds them back
* 5. Per-stream `incoming.toList()` must yield the expected bytes
*
* If any step in that loop regresses — the writer drops streams, the
* parser routes STREAM frames wrong under load, the server-side state
* forgets which stream sent which request — the assertion fails with a
* specific stream id, not a vague timeout.
*
* Stream-count notes: the matrix uses up to 1999 files; we use 64 to keep
* the test under a second on CI. The *bug class* the test guards against
* is invariant to the count — any "loses a stream's response" regression
* will fire at 64 too because it's per-stream, not aggregate.
*/
class MultiplexingRoundTripTest {
@Test
fun parallel_streams_each_get_their_own_response_back() =
runBlocking {
val (client, pipe) = newConnectedClient()
val n = 64
val streams =
(0 until n).map { i ->
val s = client.openBidiStream()
// Tiny request body. Real H3/HQ would be a `GET /file-i\r\n`;
// the matrix doesn't care WHAT the bytes are, only that the
// client emitted them on a distinct stream and the server's
// response surfaces back.
s.send.enqueue("req-$i".encodeToByteArray())
s.send.finish()
s
}
// Drain everything the client wants to send. drainOutbound returns
// one datagram per call (up to ~1200 bytes); on each, decrypt the
// 1-RTT short-header packet and collect STREAM frames. The writer
// is supposed to coalesce many streams' frames into one datagram
// (MultiplexingCoalescingTest pins that contract); here we just
// walk every emitted datagram until the client has nothing left.
val seenRequests = mutableMapOf<Long, MutableList<ByteArray>>()
val seenFin = mutableSetOf<Long>()
var totalDatagrams = 0
while (true) {
val out = drainOutbound(client, nowMillis = 0L) ?: break
if (out.isEmpty()) break
totalDatagrams += 1
val frames = pipe.decryptClientApplicationFrames(out) ?: break
for (frame in frames) {
if (frame is StreamFrame) {
// Append to per-stream chunk list — joined once at the
// end. Earlier shape merged eagerly per frame which is
// O(N²) over byte count.
seenRequests.getOrPut(frame.streamId) { mutableListOf() } += frame.data
if (frame.fin) seenFin += frame.streamId
}
}
if (seenFin.size == n) break
}
assertEquals(
n,
seenRequests.size,
"server saw STREAM frames from only ${seenRequests.size}/$n streams " +
"after $totalDatagrams datagrams — writer dropped streams under multiplex load",
)
assertEquals(
n,
seenFin.size,
"server saw FINs from only ${seenFin.size}/$n streams — client failed " +
"to deliver request FIN on every stream",
)
// End-to-end coalescing contract: 64 streams × ~10-byte
// requests fit in a handful of datagrams. The aioquic
// 2026-05-06 regression hit when the writer emitted ONE
// STREAM per datagram (so 64 datagrams for this batch).
// Threshold of 12 leaves headroom for the writer's choice
// of when to coalesce + any per-drain ACK frames.
assertTrue(
totalDatagrams <= 12,
"expected ≤12 datagrams for 64 streams, got $totalDatagrams — " +
"writer regressed to one-stream-per-datagram (the aioquic " +
"interop 2026-05-06 multiplexing failure mode)",
)
// Sanity-check the request payloads match what each stream sent.
for ((i, stream) in streams.withIndex()) {
val expected = "req-$i"
val chunks = seenRequests[stream.streamId]
assertNotNull(chunks, "stream ${stream.streamId} (i=$i): no STREAM frames received")
val joined = ByteArray(chunks.sumOf { it.size })
var p = 0
for (c in chunks) {
c.copyInto(joined, p)
p += c.size
}
val actual = joined.decodeToString()
assertEquals(
expected,
actual,
"stream ${stream.streamId} (i=$i): server received '$actual', expected '$expected'",
)
}
// Server responds: one STREAM frame per stream with a known body
// + FIN, packed into one datagram each (the in-memory pipe doesn't
// coalesce frames into a single packet across stream-ids by
// design, since the writer side of the pipe is intentionally
// minimal). Client must surface every body via stream.incoming
// and terminate each Flow on FIN.
for (stream in streams) {
val body = "response-${stream.streamId}".encodeToByteArray()
val frame =
StreamFrame(
streamId = stream.streamId,
offset = 0L,
data = body,
fin = true,
)
val datagram = pipe.buildServerApplicationDatagram(listOf(frame))!!
feedDatagram(client, datagram, nowMillis = 0L)
}
val collected =
coroutineScope {
streams
.map { stream ->
async {
val chunks =
withTimeoutOrNull(5_000L) { stream.incoming.toList() }
stream.streamId to chunks
}
}.awaitAll()
}
val hung = collected.firstOrNull { it.second == null }
assertTrue(
hung == null,
"stream ${hung?.first} never received FIN — Flow.toList() timed out " +
"(this is the matrix's 'incomplete transfer' symptom)",
)
for ((streamId, chunks) in collected) {
val joined = ByteArray(chunks!!.sumOf { it.size })
var p = 0
for (c in chunks) {
c.copyInto(joined, p)
p += c.size
}
assertEquals(
"response-$streamId",
joined.decodeToString(),
"stream $streamId: response bytes mismatch — wrong content delivered",
)
}
assertEquals(
QuicConnection.Status.CONNECTED,
client.status,
"connection must remain CONNECTED through the full round-trip",
)
}
private fun newConnectedClient(): Pair<QuicConnection, InMemoryQuicPipe> =
runBlocking {
val client =
QuicConnection(
serverName = "example.test",
config =
QuicConnectionConfig(
initialMaxStreamsBidi = 1024,
initialMaxStreamsUni = 1024,
initialMaxData = 16L * 1024 * 1024,
initialMaxStreamDataBidiLocal = 64L * 1024,
initialMaxStreamDataBidiRemote = 64L * 1024,
initialMaxStreamDataUni = 64L * 1024,
),
tlsCertificateValidator =
com.vitorpamplona.quic.tls
.PermissiveCertificateValidator(),
)
val serverScid = ConnectionId.random(8)
val tlsServer =
InProcessTlsServer(
transportParameters =
TransportParameters(
initialMaxData = 16L * 1024 * 1024,
initialMaxStreamDataBidiLocal = 64L * 1024,
initialMaxStreamDataBidiRemote = 64L * 1024,
initialMaxStreamDataUni = 64L * 1024,
initialMaxStreamsBidi = 1024,
initialMaxStreamsUni = 1024,
initialSourceConnectionId = serverScid.bytes,
originalDestinationConnectionId = client.destinationConnectionId.bytes,
).encode(),
)
val pipe =
InMemoryQuicPipe(
client = client,
initialDcid = client.destinationConnectionId.bytes,
serverScid = serverScid,
tlsServer = tlsServer,
)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
client to pipe
}
}
@@ -0,0 +1,139 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.tls.InProcessTlsServer
import com.vitorpamplona.quic.tls.PermissiveCertificateValidator
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.runBlocking
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Throughput contract for the lock-split refactor (2026-05-08): opening
* many parallel bidi streams + queueing requests must not be serialised
* by a single connection-wide mutex. Phase 1 of the split (separate
* `streamsLock` / `lifecycleLock` / per-level `levelLock`) targets the
* multiplexing testcase that drove this refactor — see
* `quic/plans/2026-05-08-lock-split-design.md`.
*
* The test stands up an in-memory client (no socket I/O), opens 1000
* client-bidi streams concurrently, enqueues a small request body + FIN
* on each, and asserts the operation completes within a generous wall-
* clock budget. The number is deliberately loose: this is a contract
* for "lock contention isn't pathological", not a microbenchmark.
*
* NOTE: the in-memory pipe doesn't drive a concurrent send loop, so
* this test exercises the lock-acquisition cost of `openBidiStream`
* itself rather than full multiplexing throughput. The interop runner
* provides the end-to-end measurement.
*/
class MultiplexingThroughputTest {
@Test
fun open_1000_bidi_streams_completes_quickly() {
runBlocking {
val client =
QuicConnection(
serverName = "example.test",
config =
QuicConnectionConfig(
initialMaxStreamsBidi = 2_000,
initialMaxStreamsUni = 2_000,
initialMaxData = 100_000_000,
initialMaxStreamDataBidiLocal = 100_000,
initialMaxStreamDataBidiRemote = 100_000,
initialMaxStreamDataUni = 100_000,
),
tlsCertificateValidator = PermissiveCertificateValidator(),
)
val serverScid = ConnectionId.random(8)
val tlsServer =
InProcessTlsServer(
transportParameters =
TransportParameters(
initialMaxData = 100_000_000,
initialMaxStreamDataBidiLocal = 100_000,
initialMaxStreamDataBidiRemote = 100_000,
initialMaxStreamDataUni = 100_000,
initialMaxStreamsBidi = 2_000,
initialMaxStreamsUni = 2_000,
initialSourceConnectionId = serverScid.bytes,
originalDestinationConnectionId = client.destinationConnectionId.bytes,
).encode(),
)
val pipe =
InMemoryQuicPipe(
client = client,
initialDcid = client.destinationConnectionId.bytes,
serverScid = serverScid,
tlsServer = tlsServer,
)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
val request = ByteArray(50) { it.toByte() }
val streamCount = 1_000
// Open all streams in parallel — each launch contends for
// streamsLock briefly. Pre-refactor this serialised against
// any in-flight drainOutbound call; phase 1 keeps openBidi
// contention scoped to streamsLock-only.
val started =
kotlin.time.TimeSource.Monotonic
.markNow()
val opens =
(0 until streamCount).map {
async {
val stream = client.openBidiStream()
stream.send.enqueue(request)
stream.send.finish()
stream.streamId
}
}
val ids = opens.awaitAll()
val elapsed = started.elapsedNow()
// Useful diagnostic for measuring future regressions: stdout
// shows up in the test report so phase-1 vs phase-2 can be
// compared against the same test.
println(
"[MultiplexingThroughputTest] opened $streamCount bidi streams in " +
"${elapsed.inWholeMilliseconds}ms " +
"(${(streamCount * 1000.0 / elapsed.inWholeMilliseconds.coerceAtLeast(1L)).toLong()} streams/sec)",
)
assertEquals(streamCount, ids.size)
assertEquals(streamCount, ids.toSet().size, "stream ids must be unique")
// Generous bound; in-process opens of 1000 streams should
// complete in well under half a second on any developer
// machine — pre-refactor this was minutes due to lock
// contention against the (idle) send-loop drain. The looser
// 2-second bound is still 100x what's expected on actual
// hardware while accounting for slow CI workers.
assertTrue(
elapsed.inWholeMilliseconds < 2_000L,
"1000 parallel openBidiStream calls took ${elapsed.inWholeMilliseconds}ms; expected <2000ms",
)
}
}
}
@@ -47,11 +47,11 @@ class OnTokensLostTest {
fun ackToken_doesNotPopulateAnyPending() =
runBlocking {
val conn = newConn()
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.onTokensLost(listOf(RecoveryToken.Ack(level = EncryptionLevel.APPLICATION, largestAcked = 0L)))
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
assertNull(conn.pendingMaxStreamsUni)
assertNull(conn.pendingMaxStreamsBidi)
@@ -64,12 +64,12 @@ class OnTokensLostTest {
runBlocking {
val conn = newConn()
// Simulate the writer having advertised a higher cap.
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.advertisedMaxStreamsUni = 150L
conn.onTokensLost(listOf(RecoveryToken.MaxStreamsUni(maxStreams = 150L)))
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
assertEquals(150L, conn.pendingMaxStreamsUni)
}
@@ -82,12 +82,12 @@ class OnTokensLostTest {
// the value carried by the lost token (150). The lost
// frame is irrelevant — re-emitting 150 would not extend
// the cap. neqo's fc.rs line 322 supersede check.
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.advertisedMaxStreamsUni = 200L
conn.onTokensLost(listOf(RecoveryToken.MaxStreamsUni(maxStreams = 150L)))
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
assertNull(conn.pendingMaxStreamsUni, "stale lost extension must not be re-emitted")
}
@@ -96,12 +96,12 @@ class OnTokensLostTest {
fun lostMaxStreamsBidi_matchingAdvertised_setsPending() =
runBlocking {
val conn = newConn()
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.advertisedMaxStreamsBidi = 200L
conn.onTokensLost(listOf(RecoveryToken.MaxStreamsBidi(maxStreams = 200L)))
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
assertEquals(200L, conn.pendingMaxStreamsBidi)
}
@@ -110,12 +110,12 @@ class OnTokensLostTest {
fun lostMaxData_matchingAdvertised_setsPending() =
runBlocking {
val conn = newConn()
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.advertisedMaxData = 1_000_000L
conn.onTokensLost(listOf(RecoveryToken.MaxData(maxData = 1_000_000L)))
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
assertEquals(1_000_000L, conn.pendingMaxData)
}
@@ -124,12 +124,12 @@ class OnTokensLostTest {
fun lostMaxData_supersededIsDropped() =
runBlocking {
val conn = newConn()
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.advertisedMaxData = 2_000_000L
conn.onTokensLost(listOf(RecoveryToken.MaxData(maxData = 1_000_000L)))
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
assertNull(conn.pendingMaxData)
}
@@ -138,13 +138,13 @@ class OnTokensLostTest {
fun lostMaxStreamData_unknownStream_dropped() =
runBlocking {
val conn = newConn()
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.onTokensLost(
listOf(RecoveryToken.MaxStreamData(streamId = 999L, maxData = 1024L)),
)
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
// No stream with id 999 exists ⇒ token is dropped silently.
assertEquals(emptyMap<Long, Long>(), conn.pendingMaxStreamData)
@@ -154,7 +154,7 @@ class OnTokensLostTest {
fun multipleLostTokens_dispatchAll() =
runBlocking {
val conn = newConn()
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.advertisedMaxStreamsUni = 150L
conn.advertisedMaxStreamsBidi = 200L
@@ -168,7 +168,7 @@ class OnTokensLostTest {
),
)
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
assertEquals(150L, conn.pendingMaxStreamsUni)
assertEquals(200L, conn.pendingMaxStreamsBidi)
@@ -183,7 +183,7 @@ class OnTokensLostTest {
// most one value (the last setter wins; the supersede
// check filters older losses).
val conn = newConn()
conn.lock.lock()
conn.streamsLock.lock()
try {
conn.advertisedMaxStreamsUni = 200L
// First lost packet had MaxStreamsUni(150) — stale, dropped.
@@ -193,7 +193,7 @@ class OnTokensLostTest {
conn.onTokensLost(listOf(RecoveryToken.MaxStreamsUni(maxStreams = 200L)))
assertEquals(200L, conn.pendingMaxStreamsUni)
} finally {
conn.lock.unlock()
conn.streamsLock.unlock()
}
}
}
@@ -94,7 +94,7 @@ class PeerStreamCreditExtensionTest {
// Simulate the relay opening uni streams to us. SERVER_UNI
// stream IDs use the encoding `index << 2 | 0x3`. Two streams
// (cap=4, half-window=2) is the threshold for a refresh.
client.lock
client.streamsLock
.let {
// Acquire under lock since getOrCreatePeerStreamLocked requires it.
it
@@ -103,7 +103,7 @@ class PeerStreamCreditExtensionTest {
kotlinx.coroutines.sync
.Mutex()
.let { /* noop: silence unused-import linter */ }
client.lock.let { l ->
client.streamsLock.let { l ->
kotlinx.coroutines.runBlocking {
l.lock()
try {
@@ -179,7 +179,7 @@ class PeerStreamCreditExtensionTest {
// Open 10 peer streams — half-window for cap=100 is 50, so
// we're well below the threshold.
client.lock.let { l ->
client.streamsLock.let { l ->
kotlinx.coroutines.runBlocking {
l.lock()
try {
@@ -0,0 +1,202 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.frame.StreamFrame
import com.vitorpamplona.quic.stream.StreamId
import com.vitorpamplona.quic.tls.InProcessTlsServer
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotEquals
/**
* Regression coverage for the multiplexing-interop tear-down (audit-4 #3
* "slow consumer" overflow on peer-initiated uni streams).
*
* Reproduction of the production symptom:
*
* The interop runner's `multiplexing` testcase opens many parallel
* client-bidi GET streams via an H3 client. The H3 client opens its
* three local uni streams (control + QPACK encoder + QPACK decoder)
* per RFC 9114 §6.2.1 but **does not consume the server's three
* counterpart peer-initiated uni streams**. Their bytes
* (SETTINGS, dynamic-table inserts, ack signals) are routed by
* [QuicConnectionParser] into each stream's bounded
* `incomingChannel` (capacity 64). Once the QPACK encoder
* stream's burst of dynamic-table inserts saturates it, the next
* delivery trips the audit-4 #3 escape hatch:
*
* conn.markClosedExternally("INTERNAL_ERROR: stream … consumer
* overflowed incoming channel
* (slow consumer)")
*
* and the entire connection dies after ~4.5 s with zero requests
* completed.
*
* The test pair below pins both halves of the contract:
*
* - [pre_fix_no_consumer_overflows_and_tears_down_connection] — without
* a consumer the connection MUST close with the documented reason.
* - [drainPeerInitiatedUniStreamsIntoBlackHole_keeps_connection_alive] —
* with [drainPeerInitiatedUniStreamsIntoBlackHole] running the
* connection MUST stay CONNECTED and absorb the same byte volume.
*
* The fix philosophy (variant B from the prompt's three-way menu): the
* `:quic` library stays strict about backpressure — silently dropping
* app-data bytes is worse than failing fast. The new public helper is
* the explicit "I do not care about these particular peer streams"
* opt-in for an H3 GET client that runs with the QPACK dynamic table
* off. Default behaviour is unchanged.
*/
class PeerUniStreamDrainTest {
@Test
fun pre_fix_no_consumer_overflows_and_tears_down_connection() {
runBlocking {
val client = buildClient()
val pipe = buildPipe(client)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
// Push 65 chunks on a server-initiated uni stream — one more
// than the per-stream incomingChannel capacity (64). The
// first 64 land; the 65th overflows trySend, sets
// QuicStream.overflowed, and the parser maps that to
// markClosedExternally with the documented reason.
//
// Each chunk fits comfortably under the per-stream receive
// limit (initialMaxStreamDataUni = 1 MiB) so the prior
// receive-limit guard does NOT fire — this test is about
// the *channel* overflow specifically, not flow control.
val streamId = StreamId.build(StreamId.Kind.SERVER_UNI, 0)
var offset = 0L
for (i in 0 until 65) {
val chunk = ByteArray(8) { (i + it).toByte() }
val frame = StreamFrame(streamId = streamId, offset = offset, data = chunk, fin = false)
offset += chunk.size.toLong()
val packet = pipe.buildServerApplicationDatagram(listOf(frame))
assertNotEquals(null, packet, "server has app keys after handshake")
feedDatagram(client, packet!!, nowMillis = 0L)
}
// The 65th chunk must have torn the connection down.
assertEquals(
QuicConnection.Status.CLOSED,
client.status,
"without a peer-uni-stream consumer the audit-4 #3 escape " +
"hatch must fire on the 65th chunk",
)
}
}
@Test
fun drainPeerInitiatedUniStreamsIntoBlackHole_keeps_connection_alive() {
runBlocking {
val client = buildClient()
val pipe = buildPipe(client)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
// Wire the explicit drainer BEFORE pushing the bytes.
val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
try {
client.drainPeerInitiatedUniStreamsIntoBlackHole(scope)
// Same volume as the pre-fix test, except this time we go
// a long way past the channel capacity to prove sustained
// operation (4× the bound). If the drainer were absent the
// connection would have died at chunk 65; with the
// drainer reading them as fast as the parser delivers, no
// backpressure builds up.
val streamId = StreamId.build(StreamId.Kind.SERVER_UNI, 0)
var offset = 0L
for (i in 0 until 256) {
val chunk = ByteArray(8) { (i + it).toByte() }
val frame = StreamFrame(streamId = streamId, offset = offset, data = chunk, fin = false)
offset += chunk.size.toLong()
val packet = pipe.buildServerApplicationDatagram(listOf(frame))!!
feedDatagram(client, packet, nowMillis = 0L)
// Yield occasionally so the drainer coroutine actually
// gets a chance to consume — feedDatagram is synchronous
// and the drainer launched with Dispatchers.Default needs
// a scheduling tick.
if (i % 16 == 15) {
withTimeout(2_000) { delay(1) }
}
}
// Ensure the drainer has caught up before we sample status.
withTimeout(2_000) { delay(50) }
assertEquals(
QuicConnection.Status.CONNECTED,
client.status,
"with drainPeerInitiatedUniStreamsIntoBlackHole the " +
"connection must absorb arbitrary peer-uni traffic " +
"without overflowing the channel; saw closeReason=" +
"${client.closeReason}",
)
} finally {
scope.cancel()
}
}
}
private fun buildClient(): QuicConnection =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator =
com.vitorpamplona.quic.tls
.PermissiveCertificateValidator(),
)
private fun buildPipe(client: QuicConnection): InMemoryQuicPipe {
val serverScid = ConnectionId.random(8)
val tlsServer =
InProcessTlsServer(
transportParameters =
TransportParameters(
initialMaxData = 10_000_000,
initialMaxStreamDataBidiLocal = 1_000_000,
initialMaxStreamDataBidiRemote = 1_000_000,
initialMaxStreamDataUni = 1_000_000,
initialMaxStreamsBidi = 16,
initialMaxStreamsUni = 16,
initialSourceConnectionId = serverScid.bytes,
originalDestinationConnectionId = client.destinationConnectionId.bytes,
).encode(),
)
return InMemoryQuicPipe(
client = client,
initialDcid = client.destinationConnectionId.bytes,
serverScid = serverScid,
tlsServer = tlsServer,
)
}
}
@@ -46,11 +46,11 @@ class PendingFlowControlEmitTest {
fun pendingMaxStreamsUni_drainEmitsFrameAndToken() =
runBlocking {
val client = handshakedClient()
client.lock.lock()
client.streamsLock.lock()
try {
client.pendingMaxStreamsUni = 150L
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
val sizeBefore = client.application.sentPackets.size
@@ -79,11 +79,11 @@ class PendingFlowControlEmitTest {
fun pendingMaxStreamsBidi_drainEmitsFrameAndToken(): Unit =
runBlocking {
val client = handshakedClient()
client.lock.lock()
client.streamsLock.lock()
try {
client.pendingMaxStreamsBidi = 200L
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
val sizeBefore = client.application.sentPackets.size
runCatching { drainOutbound(client, nowMillis = 1L) }
@@ -103,11 +103,11 @@ class PendingFlowControlEmitTest {
fun pendingMaxData_drainEmitsFrameAndToken(): Unit =
runBlocking {
val client = handshakedClient()
client.lock.lock()
client.streamsLock.lock()
try {
client.pendingMaxData = 5_000_000L
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
val sizeBefore = client.application.sentPackets.size
runCatching { drainOutbound(client, nowMillis = 1L) }
@@ -127,12 +127,12 @@ class PendingFlowControlEmitTest {
fun pendingMaxStreamData_perStreamDrain() =
runBlocking {
val client = handshakedClient()
client.lock.lock()
client.streamsLock.lock()
try {
client.pendingMaxStreamData[3L] = 1_024L
client.pendingMaxStreamData[7L] = 2_048L
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
val sizeBefore = client.application.sentPackets.size
runCatching { drainOutbound(client, nowMillis = 1L) }
@@ -159,13 +159,13 @@ class PendingFlowControlEmitTest {
fun multiplePending_drainEmitsAllInOnePacket(): Unit =
runBlocking {
val client = handshakedClient()
client.lock.lock()
client.streamsLock.lock()
try {
client.pendingMaxStreamsUni = 150L
client.pendingMaxStreamsBidi = 200L
client.pendingMaxData = 1_000_000L
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
val sizeBefore = client.application.sentPackets.size
runCatching { drainOutbound(client, nowMillis = 1L) }
@@ -216,11 +216,11 @@ class PendingFlowControlEmitTest {
// advertised cap. The writer drains it as-is — supersede check
// is in step 6 (the setter side), not here.
val client = handshakedClient()
client.lock.lock()
client.streamsLock.lock()
try {
client.pendingMaxStreamsUni = 50L
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
val sizeBefore = client.application.sentPackets.size
runCatching { drainOutbound(client, nowMillis = 1L) }
@@ -242,11 +242,11 @@ class PendingFlowControlEmitTest {
fun pendingClearedAcrossDrains() =
runBlocking {
val client = handshakedClient()
client.lock.lock()
client.streamsLock.lock()
try {
client.pendingMaxStreamsUni = 150L
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
// Drain once: pending consumed.
runCatching { drainOutbound(client, nowMillis = 1L) }
@@ -0,0 +1,210 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.connection.recovery.RecoveryToken
import com.vitorpamplona.quic.frame.CryptoFrame
import com.vitorpamplona.quic.frame.Frame
import com.vitorpamplona.quic.packet.LongHeaderPacket
import kotlinx.coroutines.runBlocking
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
/**
* RFC 9002 §6.2.4 spec-correct PTO probe behavior: when the timer fires
* pre-handshake the client MUST send an ack-eliciting packet at the
* encryption level with unacknowledged data, and SHOULD retransmit the
* lost data rather than emit a bare PING.
*
* Regression scenario from the quic-interop-runner ns-3 transfer test
* against aioquic: the first ClientHello datagram is dropped because
* the simulated server isn't fully started at t≈0.5s. Our PTO at
* t≈1.5s previously sent only a PING with the same DCID; the server
* had no state for that DCID (never saw the original Initial), so
* the PING was silently ignored and the connection died.
*
* The fix has two cooperating pieces:
* 1. [com.vitorpamplona.quic.stream.SendBuffer.requeueAllInflight]
* moves every sent-but-not-ACK'd byte range from the inflight
* list back onto the retransmit queue.
* 2. [QuicConnection.requeueAllInflightCrypto] exposes that to the
* driver, which calls it from the PTO branch when 1-RTT keys
* aren't installed yet (i.e. handshake not done).
*
* After the call, the next [drainOutbound] naturally emits a CRYPTO
* frame at the original offset — the server actually sees a fresh
* ClientHello attempt and can advance the handshake.
*/
class PtoCryptoRetransmitTest {
@Test
fun ptoPreHandshake_retransmitsClientHelloCryptoBytes() =
runBlocking {
val client = newClientWithStartedHandshake()
// First drain — emits the ClientHello inside an Initial datagram.
val firstDrain = drainOutbound(client, nowMillis = 1L)
assertNotNull(firstDrain, "first drain must produce the ClientHello datagram")
assertTrue(
firstDrain.size >= 1200,
"RFC 9000 §14.1 — Initial-bearing datagram pads to ≥ 1200 bytes (got ${firstDrain.size})",
)
// Capture the original ClientHello bytes and offset by
// pulling them out of the SentPacket bookkeeping.
val firstSent =
client.initial.sentPackets.entries.firstOrNull { entry ->
entry.value.tokens.any { it is RecoveryToken.Crypto }
}
assertNotNull(firstSent, "Initial-level SentPacket must carry a Crypto token after first drain")
val firstCrypto =
firstSent.value.tokens
.filterIsInstance<RecoveryToken.Crypto>()
.single()
assertEquals(0L, firstCrypto.offset, "ClientHello starts at offset 0")
assertTrue(firstCrypto.length > 0L, "ClientHello has non-zero length")
// Sanity: the second drain (immediately, no PTO yet) must
// produce nothing — bytes are inflight, not unsent.
val emptyDrain = drainOutbound(client, nowMillis = 2L)
assertTrue(
emptyDrain == null || emptyDrain.isEmpty(),
"no PTO yet, no fresh data → second drain must be empty (got ${emptyDrain?.size} bytes)",
)
// Drive the EXACT helper QuicConnectionDriver.sendLoop
// calls when its PTO timer fires. Earlier versions of
// this test inlined the simulation (set pendingPing,
// call requeueAllInflightCrypto manually) — but that
// hid the regression where the driver itself stopped
// calling requeueAllInflightCrypto. Calling
// [handlePtoFired] keeps the test in lockstep with the
// production code path: if anyone unwires the requeue
// again, this test breaks.
handlePtoFired(client)
// Next drain must emit a fresh Initial packet carrying
// the ClientHello CRYPTO at the original offset.
val ptoDrain = drainOutbound(client, nowMillis = 3L)
assertNotNull(ptoDrain, "PTO drain must produce a retransmit datagram")
assertTrue(
ptoDrain.size >= 1200,
"RFC 9000 §14.1 — Initial-bearing datagram still pads to ≥ 1200 bytes on PTO retransmit (got ${ptoDrain.size})",
)
// The retransmit packet must carry a Crypto token at the
// original offset and length — that's how we know the
// CRYPTO frame went out (not a PING-only probe).
val replayEntries =
client.initial.sentPackets.entries
.filter { it.key != firstSent.key }
val replaySent =
replayEntries.firstOrNull { entry ->
entry.value.tokens.any { it is RecoveryToken.Crypto }
}
assertNotNull(
replaySent,
"PTO drain must produce a fresh Initial SentPacket carrying Crypto " +
"(saw ${replayEntries.map { it.value.tokens.map { t -> t::class.simpleName } }})",
)
val replayCrypto =
replaySent.value.tokens
.filterIsInstance<RecoveryToken.Crypto>()
.single()
assertEquals(EncryptionLevel.INITIAL, replayCrypto.level)
assertEquals(firstCrypto.offset, replayCrypto.offset, "PTO retransmit replays original offset")
assertEquals(firstCrypto.length, replayCrypto.length, "PTO retransmit replays original length")
// Decode the retransmit packet and assert the CRYPTO
// frame's payload bytes match the original.
val firstFrames = decodeInitialFrames(firstDrain, client)
val firstHello =
firstFrames.filterIsInstance<CryptoFrame>().firstOrNull {
it.offset == firstCrypto.offset && it.data.size.toLong() == firstCrypto.length
}
assertNotNull(firstHello, "first drain's Initial must contain the ClientHello CRYPTO")
val ptoFrames = decodeInitialFrames(ptoDrain, client)
val replayHello =
ptoFrames.filterIsInstance<CryptoFrame>().firstOrNull {
it.offset == firstCrypto.offset
}
assertNotNull(
replayHello,
"PTO drain's Initial must contain a CRYPTO frame at offset 0 — bare PING is not enough " +
"(saw frames ${ptoFrames.map { it::class.simpleName }})",
)
assertTrue(
replayHello.data.contentEquals(firstHello.data),
"PTO retransmit must carry the same ClientHello bytes (size first=${firstHello.data.size} replay=${replayHello.data.size})",
)
// pendingPing should be cleared since the CRYPTO frame
// satisfied the ack-eliciting requirement at the level.
assertEquals(
false,
client.pendingPing,
"pendingPing must be consumed once the PTO emit went out (CRYPTO covered the probe)",
)
}
/**
* Decode an Initial-level long-header packet from a freshly-drained
* datagram and return its frames. We open with the client's own
* SEND protection (client-side keys) — symmetric AEAD opens with
* the same key/iv that sealed it.
*/
private fun decodeInitialFrames(
datagram: ByteArray,
client: QuicConnection,
): List<Frame> {
val send = client.initial.sendProtection!!
val parsed =
LongHeaderPacket.parseAndDecrypt(
bytes = datagram,
offset = 0,
aead = send.aead,
key = send.key,
iv = send.iv,
hp = send.hp,
hpKey = send.hpKey,
largestReceivedInSpace = -1L,
)
assertNotNull(parsed, "must parse the Initial packet header (datagram size=${datagram.size})")
return com.vitorpamplona.quic.frame
.decodeFrames(parsed.packet.payload)
}
private fun newClientWithStartedHandshake(): QuicConnection =
runBlocking {
val client =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator =
com.vitorpamplona.quic.tls
.PermissiveCertificateValidator(),
)
client.start()
client
}
}
@@ -93,12 +93,12 @@ class ResetStopSendingEmitTest {
.single()
// Simulate loss.
client.lock.lock()
client.streamsLock.lock()
try {
client.onTokensLost(listOf(token))
client.application.sentPackets.remove(firstEntry.key)
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
// Per-stream emit-pending should be re-flagged.
assertTrue(stream.resetEmitPending, "loss must re-flag resetEmitPending")
@@ -137,21 +137,21 @@ class ResetStopSendingEmitTest {
.single()
// ACK first.
client.lock.lock()
client.streamsLock.lock()
try {
client.onTokensAcked(listOf(token))
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
assertEquals(true, stream.resetAcked)
assertEquals(false, stream.resetEmitPending)
// Now a stale loss notification arrives. Defensive: drop.
client.lock.lock()
client.streamsLock.lock()
try {
client.onTokensLost(listOf(token))
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
assertEquals(false, stream.resetEmitPending, "stale loss after ACK must not re-flag emit-pending")
}
@@ -248,11 +248,11 @@ class ResetStopSendingEmitTest {
connectionId = byteArrayOf(1, 2, 3, 4),
statelessResetToken = ByteArray(16) { it.toByte() },
)
client.lock.lock()
client.streamsLock.lock()
try {
client.onTokensLost(listOf(token))
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
assertEquals(token, client.pendingNewConnectionId[1L])
@@ -69,7 +69,7 @@ class RetransmitIntegrationTest {
// ACK'd by reordering — its PN < largestAckedPn -
// PACKET_THRESHOLD ⇒ declared lost.
val futurePn = msuPn + 4L
client.lock.lock()
client.streamsLock.lock()
try {
// Inject a phantom SentPacket at futurePn so the loss
// detector has a credible "newly acked" reference, then
@@ -102,7 +102,7 @@ class RetransmitIntegrationTest {
// 5. Dispatch lost tokens — pendingMaxStreamsUni gets set.
client.onTokensLost(lostMsuPacket.tokens)
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
assertEquals(
capAfterFirstDrain,
@@ -148,24 +148,24 @@ class RetransmitIntegrationTest {
// Second bump: open more peer-uni streams to cross the
// (already extended) threshold again.
client.lock.lock()
client.streamsLock.lock()
try {
client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 2))
client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 3))
client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 4))
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
runCatching { drainOutbound(client, nowMillis = 2L) }
val secondCap = client.advertisedMaxStreamsUni
assertTrue(secondCap > firstCap, "second drain must advertise a still-higher cap; saw $firstCap → $secondCap")
// Now declare the FIRST emit lost via direct dispatch.
client.lock.lock()
client.streamsLock.lock()
try {
client.onTokensLost(listOf(RecoveryToken.MaxStreamsUni(maxStreams = firstCap)))
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
// Supersede check: firstCap != advertisedMaxStreamsUni (now == secondCap),
// so pending must remain null.
@@ -216,12 +216,12 @@ class RetransmitIntegrationTest {
private fun crossPeerUniHalfWindow(client: QuicConnection) =
runBlocking {
client.lock.lock()
client.streamsLock.lock()
try {
client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 0))
client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 1))
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
}
}
@@ -0,0 +1,260 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.QuicReader
import com.vitorpamplona.quic.QuicWriter
import com.vitorpamplona.quic.packet.LongHeaderPacket
import com.vitorpamplona.quic.packet.LongHeaderType
import com.vitorpamplona.quic.packet.QuicVersion
import com.vitorpamplona.quic.packet.RetryPacket
import com.vitorpamplona.quic.tls.PermissiveCertificateValidator
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
/**
* Retry packet handling end-to-end through [QuicConnection], per RFC 9000
* §17.2.5 (semantics) + RFC 9001 §5.8 (integrity tag).
*
* Synthesizes valid-and-invalid Retry packets, feeds them through
* [feedDatagram], and asserts the resulting connection state:
*
* 1. Happy path: DCID swaps, retryToken stored, Initial PN reset to 0,
* next outbound Initial carries the token in its header, contains the
* ClientHello CRYPTO, and the datagram is padded to ≥ 1200 bytes.
* 2. Bad-tag path: corrupting the integrity tag must be silently dropped;
* no state advances.
* 3. Second-Retry path: a second valid Retry after a first one is dropped
* (RFC 9000 §17.2.5.2 — at most one Retry per connection).
*/
class RetryHandlingTest {
private fun newClient(): QuicConnection =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
)
/**
* Build the on-wire bytes of a valid Retry packet for [client], with the
* given [retryScid] and [retryToken]. Computes the integrity tag using
* the client's [QuicConnection.originalDestinationConnectionId] so the
* client's [RetryPacket.verifyIntegrityTag] check passes.
*
* The Retry packet's DCID is the client's source CID (servers echo it
* even though it's unused — RFC 9000 §17.2.5.1). The high 4 bits of
* the first byte are 1100 (long header + RETRY type); the low 4 bits
* are unused — we set them to 0.
*/
private fun buildRetry(
client: QuicConnection,
retryScid: ConnectionId,
retryToken: ByteArray,
): ByteArray {
val w = QuicWriter()
// Header form (1) | fixed bit (1) | long packet type RETRY (11) | unused (0000)
w.writeByte(0xC0 or (LongHeaderType.RETRY.code shl 4))
w.writeUint32(QuicVersion.V1)
w.writeByte(client.sourceConnectionId.length)
w.writeBytes(client.sourceConnectionId.bytes)
w.writeByte(retryScid.length)
w.writeBytes(retryScid.bytes)
w.writeBytes(retryToken)
val withoutTag = w.toByteArray()
val tag =
RetryPacket.computeIntegrityTag(
retryPacketWithoutTag = withoutTag,
originalDestinationConnectionId = client.originalDestinationConnectionId.bytes,
)
return withoutTag + tag
}
/**
* Pull the Initial packet's Token field out of an on-wire datagram so
* we can assert on it. [LongHeaderPacket.parseAndDecrypt] decrypts the
* payload but doesn't surface the unprotected Token; we re-walk the
* header here to extract it without crypto.
*/
private fun extractInitialToken(datagram: ByteArray): ByteArray {
val r = QuicReader(datagram, 0)
val first = r.readByte()
require((first and 0x80) != 0) { "expected long header" }
val type = (first ushr 4) and 0x03
require(type == LongHeaderType.INITIAL.code) { "expected INITIAL, got type=$type" }
r.readUint32() // version
val dcidLen = r.readByte()
r.readBytes(dcidLen)
val scidLen = r.readByte()
r.readBytes(scidLen)
val tokenLen = r.readVarint().toInt()
return r.readBytes(tokenLen)
}
@Test
fun valid_retry_swaps_dcid_resets_pn_and_threads_token_into_next_initial() {
val client = newClient()
val originalDcid = client.originalDestinationConnectionId.bytes.copyOf()
client.start()
// Drain the initial datagram (carries ClientHello at PN=0 with empty
// token field) so we can assert the pre-Retry state.
val firstDatagram = drainOutbound(client, nowMillis = 0L)
assertNotNull(firstDatagram, "client.start() should produce an Initial datagram")
assertEquals(0, extractInitialToken(firstDatagram).size, "pre-Retry Initial must have empty token")
// Server picks a fresh source connection id and a token of its choice.
val retryScid = ConnectionId(byteArrayOf(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x78))
val retryToken = "server-issued-retry-token".encodeToByteArray()
val retryDatagram = buildRetry(client, retryScid, retryToken)
feedDatagram(client, retryDatagram, nowMillis = 1L)
// DCID is now the Retry's SCID, originalDcid is unchanged.
assertContentEquals(retryScid.bytes, client.destinationConnectionId.bytes)
assertContentEquals(originalDcid, client.originalDestinationConnectionId.bytes)
assertNotEquals(originalDcid.toList(), retryScid.bytes.toList())
// Retry token captured.
assertContentEquals(retryToken, client.retryToken)
assertTrue(client.retryConsumed)
// RFC 9001 §5.7 + RFC 9000 §17.2.5: the Initial PN namespace
// CONTINUES across the Retry boundary. The first ClientHello at
// PN=0 already consumed PN=0 (it was sent on the wire even though
// the server discarded it in favor of replying with Retry). The
// post-Retry Initial uses PN=1, not PN=0. The runner's retry
// testcase explicitly checks this — a client that resets PN to 0
// gets "Client reset the packet number. Check failed for PN 0".
assertEquals(1L, client.initial.pnSpace.nextPacketNumber)
assertEquals(-1L, client.initial.pnSpace.largestReceived)
// Next drain produces the retried Initial: token in header, ClientHello
// CRYPTO inside, datagram padded to ≥ 1200 (RFC 9000 §14.1).
val secondDatagram = drainOutbound(client, nowMillis = 2L)
assertNotNull(secondDatagram, "post-Retry drain must produce another Initial")
assertContentEquals(retryToken, extractInitialToken(secondDatagram))
assertTrue(
secondDatagram.size >= 1200,
"retried Initial datagram must be padded to >= 1200 bytes (was ${secondDatagram.size})",
)
// The Initial is encrypted under the new keys derived from the retryScid
// DCID. Decrypt + verify it carries CRYPTO with the captured ClientHello
// bytes (== the prefix of the original ClientHello — drained ALL the
// bytes from cryptoSend on Retry replay).
val newSecrets =
com.vitorpamplona.quic.crypto.InitialSecrets
.derive(retryScid.bytes)
val proto = client.initial.sendProtection!!
val parsed =
LongHeaderPacket.parseAndDecrypt(
bytes = secondDatagram,
offset = 0,
aead = proto.aead,
key = newSecrets.clientKey,
iv = newSecrets.clientIv,
hp =
com.vitorpamplona.quic.crypto.AesEcbHeaderProtection(
com.vitorpamplona.quic.crypto.PlatformAesOneBlock,
),
hpKey = newSecrets.clientHp,
largestReceivedInSpace = -1L,
)
assertNotNull(parsed, "retried Initial must decrypt under keys derived from new DCID")
// RFC 9001 §5.7: Initial PN namespace continues across Retry. The
// pre-Retry Initial in this test already consumed PN=0, so the
// retried Initial uses PN=1.
assertEquals(1L, parsed.packet.packetNumber, "retried Initial PN continues from pre-Retry counter (RFC 9001 §5.7)")
// Decoded payload starts with at least one CRYPTO frame (frame type 0x06).
val frames =
com.vitorpamplona.quic.frame
.decodeFrames(parsed.packet.payload)
val cryptoFrames = frames.filterIsInstance<com.vitorpamplona.quic.frame.CryptoFrame>()
assertTrue(cryptoFrames.isNotEmpty(), "retried Initial payload must contain CRYPTO frames (the ClientHello)")
assertEquals(0L, cryptoFrames.first().offset, "CRYPTO must restart at offset 0 on the new keys")
}
@Test
fun retry_with_corrupted_integrity_tag_is_silently_dropped() {
val client = newClient()
val originalDcid = client.destinationConnectionId.bytes.copyOf()
client.start()
// Drain pre-Retry datagram so the test mirrors a realistic ordering.
drainOutbound(client, nowMillis = 0L)
val retryScid = ConnectionId(byteArrayOf(0xAA.toByte(), 0xBB.toByte(), 0xCC.toByte(), 0xDD.toByte()))
val good = buildRetry(client, retryScid, "tk".encodeToByteArray())
// Flip a bit in the last byte — the integrity tag.
val corrupted = good.copyOf()
corrupted[corrupted.size - 1] = (corrupted[corrupted.size - 1].toInt() xor 0x01).toByte()
feedDatagram(client, corrupted, nowMillis = 1L)
// No state advanced.
assertNull(client.retryToken)
assertFalse(client.retryConsumed)
assertContentEquals(originalDcid, client.destinationConnectionId.bytes)
}
@Test
fun second_valid_retry_after_one_is_consumed_is_dropped() {
val client = newClient()
client.start()
drainOutbound(client, nowMillis = 0L)
val firstScid = ConnectionId(byteArrayOf(0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08))
val firstToken = "first".encodeToByteArray()
feedDatagram(client, buildRetry(client, firstScid, firstToken), nowMillis = 1L)
// Sanity: first applied.
assertTrue(client.retryConsumed)
assertContentEquals(firstToken, client.retryToken)
assertContentEquals(firstScid.bytes, client.destinationConnectionId.bytes)
// Build a second VALID Retry. The integrity tag is computed against
// [originalDestinationConnectionId] (still the very first random one,
// unchanged), so this packet's tag genuinely verifies.
val secondScid = ConnectionId(byteArrayOf(0x99.toByte(), 0x88.toByte(), 0x77.toByte(), 0x66.toByte()))
val secondToken = "second-should-be-ignored".encodeToByteArray()
val secondRetry = buildRetry(client, secondScid, secondToken)
// Confirm the integrity tag really would verify in isolation —
// otherwise this test would conflate "bad tag" with "second retry".
val parsedSecond = RetryPacket.parse(secondRetry)
assertNotNull(parsedSecond)
assertTrue(
parsedSecond.verifyIntegrityTag(secondRetry, client.originalDestinationConnectionId.bytes),
"second retry's tag must be valid in isolation; otherwise this test is meaningless",
)
feedDatagram(client, secondRetry, nowMillis = 2L)
// State unchanged from after the first retry.
assertContentEquals(firstToken, client.retryToken)
assertContentEquals(firstScid.bytes, client.destinationConnectionId.bytes)
}
}
@@ -200,12 +200,12 @@ class SentPacketTrackingTest {
/** Cross the half-window threshold (cap=4, two peer-uni streams ⇒ count >= cap-half=2). */
private fun crossPeerUniHalfWindow(client: QuicConnection) =
runBlocking {
client.lock.lock()
client.streamsLock.lock()
try {
client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 0))
client.getOrCreatePeerStreamLocked(StreamId.build(StreamId.Kind.SERVER_UNI, 1))
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
}
}
@@ -82,7 +82,7 @@ class StreamRetransmitTest {
val firstPn = firstPacketEntry.key
// Simulate loss via direct dispatch.
client.lock.lock()
client.streamsLock.lock()
try {
val streamToken =
firstPacketEntry.value.tokens
@@ -93,7 +93,7 @@ class StreamRetransmitTest {
// detector would have done this).
client.application.sentPackets.remove(firstPn)
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
// SendBuffer should have re-queued the bytes for retransmit.
@@ -133,11 +133,11 @@ class StreamRetransmitTest {
val packet =
client.application.sentPackets.entries
.first { it.value.tokens.any { t -> t is RecoveryToken.Stream } }
client.lock.lock()
client.streamsLock.lock()
try {
client.onTokensAcked(packet.value.tokens)
} finally {
client.lock.unlock()
client.streamsLock.unlock()
}
// After ACK: enqueue more, observe that the buffer
@@ -0,0 +1,272 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.connection
import com.vitorpamplona.quic.packet.QuicVersion
import com.vitorpamplona.quic.tls.PermissiveCertificateValidator
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotEquals
import kotlin.test.assertNull
import kotlin.test.assertTrue
/**
* Version Negotiation flow per RFC 9000 §6.
*
* The runner's `versionnegotiation` testcase has the server reply to the
* client's first Initial with a VN packet whose supported_versions list
* contains v1 (and not the version we offered). The client must:
*
* 1. Validate the VN packet (DCID echoed = our SCID, list does NOT
* include the version we offered).
* 2. Pick a version it can speak from the list (we only support v1).
* 3. Generate a fresh DCID, re-derive Initial keys, reset Initial PN
* space, re-emit the cached ClientHello, and switch the writer's
* stamped version to v1.
* 4. Latch `vnConsumed` so a second VN is dropped.
*
* Failure modes also covered:
*
* - Downgrade defense: VN that lists the version we offered is dropped
* (RFC 9000 §6.2 — anti-replay).
* - Unsupported list: VN whose supported_versions doesn't include any
* version we can speak fails the handshake with
* [QuicVersionNegotiationException].
* - Second-VN: after one consumed VN, any subsequent VN is dropped
* even if otherwise valid.
*/
class VersionNegotiationTest {
/**
* Synthesize a VN packet on the wire (RFC 9000 §17.2.1):
* first byte : 0x80 | <random low bits, set to 0 here>
* version : 0x00000000
* dcid_len + dcid (echoes the client's source CID per §6.1)
* scid_len + scid (server picks)
* supported_versions: 32-bit big-endian numbers, one per offered version
*/
private fun encodeVnPacket(
echoedDcid: ConnectionId,
serverScid: ConnectionId,
supportedVersions: List<Int>,
): ByteArray {
val out = ArrayList<Byte>()
out += 0x80.toByte() // form bit set; remaining bits unused / random
// version=0
out += 0x00.toByte()
out += 0x00.toByte()
out += 0x00.toByte()
out += 0x00.toByte()
out += echoedDcid.length.toByte()
for (b in echoedDcid.bytes) out += b
out += serverScid.length.toByte()
for (b in serverScid.bytes) out += b
for (v in supportedVersions) {
out += ((v ushr 24) and 0xFF).toByte()
out += ((v ushr 16) and 0xFF).toByte()
out += ((v ushr 8) and 0xFF).toByte()
out += (v and 0xFF).toByte()
}
return out.toByteArray()
}
private fun newClient(initialVersion: Int) =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
initialVersion = initialVersion,
)
@Test
fun happy_path_vn_switches_to_v1_and_resets_dcid_pn_keys() {
val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION)
client.start()
// The first Initial would be stamped with the forced version.
assertEquals(QuicVersion.FORCE_VERSION_NEGOTIATION, client.currentVersion)
val originalDcid = client.destinationConnectionId
val serverScid = ConnectionId.random(8)
val vn =
encodeVnPacket(
echoedDcid = client.sourceConnectionId,
serverScid = serverScid,
supportedVersions = listOf(QuicVersion.V1),
)
feedDatagram(client, vn, nowMillis = 0L)
assertTrue(client.vnConsumed, "valid VN must latch vnConsumed")
assertEquals(QuicVersion.V1, client.currentVersion, "currentVersion must switch to v1")
assertNotEquals(
originalDcid,
client.destinationConnectionId,
"DCID must be regenerated (RFC 9000 §6 fresh handshake)",
)
// Initial PN space is fresh — next outbound allocate returns 0.
assertEquals(
0L,
client.initial.pnSpace.nextPacketNumber,
"Initial PN space must reset to 0 after VN",
)
// The cached ClientHello must be re-queued so the next drain emits a v1
// Initial with the same handshake bytes.
val drained = drainOutbound(client, nowMillis = 1L)
assertTrue(drained != null && drained.isNotEmpty(), "post-VN drain must emit a fresh Initial")
// Wire-level check: bytes 1..4 of the long-header packet are the version.
val versionOnWire =
((drained[1].toInt() and 0xFF) shl 24) or
((drained[2].toInt() and 0xFF) shl 16) or
((drained[3].toInt() and 0xFF) shl 8) or
(drained[4].toInt() and 0xFF)
assertEquals(
QuicVersion.V1,
versionOnWire,
"post-VN Initial datagram must carry v1 in the long-header version field",
)
}
@Test
fun downgrade_defense_vn_listing_offered_version_is_dropped() {
val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION)
client.start()
val originalDcid = client.destinationConnectionId
val originalVersion = client.currentVersion
val vn =
encodeVnPacket(
echoedDcid = client.sourceConnectionId,
serverScid = ConnectionId.random(8),
// The list MUST NOT contain the version we offered. If it does,
// it's a probable replay/spoof — RFC 9000 §6.2 says drop.
supportedVersions = listOf(QuicVersion.V1, QuicVersion.FORCE_VERSION_NEGOTIATION),
)
feedDatagram(client, vn, nowMillis = 0L)
assertFalse(client.vnConsumed, "anti-replay: VN containing offered version must be dropped")
assertEquals(originalVersion, client.currentVersion, "currentVersion unchanged")
assertEquals(originalDcid, client.destinationConnectionId, "DCID unchanged")
}
@Test
fun unsupported_list_fails_handshake() {
val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION)
client.start()
// quic-go's force-VN test version. We don't support it, so the
// handshake must fail.
val vn =
encodeVnPacket(
echoedDcid = client.sourceConnectionId,
serverScid = ConnectionId.random(8),
supportedVersions = listOf(0x6b3343cf),
)
feedDatagram(client, vn, nowMillis = 0L)
// Unsupported list ⇒ handshake fails BEFORE the latch is set.
// vnConsumed therefore stays false; the connection is forced
// closed via signalHandshakeFailed → markClosedExternally.
assertFalse(client.vnConsumed, "vnConsumed only latches on successful version pick")
assertEquals(
QuicConnection.Status.CLOSED,
client.status,
"no mutually-supported version must close the connection",
)
}
@Test
fun second_vn_is_ignored_after_first() {
val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION)
client.start()
// First VN: valid, switches to v1.
val serverScid1 = ConnectionId.random(8)
feedDatagram(
client,
encodeVnPacket(
echoedDcid = client.sourceConnectionId,
serverScid = serverScid1,
supportedVersions = listOf(QuicVersion.V1),
),
nowMillis = 0L,
)
assertTrue(client.vnConsumed)
assertEquals(QuicVersion.V1, client.currentVersion)
val dcidAfterFirstVn = client.destinationConnectionId
// Second VN: even if structurally fine, must be dropped. We craft
// one whose supported list does NOT include the version we
// ORIGINALLY offered — so it would otherwise look valid.
feedDatagram(
client,
encodeVnPacket(
echoedDcid = client.sourceConnectionId,
serverScid = ConnectionId.random(8),
supportedVersions = listOf(QuicVersion.V1),
),
nowMillis = 1L,
)
assertEquals(
dcidAfterFirstVn,
client.destinationConnectionId,
"second VN must NOT regenerate the DCID",
)
assertEquals(QuicVersion.V1, client.currentVersion, "current version stays at v1")
}
@Test
fun vn_with_dcid_mismatch_is_dropped() {
// Defensive: a VN whose echoed DCID doesn't equal our SCID is
// probably an off-path attacker's spoof — drop without state change.
val client = newClient(QuicVersion.FORCE_VERSION_NEGOTIATION)
client.start()
val vn =
encodeVnPacket(
echoedDcid = ConnectionId.random(8), // wrong
serverScid = ConnectionId.random(8),
supportedVersions = listOf(QuicVersion.V1),
)
feedDatagram(client, vn, nowMillis = 0L)
assertFalse(client.vnConsumed, "DCID mismatch ⇒ no state change")
assertEquals(QuicVersion.FORCE_VERSION_NEGOTIATION, client.currentVersion)
}
@Test
fun default_initial_version_is_v1_for_existing_callers() {
// Backward-compat: not passing initialVersion must keep the writer
// emitting v1 (no behavior change for existing tests).
val client =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
)
assertEquals(QuicVersion.V1, client.currentVersion)
assertFalse(client.vnConsumed)
assertNull(client.peerTransportParameters)
}
}
@@ -0,0 +1,286 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.observability
import com.vitorpamplona.quic.connection.ConnectionId
import com.vitorpamplona.quic.connection.EncryptionLevel
import com.vitorpamplona.quic.connection.InMemoryQuicPipe
import com.vitorpamplona.quic.connection.QuicConnection
import com.vitorpamplona.quic.connection.QuicConnectionConfig
import com.vitorpamplona.quic.connection.TransportParameters
import com.vitorpamplona.quic.connection.feedDatagram
import com.vitorpamplona.quic.tls.InProcessTlsServer
import com.vitorpamplona.quic.tls.PermissiveCertificateValidator
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withLock
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* End-to-end smoke for [QlogObserver]:
*
* 1. NoOp safety — a connection driven through start → handshake →
* close with [QlogObserver.NoOp] must complete without throwing.
* 2. RecordingQlogObserver captures the expected events (start,
* local transport_params, packet_sent for ClientHello, close).
* 3. Malformed inbound datagram surfaces a `packet_dropped` event.
*
* The recorded-event list documented in the implementation report
* comes from the second test below.
*/
class QlogObserverTest {
@Test
fun noOpObserver_handshakeAndCloseDoNotThrow(): Unit =
runBlocking {
val client = handshakedClient(QlogObserver.NoOp)
// Drive a tiny operation post-handshake.
client.lock.withLock {
// No-op — just confirm we can still take the lock.
assertEquals(QuicConnection.Status.CONNECTED, client.status)
}
client.close(0L, "done")
// close() flips status to CLOSING; the writer's next
// drainOutbound emits CONNECTION_CLOSE and transitions to
// CLOSED. We don't run the driver here, so stop at CLOSING
// and assert that — the point of this test is just that
// the no-op observer doesn't throw on any call site.
assertTrue(
client.status == QuicConnection.Status.CLOSING ||
client.status == QuicConnection.Status.CLOSED,
"expected CLOSING or CLOSED, was ${client.status}",
)
}
@Test
fun recordingObserver_capturesStartParametersSentAndClose(): Unit =
runBlocking {
val recorder = RecordingQlogObserver()
val client = handshakedClient(recorder)
client.close(0L, "shutdown")
val names = recorder.events.map { it.name }
assertTrue(
names.contains("connectionStarted"),
"expected connectionStarted in $names",
)
// Local transport parameters set at start.
val localTps =
recorder.events
.filter { it.name == "transportParametersSet" }
.map { it.payload["initiator"] }
assertTrue(
localTps.contains("local"),
"expected local transportParametersSet in $localTps",
)
// At least one packet_sent for the ClientHello at INITIAL level.
val initialSends =
recorder.events.filter {
it.name == "packetSent" && it.payload["level"] == EncryptionLevel.INITIAL
}
assertTrue(
initialSends.isNotEmpty(),
"expected at least one INITIAL-level packetSent (ClientHello) " +
"but saw ${recorder.events.map { it.name to it.payload["level"] }}",
)
assertTrue(
names.contains("connectionClosed"),
"expected connectionClosed in $names",
)
}
@Test
fun malformedDatagram_recordsPacketDropped(): Unit =
runBlocking {
val recorder = RecordingQlogObserver()
val client =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
qlogObserver = recorder,
)
client.start()
// Long-header packet bytes that look parseable enough to peek
// but won't decrypt — the receive keys for HANDSHAKE/APPLICATION
// aren't installed yet, so feedDatagram drops with "no receive
// keys at level …". 0xC0 = long header, INITIAL with the
// smallest valid layout: 0xC0 | version(0x00000001) | dcil=0
// | scil=0 | token_len=0 | length=2 | pn=0x00 | one byte of
// garbage. AEAD-decrypt will fail on this Initial too, since
// the keys are derived from a different DCID.
val garbage =
byteArrayOf(
0xC0.toByte(), // long header initial
0x00,
0x00,
0x00,
0x01, // version v1
0x00, // dcil = 0
0x00, // scil = 0
0x00, // token length varint = 0
0x02, // length = 2
0x00, // pn byte
0x00, // payload byte (will fail AEAD)
)
client.lock.withLock {
feedDatagram(client, garbage, nowMillis = 1L)
}
val drops = recorder.events.filter { it.name == "packetDropped" }
assertTrue(
drops.isNotEmpty(),
"expected at least one packetDropped event but saw ${recorder.events.map { it.name }}",
)
}
private fun handshakedClient(observer: QlogObserver): QuicConnection =
runBlocking {
val client =
QuicConnection(
serverName = "example.test",
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
qlogObserver = observer,
)
val serverScid = ConnectionId.random(8)
val tlsServer =
InProcessTlsServer(
transportParameters =
TransportParameters(
initialMaxData = 1_000_000,
initialMaxStreamDataBidiLocal = 100_000,
initialMaxStreamDataBidiRemote = 100_000,
initialMaxStreamDataUni = 100_000,
initialMaxStreamsBidi = 100,
initialMaxStreamsUni = 100,
initialSourceConnectionId = serverScid.bytes,
originalDestinationConnectionId = client.destinationConnectionId.bytes,
).encode(),
)
val pipe =
InMemoryQuicPipe(
client = client,
initialDcid = client.destinationConnectionId.bytes,
serverScid = serverScid,
tlsServer = tlsServer,
)
client.start()
pipe.drive(maxRounds = 16)
assertEquals(QuicConnection.Status.CONNECTED, client.status)
client
}
}
/**
* Test-only [QlogObserver] that appends every callback into a list,
* with a tiny serialized-payload shape so assertions read like
* structured pattern-matches rather than a soup of positional args.
*/
internal class RecordingQlogObserver : QlogObserver {
data class Event(
val name: String,
val payload: Map<String, Any?>,
)
val events: MutableList<Event> = mutableListOf()
private fun add(
name: String,
payload: Map<String, Any?>,
) {
events += Event(name, payload)
}
override fun onConnectionStarted(
serverName: String,
dcid: ByteArray,
scid: ByteArray,
) = add(
"connectionStarted",
mapOf("serverName" to serverName, "dcid_size" to dcid.size, "scid_size" to scid.size),
)
override fun onConnectionClosed(
initiator: String,
errorCode: Long,
reason: String,
) = add(
"connectionClosed",
mapOf("initiator" to initiator, "errorCode" to errorCode, "reason" to reason),
)
override fun onPacketSent(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
) = add(
"packetSent",
mapOf("level" to level, "pn" to packetNumber, "size" to sizeBytes, "frames" to frames),
)
override fun onPacketReceived(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
) = add(
"packetReceived",
mapOf("level" to level, "pn" to packetNumber, "size" to sizeBytes, "frames" to frames),
)
override fun onPacketDropped(
reason: String,
sizeBytes: Int,
) = add("packetDropped", mapOf("reason" to reason, "size" to sizeBytes))
override fun onKeyUpdated(
keyType: String,
level: EncryptionLevel,
) = add("keyUpdated", mapOf("keyType" to keyType, "level" to level))
override fun onLossDetected(
level: EncryptionLevel,
lostPacketNumbers: List<Long>,
) = add("lossDetected", mapOf("level" to level, "lost" to lostPacketNumbers))
override fun onPtoFired(
consecutivePtoCount: Int,
ptoMillis: Long,
) = add("ptoFired", mapOf("count" to consecutivePtoCount, "ptoMillis" to ptoMillis))
override fun onCongestionStateUpdated(newState: String) = add("congestionStateUpdated", mapOf("newState" to newState))
override fun onTransportParametersSet(
initiator: String,
params: Map<String, String>,
) = add("transportParametersSet", mapOf("initiator" to initiator, "params" to params))
override fun onAlpnNegotiated(alpn: String) = add("alpnNegotiated", mapOf("alpn" to alpn))
override fun onVersionInformation(
chosenVersion: String,
otherVersionsOffered: List<String>,
) = add(
"versionInformation",
mapOf("chosen" to chosenVersion, "offered" to otherVersionsOffered),
)
}
@@ -23,6 +23,8 @@ package com.vitorpamplona.quic.interop
import com.vitorpamplona.quic.connection.QuicConnection
import com.vitorpamplona.quic.connection.QuicConnectionConfig
import com.vitorpamplona.quic.connection.QuicConnectionDriver
import com.vitorpamplona.quic.observability.QlogObserver
import com.vitorpamplona.quic.packet.QuicVersion
import com.vitorpamplona.quic.tls.PermissiveCertificateValidator
import com.vitorpamplona.quic.transport.UdpSocket
import kotlinx.coroutines.CoroutineScope
@@ -31,6 +33,7 @@ import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeoutOrNull
import java.io.File
/**
* Standalone interop runner. Drives [QuicConnection] against a real QUIC
@@ -52,12 +55,41 @@ fun main(args: Array<String>) {
val host = System.getProperty("interopHost") ?: args.getOrNull(0) ?: "127.0.0.1"
val port = (System.getProperty("interopPort") ?: args.getOrNull(1) ?: "4433").toInt()
val timeoutSec = (System.getProperty("interopTimeoutSec") ?: "10").toLong()
// Public quic-interop-runner contract: TESTCASE env names the scenario.
// We currently only special-case `versionnegotiation`; everything else
// falls through to the default v1-handshake path, which is enough for
// the `transfer` / `handshake` / `multiconnect` testcases.
val testcase =
System.getProperty("interopTestcase")
?: System.getenv("TESTCASE")
?: args.getOrNull(2)
val initialVersion =
when (testcase) {
"versionnegotiation" -> QuicVersion.FORCE_VERSION_NEGOTIATION
else -> QuicVersion.V1
}
println("== :quic interop runner ==")
println("target: $host:$port")
println("timeout: ${timeoutSec}s")
println("target: $host:$port")
println("testcase: ${testcase ?: "(default)"}")
println("init ver: 0x${initialVersion.toUInt().toString(16)}")
println("timeout: ${timeoutSec}s")
println()
// qlog: if QLOGDIR is set, drop a `client.sqlog` file at
// <QLOGDIR>/client.sqlog so the operator can hand the failed run
// to qvis. The interop-runner contract from the IETF QUIC team's
// Docker harness (quic-interop-runner) sets this env var on every
// test invocation.
val qlogDir =
(System.getenv("QLOGDIR") ?: System.getProperty("QLOGDIR"))?.takeIf { it.isNotBlank() }
val qlogWriter: QlogWriter? =
qlogDir?.let { dir ->
val parent = File(dir).also { it.mkdirs() }
QlogWriter(File(parent, "client.sqlog"), odcidHex = "00")
}
val qlogObserver: QlogObserver = qlogWriter ?: QlogObserver.NoOp
val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val outcome =
runBlocking {
@@ -73,6 +105,7 @@ fun main(args: Array<String>) {
serverName = host,
config = QuicConnectionConfig(),
tlsCertificateValidator = PermissiveCertificateValidator(),
qlogObserver = qlogObserver,
)
val driver = QuicConnectionDriver(conn, socket, scope)
driver.start()
@@ -110,6 +143,10 @@ fun main(args: Array<String>) {
// are torn down before main() exits. Without this the JVM hangs on stray
// IO-dispatcher threads.
scope.cancel()
// Flush + close the qlog file before exiting. Without this, an
// exitProcess() call below could leave the trailing events
// unflushed.
runCatching { qlogWriter?.close() }
when (outcome) {
is InteropOutcome.Connected -> {
@@ -0,0 +1,319 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.vitorpamplona.quic.connection.EncryptionLevel
import com.vitorpamplona.quic.observability.QlogObserver
import java.io.BufferedWriter
import java.io.Closeable
import java.io.File
import java.io.FileWriter
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
/**
* JSON-NDJSON qlog writer (qlog 0.3 / JSON-SEQ format) used by the
* `:quic` interop runner. One JSON object per line; first line is the
* qlog header, subsequent lines are events.
*
* Tools like qvis (https://qvis.quictools.info/) consume the resulting
* `.sqlog` file to render sequence diagrams + RTT graphs + recovery
* timelines.
*
* **Goal: every interop-runner test failure produces a qlog file the
* caller can drop into qvis to see exactly what we did differently
* from the spec.**
*
* Threading: [java.io.BufferedWriter] is not safe for concurrent
* writers; we hold a [ReentrantLock] around each line emit so the
* read + send loops can fire events concurrently without interleaving
* partial JSON.
*/
class QlogWriter(
file: File,
private val odcidHex: String,
private val mapper: ObjectMapper = DEFAULT_MAPPER,
/** Wall-clock provider; tests inject a deterministic source. */
private val nowMillis: () -> Long = { System.currentTimeMillis() },
) : QlogObserver,
Closeable {
// append=false → truncate any prior trace at this path so a reused
// QLOGDIR doesn't accumulate stale events from a previous run.
private val writer: BufferedWriter = BufferedWriter(FileWriter(file, false))
private val lock = ReentrantLock()
private val startMillis: Long = nowMillis()
init {
// qlog 0.3 JSON-SEQ header. qvis tolerates both `qlog_format`
// values "JSON-SEQ" and "NDJSON"; we use JSON-SEQ to match the
// most-common production qlog files (Chromium, mvfst).
val header =
mapOf(
"qlog_version" to "0.3",
"qlog_format" to "JSON-SEQ",
"title" to "amethyst :quic client trace",
"trace" to
mapOf(
"vantage_point" to mapOf("type" to "client", "name" to "amethyst-quic"),
"common_fields" to
mapOf(
"ODCID" to odcidHex,
"reference_time" to startMillis,
"time_format" to "relative",
),
),
)
writeLineLocked(mapper.writeValueAsString(header))
}
override fun onConnectionStarted(
serverName: String,
dcid: ByteArray,
scid: ByteArray,
) {
emit(
"transport:connection_started",
mapOf(
"ip_version" to "v4_or_v6",
"server_name" to serverName,
"dst_cid" to hex(dcid),
"src_cid" to hex(scid),
),
)
}
override fun onConnectionClosed(
initiator: String,
errorCode: Long,
reason: String,
) {
emit(
"transport:connection_closed",
mapOf(
"owner" to initiator,
"application_code" to errorCode,
"reason" to reason,
),
)
}
override fun onPacketSent(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
) {
emit(
"transport:packet_sent",
mapOf(
"header" to
mapOf(
"packet_type" to packetTypeFor(level),
"packet_number" to packetNumber,
),
"raw" to mapOf("length" to sizeBytes),
"frames" to frames.map { mapOf("frame_type" to it) },
),
)
}
override fun onPacketReceived(
level: EncryptionLevel,
packetNumber: Long,
sizeBytes: Int,
frames: List<String>,
) {
emit(
"transport:packet_received",
mapOf(
"header" to
mapOf(
"packet_type" to packetTypeFor(level),
"packet_number" to packetNumber,
),
"raw" to mapOf("length" to sizeBytes),
"frames" to frames.map { mapOf("frame_type" to it) },
),
)
}
override fun onPacketDropped(
reason: String,
sizeBytes: Int,
) {
emit(
"transport:packet_dropped",
mapOf(
"trigger" to reason,
"raw" to mapOf("length" to sizeBytes),
),
)
}
override fun onKeyUpdated(
keyType: String,
level: EncryptionLevel,
) {
emit(
"security:key_updated",
mapOf(
"key_type" to "${keyType}_${packetTypeFor(level)}_secret",
"trigger" to "tls",
),
)
}
override fun onLossDetected(
level: EncryptionLevel,
lostPacketNumbers: List<Long>,
) {
for (pn in lostPacketNumbers) {
emit(
"recovery:packet_lost",
mapOf(
"header" to
mapOf(
"packet_type" to packetTypeFor(level),
"packet_number" to pn,
),
"trigger" to "reordering_threshold_or_time_threshold",
),
)
}
}
override fun onPtoFired(
consecutivePtoCount: Int,
ptoMillis: Long,
) {
emit(
"recovery:loss_timer_updated",
mapOf(
"event_type" to "expired",
"timer_type" to "pto",
"pto_count" to consecutivePtoCount,
"delta" to ptoMillis,
),
)
}
override fun onCongestionStateUpdated(newState: String) {
emit(
"recovery:congestion_state_updated",
mapOf("new" to newState),
)
}
override fun onTransportParametersSet(
initiator: String,
params: Map<String, String>,
) {
emit(
"transport:parameters_set",
mapOf(
"owner" to initiator,
"params" to params,
),
)
}
override fun onAlpnNegotiated(alpn: String) {
emit(
"transport:alpn_information",
mapOf("chosen_alpn" to alpn),
)
}
override fun onVersionInformation(
chosenVersion: String,
otherVersionsOffered: List<String>,
) {
emit(
"transport:version_information",
mapOf(
"chosen_version" to chosenVersion,
"client_versions" to otherVersionsOffered,
),
)
}
override fun close() {
lock.withLock {
try {
writer.flush()
} finally {
writer.close()
}
}
}
private fun emit(
name: String,
data: Map<String, Any?>,
) {
val event =
linkedMapOf<String, Any?>(
"time" to (nowMillis() - startMillis),
"name" to name,
"data" to data,
)
// Serialize OUTSIDE the lock so concurrent emitters don't
// serialize their JSON serially. The lock is only held while
// appending the line to the file.
val line = mapper.writeValueAsString(event)
writeLineLocked(line)
}
private fun writeLineLocked(line: String) {
lock.withLock {
writer.write(line)
writer.write("\n")
// Flush every line so a hard-killed process still leaves a
// partial-but-parseable trace.
writer.flush()
}
}
companion object {
private val DEFAULT_MAPPER: ObjectMapper = jacksonObjectMapper()
private fun packetTypeFor(level: EncryptionLevel): String =
when (level) {
EncryptionLevel.INITIAL -> "initial"
EncryptionLevel.HANDSHAKE -> "handshake"
EncryptionLevel.APPLICATION -> "1RTT"
}
private val HEX_CHARS = "0123456789abcdef".toCharArray()
fun hex(bytes: ByteArray): String {
val sb = StringBuilder(bytes.size * 2)
for (b in bytes) {
val v = b.toInt() and 0xFF
sb.append(HEX_CHARS[v ushr 4])
sb.append(HEX_CHARS[v and 0x0F])
}
return sb.toString()
}
}
}
@@ -0,0 +1,161 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quic.interop
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.vitorpamplona.quic.connection.EncryptionLevel
import org.junit.Test
import java.io.File
import java.nio.file.Files
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
/**
* Validates the JSON-NDJSON shape qvis (https://qvis.quictools.info/)
* expects: header line + one event per line, every line independently
* parseable as JSON.
*
* Drives [QlogWriter] through one event of each type to confirm we
* don't emit anything that breaks the format.
*/
class QlogWriterTest {
@Test
fun headerThenOneEventPerLine_allParseable() {
val tmp = Files.createTempFile("amethyst-qlog-test", ".sqlog").toFile()
tmp.deleteOnExit()
var clock = 0L
QlogWriter(tmp, odcidHex = "deadbeef", nowMillis = { clock }).use { w ->
clock = 5L
w.onConnectionStarted("example.test", byteArrayOf(1, 2, 3), byteArrayOf(4, 5, 6))
clock = 7L
w.onTransportParametersSet("local", mapOf("initial_max_data" to "1000000"))
clock = 10L
w.onPacketSent(EncryptionLevel.INITIAL, packetNumber = 0, sizeBytes = 1200, frames = listOf("crypto"))
clock = 15L
w.onPacketReceived(EncryptionLevel.INITIAL, packetNumber = 0, sizeBytes = 800, frames = listOf("crypto", "ack"))
clock = 20L
w.onPacketDropped("AEAD auth failed", sizeBytes = 80)
clock = 25L
w.onKeyUpdated("server", EncryptionLevel.HANDSHAKE)
clock = 30L
w.onLossDetected(EncryptionLevel.APPLICATION, lostPacketNumbers = listOf(3L, 5L))
clock = 35L
w.onPtoFired(consecutivePtoCount = 1, ptoMillis = 333)
clock = 40L
w.onCongestionStateUpdated("recovery")
clock = 45L
w.onAlpnNegotiated("h3")
clock = 50L
w.onVersionInformation("v1", emptyList())
clock = 55L
w.onConnectionClosed("local", errorCode = 0, reason = "done")
}
val lines = tmp.readLines().filter { it.isNotBlank() }
assertTrue(lines.size >= 12, "expected >= 12 lines (header + at least 11 events) but got ${lines.size}")
val mapper = jacksonObjectMapper()
// Line 1: qlog header.
val header = mapper.readTree(lines[0])
assertEquals("0.3", header.get("qlog_version").asText(), "qlog_version must be 0.3")
assertEquals("JSON-SEQ", header.get("qlog_format").asText(), "qlog_format must be JSON-SEQ")
val vp = header.get("trace").get("vantage_point")
assertEquals("client", vp.get("type").asText())
assertEquals(
"deadbeef",
header
.get("trace")
.get("common_fields")
.get("ODCID")
.asText(),
)
// Lines 2..N: event objects with `time`, `name`, `data`.
for (i in 1 until lines.size) {
val node = mapper.readTree(lines[i])
assertNotNull(node.get("time"), "line $i missing 'time': ${lines[i]}")
val name = node.get("name")
assertNotNull(name, "line $i missing 'name': ${lines[i]}")
assertTrue(
name.asText().contains(":"),
"name '${name.asText()}' must be in '<category>:<event>' form",
)
assertNotNull(node.get("data"), "line $i missing 'data': ${lines[i]}")
}
// Spot-check specific events made it through.
val names = lines.drop(1).map { mapper.readTree(it).get("name").asText() }
assertTrue(names.contains("transport:connection_started"), names.toString())
assertTrue(names.contains("transport:packet_sent"), names.toString())
assertTrue(names.contains("transport:packet_received"), names.toString())
assertTrue(names.contains("transport:packet_dropped"), names.toString())
assertTrue(names.contains("security:key_updated"), names.toString())
assertTrue(names.contains("recovery:packet_lost"), names.toString())
assertTrue(names.contains("recovery:loss_timer_updated"), names.toString())
assertTrue(names.contains("transport:parameters_set"), names.toString())
assertTrue(names.contains("transport:alpn_information"), names.toString())
assertTrue(names.contains("transport:version_information"), names.toString())
assertTrue(names.contains("transport:connection_closed"), names.toString())
}
@Test
fun timesAreRelativeToConstructorTime() {
val tmp = Files.createTempFile("amethyst-qlog-rel", ".sqlog").toFile()
tmp.deleteOnExit()
var clock = 1_000L
QlogWriter(tmp, odcidHex = "00", nowMillis = { clock }).use { w ->
clock = 1_050L
w.onAlpnNegotiated("h3")
}
val lines = tmp.readLines().filter { it.isNotBlank() }
val mapper = jacksonObjectMapper()
val event = mapper.readTree(lines[1])
assertEquals(50L, event.get("time").asLong(), "time must be relative to constructor (1050 - 1000)")
}
@Test
fun fileEndsWithNewline_qvisCompatible() {
val tmp = Files.createTempFile("amethyst-qlog-nl", ".sqlog").toFile()
tmp.deleteOnExit()
QlogWriter(tmp, odcidHex = "00").use { w ->
w.onAlpnNegotiated("h3")
}
val bytes = tmp.readBytes()
assertTrue(bytes.isNotEmpty(), "file must not be empty")
assertEquals('\n'.code.toByte(), bytes.last(), "file must end with '\\n' so trailing event parses")
}
@Test
fun handlesEmptyFramesList(): Unit =
File.createTempFile("amethyst-qlog-empty", ".sqlog").let { tmp ->
tmp.deleteOnExit()
QlogWriter(tmp, odcidHex = "00").use { w ->
w.onPacketSent(EncryptionLevel.INITIAL, 0, 1200, emptyList())
}
val mapper = jacksonObjectMapper()
val lines = tmp.readLines().filter { it.isNotBlank() }
val frames = mapper.readTree(lines[1]).get("data").get("frames")
assertTrue(frames.isArray, "frames must be an array even when empty")
assertEquals(0, frames.size())
}
}
+2
View File
@@ -41,3 +41,5 @@ include ':quic'
include ':nestsClient'
include ':desktopApp'
include ':cli'
include ':quic-interop'
project(':quic-interop').projectDir = file('quic/interop')