From 4f52027ccbb577fdb9fddf962f15dfef7374ed8b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 7 May 2026 01:44:37 +0000 Subject: [PATCH] =?UTF-8?q?fix(quic-interop):=20wake=20send=20loop=20on=20?= =?UTF-8?q?every=20queued=20request=20=E2=80=94=20fixes=20throughput?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis (qlog): 1361 GETs in 58s = 23/sec, but only 2618 packets sent in that window — 0.5 GETs per packet. We were sending nearly empty packets one at a time, not coalescing requests. Root cause: stream.send.enqueue + stream.send.finish() don't wake the send loop. The send loop suspends on sendWakeup until either an inbound packet arrives or the PTO timer fires (~1s). For our chunked parallel multiplexing path: 1. enqueue 64 GETs into 64 streams 2. send loop is asleep (last drain happened on previous chunk) 3. wait ~1s for PTO before any of them go on the wire 4. server processes, replies, our read loop wakes the send loop 5. send loop drains ACKs (no new requests yet) Each chunk wasted ~1s of PTO wait. With ~21 chunks at 1s each plus RTTs and server processing, throughput floored at ~23 streams/sec. Fix: pass QuicConnectionDriver to Http3GetClient + HqInteropGetClient constructors. After stream.send.finish(), call driver.wakeup() to nudge the send loop. The first request of each chunk now leaves within microseconds instead of waiting for PTO. Predicted throughput: 600+ streams/sec (RTT-bound at 30ms per chunk of 64 = 2100/sec ceiling; CPU and lock contention drop it to ~600). 1999 streams in 3-5s instead of timing out at 60s. The architectural fix would be having stream.send.enqueue auto-wake via a callback set during stream creation. That's the cleaner shape; this commit takes the minimal path: explicit driver-passed wakeup from interop client code where the throughput matters. https://claude.ai/code/session_01HcvfQq1ttPV9PkRoJb4nyT --- .../quic/interop/runner/HqInteropGetClient.kt | 4 ++++ .../vitorpamplona/quic/interop/runner/Http3GetClient.kt | 7 +++++++ .../com/vitorpamplona/quic/interop/runner/InteropClient.kt | 6 +++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/HqInteropGetClient.kt b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/HqInteropGetClient.kt index a1745efb06..10c249c9f8 100644 --- a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/HqInteropGetClient.kt +++ b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/HqInteropGetClient.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.quic.interop.runner import com.vitorpamplona.quic.connection.QuicConnection +import com.vitorpamplona.quic.connection.QuicConnectionDriver import kotlinx.coroutines.flow.toList /** @@ -38,6 +39,7 @@ import kotlinx.coroutines.flow.toList */ class HqInteropGetClient( private val conn: QuicConnection, + private val driver: QuicConnectionDriver, ) : GetClient { override suspend fun get( @Suppress("UNUSED_PARAMETER") authority: String, @@ -47,6 +49,8 @@ class HqInteropGetClient( val request = "GET $path\r\n".encodeToByteArray() stream.send.enqueue(request) stream.send.finish() + // Nudge the send loop — see Http3GetClient.get for rationale. + driver.wakeup() val chunks = stream.incoming.toList() val total = chunks.sumOf { it.size } diff --git a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClient.kt b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClient.kt index 734be314a0..ca2571d840 100644 --- a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClient.kt +++ b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/Http3GetClient.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quic.interop.runner import com.vitorpamplona.quic.QuicWriter import com.vitorpamplona.quic.connection.QuicConnection +import com.vitorpamplona.quic.connection.QuicConnectionDriver import com.vitorpamplona.quic.connection.drainPeerInitiatedUniStreamsIntoBlackHole import com.vitorpamplona.quic.http3.Http3Frame import com.vitorpamplona.quic.http3.Http3FrameReader @@ -60,6 +61,7 @@ data class GetResponse( */ class Http3GetClient( private val conn: QuicConnection, + private val driver: QuicConnectionDriver, ) : GetClient { suspend fun init(scope: CoroutineScope) { // Control stream: type-0x00 prefix followed by a SETTINGS frame @@ -111,6 +113,11 @@ class Http3GetClient( val stream = conn.openBidiStream() stream.send.enqueue(encodeRequest(authority, path)) stream.send.finish() + // Nudge the send loop. Without this it suspends until PTO (~1s) + // or until an inbound packet arrives. For the multiplexing path + // this was the dominant throughput bottleneck — chunks of 64 + // requests sat idle for ~1s each waiting to be drained. + driver.wakeup() val reader = Http3FrameReader() var status = 0 diff --git a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/InteropClient.kt b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/InteropClient.kt index 5532186659..40bab3887a 100644 --- a/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/InteropClient.kt +++ b/quic/interop/src/main/kotlin/com/vitorpamplona/quic/interop/runner/InteropClient.kt @@ -291,16 +291,16 @@ private fun runTransferTest( val client: GetClient = when (negotiated) { "h3" -> { - Http3GetClient(conn).also { it.init(scope) } + Http3GetClient(conn, driver).also { it.init(scope) } } "hq-interop" -> { - HqInteropGetClient(conn) + HqInteropGetClient(conn, driver) } else -> { System.err.println("unrecognized negotiated ALPN '$negotiated'; defaulting to hq-interop") - HqInteropGetClient(conn) + HqInteropGetClient(conn, driver) } }