From e595d42c49e43a2f4d17a4f97896fec074d88b5f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 03:02:51 +0000 Subject: [PATCH] perf(graperank): run the crawl on Dispatchers.IO, not the caller's loop The CLI calls crawl() from runBlocking's single-threaded event loop, so Phase B's thousands of concurrent drain-unit coroutines (timeout timers, channels, REQ JSON encoding, signature verifies, SQLite writes) all queued on ONE thread: timers fired late, batch walls inflated ~5x, and the --diagnose ticker showed 24/24 workers pinned while completing ~1 user/s with one core pegged and three idle. It also explains why the old 64-worker A/B ran slower - more coroutines on the same thread. Hop the whole run onto Dispatchers.IO inside the crawler so every front end gets real parallelism; IO (not Default) because the store's blocking SQLite calls must not starve the cores-sized pool. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_013zEYRGKF943RgLaHTViJaB --- .../experimental/graperank/GrapeRankCrawler.kt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt index 5aaa544c52..32a9ed36f1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt @@ -44,6 +44,8 @@ import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap import com.vitorpamplona.quartz.utils.concurrent.ConcurrentSet import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll @@ -57,6 +59,7 @@ import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.selects.select import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull import kotlin.concurrent.atomics.AtomicLong import kotlin.concurrent.atomics.ExperimentalAtomicApi @@ -278,7 +281,15 @@ class GrapeRankCrawler( verifyNanos.store(0) insertNanos.store(0) eventsStored.store(0) - return CrawlRun(observer, builder).run() + // Hop off the caller's dispatcher: a CLI calls this from runBlocking's + // SINGLE-THREADED event loop, and Phase B runs thousands of concurrent + // drain-unit coroutines (each with a timeout timer, a channel, REQ JSON + // encoding, signature verifies and store writes). On one thread those + // timers fire late and batch walls inflate ~5x — measured 111 users/s + // with the lone thread pegged at 100% while 3 cores idled. IO (not + // Default): the store's blocking SQLite calls must not starve the + // small cores-sized pool. + return withContext(Dispatchers.IO) { CrawlRun(observer, builder).run() } } /**