mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-10 08:27:04 +00:00
fix(graperank): run reachability probe on an isolated thread pool
The probe does blocking DNS + TCP connect, and dead-domain DNS lookups hang well past the connect timeout. On the shared Dispatchers.IO those hanging lookups starved the crawl's own IO: an A/B at hop-3 showed probe-on 981s vs probe-off 517s, the entire +464s landing on the finishing drain (rounds were identical). Coverage was unchanged (91.84% vs 91.74%), so the probe classification is correct — it was purely IO contention. Give the probe its own fixed daemon pool (128 threads) so its blocking work can never touch the crawl's IO, and align the culler's concurrency to it. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MSW59hJtP4Yn8fnRUxc7F5
This commit is contained in:
@@ -54,6 +54,7 @@ import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.asCoroutineDispatcher
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.awaitAll
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
@@ -61,6 +62,7 @@ import kotlinx.coroutines.withContext
|
||||
import java.net.InetSocketAddress
|
||||
import java.net.Socket
|
||||
import java.net.URI
|
||||
import java.util.concurrent.Executors
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
/**
|
||||
@@ -129,6 +131,15 @@ object GrapeRankCommand {
|
||||
|
||||
private const val PROBE_TIMEOUT_MS = 2000
|
||||
|
||||
// The probe does BLOCKING DNS + TCP connect, and dead-domain DNS lookups can hang
|
||||
// far past the connect timeout. On the shared Dispatchers.IO those hanging lookups
|
||||
// starve the crawl's own IO — measured +462s on the finishing drain at hop-3. Run
|
||||
// them on a dedicated, isolated daemon pool instead so the crawl's IO is untouched.
|
||||
private val probeDispatcher =
|
||||
Executors
|
||||
.newFixedThreadPool(128) { r -> Thread(r, "relay-probe").apply { isDaemon = true } }
|
||||
.asCoroutineDispatcher()
|
||||
|
||||
/**
|
||||
* Cheap reachability pre-probe: a raw TCP connect (one round trip) with a tight
|
||||
* timeout. Returns false only when the port won't even accept a socket — a dead
|
||||
@@ -139,7 +150,7 @@ object GrapeRankCommand {
|
||||
* so an odd URL is never culled on a parse quirk — let the WS decide.
|
||||
*/
|
||||
private suspend fun tcpReachable(relay: NormalizedRelayUrl): Boolean =
|
||||
withContext(Dispatchers.IO) {
|
||||
withContext(probeDispatcher) {
|
||||
val hostPort = relayHostPort(relay) ?: return@withContext true
|
||||
try {
|
||||
Socket().use { it.connect(InetSocketAddress(hostPort.first, hostPort.second), PROBE_TIMEOUT_MS) }
|
||||
|
||||
Reference in New Issue
Block a user