mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-09 08:04:45 +00:00
fix(graperank): run reachability probe on an isolated thread pool
The probe does blocking DNS + TCP connect, and dead-domain DNS lookups hang well past the connect timeout. On the shared Dispatchers.IO those hanging lookups starved the crawl's own IO: an A/B at hop-3 showed probe-on 981s vs probe-off 517s, the entire +464s landing on the finishing drain (rounds were identical). Coverage was unchanged (91.84% vs 91.74%), so the probe classification is correct — it was purely IO contention. Give the probe its own fixed daemon pool (128 threads) so its blocking work can never touch the crawl's IO, and align the culler's concurrency to it. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MSW59hJtP4Yn8fnRUxc7F5
This commit is contained in:
@@ -54,6 +54,7 @@ import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.asCoroutineDispatcher
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.awaitAll
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
@@ -61,6 +62,7 @@ import kotlinx.coroutines.withContext
|
||||
import java.net.InetSocketAddress
|
||||
import java.net.Socket
|
||||
import java.net.URI
|
||||
import java.util.concurrent.Executors
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
/**
|
||||
@@ -129,6 +131,15 @@ object GrapeRankCommand {
|
||||
|
||||
private const val PROBE_TIMEOUT_MS = 2000
|
||||
|
||||
// The probe does BLOCKING DNS + TCP connect, and dead-domain DNS lookups can hang
|
||||
// far past the connect timeout. On the shared Dispatchers.IO those hanging lookups
|
||||
// starve the crawl's own IO — measured +462s on the finishing drain at hop-3. Run
|
||||
// them on a dedicated, isolated daemon pool instead so the crawl's IO is untouched.
|
||||
private val probeDispatcher =
|
||||
Executors
|
||||
.newFixedThreadPool(128) { r -> Thread(r, "relay-probe").apply { isDaemon = true } }
|
||||
.asCoroutineDispatcher()
|
||||
|
||||
/**
|
||||
* Cheap reachability pre-probe: a raw TCP connect (one round trip) with a tight
|
||||
* timeout. Returns false only when the port won't even accept a socket — a dead
|
||||
@@ -139,7 +150,7 @@ object GrapeRankCommand {
|
||||
* so an odd URL is never culled on a parse quirk — let the WS decide.
|
||||
*/
|
||||
private suspend fun tcpReachable(relay: NormalizedRelayUrl): Boolean =
|
||||
withContext(Dispatchers.IO) {
|
||||
withContext(probeDispatcher) {
|
||||
val hostPort = relayHostPort(relay) ?: return@withContext true
|
||||
try {
|
||||
Socket().use { it.connect(InetSocketAddress(hostPort.first, hostPort.second), PROBE_TIMEOUT_MS) }
|
||||
|
||||
+1
-1
@@ -1666,7 +1666,7 @@ class GrapeRankDataCrawler(
|
||||
// Concurrent TCP reachability probes in the background culler. Raw sockets are
|
||||
// cheap and short-lived; the per-relay WS limiter is unaffected (this never
|
||||
// opens a REQ), so this only bounds file descriptors during the cull.
|
||||
private const val PROBE_CONCURRENCY = 256
|
||||
private const val PROBE_CONCURRENCY = 128
|
||||
|
||||
// Re-scan interval for the culler when it has probed everything learned so far
|
||||
// and is waiting for new relays to be discovered.
|
||||
|
||||
Reference in New Issue
Block a user