mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-09 08:04:45 +00:00
feat(graperank): TCP reachability pre-probe + .onion skip to cull the dead graveyard
At hop-8 the crawl dials into thousands of dead relay hints from old accounts. Most fail slowly: a silently-dropping host has no RST to receive, so the WS connect just hangs to the 7s connectTimeout. First-strike eviction pays that once per host, but with ~3,000 dead hosts that's ~80s of connect-setup serialized through the dispatcher. Add a background reachability culler: a cheap raw TCP connect (one round trip, 2s timeout) over the learned relays COLD-TAIL FIRST, dropping the unreachable ones into deadHosts before the WS path pays its 7s. The key property is that a tight TCP timeout is safe where a tight WS timeout is not — a busy-but-alive relay accepts the SYN instantly at the kernel level and only stalls at the app layer, so the probe separates "unreachable" from "slow" and never false-kills the busy. It only ever marks dead and probes each authority once; a host the WS path already resolved (isDead) is skipped, and a live host passes the probe, so the WS verdict always wins. Injected as an optional Config.reachabilityProbe (JVM: java.net.Socket in the CLI; --no-probe disables); writeRelayFreq becomes concurrent so the culler can read it while routeByOutbox writes. Also: when there's no Tor transport (Config.torEnabled=false), isDead skips every .onion relay on sight — no socket, no wasted connect. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MSW59hJtP4Yn8fnRUxc7F5
This commit is contained in:
@@ -52,10 +52,15 @@ import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProvider
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceType
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.awaitAll
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.net.InetSocketAddress
|
||||
import java.net.Socket
|
||||
import java.net.URI
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
/**
|
||||
@@ -122,6 +127,46 @@ object GrapeRankCommand {
|
||||
"wss://nos.lol",
|
||||
).mapNotNull { RelayUrlNormalizer.normalizeOrNull(it) }.toSet()
|
||||
|
||||
private const val PROBE_TIMEOUT_MS = 2000
|
||||
|
||||
/**
|
||||
* Cheap reachability pre-probe: a raw TCP connect (one round trip) with a tight
|
||||
* timeout. Returns false only when the port won't even accept a socket — a dead
|
||||
* dropper, refusal, or unroutable/onion/LAN host — which the crawler drops into
|
||||
* deadHosts before the WS path pays its 7s connectTimeout. A busy-but-alive relay
|
||||
* accepts the SYN instantly at the kernel level (its slowness is at the app layer),
|
||||
* so it passes here and is left for the real WS attempt. Unparseable host → true,
|
||||
* so an odd URL is never culled on a parse quirk — let the WS decide.
|
||||
*/
|
||||
private suspend fun tcpReachable(relay: NormalizedRelayUrl): Boolean =
|
||||
withContext(Dispatchers.IO) {
|
||||
val hostPort = relayHostPort(relay) ?: return@withContext true
|
||||
try {
|
||||
Socket().use { it.connect(InetSocketAddress(hostPort.first, hostPort.second), PROBE_TIMEOUT_MS) }
|
||||
true
|
||||
} catch (e: Exception) {
|
||||
if (e is CancellationException) throw e
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
private fun relayHostPort(relay: NormalizedRelayUrl): Pair<String, Int>? =
|
||||
try {
|
||||
val uri = URI(relay.url)
|
||||
val host = uri.host ?: return null
|
||||
val port =
|
||||
if (uri.port > 0) {
|
||||
uri.port
|
||||
} else if (relay.url.startsWith("wss://", ignoreCase = true)) {
|
||||
443
|
||||
} else {
|
||||
80
|
||||
}
|
||||
host to port
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
|
||||
suspend fun dispatch(
|
||||
dataDir: DataDir,
|
||||
tail: Array<String>,
|
||||
@@ -396,6 +441,10 @@ object GrapeRankCommand {
|
||||
insertBatchSize = args.intFlag("insert-batch", 500),
|
||||
drainConcurrency = args.intFlag("drain-concurrency", 24),
|
||||
timeoutEvictStrikes = args.intFlag("timeout-evict", 3),
|
||||
// Cheap TCP reachability pre-probe (--no-probe to disable). No Tor
|
||||
// transport here, so .onion relays are skipped on sight.
|
||||
reachabilityProbe = if (args.bool("no-probe")) null else ::tcpReachable,
|
||||
torEnabled = false,
|
||||
// shedDeadDiscovery / shardRotations keep their benchmarked-best
|
||||
// Config defaults.
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user