mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-10 16:33:27 +00:00
perf(graperank): mass pre-connect, cached DNS, and an amy relay census probe
Most of a from-scratch crawl's wall clock was connection setup, re-paid serially: the relay pool tears down a relay's socket ~300ms after its last drain unsubscribes, OkHttp allowed only 256 concurrent WS handshakes, and every dial re-ran an uncached blocking getaddrinfo (10-30s per dead domain, once per per-user path URL of the same host). Open the connections once, in parallel, and keep them: - GrapeRankCrawler: the warm-pool trick (never-matching REQ that only holds the socket) now covers the whole candidate universe instead of the top 20 - seeded at crawl start from the reachability cache's live set (one parallel connection storm, Config.knownLiveRelays) and refreshed each round with newly learned outbox relays, capped by Config.preconnectCap (FD-budget aware, --preconnect-cap / --no-preconnect). - CachingDns (quartz jvmAndroid): 10-min positive + negative DNS cache with in-flight per-host dedup; dead domains fail in microseconds instead of re-burning resolver timeouts, path URLs of one host resolve once. - cli Context: dispatcher and pre-connect caps derived from the process's open-files limit (UnixOperatingSystemMXBean), warning when ulimit is low. - amy graperank probe: relay census - mass-connects every relay the store knows (kind:10002 universe deduped per authority + cached verdicts) in waves, records live/dead with real measured rtt-open into the NIP-66 reachability cache (RelayProber + RelayReachabilityStore.recordProbed), so the next crawl skips dead relays and waits once for the slow-but-alive. Single-server limits (FDs, ephemeral ports, DNS, threads, conntrack) and the design are documented in quartz/plans/2026-07-10-graperank-connect-storm.md. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013zEYRGKF943RgLaHTViJaB
This commit is contained in:
Claude
@@ -20,6 +20,7 @@
|
||||
*/
|
||||
package com.vitorpamplona.amethyst.cli
|
||||
|
||||
import com.sun.management.UnixOperatingSystemMXBean
|
||||
import com.vitorpamplona.amethyst.cli.stores.FileCashuKeysetCounterStore
|
||||
import com.vitorpamplona.amethyst.cli.stores.FileKeyPackageBundleStore
|
||||
import com.vitorpamplona.amethyst.cli.stores.FileMarmotMessageStore
|
||||
@@ -53,6 +54,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.BasicOkHttpWebSocket
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.CachingDns
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.TcpNoDelaySocketFactory
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal
|
||||
@@ -83,6 +85,7 @@ import kotlinx.coroutines.selects.select
|
||||
import kotlinx.coroutines.withTimeoutOrNull
|
||||
import okhttp3.Dispatcher
|
||||
import okhttp3.OkHttpClient
|
||||
import java.lang.management.ManagementFactory
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
@@ -144,14 +147,24 @@ class Context(
|
||||
// already-open sockets, bounded by AdaptiveRelayLimiter), so it can't
|
||||
// trip a relay's REQ rate-limit — it only speeds connection setup. The
|
||||
// executor thread pool is unbounded on demand, so raising maxRequests
|
||||
// just lets more of those short-lived handshakes proceed at once. 7s
|
||||
// just lets more of those short-lived handshakes proceed at once
|
||||
// (1 platform thread per in-flight handshake — ~1k is fine on a JVM,
|
||||
// 10k is not). Sized against the process FD budget: every pending
|
||||
// handshake and every open socket is one file descriptor. 7s
|
||||
// (not 5s): a 5s cap struck too many merely-busy relays as connect
|
||||
// failures — the crawl treats a connect *timeout* as retryable anyway,
|
||||
// but the extra headroom lets slow-but-alive relays finish the handshake.
|
||||
.connectTimeout(7, TimeUnit.SECONDS)
|
||||
// DNS dominates a mass connection ramp without help: Dns.SYSTEM blocks a
|
||||
// dispatcher thread per lookup, the JVM's own cache lasts ~30s (useless
|
||||
// over a 30-minute crawl), a dead domain burns 10-30s of resolver
|
||||
// timeouts on EVERY re-dial, and the outbox model mints hundreds of
|
||||
// per-user URLs on one host — each a fresh lookup. The cache (10-min
|
||||
// positive + negative TTL, in-flight dedup per host) collapses all of it.
|
||||
.dns(CachingDns())
|
||||
.dispatcher(
|
||||
Dispatcher().apply {
|
||||
maxRequests = 256
|
||||
maxRequests = maxParallelHandshakes
|
||||
maxRequestsPerHost = 16
|
||||
},
|
||||
).build()
|
||||
@@ -1033,6 +1046,39 @@ class Context(
|
||||
*/
|
||||
private const val GIFT_WRAP_LOOKBACK_SECS: Long = 2L * 24 * 60 * 60
|
||||
|
||||
/** FDs reserved for everything that isn't a relay socket (store, jars, pipes, DNS). */
|
||||
private const val FD_RESERVE = 256L
|
||||
|
||||
/**
|
||||
* The process's max-open-files limit (`ulimit -n`), the hard ceiling on
|
||||
* concurrent sockets: every open WebSocket AND every in-flight handshake is
|
||||
* one file descriptor, and the JVM cannot raise its own rlimit. Falls back
|
||||
* to the conservative 1024 (the common soft default) when the platform bean
|
||||
* doesn't expose it.
|
||||
*/
|
||||
val maxFileDescriptors: Long =
|
||||
(ManagementFactory.getOperatingSystemMXBean() as? UnixOperatingSystemMXBean)
|
||||
?.maxFileDescriptorCount ?: 1024L
|
||||
|
||||
/**
|
||||
* Concurrent WS-upgrade handshakes (OkHttp Dispatcher.maxRequests): a quarter
|
||||
* of the FD budget, so pending dials can never crowd out the sockets already
|
||||
* held open (warm pool, drains, parked subs). Also bounds the transient
|
||||
* platform threads OkHttp spawns — one per in-flight handshake.
|
||||
*/
|
||||
val maxParallelHandshakes: Int =
|
||||
((maxFileDescriptors - FD_RESERVE) / 4).coerceIn(64, 1024).toInt()
|
||||
|
||||
/**
|
||||
* Default cap for the crawl's mass pre-connect (warm pool): half the FD
|
||||
* budget goes to held-open relay sockets, leaving the other half for
|
||||
* in-flight handshakes, drain/parked subscriptions and headroom. At the
|
||||
* common 1024-FD soft limit this is ~384; `ulimit -n 16384` unlocks the
|
||||
* full 4000. Overridable per-run with --preconnect-cap.
|
||||
*/
|
||||
val defaultPreconnectCap: Int =
|
||||
((maxFileDescriptors - FD_RESERVE) / 2).coerceIn(100, 4000).toInt()
|
||||
|
||||
/**
|
||||
* Build a Context but require an account with a usable identity —
|
||||
* signing verbs can't run without one. Throws [IllegalArgumentException]
|
||||
|
||||
@@ -609,6 +609,14 @@ private fun printUsage() {
|
||||
| new/changed ranks >= --min-rank (default 2), skips
|
||||
| unchanged, and retracts (kind:5) any card whose
|
||||
| target left the graph or fell below the cutoff.
|
||||
| graperank crawl [OBSERVER] network only: crawl the WoT graph (kind 3/10000/
|
||||
| [--max-hops N] [--preconnect-cap N] 1984/10002) into the local store without scoring.
|
||||
| [--no-preconnect] Pre-connects every known-live relay in one parallel
|
||||
| storm (seeded from the reachability cache).
|
||||
| graperank probe [--timeout SECS] relay census: mass-connect every relay the store
|
||||
| [--concurrency N] knows and record live/dead + measured rtt-open into
|
||||
| the reachability cache (NIP-66 kind:30166), so the
|
||||
| next crawl skips dead relays and waits once.
|
||||
| graperank update [--down] [--up] refresh every locally-known author's WoT record kinds
|
||||
| [--no-sync-deletions] [--timeout SECS] (0/3/10002/1984) from their own outbox: reads all
|
||||
| [--relay-concurrency N] [--author-chunk N] kind:10002 in the store, groups authors by write
|
||||
|
||||
@@ -46,6 +46,7 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
|
||||
import com.vitorpamplona.quartz.nip09Deletions.DeletionIndex
|
||||
import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent
|
||||
import com.vitorpamplona.quartz.nip56Reports.ReportEvent
|
||||
import com.vitorpamplona.quartz.nip66RelayMonitor.reachability.RelayProber
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.list.TrustProviderListEvent
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.list.serviceProviders
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ProviderTypes
|
||||
@@ -92,6 +93,9 @@ import kotlin.math.roundToInt
|
||||
* - `amy graperank score [OBSERVER]` — local only: build the graph from the store
|
||||
* and score (same as bare `--offline`). Instant and param-tunable; repeat with
|
||||
* different `--rigor`/`--attenuation`/cutoffs without re-crawling.
|
||||
* - `amy graperank probe` — the relay census: mass-connect every relay the store
|
||||
* knows and record live/dead + measured RTT into the reachability cache, so the
|
||||
* next crawl skips the dead and pre-connects the living in one parallel storm.
|
||||
* - bare `amy graperank [OBSERVER]` — the convenience combo: crawl then score.
|
||||
*
|
||||
* Sub-verbs complete the NIP-85 provider experience — the discovery layer that
|
||||
@@ -193,6 +197,7 @@ object GrapeRankCommand {
|
||||
// `sync` is the pre-rename name kept as a back-compat alias; `crawl` is
|
||||
// canonical (disambiguates from negentropy `amy sync` / `graperank update`).
|
||||
"crawl", "sync" -> crawl(dataDir, tail.drop(1).toTypedArray())
|
||||
"probe" -> probe(dataDir, tail.drop(1).toTypedArray())
|
||||
"update" -> update(dataDir, tail.drop(1).toTypedArray())
|
||||
"score" -> run(dataDir, tail.drop(1).toTypedArray(), forceOffline = true)
|
||||
else -> run(dataDir, tail)
|
||||
@@ -441,11 +446,25 @@ object GrapeRankCommand {
|
||||
// Aggregator kind:3 recovery for stragglers is on by default; --no-aggregators
|
||||
// disables it for A/B comparison.
|
||||
val aggregators = if (args.bool("no-aggregators")) emptySet() else CONTENT_AGGREGATOR_RELAYS
|
||||
// Seed the crawl with relays a prior run/monitor proved dead within the cache's
|
||||
// TTL, so the WS path never re-pays their connect timeouts (--no-reachability-cache
|
||||
// to skip). The crawl's own final live/dead set is flushed back by the caller.
|
||||
val knownDead =
|
||||
if (args.bool("no-reachability-cache")) emptySet() else ctx.reachability.snapshot().dead
|
||||
// Seed the crawl from the reachability cache (--no-reachability-cache to skip):
|
||||
// proven-dead relays within the TTL are never dialed (their connect timeouts
|
||||
// aren't re-paid), and the proven-live universe is pre-connected in one
|
||||
// parallel storm at crawl start so the connect wait is paid once, up front.
|
||||
// The crawl's own final live/dead set is flushed back by the caller.
|
||||
val reachability =
|
||||
if (args.bool("no-reachability-cache")) null else ctx.reachability.snapshot()
|
||||
val knownDead = reachability?.dead ?: emptySet()
|
||||
// Mass pre-connect sizing: every warm socket is one FD, so the default is
|
||||
// derived from the process's ulimit (--preconnect-cap to override,
|
||||
// --no-preconnect to fall back to the top-20 warm pool).
|
||||
val preconnectCap =
|
||||
if (args.bool("no-preconnect")) 0 else args.intFlag("preconnect-cap", Context.defaultPreconnectCap)
|
||||
if (preconnectCap > 0 && Context.maxFileDescriptors < 4096) {
|
||||
System.err.println(
|
||||
"[graperank] open-files limit is ${Context.maxFileDescriptors} → pre-connect capped at " +
|
||||
"$preconnectCap sockets; `ulimit -n 16384` before running unlocks a faster crawl",
|
||||
)
|
||||
}
|
||||
return GrapeRankCrawler(
|
||||
client = ctx.client,
|
||||
store = ctx.store,
|
||||
@@ -454,6 +473,8 @@ object GrapeRankCommand {
|
||||
GrapeRankCrawler.Config(
|
||||
relayListDiscoveryRelays = discoveryRelays,
|
||||
knownDeadRelays = knownDead,
|
||||
knownLiveRelays = if (preconnectCap > 0) reachability?.live.orEmpty() else emptySet(),
|
||||
preconnectCap = preconnectCap,
|
||||
contentFallbackRelays = contentFallback,
|
||||
contentAggregatorRelays = aggregators,
|
||||
maxRounds = args.intFlag("max-rounds", Int.MAX_VALUE),
|
||||
@@ -548,6 +569,86 @@ object GrapeRankCommand {
|
||||
return 0
|
||||
}
|
||||
|
||||
/**
|
||||
* `amy graperank probe [--timeout SECS] [--concurrency N]` —
|
||||
* the relay census. Mass-connects the ENTIRE relay universe the local store knows
|
||||
* (every relay advertised in any stored kind:10002, deduped per host, plus
|
||||
* everything already in the reachability cache) in parallel waves with a no-op
|
||||
* REQ, so the "is this relay alive, and how slow?" wait is paid once, up front,
|
||||
* concurrently — then records per-relay verdicts with real measured `rtt-open`
|
||||
* into the NIP-66 reachability cache (kind:30166).
|
||||
*
|
||||
* The next `graperank crawl` reads that cache to (a) skip the dead set without
|
||||
* dialing it and (b) pre-connect the live set in one storm — separating "working
|
||||
* but slow" (kept; the crawler's patient park path waits for them) from "not
|
||||
* working" (skipped entirely). Typical flow the first time:
|
||||
* `graperank crawl --max-hops 2` (cheap, saves the relay lists) → `graperank
|
||||
* probe` → full `graperank crawl`.
|
||||
*/
|
||||
private suspend fun probe(
|
||||
dataDir: DataDir,
|
||||
rest: Array<String>,
|
||||
): Int {
|
||||
val args = Args(rest)
|
||||
val timeoutMs = args.longFlag("timeout", 15L) * 1000
|
||||
val waveSize = args.intFlag("concurrency", Context.defaultPreconnectCap)
|
||||
|
||||
Context.openOrAnonymous(dataDir).use { ctx ->
|
||||
ctx.prepare()
|
||||
val cached = ctx.reachability.snapshot()
|
||||
val universe = RelayProber.knownRelayUniverse(ctx.store) + cached.live + cached.dead
|
||||
if (universe.isEmpty()) {
|
||||
Output.emit(
|
||||
linkedMapOf<String, Any?>(
|
||||
"probed" to 0,
|
||||
"note" to "no relays known locally — run `amy graperank crawl` first to gather kind:10002 relay lists",
|
||||
),
|
||||
)
|
||||
return 0
|
||||
}
|
||||
|
||||
System.err.println(
|
||||
"[relay-probe] probing ${universe.size} relays in waves of $waveSize " +
|
||||
"(${timeoutMs / 1000}s per wave; open-files limit ${Context.maxFileDescriptors})",
|
||||
)
|
||||
val result =
|
||||
RelayProber(ctx.client) { System.err.println(it) }
|
||||
.probe(universe, timeoutMs, waveSize)
|
||||
|
||||
ctx.reachability.recordProbed(result.reachableRttMs(), result.deadRelays())
|
||||
|
||||
val rtts =
|
||||
result.reachable
|
||||
.map { it.rttOpenMs }
|
||||
.filter { it >= 0 }
|
||||
.sorted()
|
||||
|
||||
fun pct(p: Int): Long? = if (rtts.isEmpty()) null else rtts[(rtts.size - 1) * p / 100]
|
||||
val slowest =
|
||||
result.reachable
|
||||
.filter { it.rttOpenMs >= 0 }
|
||||
.sortedByDescending { it.rttOpenMs }
|
||||
.take(10)
|
||||
.map { mapOf("relay" to it.relay.url, "rtt_open_ms" to it.rttOpenMs) }
|
||||
val authWalled = result.reachable.count { it.error?.startsWith("closed:") == true }
|
||||
|
||||
Output.emit(
|
||||
linkedMapOf<String, Any?>(
|
||||
"probed" to result.verdicts.size,
|
||||
"reachable" to result.reachable.size,
|
||||
"dead" to result.dead.size,
|
||||
"closed_by_policy" to authWalled,
|
||||
"elapsed_ms" to result.elapsedMs,
|
||||
"rtt_open_p50_ms" to pct(50),
|
||||
"rtt_open_p90_ms" to pct(90),
|
||||
"rtt_open_p99_ms" to pct(99),
|
||||
"slowest" to slowest,
|
||||
),
|
||||
)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
/**
|
||||
* `amy graperank update [flags]` — refresh every locally-known author's WoT
|
||||
* record kinds (0 / 3 / 10002 / 1984) straight from their own outbox, so the
|
||||
|
||||
Reference in New Issue
Block a user