mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-09 08:04:45 +00:00
fix(graperank): page through ALL followers, not just the first limit
FollowerCrawler set the reverse-lookup filter's `limit` to the page size, but fetchAllPages treats a filter `limit` as the TOTAL cap across all pages and stops paging once it's reached — so the crawl silently capped at ~500 followers per relay (verified live: relay.damus.io returned exactly 500 for a many-thousand-follower observer). Leave the filter limit null by default so pagination walks the whole result set (the same observer now returns 12,823 followers from damus alone); Config gains `maxPerRelay` and the CLI a `--max N` flag to opt back into a bounded spot check. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Xc3Wm4qVCrAGvSAotTUVt4
This commit is contained in:
@@ -607,9 +607,10 @@ object GrapeRankCommand {
|
||||
* observer. Idempotent and cumulative; run it a few times for completeness.
|
||||
*
|
||||
* Flags: `--relay URL[,URL…]` (query only these instead of the whole universe),
|
||||
* `--page-limit N` (per-page REQ limit, default 500), `--timeout SECS` (per-page
|
||||
* EOSE watchdog, default 15), `--relay-concurrency N` (relays paged at once,
|
||||
* default 16), `--insert-batch N` (events per store commit, default 500).
|
||||
* `--max N` (cap followers pulled per relay; default: pull every follower each
|
||||
* relay holds), `--timeout SECS` (per-page EOSE watchdog, default 15),
|
||||
* `--relay-concurrency N` (relays paged at once, default 16), `--insert-batch N`
|
||||
* (events per store commit, default 500).
|
||||
*/
|
||||
private suspend fun followers(
|
||||
dataDir: DataDir,
|
||||
@@ -649,7 +650,9 @@ object GrapeRankCommand {
|
||||
config =
|
||||
FollowerCrawler.Config(
|
||||
relays = relays,
|
||||
pageLimit = args.intFlag("page-limit", 500),
|
||||
// Default null → pull EVERY follower each relay holds; --max
|
||||
// N caps the total per relay for a quick spot check.
|
||||
maxPerRelay = args.flag("max")?.toIntOrNull(),
|
||||
timeoutMs = args.longFlag("timeout", 15L) * 1000,
|
||||
maxConcurrentRelays = relayConcurrency,
|
||||
insertBatchSize = args.intFlag("insert-batch", 500),
|
||||
|
||||
+10
-4
@@ -68,15 +68,19 @@ class FollowerCrawler(
|
||||
* @param relays the relay universe to query — "all possible relays" is the
|
||||
* caller's policy (reachability-cache live set + every kind:10002 relay in the
|
||||
* store + the index/aggregator relays). Empty means nothing to do.
|
||||
* @param pageLimit per-page `limit` handed to each relay's paged REQ; the cursor
|
||||
* walks past it, so this only bounds one page, not the total.
|
||||
* @param maxPerRelay TOTAL cap on followers pulled from each relay, or `null`
|
||||
* (default) to pull **every** follower a relay holds. This is NOT a page size:
|
||||
* [fetchAllPages] treats a filter's `limit` as the total across all pages and
|
||||
* stops paging once it's reached, so a non-null value cuts the crawl short at
|
||||
* that many per relay. Leave it null for completeness; set it only to bound a
|
||||
* spot check. The per-page size is the relay's own default either way.
|
||||
* @param timeoutMs per-page EOSE timeout for a relay before its next page fires.
|
||||
* @param maxConcurrentRelays how many relays page at once (a global fan-out cap).
|
||||
* @param insertBatchSize verified events group-committed per [IEventStore.batchInsert].
|
||||
*/
|
||||
class Config(
|
||||
val relays: Set<NormalizedRelayUrl>,
|
||||
val pageLimit: Int = 500,
|
||||
val maxPerRelay: Int? = null,
|
||||
val timeoutMs: Long = 15_000,
|
||||
val maxConcurrentRelays: Int = 16,
|
||||
val insertBatchSize: Int = 500,
|
||||
@@ -105,7 +109,9 @@ class FollowerCrawler(
|
||||
if (config.relays.isEmpty()) return@withContext Stats(0, 0, 0, 0L, 0L)
|
||||
|
||||
val mark = TimeSource.Monotonic.markNow()
|
||||
val filter = Filter(kinds = FOLLOW_KINDS, tags = mapOf("p" to listOf(observer)), limit = config.pageLimit)
|
||||
// No limit by default → fetchAllPages walks the whole result set page by
|
||||
// page; a non-null maxPerRelay caps the total per relay (see Config).
|
||||
val filter = Filter(kinds = FOLLOW_KINDS, tags = mapOf("p" to listOf(observer)), limit = config.maxPerRelay)
|
||||
val perRelay = config.relays.associateWith { listOf(filter) }
|
||||
|
||||
// These three are read/written ONLY by the verifier's single drain
|
||||
|
||||
Reference in New Issue
Block a user