From dc157e32d23f8330b9431d4ec6407f251f591ea2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 21:32:16 +0000 Subject: [PATCH] fix(graperank): page through ALL followers, not just the first limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FollowerCrawler set the reverse-lookup filter's `limit` to the page size, but fetchAllPages treats a filter `limit` as the TOTAL cap across all pages and stops paging once it's reached — so the crawl silently capped at ~500 followers per relay (verified live: relay.damus.io returned exactly 500 for a many-thousand-follower observer). Leave the filter limit null by default so pagination walks the whole result set (the same observer now returns 12,823 followers from damus alone); Config gains `maxPerRelay` and the CLI a `--max N` flag to opt back into a bounded spot check. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Xc3Wm4qVCrAGvSAotTUVt4 --- .../amethyst/cli/commands/GrapeRankCommand.kt | 11 +++++++---- .../experimental/graperank/FollowerCrawler.kt | 14 ++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index e028d0335e..3b1a8fa95a 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -607,9 +607,10 @@ object GrapeRankCommand { * observer. Idempotent and cumulative; run it a few times for completeness. * * Flags: `--relay URL[,URL…]` (query only these instead of the whole universe), - * `--page-limit N` (per-page REQ limit, default 500), `--timeout SECS` (per-page - * EOSE watchdog, default 15), `--relay-concurrency N` (relays paged at once, - * default 16), `--insert-batch N` (events per store commit, default 500). + * `--max N` (cap followers pulled per relay; default: pull every follower each + * relay holds), `--timeout SECS` (per-page EOSE watchdog, default 15), + * `--relay-concurrency N` (relays paged at once, default 16), `--insert-batch N` + * (events per store commit, default 500). */ private suspend fun followers( dataDir: DataDir, @@ -649,7 +650,9 @@ object GrapeRankCommand { config = FollowerCrawler.Config( relays = relays, - pageLimit = args.intFlag("page-limit", 500), + // Default null → pull EVERY follower each relay holds; --max + // N caps the total per relay for a quick spot check. + maxPerRelay = args.flag("max")?.toIntOrNull(), timeoutMs = args.longFlag("timeout", 15L) * 1000, maxConcurrentRelays = relayConcurrency, insertBatchSize = args.intFlag("insert-batch", 500), diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/FollowerCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/FollowerCrawler.kt index 4456015dae..c1f97e2989 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/FollowerCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/FollowerCrawler.kt @@ -68,15 +68,19 @@ class FollowerCrawler( * @param relays the relay universe to query — "all possible relays" is the * caller's policy (reachability-cache live set + every kind:10002 relay in the * store + the index/aggregator relays). Empty means nothing to do. - * @param pageLimit per-page `limit` handed to each relay's paged REQ; the cursor - * walks past it, so this only bounds one page, not the total. + * @param maxPerRelay TOTAL cap on followers pulled from each relay, or `null` + * (default) to pull **every** follower a relay holds. This is NOT a page size: + * [fetchAllPages] treats a filter's `limit` as the total across all pages and + * stops paging once it's reached, so a non-null value cuts the crawl short at + * that many per relay. Leave it null for completeness; set it only to bound a + * spot check. The per-page size is the relay's own default either way. * @param timeoutMs per-page EOSE timeout for a relay before its next page fires. * @param maxConcurrentRelays how many relays page at once (a global fan-out cap). * @param insertBatchSize verified events group-committed per [IEventStore.batchInsert]. */ class Config( val relays: Set, - val pageLimit: Int = 500, + val maxPerRelay: Int? = null, val timeoutMs: Long = 15_000, val maxConcurrentRelays: Int = 16, val insertBatchSize: Int = 500, @@ -105,7 +109,9 @@ class FollowerCrawler( if (config.relays.isEmpty()) return@withContext Stats(0, 0, 0, 0L, 0L) val mark = TimeSource.Monotonic.markNow() - val filter = Filter(kinds = FOLLOW_KINDS, tags = mapOf("p" to listOf(observer)), limit = config.pageLimit) + // No limit by default → fetchAllPages walks the whole result set page by + // page; a non-null maxPerRelay caps the total per relay (see Config). + val filter = Filter(kinds = FOLLOW_KINDS, tags = mapOf("p" to listOf(observer)), limit = config.maxPerRelay) val perRelay = config.relays.associateWith { listOf(filter) } // These three are read/written ONLY by the verifier's single drain