fix(graperank): page through ALL followers, not just the first limit

FollowerCrawler set the reverse-lookup filter's `limit` to the page size, but
fetchAllPages treats a filter `limit` as the TOTAL cap across all pages and
stops paging once it's reached — so the crawl silently capped at ~500 followers
per relay (verified live: relay.damus.io returned exactly 500 for a
many-thousand-follower observer).

Leave the filter limit null by default so pagination walks the whole result set
(the same observer now returns 12,823 followers from damus alone); Config gains
`maxPerRelay` and the CLI a `--max N` flag to opt back into a bounded spot check.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Xc3Wm4qVCrAGvSAotTUVt4
This commit is contained in:
Claude
2026-07-17 21:32:16 +00:00
parent 5f65405b08
commit dc157e32d2
2 changed files with 17 additions and 8 deletions
@@ -607,9 +607,10 @@ object GrapeRankCommand {
* observer. Idempotent and cumulative; run it a few times for completeness.
*
* Flags: `--relay URL[,URL…]` (query only these instead of the whole universe),
* `--page-limit N` (per-page REQ limit, default 500), `--timeout SECS` (per-page
* EOSE watchdog, default 15), `--relay-concurrency N` (relays paged at once,
* default 16), `--insert-batch N` (events per store commit, default 500).
* `--max N` (cap followers pulled per relay; default: pull every follower each
* relay holds), `--timeout SECS` (per-page EOSE watchdog, default 15),
* `--relay-concurrency N` (relays paged at once, default 16), `--insert-batch N`
* (events per store commit, default 500).
*/
private suspend fun followers(
dataDir: DataDir,
@@ -649,7 +650,9 @@ object GrapeRankCommand {
config =
FollowerCrawler.Config(
relays = relays,
pageLimit = args.intFlag("page-limit", 500),
// Default null → pull EVERY follower each relay holds; --max
// N caps the total per relay for a quick spot check.
maxPerRelay = args.flag("max")?.toIntOrNull(),
timeoutMs = args.longFlag("timeout", 15L) * 1000,
maxConcurrentRelays = relayConcurrency,
insertBatchSize = args.intFlag("insert-batch", 500),
@@ -68,15 +68,19 @@ class FollowerCrawler(
* @param relays the relay universe to query — "all possible relays" is the
* caller's policy (reachability-cache live set + every kind:10002 relay in the
* store + the index/aggregator relays). Empty means nothing to do.
* @param pageLimit per-page `limit` handed to each relay's paged REQ; the cursor
* walks past it, so this only bounds one page, not the total.
* @param maxPerRelay TOTAL cap on followers pulled from each relay, or `null`
* (default) to pull **every** follower a relay holds. This is NOT a page size:
* [fetchAllPages] treats a filter's `limit` as the total across all pages and
* stops paging once it's reached, so a non-null value cuts the crawl short at
* that many per relay. Leave it null for completeness; set it only to bound a
* spot check. The per-page size is the relay's own default either way.
* @param timeoutMs per-page EOSE timeout for a relay before its next page fires.
* @param maxConcurrentRelays how many relays page at once (a global fan-out cap).
* @param insertBatchSize verified events group-committed per [IEventStore.batchInsert].
*/
class Config(
val relays: Set<NormalizedRelayUrl>,
val pageLimit: Int = 500,
val maxPerRelay: Int? = null,
val timeoutMs: Long = 15_000,
val maxConcurrentRelays: Int = 16,
val insertBatchSize: Int = 500,
@@ -105,7 +109,9 @@ class FollowerCrawler(
if (config.relays.isEmpty()) return@withContext Stats(0, 0, 0, 0L, 0L)
val mark = TimeSource.Monotonic.markNow()
val filter = Filter(kinds = FOLLOW_KINDS, tags = mapOf("p" to listOf(observer)), limit = config.pageLimit)
// No limit by default → fetchAllPages walks the whole result set page by
// page; a non-null maxPerRelay caps the total per relay (see Config).
val filter = Filter(kinds = FOLLOW_KINDS, tags = mapOf("p" to listOf(observer)), limit = config.maxPerRelay)
val perRelay = config.relays.associateWith { listOf(filter) }
// These three are read/written ONLY by the verifier's single drain