mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-10 16:33:27 +00:00
fix(graperank): page through ALL followers, not just the first limit
FollowerCrawler set the reverse-lookup filter's `limit` to the page size, but fetchAllPages treats a filter `limit` as the TOTAL cap across all pages and stops paging once it's reached — so the crawl silently capped at ~500 followers per relay (verified live: relay.damus.io returned exactly 500 for a many-thousand-follower observer). Leave the filter limit null by default so pagination walks the whole result set (the same observer now returns 12,823 followers from damus alone); Config gains `maxPerRelay` and the CLI a `--max N` flag to opt back into a bounded spot check. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Xc3Wm4qVCrAGvSAotTUVt4
This commit is contained in:
@@ -607,9 +607,10 @@ object GrapeRankCommand {
|
||||
* observer. Idempotent and cumulative; run it a few times for completeness.
|
||||
*
|
||||
* Flags: `--relay URL[,URL…]` (query only these instead of the whole universe),
|
||||
* `--page-limit N` (per-page REQ limit, default 500), `--timeout SECS` (per-page
|
||||
* EOSE watchdog, default 15), `--relay-concurrency N` (relays paged at once,
|
||||
* default 16), `--insert-batch N` (events per store commit, default 500).
|
||||
* `--max N` (cap followers pulled per relay; default: pull every follower each
|
||||
* relay holds), `--timeout SECS` (per-page EOSE watchdog, default 15),
|
||||
* `--relay-concurrency N` (relays paged at once, default 16), `--insert-batch N`
|
||||
* (events per store commit, default 500).
|
||||
*/
|
||||
private suspend fun followers(
|
||||
dataDir: DataDir,
|
||||
@@ -649,7 +650,9 @@ object GrapeRankCommand {
|
||||
config =
|
||||
FollowerCrawler.Config(
|
||||
relays = relays,
|
||||
pageLimit = args.intFlag("page-limit", 500),
|
||||
// Default null → pull EVERY follower each relay holds; --max
|
||||
// N caps the total per relay for a quick spot check.
|
||||
maxPerRelay = args.flag("max")?.toIntOrNull(),
|
||||
timeoutMs = args.longFlag("timeout", 15L) * 1000,
|
||||
maxConcurrentRelays = relayConcurrency,
|
||||
insertBatchSize = args.intFlag("insert-batch", 500),
|
||||
|
||||
Reference in New Issue
Block a user