From dd86b617c1fcd1bd4cccb307f555c9b9ac75f26d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 18:05:17 +0000 Subject: [PATCH] fix(cli): route graperank content to outboxes, indexers only for 10002 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correct the injector's relay model: indexer relays (purplepag.es, coracle, …) aggregate kind:10002 (and kind:0) for the whole network but do NOT serve kind:3/10000/1984. Those live only on each user's own outbox. - Split the relay sets: `relayListDiscoveryRelays` (bootstrap + event-finder + indexers) is used only to locate kind:10002; `contentFallbackRelays` (bootstrap + event-finder, no indexers) is the best-effort fallback for content when a user's outbox is unknown/down. - Content is fetched from each user's outbox write relays, with harvested relay hints and general relays as fallback — never indexers. Also add progress status (all on stderr, stdout stays the JSON contract): - loading already logs per-hop frontier/recovered/new/total counts; - "graph built: N users, E edges; scoring…" and "scored N users" bracket the calculation; - GrapeRank.compute gains an optional (visited, scored, queued) progress callback, wired to emit a scoring line every 5000 worklist visits so a large graph shows movement instead of hanging silently. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../2026-07-06-graperank-brainstorm-parity.md | 22 ++++--- .../amethyst/cli/commands/GrapeRankCommand.kt | 57 ++++++++++++++----- .../amethyst/commons/wot/GrapeRank.kt | 9 +++ 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/cli/plans/2026-07-06-graperank-brainstorm-parity.md b/cli/plans/2026-07-06-graperank-brainstorm-parity.md index 1383a5defb..3571de22ec 100644 --- a/cli/plans/2026-07-06-graperank-brainstorm-parity.md +++ b/cli/plans/2026-07-06-graperank-brainstorm-parity.md @@ -90,16 +90,20 @@ It is **data**, not math: So the effective scoring input is the same, provided the crawl runs to convergence (our default) rather than a shallow `--max-depth`. 2. **Fringe users / crawl gaps.** Relay timeouts that drop a contact list, or a - `--max-users` cap, remove edges and shift nearby scores. The injector now - mitigates this with a three-tier discovery model mirroring the app's - `pickRelaysToLoadUsers`: each user's kind:10002 **outbox**, then harvested - **relay hints** (from `p`-tag hints in the contact lists we crawl), then the - broad **discovery set** — bootstrap + event-finder + **indexer relays** - (purplepag.es, coracle, …) that serve kind:0/3/10002 for the whole network. + `--max-users` cap, remove edges and shift nearby scores. The injector mitigates + this with a two-stage model mirroring the app's `pickRelaysToLoadUsers`: + - **Relay-list discovery** (kind:10002) queries the account's relays + + bootstrap + event-finder + **indexer relays** (purplepag.es, coracle, …). + Indexers aggregate kind:10002 (and kind:0) for the whole network, so this is + where a stranger's outbox is found — the biggest completeness lever. + - **Content** (kind:3/10000/1984/0) is fetched from each user's **own outbox** + write relays, with harvested **relay hints** (from the `p`-tag hints in + contact lists we crawl) and general-purpose relays as a best-effort fallback + when the outbox is unknown/down. **Indexers are not used for content** — they + don't serve those kinds; kind:3/mutes/reports live only on the user's outbox. A per-hop **retry pass** re-queries any member whose contact list still didn't - arrive against that indexer + hint set, recovering users the outbox model - alone would miss. Remaining mitigation levers: a full crawl (default) and a - generous `--timeout`. + arrive against that hint + general-relay set. Remaining mitigation levers: a + full crawl (default) and a generous `--timeout`. 3. **Convergence precision.** Both stop at delta 0.0001; residual error is < ~0.0001 in influence ⇒ < ~0.01 rank points ⇒ identical integer `rank`. 4. **Seeding.** Their hop-distance seed vs our zero seed — same fixed point, no diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index ec5136874f..4d125e3592 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -79,6 +79,9 @@ object GrapeRankCommand { // Concurrent publishes when writing NIP-85 cards. private const val PUBLISH_CONCURRENCY = 16 + // Emit a scoring-progress line every this many worklist visits. + private const val SCORE_PROGRESS_STEP = 5_000 + suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -148,12 +151,14 @@ object GrapeRankCommand { // 3. Completeness retry: any member whose contact list still // didn't arrive (no kind:10002, or its outbox was down) gets - // re-queried against the broad indexer + hint set. Indexers - // like purplepag.es serve kind:3 for the whole network, so - // this recovers users the outbox model alone would miss. + // re-queried against its relay hints plus the general-purpose + // fallback relays. kind:3/10000/1984 live on the user's own + // outbox — NOT on indexers (those only aggregate kind:10002) — + // so this is best-effort recovery from general relays that may + // hold a copy, not a guaranteed find. val stillMissing = frontier.filter { ctx.contactsOf(it) == null } if (stillMissing.isNotEmpty()) { - val retryRelays = discoveryRelays(ctx) + stillMissing.flatMap { relayHints[it].orEmpty() } + val retryRelays = contentFallbackRelays(ctx) + stillMissing.flatMap { relayHints[it].orEmpty() } val retry = retryRelays.associateWith { stillMissing.chunked(AUTHORS_PER_FILTER).map { chunk -> Filter(kinds = graphKinds, authors = chunk) } @@ -185,7 +190,20 @@ object GrapeRankCommand { } val graph = TrustGraphBuilder.build(events) - val scores = GrapeRank(params).compute(graph, observer) + System.err.println( + "[graperank] graph built: ${graph.users.size} users, ${graph.edgeCount()} edges from ${events.size} events; scoring…", + ) + + // Live scoring progress: the worklist visits each reachable user once + // per relaxation; report every PROGRESS_STEP visits so a large graph + // shows movement instead of hanging silently. + val scores = + GrapeRank(params).compute(graph, observer) { visited, scored, queued -> + if (visited % SCORE_PROGRESS_STEP == 0) { + System.err.println("[graperank] scoring: $visited visited, $scored scored, $queued queued") + } + } + System.err.println("[graperank] scored ${scores.size} users") fun rankOf(score: Double) = (score * 100).roundToInt() @@ -423,19 +441,28 @@ object GrapeRankCommand { } /** - * The broad, network-wide discovery set: the account's own relays + Amethyst's - * bootstrap defaults + the event-finder relays + the **indexer relays** - * (purplepag.es, coracle, …). Indexers aggregate kind:0 / kind:3 / kind:10002 - * for the whole network, so they are where a stranger's relay list and contact - * list are actually found — the single biggest lever on crawl completeness. + * Relays to query for **kind:10002 relay lists** — the account's own relays + + * bootstrap defaults + event-finder relays + the **indexer relays** + * (purplepag.es, coracle, …). Indexers aggregate kind:10002 (and kind:0) for + * the whole network, so this is where a stranger's relay list is found. They + * do NOT hold kind:3/10000/1984 — see [contentFallbackRelays]. */ - private suspend fun discoveryRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + private suspend fun relayListDiscoveryRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + + /** + * Best-effort fallback relays for **content** (kind:3/10000/1984/0) when a + * user's outbox is unknown or unreachable. Content lives on each user's own + * outbox, so this is only general-purpose relays that *might* hold a copy — + * bootstrap + event-finder. **No indexers**: they don't serve these kinds. + */ + private suspend fun contentFallbackRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays /** * Fetch kind:10002 relay lists for any frontier member we don't already know, * so [routeByOutbox] can route their content query to their own write relays. - * Queries the broad discovery set (incl. indexers) plus each user's harvested - * relay [hints] — the CLI analog of the app's tiered `pickRelaysToLoadUsers`. + * Queries the relay-list discovery set (incl. indexers) plus each user's + * harvested relay [hints] — the CLI analog of the app's tiered + * `pickRelaysToLoadUsers`. */ private suspend fun ensureRelayLists( ctx: Context, @@ -446,7 +473,7 @@ object GrapeRankCommand { val missing = pubkeys.filter { ctx.relaysOf(it) == null } if (missing.isEmpty()) return - val base = discoveryRelays(ctx) + val base = relayListDiscoveryRelays(ctx) val perRelay = HashMap>() for (pk in missing) { for (relay in base + hints[pk].orEmpty()) perRelay.getOrPut(relay) { HashSet() }.add(pk) @@ -474,7 +501,7 @@ object GrapeRankCommand { hints: Map>, kinds: List, ): Map> { - val fallback = discoveryRelays(ctx) + val fallback = contentFallbackRelays(ctx) val perRelay = HashMap>() for (pk in pubkeys) { diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt index 848d91cb0c..fd4859cade 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt @@ -86,10 +86,15 @@ class GrapeRank( * Score every user reachable from [observer]. The returned map excludes the * observer itself (its score is a pinned `1.0` and not part of a ranking). * Users with no positive path from the observer are absent (equivalently, 0). + * + * [onProgress] is invoked once per worklist visit with + * `(visited, scored, queued)` running counts, so a caller can report progress + * on a large graph; it defaults to a no-op. */ fun compute( graph: TrustGraph, observer: HexKey, + onProgress: ((visited: Int, scored: Int, queued: Int) -> Unit)? = null, ): Map { val scores = HashMap() scores[observer] = 1.0 @@ -103,6 +108,7 @@ class GrapeRank( graph.outgoing[observer]?.forEach(::enqueue) + var visited = 0 while (queue.isNotEmpty()) { val target = queue.removeFirst() queued.remove(target) @@ -113,6 +119,9 @@ class GrapeRank( if (abs(newScore - oldScore) > params.convergence) { graph.outgoing[target]?.forEach(::enqueue) } + + visited++ + onProgress?.invoke(visited, scores.size, queue.size) } scores.remove(observer)