From e6291fa912a1110567bf206740fb364b3788d121 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 13:06:19 +0000 Subject: [PATCH 01/58] feat(cli): add GrapeRank web-of-trust calculator (amy graperank) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the GrapeRank algorithm into Amethyst as a WoT service calculator on the CLI. commons/wot (protocol-agnostic, CLI-safe, reusable by the apps): - TrustGraph / TrustEdge / TrustRelation — pubkey-keyed graph model. - GrapeRank — single-observer scoring engine, a faithful port of the reference v3 TargetedBFS variant using a worklist that reaches the same fixed point a full sweep would while only touching reachable users. - TrustGraphBuilder — pure kind:3 / kind:10000 / kind:1984 events -> graph (latest-replaceable-per-author, dedup, self-edge drop). - Unit tests: hand-computed values plus an adversarial full-sweep cross-check over 50 random graphs. cli: `amy graperank [OBSERVER]` crawls the follow/mute/report graph via the outbox model (locate each user's kind:10002 write relays, then fetch their lists from their own relays, with a broad event-finder fallback) until no new users appear, scores it, and prints a ranked list (text / --json). --target queries one user, --offline scores from the local store, and --publish writes NIP-85 kind:30382 ContactCard assertions (rank = round(score*100)) per user at or above --min-rank. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- cli/README.md | 2 + cli/ROADMAP.md | 1 + .../com/vitorpamplona/amethyst/cli/Main.kt | 13 + .../amethyst/cli/commands/GrapeRankCommand.kt | 299 ++++++++++++++++++ .../amethyst/commons/wot/GrapeRank.kt | 143 +++++++++ .../amethyst/commons/wot/TrustGraph.kt | 83 +++++ .../amethyst/commons/wot/TrustGraphBuilder.kt | 105 ++++++ .../amethyst/commons/wot/GrapeRankTest.kt | 220 +++++++++++++ .../commons/wot/TrustGraphBuilderTest.kt | 150 +++++++++ 9 files changed, 1016 insertions(+) create mode 100644 cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt diff --git a/cli/README.md b/cli/README.md index 13da5dd96d..0d678a8769 100644 --- a/cli/README.md +++ b/cli/README.md @@ -383,6 +383,8 @@ HTTP endpoint. Reuses quartz's `Nip86Client` and the shared `Nip86Retriever` | `amy notes feed [--author USER \| --following] [--limit N]` | Read recent kind:1 notes (yours, one user's, or your follow set). | | `amy profile show [USER]` | Print kind:0 metadata. USER accepts npub/nprofile/hex/NIP-05; defaults to self. | | `amy profile edit --name … --about … --picture URL …` | Patch and re-publish your kind:0. | +| `amy follow USER` / `amy unfollow USER` | Add/remove USER from your kind:3 contact list (fetches the freshest list first). | +| `amy graperank [OBSERVER] [--max-depth N] [--target USER] [--offline] [--publish]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph, crawled via the outbox model; optionally publish results as NIP-85 kind:30382 cards. | ### Direct messages (NIP-17) diff --git a/cli/ROADMAP.md b/cli/ROADMAP.md index 6e5a7cd979..d31deebf08 100644 --- a/cli/ROADMAP.md +++ b/cli/ROADMAP.md @@ -58,6 +58,7 @@ Status legend: ✅ shipped · 📦 logic lives in `commons/`, needs a command · | NIP-51 lists (bookmarks, mute, follow sets) | 🆕 | `amethyst/model/nip51Lists/` | | NIP-57 zaps (send + verify) | 🆕 | Needs LN-URL plumbing; `amethyst/service/lnurl/`. | | NIP-65 outbox model queries | 🆕 | | +| NIP-85 GrapeRank web-of-trust (`amy graperank`) | ✅ | `GrapeRankCommand` — outbox-model crawl + scoring engine in `commons/wot/` (`GrapeRank`, `TrustGraph`, `TrustGraphBuilder`); publishes kind:30382 `ContactCardEvent`. | | NIP-72 communities | 🆕 | | | NIP-78 app-specific data (settings sync) | 🆕 | | | Long-form (NIP-23) publish / read | 🆕 | | diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index fc85fa7494..027483ad87 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -38,6 +38,7 @@ import com.vitorpamplona.amethyst.cli.commands.FilterCommand import com.vitorpamplona.amethyst.cli.commands.FollowCommand import com.vitorpamplona.amethyst.cli.commands.GiftCommands import com.vitorpamplona.amethyst.cli.commands.GitCommands +import com.vitorpamplona.amethyst.cli.commands.GrapeRankCommand import com.vitorpamplona.amethyst.cli.commands.GroupCommands import com.vitorpamplona.amethyst.cli.commands.InitCommands import com.vitorpamplona.amethyst.cli.commands.KeyCommands @@ -214,6 +215,7 @@ private suspend fun dispatch(argv: Array): Int { "store" -> StoreCommands.dispatch(dataDir, tail) "follow" -> FollowCommand.follow(dataDir, tail) "unfollow" -> FollowCommand.unfollow(dataDir, tail) + "graperank" -> GrapeRankCommand.run(dataDir, tail) "search" -> SearchCommand.dispatch(dataDir, tail) "zap" -> ZapCommand.dispatch(dataDir, tail) "offer" -> OfferCommands.dispatch(dataDir, tail) @@ -524,6 +526,17 @@ private fun printUsage() { | unfollow USER [--timeout SECS] remove USER from your contact list | (USER: npub|nprofile|hex|name@domain) | + |Web of Trust (GrapeRank): + | graperank [OBSERVER] compute subjective trust scores (0..1) for every + | [--max-depth N] [--max-users N] user reachable in the follow/mute/report graph, + | [--limit N] [--min-score X] crawled via the outbox model until no new users + | [--target USER] appear (OBSERVER: npub|nprofile|hex|name@domain, + | [--no-mutes] [--no-reports] default: active account). --target prints one + | [--rigor X] [--attenuation X] user's score; --offline scores from the local + | [--offline] [--timeout SECS] store only. --publish writes NIP-85 kind:30382 + | [--publish] [--min-rank N] trusted-assertion cards (rank = round(score*100)) + | [--publish-limit N] [--publish-relay URL] for each user at or above --min-rank. + | |Zaps (NIP-57): | zap user USER SATS build a profile zap-request, fetch a BOLT11 | [--comment X] [--anon|--private] invoice from the recipient's LN service diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt new file mode 100644 index 0000000000..fb3e65c5ec --- /dev/null +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.cli.commands + +import com.vitorpamplona.amethyst.cli.Args +import com.vitorpamplona.amethyst.cli.Context +import com.vitorpamplona.amethyst.cli.DataDir +import com.vitorpamplona.amethyst.cli.Output +import com.vitorpamplona.amethyst.commons.defaults.Constants +import com.vitorpamplona.amethyst.commons.wot.GrapeRank +import com.vitorpamplona.amethyst.commons.wot.GrapeRankParams +import com.vitorpamplona.amethyst.commons.wot.TrustGraphBuilder +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent +import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent +import com.vitorpamplona.quartz.nip56Reports.ReportEvent +import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent +import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlin.math.roundToInt + +/** + * `amy graperank [OBSERVER] [flags]` — compute GrapeRank web-of-trust scores. + * + * GrapeRank assigns every user reachable in the follow/mute/report graph a + * subjective trust score in `[0, 1]` from the observer's point of view (the + * observer has full self-trust). It crawls the follow graph outward using the + * outbox model — each user's kind:10002 write relays are located first, then + * their kind:3 / kind:10000 / kind:1984 events are fetched from *their own* + * relays — until no new users appear (typically ~8 hops), then runs the scoring + * engine in `commons/wot`. + * + * Prints a ranked list (text, or one JSON object under `--json`). With + * `--publish`, results are also published as NIP-85 kind:30382 `ContactCardEvent` + * trusted assertions (one per scored user, `rank = round(score*100)`). + */ +object GrapeRankCommand { + // Authors per REQ filter — keeps individual subscriptions within relay limits. + private const val AUTHORS_PER_FILTER = 300 + + // Concurrent publishes when writing NIP-85 cards. + private const val PUBLISH_CONCURRENCY = 16 + + suspend fun run( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val observerArg = args.positionalOrNull(0) + val maxDepth = args.intFlag("max-depth", 8) + val maxUsers = args.intFlag("max-users", 50_000) + val limit = args.intFlag("limit", 100) + val minScore = args.flag("min-score")?.toDoubleOrNull() ?: 0.0 + val targetArg = args.flag("target") + val includeMutes = !args.bool("no-mutes") + val includeReports = !args.bool("no-reports") + val offline = args.bool("offline") + val timeoutMs = args.longFlag("timeout", 10L) * 1000 + val doPublish = args.bool("publish") + val minRank = args.intFlag("min-rank", 1) + val publishLimit = args.intFlag("publish-limit", 500) + val publishRelaysArg = args.flag("publish-relay") + + val params = + GrapeRankParams( + attenuation = args.flag("attenuation")?.toDoubleOrNull() ?: GrapeRankParams().attenuation, + rigor = args.flag("rigor")?.toDoubleOrNull() ?: GrapeRankParams().rigor, + ) + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val observer = observerArg?.let { ctx.requireUserHex(it) } ?: ctx.identity.pubKeyHex + + val graphKinds = + buildList { + add(ContactListEvent.KIND) + if (includeMutes) add(MuteListEvent.KIND) + if (includeReports) add(ReportEvent.KIND) + } + + var depthReached = 0 + val events: List + + if (offline) { + events = ctx.store.query(Filter(kinds = graphKinds)) + System.err.println("[graperank] offline: ${events.size} events from local store") + } else { + val collected = mutableListOf() + val discovered = hashSetOf(observer) + var frontier: Set = setOf(observer) + + for (hop in 0 until maxDepth) { + if (frontier.isEmpty()) break + depthReached = hop + 1 + + ensureRelayLists(ctx, frontier, timeoutMs) + + val filters = routeByOutbox(ctx, frontier, graphKinds) + collected += ctx.drain(filters, timeoutMs).map { it.second } + + val next = hashSetOf() + for (pk in frontier) { + ctx.contactsOf(pk)?.verifiedFollowKeySet()?.forEach { followed -> + if (discovered.size < maxUsers && discovered.add(followed)) next += followed + } + } + System.err.println("[graperank] hop ${hop + 1}: fetched frontier=${frontier.size}, new=${next.size}, total=${discovered.size}") + + if (discovered.size >= maxUsers) { + System.err.println("[graperank] reached --max-users=$maxUsers cap; stopping crawl") + break + } + frontier = next + } + events = collected + } + + val graph = TrustGraphBuilder.build(events, includeMutes = includeMutes, includeReports = includeReports) + val scores = GrapeRank(params).compute(graph, observer) + + fun rankOf(score: Double) = (score * 100).roundToInt() + + if (targetArg != null) { + val target = ctx.requireUserHex(targetArg) + // The observer trusts itself fully by definition; it is excluded + // from the ranking map, so answer it directly. + val score = if (target == observer) 1.0 else scores[target] ?: 0.0 + Output.emit( + mapOf( + "observer" to observer, + "target" to target, + "score" to score, + "rank" to rankOf(score), + "users_scored" to scores.size, + "depth_reached" to depthReached, + ), + ) + return 0 + } + + val ranked = + scores.entries + .filter { it.value >= minScore } + .sortedByDescending { it.value } + + val result = + linkedMapOf( + "observer" to observer, + "depth_reached" to depthReached, + "graph_users" to graph.users.size, + "graph_edges" to graph.edgeCount(), + "users_scored" to scores.size, + "scores" to + ranked.take(limit).map { + mapOf("pubkey" to it.key, "score" to it.value, "rank" to rankOf(it.value)) + }, + ) + + if (doPublish) { + val relays = + publishRelaysArg + ?.split(",") + ?.mapNotNull { RelayUrlNormalizer.normalizeOrNull(it.trim()) } + ?.toSet() + ?.takeIf { it.isNotEmpty() } + ?: ctx.outboxRelays() + + val toPublish = + ranked + .filter { rankOf(it.value) >= minRank } + .take(publishLimit) + .map { it.key to rankOf(it.value) } + + if (relays.isEmpty()) { + result["published"] = 0 + result["publish_error"] = "no publish relays configured" + } else { + val (ok, rejected) = publishCards(ctx, toPublish, relays) + result["published"] = ok + result["publish_rejected"] = rejected + result["published_kind"] = ContactCardEvent.KIND + result["published_to"] = relays.map { it.url } + } + } + + Output.emit(result) + return 0 + } + } + + /** + * Fetch kind:10002 relay lists for any frontier member we don't already know, + * so [routeByOutbox] can route their content query to their own write relays. + * Uses the broad bootstrap + event-finder relay set as the discovery seed — + * the CLI analog of the app's tiered outbox lookup. + */ + private suspend fun ensureRelayLists( + ctx: Context, + pubkeys: Set, + timeoutMs: Long, + ) { + val missing = pubkeys.filter { ctx.relaysOf(it) == null } + if (missing.isEmpty()) return + + val seedRelays = ctx.bootstrapRelays() + Constants.eventFinderRelays + if (seedRelays.isEmpty()) return + + val filters = + seedRelays.associateWith { + missing.chunked(AUTHORS_PER_FILTER).map { chunk -> + Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) + } + } + ctx.drain(filters, timeoutMs) + } + + /** + * Group [pubkeys] by the relays we should query for their events: each user's + * kind:10002 write relays (the outbox model), falling back to the broad + * event-finder set for users with no advertised relay list. Authors are + * chunked per relay to respect relay REQ limits. + */ + private suspend fun routeByOutbox( + ctx: Context, + pubkeys: Set, + kinds: List, + ): Map> { + val fallback = ctx.bootstrapRelays() + Constants.eventFinderRelays + val perRelay = HashMap>() + + for (pk in pubkeys) { + val write = ctx.relaysOf(pk)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() } + val relays = write ?: fallback + for (relay in relays) perRelay.getOrPut(relay) { HashSet() }.add(pk) + } + + return perRelay.mapValues { (_, authors) -> + authors.chunked(AUTHORS_PER_FILTER).map { chunk -> + Filter(kinds = kinds, authors = chunk) + } + } + } + + /** Build + publish one NIP-85 kind:30382 card per user, bounded-concurrently. */ + private suspend fun publishCards( + ctx: Context, + cards: List>, + relays: Set, + ): Pair { + var published = 0 + var rejected = 0 + for (batch in cards.chunked(PUBLISH_CONCURRENCY)) { + val acks = + coroutineScope { + batch + .map { (pubkey, rank) -> + async { + val card = + ContactCardEvent.create( + targetUser = pubkey, + signer = ctx.signer, + publicInitializer = { add(RankTag.assemble(rank)) }, + ) + ctx.publish(card, relays) + } + }.awaitAll() + } + for (ack in acks) { + if (ack.values.any { it }) published++ else rejected++ + } + } + return published to rejected + } +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt new file mode 100644 index 0000000000..848d91cb0c --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.wot + +import androidx.compose.runtime.Immutable +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import kotlin.math.abs +import kotlin.math.exp +import kotlin.math.ln + +/** + * Tunable GrapeRank parameters. Defaults mirror the reference implementation at + * . + * + * A follow from the observer themselves counts far more than a follow from a + * stranger deep in the graph ([directFollowConfidence] vs + * [indirectFollowConfidence]); mutes and reports are trusted more heavily than + * an indirect follow because negative signals are rarer and more deliberate. + */ +@Immutable +data class GrapeRankParams( + val attenuation: Double = 0.85, + val rigor: Double = 0.5, + val directFollowConfidence: Double = 0.5, + val indirectFollowConfidence: Double = 0.03, + val muteConfidence: Double = 0.5, + val reportConfidence: Double = 0.5, + val convergence: Double = 0.0001, +) + +/** + * GrapeRank — a subjective, observer-centric web-of-trust score in `[0, 1]` for + * every user reachable from an observer in a [TrustGraph]. The observer has full + * self-trust (`1.0`); trust decays by roughly the attenuation factor each hop, + * so scores fall to ~0 within a handful of hops. + * + * This is a faithful single-observer port of the reference `v3TargetedBFS` + * variant. Rather than the reactive per-edge propagation the reference uses (it + * assumes edges stream in one at a time), this recomputes over a graph that is + * already fully loaded, using a worklist that: + * 1. seeds the observer at `1.0` and enqueues the users it attests about, + * 2. dequeues a target, recomputes its score over *all* its incoming edges, + * 3. re-enqueues that target's out-neighbours whenever its score moved by more + * than [GrapeRankParams.convergence]. + * + * Attenuation makes the update a contraction, so the worklist reaches the same + * fixed point a full sweep would — while only ever touching users reachable from + * the observer. See `GrapeRankTest` for the full-sweep cross-check. + */ +class GrapeRank( + val params: GrapeRankParams = GrapeRankParams(), +) { + /** Confidence weight [source]→target contributes, from [observer]'s point of view. */ + private fun confidence( + edge: TrustEdge, + observer: HexKey, + ): Double = + when (edge.relation) { + TrustRelation.FOLLOW -> if (edge.source == observer) params.directFollowConfidence else params.indirectFollowConfidence + TrustRelation.MUTE -> params.muteConfidence + TrustRelation.REPORT -> params.reportConfidence + } + + /** Exponential saturation curve turning accumulated weight into a confidence in `[0, 1)`. */ + private fun weightToConfidence(weight: Double): Double = 1.0 - exp(-weight * -ln(params.rigor)) + + /** + * Score every user reachable from [observer]. The returned map excludes the + * observer itself (its score is a pinned `1.0` and not part of a ranking). + * Users with no positive path from the observer are absent (equivalently, 0). + */ + fun compute( + graph: TrustGraph, + observer: HexKey, + ): Map { + val scores = HashMap() + scores[observer] = 1.0 + + val queue = ArrayDeque() + val queued = HashSet() + + fun enqueue(user: HexKey) { + if (user != observer && queued.add(user)) queue.addLast(user) + } + + graph.outgoing[observer]?.forEach(::enqueue) + + while (queue.isNotEmpty()) { + val target = queue.removeFirst() + queued.remove(target) + + val newScore = scoreOf(graph, scores, target, observer) + val oldScore = scores.put(target, newScore) ?: 0.0 + + if (abs(newScore - oldScore) > params.convergence) { + graph.outgoing[target]?.forEach(::enqueue) + } + } + + scores.remove(observer) + return scores + } + + private fun scoreOf( + graph: TrustGraph, + scores: Map, + target: HexKey, + observer: HexKey, + ): Double { + var sumOfWeights = 0.0 + var sumOfWeightedRatings = 0.0 + + val edges = graph.incoming[target] ?: return 0.0 + for (edge in edges) { + val sourceScore = scores[edge.source] ?: continue + val weight = confidence(edge, observer) * sourceScore * params.attenuation + sumOfWeights += weight + sumOfWeightedRatings += weight * edge.relation.rating + } + + if (abs(sumOfWeights) < 0.00001) return 0.0 + val score = weightToConfidence(sumOfWeights) * sumOfWeightedRatings / sumOfWeights + return if (score > 0.0) score else 0.0 + } +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt new file mode 100644 index 0000000000..d27e855c03 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.wot + +import androidx.compose.runtime.Immutable +import com.vitorpamplona.quartz.nip01Core.core.HexKey + +/** + * A single directed trust attestation between two Nostr users, mapped from a + * kind:3 follow / kind:10000 mute / kind:1984 report. Each carries a [rating] + * (how the relationship reflects on the target) that GrapeRank multiplies by an + * observer-relative confidence — see [GrapeRank]. + */ +@Immutable +enum class TrustRelation( + val rating: Double, +) { + FOLLOW(1.0), + MUTE(-0.1), + REPORT(-0.1), +} + +/** [source] asserts [relation] about the (implicit) target it is indexed under. */ +@Immutable +data class TrustEdge( + val source: HexKey, + val relation: TrustRelation, +) + +/** + * A protocol-agnostic web-of-trust graph keyed by pubkey hex. + * + * [incoming] maps every target user to the attestations pointing *at* it — the + * only view GrapeRank needs to score a node. [outgoing] (source → the set of + * users it attests about) is derived once and used by the propagation worklist + * to know which nodes to re-score when a source's score moves. + * + * Build one with [TrustGraphBuilder.build] from a bag of Nostr events; score it + * with [GrapeRank.compute]. + */ +class TrustGraph( + val incoming: Map>, +) { + /** source pubkey → the targets it has an outgoing edge to. */ + val outgoing: Map> by lazy { + val out = HashMap>() + for ((target, edges) in incoming) { + for (edge in edges) { + out.getOrPut(edge.source) { HashSet() }.add(target) + } + } + out + } + + /** Every user that appears in the graph, as a target or as an edge source. */ + val users: Set by lazy { + val all = HashSet(incoming.keys) + for (edges in incoming.values) { + for (edge in edges) all.add(edge.source) + } + all + } + + fun edgeCount(): Int = incoming.values.sumOf { it.size } +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt new file mode 100644 index 0000000000..4848c0bbe1 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.wot + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent +import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent +import com.vitorpamplona.quartz.nip56Reports.ReportEvent + +/** + * Turns a bag of Nostr events into a [TrustGraph]. Pure: no network, no state — + * hand it whatever kind:3 / kind:10000 / kind:1984 events you have collected. + * + * - **kind:3** [ContactListEvent] → a [TrustRelation.FOLLOW] edge per followed key. + * - **kind:10000** [MuteListEvent] → a [TrustRelation.MUTE] edge per publicly muted + * key. Private (NIP-44 encrypted) mutes are ignored — they aren't ours to + * decrypt and aren't fetchable from another user's relays anyway. + * - **kind:1984** [ReportEvent] → a [TrustRelation.REPORT] edge per reported author. + * + * kind:3 and kind:10000 are replaceable, so only the newest per author is kept. + * Reports are regular events; every distinct `(reporter → reported)` pair counts + * once. Self-edges are dropped. + */ +object TrustGraphBuilder { + fun build( + events: Collection, + includeFollows: Boolean = true, + includeMutes: Boolean = true, + includeReports: Boolean = true, + ): TrustGraph { + // Latest replaceable-per-author for kind 3 / 10000. + val latestContacts = HashMap() + val latestMutes = HashMap() + val reports = ArrayList() + + for (event in events) { + when (event) { + is ContactListEvent -> + if (includeFollows) { + val prev = latestContacts[event.pubKey] + if (prev == null || event.createdAt > prev.createdAt) latestContacts[event.pubKey] = event + } + + is MuteListEvent -> + if (includeMutes) { + val prev = latestMutes[event.pubKey] + if (prev == null || event.createdAt > prev.createdAt) latestMutes[event.pubKey] = event + } + + is ReportEvent -> if (includeReports) reports.add(event) + } + } + + // target -> distinct incoming edges (dedup identical source+relation pairs). + val incoming = HashMap>() + + fun addEdge( + source: HexKey, + target: HexKey, + relation: TrustRelation, + ) { + if (source == target) return + incoming.getOrPut(target) { LinkedHashSet() }.add(TrustEdge(source, relation)) + } + + for (contacts in latestContacts.values) { + for (target in contacts.verifiedFollowKeySet()) { + addEdge(contacts.pubKey, target, TrustRelation.FOLLOW) + } + } + + for (mutes in latestMutes.values) { + for (target in mutes.linkedPubKeys()) { + addEdge(mutes.pubKey, target, TrustRelation.MUTE) + } + } + + for (report in reports) { + for (reported in report.reportedAuthor()) { + addEdge(report.pubKey, reported.pubkey, TrustRelation.REPORT) + } + } + + return TrustGraph(incoming.mapValues { (_, edges) -> edges.toList() }) + } +} diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt new file mode 100644 index 0000000000..11bb658575 --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.wot + +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import kotlin.math.abs +import kotlin.math.exp +import kotlin.math.ln +import kotlin.math.max +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class GrapeRankTest { + private val obs = "observer" + + private fun graphOf(vararg edges: Triple): TrustGraph { + val incoming = HashMap>() + for ((source, target, relation) in edges) { + incoming.getOrPut(target) { mutableListOf() }.add(TrustEdge(source, relation)) + } + return TrustGraph(incoming) + } + + @Test + fun observerIsExcludedFromRanking() { + val scores = GrapeRank().compute(graphOf(Triple(obs, "a", TrustRelation.FOLLOW)), obs) + assertNull(scores[obs], "observer's pinned self-trust is not part of the ranking") + } + + @Test + fun directFollowMatchesHandComputedValue() { + val scores = GrapeRank().compute(graphOf(Triple(obs, "a", TrustRelation.FOLLOW)), obs) + // weight = 0.5 * 1.0 * 0.85 = 0.425 ; conf(0.425) = 1 - 2^-0.425 + // score = conf * (0.425 / 0.425) = 0.2551612... + assertEquals(0.25516127, scores.getValue("a"), 1e-6) + } + + @Test + fun trustDecaysSteeplyAcrossHops() { + val scores = + GrapeRank().compute( + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("a", "b", TrustRelation.FOLLOW), + ), + obs, + ) + val a = scores.getValue("a") + val b = scores.getValue("b") + // Indirect follow from a (conf 0.03) two hops out: ~0.0045, an ~56x drop. + assertEquals(0.004499, b, 1e-5) + assertTrue(b < a / 10.0, "two-hop trust should be far below one-hop trust") + } + + @Test + fun aMuteFromAnEndorsedUserLowersTheScore() { + val followOnly = GrapeRank().compute(graphOf(Triple(obs, "b", TrustRelation.FOLLOW)), obs) + val withMute = + GrapeRank().compute( + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple(obs, "b", TrustRelation.FOLLOW), + Triple("a", "b", TrustRelation.MUTE), + ), + obs, + ) + assertTrue( + withMute.getValue("b") < followOnly.getValue("b"), + "a mute from a trusted user should pull b's score below the follow-only baseline", + ) + } + + @Test + fun purelyReportedUserFloorsAtZero() { + val scores = + GrapeRank().compute( + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("a", "d", TrustRelation.REPORT), + ), + obs, + ) + assertEquals(0.0, scores.getValue("d"), 1e-9, "negative-only signals floor at zero") + } + + @Test + fun unreachableUsersAreNotScored() { + // x -> y exists but neither is reachable from the observer. + val scores = + GrapeRank().compute( + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("x", "y", TrustRelation.FOLLOW), + ), + obs, + ) + assertTrue("a" in scores) + assertNull(scores["y"], "a user with no path from the observer is absent from the result") + } + + @Test + fun cyclesConverge() { + // a<->b mutual follow plus observer->a. Must terminate at a fixed point. + val scores = + GrapeRank().compute( + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("a", "b", TrustRelation.FOLLOW), + Triple("b", "a", TrustRelation.FOLLOW), + ), + obs, + ) + assertTrue(scores.getValue("a") > 0.0) + assertTrue(scores.getValue("b") > 0.0) + } + + /** + * Adversarial cross-check: the worklist propagation must reach the same fixed + * point as a naive full-sweep (the reference `v1FullSweep`) on random graphs. + */ + @Test + fun worklistMatchesFullSweepOnRandomGraphs() { + // Tight convergence so both methods settle onto essentially the same + // fixed point (attenuation < 1 makes the update a contraction), leaving + // only floating-point slop to compare against. + val params = GrapeRankParams(convergence = 1e-10) + val engine = GrapeRank(params) + repeat(50) { seed -> + val rng = Random(seed) + val n = 3 + rng.nextInt(12) + val nodes = (0 until n).map { "u$it" } + val edges = ArrayList>() + for (src in nodes) { + for (dst in nodes) { + if (src == dst) continue + if (rng.nextDouble() < 0.25) { + val relation = + when (rng.nextInt(5)) { + 0 -> TrustRelation.MUTE + 1 -> TrustRelation.REPORT + else -> TrustRelation.FOLLOW + } + edges.add(Triple(src, dst, relation)) + } + } + } + val graph = graphOf(*edges.toTypedArray()) + val observer = nodes.first() + + val worklist = engine.compute(graph, observer) + val fullSweep = fullSweep(graph, observer, params) + + for (node in graph.users) { + if (node == observer) continue + val a = worklist[node] ?: 0.0 + val b = fullSweep[node] ?: 0.0 + assertEquals(b, a, 1e-5, "seed=$seed node=$node worklist=$a fullSweep=$b") + } + } + } + + // Reference implementation: blind full sweep over every user until nothing changes. + private fun fullSweep( + graph: TrustGraph, + observer: HexKey, + params: GrapeRankParams, + ): Map { + fun confidence(edge: TrustEdge): Double = + when (edge.relation) { + TrustRelation.FOLLOW -> if (edge.source == observer) params.directFollowConfidence else params.indirectFollowConfidence + TrustRelation.MUTE -> params.muteConfidence + TrustRelation.REPORT -> params.reportConfidence + } + + fun weightToConfidence(w: Double) = 1.0 - exp(-w * -ln(params.rigor)) + + val scores = HashMap() + scores[observer] = 1.0 + do { + var changed = false + for (target in graph.users) { + if (target == observer) continue + var sumW = 0.0 + var sumWR = 0.0 + for (edge in graph.incoming[target] ?: emptyList()) { + val s = scores[edge.source] ?: continue + val w = confidence(edge) * s * params.attenuation + sumW += w + sumWR += w * edge.relation.rating + } + val newScore = if (abs(sumW) < 0.00001) 0.0 else max(weightToConfidence(sumW) * sumWR / sumW, 0.0) + val old = scores.put(target, newScore) ?: 0.0 + changed = changed || abs(newScore - old) > params.convergence + } + } while (changed) + scores.remove(observer) + return scores + } +} diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt new file mode 100644 index 0000000000..ad219bebf0 --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.wot + +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent +import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent +import com.vitorpamplona.quartz.nip56Reports.ReportEvent +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class TrustGraphBuilderTest { + // Distinct valid 64-hex pubkeys. + private fun pk(n: Int): HexKey = n.toString(16).padStart(64, '0') + + private val alice = pk(0xA1) + private val bob = pk(0xB0) + private val carol = pk(0xC0) + private val dave = pk(0xD0) + + private val dummySig = "0".repeat(128) + + private fun contactList( + author: HexKey, + follows: List, + createdAt: Long = 1000, + ) = ContactListEvent( + id = pk(author.hashCode() xor createdAt.toInt()), + pubKey = author, + createdAt = createdAt, + tags = follows.map { arrayOf("p", it) }.toTypedArray(), + content = "", + sig = dummySig, + ) + + private fun muteList( + author: HexKey, + mutes: List, + createdAt: Long = 1000, + ) = MuteListEvent( + id = pk(author.hashCode() xor createdAt.toInt() xor 0x5555), + pubKey = author, + createdAt = createdAt, + tags = mutes.map { arrayOf("p", it) }.toTypedArray(), + content = "", + sig = dummySig, + ) + + private fun report( + author: HexKey, + reported: HexKey, + createdAt: Long = 1000, + ) = ReportEvent( + id = pk(author.hashCode() xor reported.hashCode() xor createdAt.toInt()), + pubKey = author, + createdAt = createdAt, + tags = arrayOf(arrayOf("p", reported, "spam")), + content = "", + sig = dummySig, + ) + + @Test + fun buildsFollowMuteAndReportEdges() { + val graph = + TrustGraphBuilder.build( + listOf( + contactList(alice, listOf(bob, carol)), + muteList(bob, listOf(dave)), + report(carol, dave), + ), + ) + + assertEquals( + setOf(TrustEdge(alice, TrustRelation.FOLLOW)), + graph.incoming[bob]?.toSet(), + ) + assertEquals( + setOf(TrustEdge(alice, TrustRelation.FOLLOW)), + graph.incoming[carol]?.toSet(), + ) + assertEquals( + setOf(TrustEdge(bob, TrustRelation.MUTE), TrustEdge(carol, TrustRelation.REPORT)), + graph.incoming[dave]?.toSet(), + ) + } + + @Test + fun keepsOnlyLatestReplaceablePerAuthor() { + val graph = + TrustGraphBuilder.build( + listOf( + contactList(alice, listOf(bob), createdAt = 1000), + contactList(alice, listOf(carol), createdAt = 2000), + ), + ) + // The newer list (follows carol) wins; the stale bob follow is gone. + assertTrue(graph.incoming[bob].isNullOrEmpty()) + assertEquals(setOf(TrustEdge(alice, TrustRelation.FOLLOW)), graph.incoming[carol]?.toSet()) + } + + @Test + fun dedupesRepeatedReports() { + val graph = + TrustGraphBuilder.build( + listOf( + report(alice, dave, createdAt = 1000), + report(alice, dave, createdAt = 2000), + ), + ) + assertEquals(listOf(TrustEdge(alice, TrustRelation.REPORT)), graph.incoming[dave]) + } + + @Test + fun dropsSelfEdges() { + val graph = TrustGraphBuilder.build(listOf(contactList(alice, listOf(alice, bob)))) + assertTrue(graph.incoming[alice].isNullOrEmpty(), "a self-follow must not become an edge") + assertEquals(setOf(TrustEdge(alice, TrustRelation.FOLLOW)), graph.incoming[bob]?.toSet()) + } + + @Test + fun disablingNegativeSignalsExcludesThem() { + val events = + listOf( + contactList(alice, listOf(bob)), + muteList(alice, listOf(bob)), + report(carol, bob), + ) + val graph = TrustGraphBuilder.build(events, includeMutes = false, includeReports = false) + assertEquals(setOf(TrustEdge(alice, TrustRelation.FOLLOW)), graph.incoming[bob]?.toSet()) + } +} From 7b7c5533316a7316fa336d2346c61e8aeecb583b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 14:30:26 +0000 Subject: [PATCH 02/58] refactor(cli): drop graperank --target and the signal-toggle flags - Remove `--target USER`: the command already emits the full ranking, and a single-user lookup is a trivial slice of it. - Remove `--no-mutes` / `--no-reports` and the include* parameters on TrustGraphBuilder.build. GrapeRank is defined over follows, mutes and reports together; scoring with a signal disabled isn't a meaningful WoT. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- cli/README.md | 2 +- .../com/vitorpamplona/amethyst/cli/Main.kt | 12 ++++---- .../amethyst/cli/commands/GrapeRankCommand.kt | 30 ++----------------- .../amethyst/commons/wot/TrustGraphBuilder.kt | 27 +++++++---------- .../commons/wot/TrustGraphBuilderTest.kt | 12 -------- 5 files changed, 19 insertions(+), 64 deletions(-) diff --git a/cli/README.md b/cli/README.md index 0d678a8769..2976bcd1b5 100644 --- a/cli/README.md +++ b/cli/README.md @@ -384,7 +384,7 @@ HTTP endpoint. Reuses quartz's `Nip86Client` and the shared `Nip86Retriever` | `amy profile show [USER]` | Print kind:0 metadata. USER accepts npub/nprofile/hex/NIP-05; defaults to self. | | `amy profile edit --name … --about … --picture URL …` | Patch and re-publish your kind:0. | | `amy follow USER` / `amy unfollow USER` | Add/remove USER from your kind:3 contact list (fetches the freshest list first). | -| `amy graperank [OBSERVER] [--max-depth N] [--target USER] [--offline] [--publish]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph, crawled via the outbox model; optionally publish results as NIP-85 kind:30382 cards. | +| `amy graperank [OBSERVER] [--max-depth N] [--offline] [--publish]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph, crawled via the outbox model; optionally publish results as NIP-85 kind:30382 cards. | ### Direct messages (NIP-17) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 027483ad87..a405f7dde6 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -530,12 +530,12 @@ private fun printUsage() { | graperank [OBSERVER] compute subjective trust scores (0..1) for every | [--max-depth N] [--max-users N] user reachable in the follow/mute/report graph, | [--limit N] [--min-score X] crawled via the outbox model until no new users - | [--target USER] appear (OBSERVER: npub|nprofile|hex|name@domain, - | [--no-mutes] [--no-reports] default: active account). --target prints one - | [--rigor X] [--attenuation X] user's score; --offline scores from the local - | [--offline] [--timeout SECS] store only. --publish writes NIP-85 kind:30382 - | [--publish] [--min-rank N] trusted-assertion cards (rank = round(score*100)) - | [--publish-limit N] [--publish-relay URL] for each user at or above --min-rank. + | [--rigor X] [--attenuation X] appear (OBSERVER: npub|nprofile|hex|name@domain, + | [--offline] [--timeout SECS] default: active account). --offline scores from + | [--publish] [--min-rank N] the local store only. --publish writes NIP-85 + | [--publish-limit N] [--publish-relay URL] kind:30382 trusted-assertion cards + | (rank = round(score*100)) for each user at or + | above --min-rank. | |Zaps (NIP-57): | zap user USER SATS build a profile zap-request, fetch a BOLT11 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index fb3e65c5ec..edf22b3087 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -76,9 +76,6 @@ object GrapeRankCommand { val maxUsers = args.intFlag("max-users", 50_000) val limit = args.intFlag("limit", 100) val minScore = args.flag("min-score")?.toDoubleOrNull() ?: 0.0 - val targetArg = args.flag("target") - val includeMutes = !args.bool("no-mutes") - val includeReports = !args.bool("no-reports") val offline = args.bool("offline") val timeoutMs = args.longFlag("timeout", 10L) * 1000 val doPublish = args.bool("publish") @@ -96,12 +93,7 @@ object GrapeRankCommand { ctx.prepare() val observer = observerArg?.let { ctx.requireUserHex(it) } ?: ctx.identity.pubKeyHex - val graphKinds = - buildList { - add(ContactListEvent.KIND) - if (includeMutes) add(MuteListEvent.KIND) - if (includeReports) add(ReportEvent.KIND) - } + val graphKinds = listOf(ContactListEvent.KIND, MuteListEvent.KIND, ReportEvent.KIND) var depthReached = 0 val events: List @@ -140,29 +132,11 @@ object GrapeRankCommand { events = collected } - val graph = TrustGraphBuilder.build(events, includeMutes = includeMutes, includeReports = includeReports) + val graph = TrustGraphBuilder.build(events) val scores = GrapeRank(params).compute(graph, observer) fun rankOf(score: Double) = (score * 100).roundToInt() - if (targetArg != null) { - val target = ctx.requireUserHex(targetArg) - // The observer trusts itself fully by definition; it is excluded - // from the ranking map, so answer it directly. - val score = if (target == observer) 1.0 else scores[target] ?: 0.0 - Output.emit( - mapOf( - "observer" to observer, - "target" to target, - "score" to score, - "rank" to rankOf(score), - "users_scored" to scores.size, - "depth_reached" to depthReached, - ), - ) - return 0 - } - val ranked = scores.entries .filter { it.value >= minScore } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt index 4848c0bbe1..485ad91eb0 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt @@ -41,12 +41,7 @@ import com.vitorpamplona.quartz.nip56Reports.ReportEvent * once. Self-edges are dropped. */ object TrustGraphBuilder { - fun build( - events: Collection, - includeFollows: Boolean = true, - includeMutes: Boolean = true, - includeReports: Boolean = true, - ): TrustGraph { + fun build(events: Collection): TrustGraph { // Latest replaceable-per-author for kind 3 / 10000. val latestContacts = HashMap() val latestMutes = HashMap() @@ -54,19 +49,17 @@ object TrustGraphBuilder { for (event in events) { when (event) { - is ContactListEvent -> - if (includeFollows) { - val prev = latestContacts[event.pubKey] - if (prev == null || event.createdAt > prev.createdAt) latestContacts[event.pubKey] = event - } + is ContactListEvent -> { + val prev = latestContacts[event.pubKey] + if (prev == null || event.createdAt > prev.createdAt) latestContacts[event.pubKey] = event + } - is MuteListEvent -> - if (includeMutes) { - val prev = latestMutes[event.pubKey] - if (prev == null || event.createdAt > prev.createdAt) latestMutes[event.pubKey] = event - } + is MuteListEvent -> { + val prev = latestMutes[event.pubKey] + if (prev == null || event.createdAt > prev.createdAt) latestMutes[event.pubKey] = event + } - is ReportEvent -> if (includeReports) reports.add(event) + is ReportEvent -> reports.add(event) } } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt index ad219bebf0..d43ecdc459 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt @@ -135,16 +135,4 @@ class TrustGraphBuilderTest { assertTrue(graph.incoming[alice].isNullOrEmpty(), "a self-follow must not become an edge") assertEquals(setOf(TrustEdge(alice, TrustRelation.FOLLOW)), graph.incoming[bob]?.toSet()) } - - @Test - fun disablingNegativeSignalsExcludesThem() { - val events = - listOf( - contactList(alice, listOf(bob)), - muteList(alice, listOf(bob)), - report(carol, bob), - ) - val graph = TrustGraphBuilder.build(events, includeMutes = false, includeReports = false) - assertEquals(setOf(TrustEdge(alice, TrustRelation.FOLLOW)), graph.incoming[bob]?.toSet()) - } } From 90452358027291dc7a7195109dfc60bc26bbbf54 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 15:08:26 +0000 Subject: [PATCH 03/58] feat(cli): skip republishing unchanged graperank cards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously `graperank --publish` rebuilt and rebroadcast a NIP-85 kind:30382 ContactCard for every scored user on every run, minting a new event id and created_at even when the rank was identical — pure churn for a parameterized- replaceable event. Read back the ranks we last published from the account's own kind:30382 cards in the local store (ctx.publish already persists them) and publish only the targets whose rank is new or changed. Report the count left alone as `skipped_unchanged`. Verified against a local geode relay: first run publishes N cards (skipped_unchanged=0); an immediate re-run with identical ranks publishes 0 (skipped_unchanged=N). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index edf22b3087..0e1b2d2d18 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -164,11 +164,22 @@ object GrapeRankCommand { ?.takeIf { it.isNotEmpty() } ?: ctx.outboxRelays() - val toPublish = + // Ranks we've already published (read back from the store, which + // holds our own prior cards) — keyed by target, newest per target. + // Lets us leave an unchanged card alone instead of churning it. + val publishedRanks = publishedCardRanks(ctx) + + val candidates = ranked .filter { rankOf(it.value) >= minRank } - .take(publishLimit) .map { it.key to rankOf(it.value) } + val changed = candidates.filter { (target, rank) -> publishedRanks[target] != rank } + val toPublish = changed.take(publishLimit) + + result["skipped_unchanged"] = candidates.size - changed.size + if (changed.size > toPublish.size) { + result["publish_truncated"] = changed.size - toPublish.size + } if (relays.isEmpty()) { result["published"] = 0 @@ -240,6 +251,25 @@ object GrapeRankCommand { } } + /** + * The rank we last published for each target, read from the active account's + * own kind:30382 cards in the local store (newest card wins per target). + * `ctx.publish` stores every card it sends, so on repeat runs this reflects + * what's already out there and lets us skip targets whose rank is unchanged. + */ + private suspend fun publishedCardRanks(ctx: Context): Map { + val self = ctx.identity.pubKeyHex + return ctx.store + .query(Filter(kinds = listOf(ContactCardEvent.KIND), authors = listOf(self))) + .filterIsInstance() + .groupBy { it.aboutUser() } + .mapNotNull { (target, cards) -> + val t = target ?: return@mapNotNull null + val rank = cards.maxByOrNull { it.createdAt }?.rank() ?: return@mapNotNull null + t to rank + }.toMap() + } + /** Build + publish one NIP-85 kind:30382 card per user, bounded-concurrently. */ private suspend fun publishCards( ctx: Context, From 1b78dfec57724973bb45d7c9e24a503766fec0ed Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 16:14:48 +0000 Subject: [PATCH 04/58] feat(cli): add NIP-85 provider discovery to graperank (kind:10040) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishing kind:30382 rank cards is only half of NIP-85 — clients also need the kind:10040 TrustProviderListEvent to discover which key provides which assertion, and where. Add the discovery layer as two sub-verbs: - `amy graperank register [PROVIDER]` — append a ServiceProviderTag (default `30382:rank`, self, first outbox relay) to the account's kind:10040, fetching the freshest list first so existing providers are preserved. Idempotent, supports `--service KIND:TAG`, `--relay`, and `--private`. - `amy graperank providers [USER]` — list a user's declared providers (cache-first; own private entries are decrypted and included). Bare `amy graperank [OBSERVER]` still computes scores; the dispatcher only peels off the `register` / `providers` words. All built on quartz's existing `TrustProviderListEvent` / `ServiceProviderTag` / `ProviderTypes`. Verified against a local geode relay: register creates the 10040 and is idempotent on re-run; providers lists both a public 30382:rank entry and a private 30382:followers entry. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- cli/README.md | 4 +- cli/ROADMAP.md | 2 +- .../com/vitorpamplona/amethyst/cli/Main.kt | 9 +- .../amethyst/cli/commands/GrapeRankCommand.kt | 196 ++++++++++++++++++ 4 files changed, 207 insertions(+), 4 deletions(-) diff --git a/cli/README.md b/cli/README.md index 2976bcd1b5..88eeeea65d 100644 --- a/cli/README.md +++ b/cli/README.md @@ -384,7 +384,9 @@ HTTP endpoint. Reuses quartz's `Nip86Client` and the shared `Nip86Retriever` | `amy profile show [USER]` | Print kind:0 metadata. USER accepts npub/nprofile/hex/NIP-05; defaults to self. | | `amy profile edit --name … --about … --picture URL …` | Patch and re-publish your kind:0. | | `amy follow USER` / `amy unfollow USER` | Add/remove USER from your kind:3 contact list (fetches the freshest list first). | -| `amy graperank [OBSERVER] [--max-depth N] [--offline] [--publish]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph, crawled via the outbox model; optionally publish results as NIP-85 kind:30382 cards. | +| `amy graperank [OBSERVER] [--max-depth N] [--offline] [--publish]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph, crawled via the outbox model; optionally publish results as NIP-85 kind:30382 cards (unchanged ranks are skipped). | +| `amy graperank register [PROVIDER] [--service KIND:TAG] [--relay URL]` | Declare a NIP-85 provider in your kind:10040 so clients can discover it (default: self as the `30382:rank` provider). | +| `amy graperank providers [USER]` | List a user's declared NIP-85 trusted providers (public + your own private entries). | ### Direct messages (NIP-17) diff --git a/cli/ROADMAP.md b/cli/ROADMAP.md index d31deebf08..8fc7609b0b 100644 --- a/cli/ROADMAP.md +++ b/cli/ROADMAP.md @@ -58,7 +58,7 @@ Status legend: ✅ shipped · 📦 logic lives in `commons/`, needs a command · | NIP-51 lists (bookmarks, mute, follow sets) | 🆕 | `amethyst/model/nip51Lists/` | | NIP-57 zaps (send + verify) | 🆕 | Needs LN-URL plumbing; `amethyst/service/lnurl/`. | | NIP-65 outbox model queries | 🆕 | | -| NIP-85 GrapeRank web-of-trust (`amy graperank`) | ✅ | `GrapeRankCommand` — outbox-model crawl + scoring engine in `commons/wot/` (`GrapeRank`, `TrustGraph`, `TrustGraphBuilder`); publishes kind:30382 `ContactCardEvent`. | +| NIP-85 GrapeRank web-of-trust (`amy graperank`) | ✅ | `GrapeRankCommand` — outbox-model crawl + scoring engine in `commons/wot/` (`GrapeRank`, `TrustGraph`, `TrustGraphBuilder`); publishes kind:30382 `ContactCardEvent` (diffed against prior ranks), plus `register` / `providers` for the kind:10040 `TrustProviderListEvent` discovery layer. | | NIP-72 communities | 🆕 | | | NIP-78 app-specific data (settings sync) | 🆕 | | | Long-form (NIP-23) publish / read | 🆕 | | diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index a405f7dde6..08667592ad 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -215,7 +215,7 @@ private suspend fun dispatch(argv: Array): Int { "store" -> StoreCommands.dispatch(dataDir, tail) "follow" -> FollowCommand.follow(dataDir, tail) "unfollow" -> FollowCommand.unfollow(dataDir, tail) - "graperank" -> GrapeRankCommand.run(dataDir, tail) + "graperank" -> GrapeRankCommand.dispatch(dataDir, tail) "search" -> SearchCommand.dispatch(dataDir, tail) "zap" -> ZapCommand.dispatch(dataDir, tail) "offer" -> OfferCommands.dispatch(dataDir, tail) @@ -535,7 +535,12 @@ private fun printUsage() { | [--publish] [--min-rank N] the local store only. --publish writes NIP-85 | [--publish-limit N] [--publish-relay URL] kind:30382 trusted-assertion cards | (rank = round(score*100)) for each user at or - | above --min-rank. + | above --min-rank (unchanged ranks are skipped). + | graperank register [PROVIDER] declare a NIP-85 provider in your kind:10040 so + | [--service KIND:TAG] [--relay URL] clients can discover it (default: self as the + | [--private] 30382:rank provider at your first outbox relay). + | graperank providers [USER] [--refresh] list a user's declared NIP-85 trusted providers + | [--timeout SECS] (default: active account). | |Zaps (NIP-57): | zap user USER SATS build a profile zap-request, fetch a BOLT11 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 0e1b2d2d18..07f71c7360 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -37,6 +37,11 @@ import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent import com.vitorpamplona.quartz.nip56Reports.ReportEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.TrustProviderListEvent +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.serviceProviders +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ProviderTypes +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProviderTag +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceType import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag import kotlinx.coroutines.async @@ -58,6 +63,13 @@ import kotlin.math.roundToInt * Prints a ranked list (text, or one JSON object under `--json`). With * `--publish`, results are also published as NIP-85 kind:30382 `ContactCardEvent` * trusted assertions (one per scored user, `rank = round(score*100)`). + * + * Sub-verbs complete the NIP-85 provider experience — the discovery layer that + * lets clients find and consume those assertions: + * - `amy graperank register` — advertise a `30382:rank` provider in the + * account's kind:10040 [TrustProviderListEvent] (defaults to self, so a + * provider publishing ranks announces where to find them). + * - `amy graperank providers [USER]` — list a user's trusted providers. */ object GrapeRankCommand { // Authors per REQ filter — keeps individual subscriptions within relay limits. @@ -66,6 +78,18 @@ object GrapeRankCommand { // Concurrent publishes when writing NIP-85 cards. private const val PUBLISH_CONCURRENCY = 16 + suspend fun dispatch( + dataDir: DataDir, + tail: Array, + ): Int = + // Sub-verbs are explicit words; anything else (npub / hex / nprofile / + // NIP-05, or nothing) is the OBSERVER positional for a score computation. + when (tail.firstOrNull()) { + "register" -> register(dataDir, tail.drop(1).toTypedArray()) + "providers" -> providers(dataDir, tail.drop(1).toTypedArray()) + else -> run(dataDir, tail) + } + suspend fun run( dataDir: DataDir, rest: Array, @@ -198,6 +222,178 @@ object GrapeRankCommand { } } + /** + * `amy graperank register [PROVIDER] [--service KIND:TAG] [--relay URL] [--private]` + * + * Add a NIP-85 provider entry to the account's kind:10040 + * [TrustProviderListEvent] — the declaration a client reads to discover which + * key publishes which assertion, and where. Defaults to declaring *self* as + * the `30382:rank` provider at the account's first outbox relay, which is the + * self-advertisement a GrapeRank provider makes so its followers can find the + * cards it publishes. Fetches the freshest list first so existing providers + * are preserved. + */ + private suspend fun register( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val providerArg = args.positionalOrNull(0) ?: args.flag("provider") + val serviceArg = args.flag("service") + val relayArg = args.flag("relay") + val isPrivate = args.bool("private") + val timeoutMs = args.longFlag("timeout", 8L) * 1000 + + val service = + serviceArg?.let { + ServiceType.parse(it) ?: return Output.error("bad_args", "--service must be KIND:TAG, e.g. 30382:rank") + } ?: ProviderTypes.rank + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val self = ctx.identity.pubKeyHex + val provider = providerArg?.let { ctx.requireUserHex(it) } ?: self + + val outbox = ctx.outboxRelays() + val relay = + relayArg?.let { RelayUrlNormalizer.normalizeOrNull(it) } + ?: outbox.firstOrNull() + ?: return Output.error("no_relays", "no relay hint; pass --relay URL or configure outbox relays") + + val latest = fetchLatestProviderList(ctx, self, outbox, timeoutMs) + val alreadyListed = + latest?.serviceProviders()?.any { + it.service == service && it.pubkey == provider && it.relayUrl == relay + } ?: false + + if (alreadyListed) { + Output.emit( + mapOf( + "service" to service.toValue(), + "provider" to provider, + "relay" to relay.url, + "changed" to false, + "based_on" to latest?.id, + ), + ) + return 0 + } + + val tag = ServiceProviderTag(service, provider, relay) + val event = + if (latest == null) { + TrustProviderListEvent.create(tag, isPrivate = isPrivate, signer = ctx.signer) + } else { + TrustProviderListEvent.add(latest, tag, isPrivate = isPrivate, signer = ctx.signer) + } + + val ack = ctx.publish(event, outbox) + Output.emit( + mapOf( + "service" to service.toValue(), + "provider" to provider, + "relay" to relay.url, + "private" to isPrivate, + "changed" to true, + "event_id" to event.id, + "based_on" to latest?.id, + "published_to" to ack.filterValues { it }.keys.map { it.url }, + "rejected_by" to ack.filterValues { !it }.keys.map { it.url }, + ), + ) + return 0 + } + } + + /** + * `amy graperank providers [USER] [--refresh] [--timeout SECS]` + * + * List the NIP-85 trusted providers a user declares in their kind:10040 + * (default: the active account). Cache-first; falls back to a relay drain on + * a miss or with `--refresh`. For the active account, private (NIP-44) + * provider entries are decrypted and included too. + */ + private suspend fun providers( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val userArg = args.positionalOrNull(0) + val refresh = args.bool("refresh") + val timeoutMs = args.longFlag("timeout", 8L) * 1000 + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val user = userArg?.let { ctx.requireUserHex(it) } ?: ctx.identity.pubKeyHex + val isSelf = user == ctx.identity.pubKeyHex + + var event = if (refresh) null else providerListOf(ctx, user) + if (event == null) { + ctx.drain( + (ctx.bootstrapRelays() + Constants.eventFinderRelays).associateWith { + listOf(Filter(kinds = listOf(TrustProviderListEvent.KIND), authors = listOf(user), limit = 1)) + }, + timeoutMs, + ) + event = providerListOf(ctx, user) + } + + if (event == null) { + Output.emit(mapOf("user" to user, "found" to false, "providers" to emptyList())) + return 0 + } + + val public = event.serviceProviders() + val private = if (isSelf) event.privateTags(ctx.signer)?.serviceProviders().orEmpty() else emptyList() + + fun render( + tag: ServiceProviderTag, + scope: String, + ) = mapOf( + "service" to tag.service.toValue(), + "provider" to tag.pubkey, + "relay" to tag.relayUrl.url, + "scope" to scope, + ) + + Output.emit( + mapOf( + "user" to user, + "found" to true, + "event_id" to event.id, + "created_at" to event.createdAt, + "providers" to public.map { render(it, "public") } + private.map { render(it, "private") }, + ), + ) + return 0 + } + } + + /** Latest known kind:10040 provider list for [pubKey] from the local store. */ + private suspend fun providerListOf( + ctx: Context, + pubKey: HexKey, + ): TrustProviderListEvent? = + ctx.store + .query(Filter(kinds = listOf(TrustProviderListEvent.KIND), authors = listOf(pubKey), limit = 1)) + .firstOrNull() as? TrustProviderListEvent + + /** + * Fetch the freshest kind:10040 for [pubKey] from [relays] so a register + * builds on top of the current provider set instead of clobbering it. + */ + private suspend fun fetchLatestProviderList( + ctx: Context, + pubKey: HexKey, + relays: Set, + timeoutMs: Long, + ): TrustProviderListEvent? { + if (relays.isEmpty()) return providerListOf(ctx, pubKey) + val filter = Filter(kinds = listOf(TrustProviderListEvent.KIND), authors = listOf(pubKey), limit = 1) + ctx.drain(relays.associateWith { listOf(filter) }, timeoutMs) + return providerListOf(ctx, pubKey) + } + /** * Fetch kind:10002 relay lists for any frontier member we don't already know, * so [routeByOutbox] can route their content query to their own write relays. From 1f6a11c59c08e73ad36ea7228b6a8328be56d23b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 17:14:24 +0000 Subject: [PATCH 05/58] docs(cli): analyse Brainstorm GrapeRank service for score parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Analysis of NosFabrica/brainstorm_graperank_algorithm (Java scoring worker) and NosFabrica/brainstorm_server (Python orchestration) to confirm amy's scores match the reference GrapeRank service. Finding: our commons/wot formula and every scoring parameter are already identical to Brainstorm's DEFAULT preset (attenuation 0.85, rigor 0.5, follow 1.0/0.03, from-observer 0.5, mute/report -0.1/0.5, delta 0.0001). Our `score` is exactly their ScoreCard `influence`. Remaining divergence is data completeness, not math — and because a signal's weight scales by the rater's influence, only in-graph raters move a score, which our outbox crawl already captures. Documents the pipeline, side-by-side params, divergence sources, and follow-ups (presets, influence/verified fields). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../2026-07-06-graperank-brainstorm-parity.md | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 cli/plans/2026-07-06-graperank-brainstorm-parity.md diff --git a/cli/plans/2026-07-06-graperank-brainstorm-parity.md b/cli/plans/2026-07-06-graperank-brainstorm-parity.md new file mode 100644 index 0000000000..d54bb0bad4 --- /dev/null +++ b/cli/plans/2026-07-06-graperank-brainstorm-parity.md @@ -0,0 +1,119 @@ +# GrapeRank score parity with NosFabrica Brainstorm + +Goal: `amy graperank` should output scores **numerically very close** to +NosFabrica's Brainstorm service, the reference GrapeRank implementation. + +Sources analysed: +- `NosFabrica/brainstorm_graperank_algorithm` — the Java scoring worker. +- `NosFabrica/brainstorm_server` — the Python orchestration server. + +## How Brainstorm builds its service + +A four-stage pipeline: + +1. **Ingest.** `app/nostr_event_transferer/nostr_event_transferer.py` copies raw + social-graph events — **kinds 0, 3, 10000, 1984** (profiles, follows, mutes, + reports) — from a strfry relay into the server. Same four kinds we crawl. +2. **Graph.** Events land in **Neo4j** as a directed graph of follow / mute / + report edges between pubkeys. Redis + Postgres back the job queue and config. +3. **Score.** The Java worker (`grape/GrapeRankAlgorithm.java`) runs GrapeRank + from an observer, producing a **`ScoreCard`** per user + (`rank/ScoreCard.java`): `observer, observee, hops, averageScore, input, + confidence, influence, verified, trustedFollowers, trustedReporters`. + **There is no `rank` field — the trust value is `influence` ∈ [0,1].** +4. **Serve / publish.** Presets are tunable per deployment + (`DEFAULT` / `PERMISSIVE` / `RESTRICTIVE`, `graperank_preset` table, validated + by `GrapeRankPresetParams`). Java `GrapeRankParams` mirrors the Python model + field-for-field; the README states Python is the source of truth and both + repos must stay in sync. + +## The algorithm (their `grape/GrapeRankAlgorithm.java`) + +``` +rigority = -log(rigor) +confidence(sumWeights) = 1 - exp(-sumWeights * rigority) # weight -> confidence +per edge: weight = edgeConfidence * influenceOfRater * attenuationFactor + wxr = weight * edgeRating +averageScore = sumWxR / sumWeights (0 if sumWeights == 0) +influence = max(averageScore * confidence(sumWeights), 0) +``` + +- Observer seeded at `influence = 1.0` (fixed authority). +- Non-observers seeded by hop distance, then **iterated until every user's + influence delta < 0.0001** (`loopBreakDelta`). Seeding only affects the + starting guess; attenuation < 1 makes the update a contraction, so the fixed + point is unique. +- The rater weight uses the rater's **`influence`**, and + `influence = max(weightToConfidence(sumW) * sumWR/sumW, 0)`. + +## Side-by-side: Brainstorm DEFAULT vs `commons/wot` + +`Constants.java` `DEFAULT_PARAMS` (== the Pydantic `GrapeRankPresetParams` +DEFAULT) against our `GrapeRankParams` defaults: + +| Brainstorm field | value | our field | value | match | +|---|---|---|---|---| +| `attenuationFactor` | 0.85 | `attenuation` | 0.85 | ✅ | +| `rigor` | 0.5 | `rigor` | 0.5 | ✅ | +| `followRating` | 1.0 | `FOLLOW.rating` | 1.0 | ✅ | +| `muteRating` | -0.1 | `MUTE.rating` | -0.1 | ✅ | +| `reportRating` | -0.1 | `REPORT.rating` | -0.1 | ✅ | +| `followConfidenceOfObserver` | 0.5 | `directFollowConfidence` | 0.5 | ✅ | +| `followConfidence` | 0.03 | `indirectFollowConfidence` | 0.03 | ✅ | +| `muteConfidence` | 0.5 | `muteConfidence` | 0.5 | ✅ | +| `reportConfidence` | 0.5 | `reportConfidence` | 0.5 | ✅ | +| `loopBreakDelta` | 0.0001 | `convergence` | 0.0001 | ✅ | + +The three `verified*InfluenceCutoff`s (followers 0.02, reporters 0.1, +muters 0.01) only flag a derived `verified` boolean; they do **not** affect the +score. + +**Conclusion: our formula is identical and every scoring parameter matches +DEFAULT.** Our `score` *is* their `influence` +(`max(weightToConfidence(sumW) * sumWR/sumW, 0)`), propagated as the rater +weight — the exact same quantity. On the same input graph the two produce the +same influence to floating-point precision. Our published `rank = round(score * +100)` is a presentation choice on top of that influence (their `ScoreCard` +exposes `influence` as a raw float via the API). + +## Where divergence can still come from — and why it's small + +It is **data**, not math: + +1. **Graph completeness.** Brainstorm ingests the whole strfry graph into Neo4j; + we crawl outward from the observer via the outbox model. **This matters less + than it seems:** a mute/report contributes `confidence * influenceOfRater * + attenuation`, so a signal from a user with **zero influence** (someone outside + the observer's trust graph) contributes **zero**. Only follows/mutes/reports + authored by users *inside* the follow graph move a score — and those are + exactly the users our crawl discovers and whose kind 3/10000/1984 we fetch. + So the effective scoring input is the same, provided the crawl runs to + convergence (our default) rather than a shallow `--max-depth`. +2. **Fringe users / crawl gaps.** Relay timeouts that drop a contact list, or a + `--max-users` cap, remove edges and shift nearby scores. Mitigate with a full + crawl and generous `--timeout`. +3. **Convergence precision.** Both stop at delta 0.0001; residual error is + < ~0.0001 in influence ⇒ < ~0.01 rank points ⇒ identical integer `rank`. +4. **Seeding.** Their hop-distance seed vs our zero seed — same fixed point, no + effect on the result. + +## Recommendations + +- **Keep the current DEFAULT params** — they are byte-for-byte the Brainstorm + DEFAULT preset. No change needed for parity. +- **Crawl to convergence** (the default) rather than a small `--max-depth`; a + shallow crawl is the single biggest source of drift. +- **Optional, for fuller parity (not required for close scores):** + - Add `--preset default|permissive|restrictive`. DEFAULT is confirmed; the + PERMISSIVE / RESTRICTIVE numbers are DB-seeded in `brainstorm_server` (an + alembic seed migration) and were not extractable from the public tree — + pull them from a running instance before hard-coding. + - Optionally expose `influence` as a raw float alongside `rank` in `--json`, + and compute the `verified` flag from the cutoffs, to mirror their + `ScoreCard` shape for interop diffing. + +## Verification idea + +Point `amy graperank --offline` at a store seeded from the same +strfry snapshot Brainstorm ingested, and diff our `score` against their +`ScoreCard.influence` for the same observer. Expect agreement to ~1e-4. From 135390ff669f7aebdd988691757255ad43d8b056 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 17:33:28 +0000 Subject: [PATCH 06/58] feat(cli): broaden graperank injector for full graph discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To match Brainstorm's full-graph ingest, the crawl now discovers data through three tiers (mirroring the app's pickRelaysToLoadUsers) instead of just the outbox + a small fallback: - Indexer relays (purplepag.es, coracle, …) join the discovery set. They serve kind:0/3/10002 for the whole network and are where a stranger's relay list and contact list are actually found — the biggest completeness lever. - Per-follow relay hints are harvested from the `p`-tag hints in every contact list we crawl and used as a discovery tier below each user's kind:10002. - A per-hop retry pass re-queries any frontier member whose contact list still didn't arrive (no kind:10002, or its outbox was unreachable) against the indexer + hint set, recovering users the outbox model alone would miss. This tightens the only real source of score divergence from Brainstorm — data completeness — since a signal's weight scales by the rater's influence, so the users that matter are exactly the in-graph ones this crawl now reaches more reliably. Local regression check: scores unchanged (rank 26 for direct follows). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../2026-07-06-graperank-brainstorm-parity.md | 12 ++- .../amethyst/cli/commands/GrapeRankCommand.kt | 75 +++++++++++++++---- 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/cli/plans/2026-07-06-graperank-brainstorm-parity.md b/cli/plans/2026-07-06-graperank-brainstorm-parity.md index d54bb0bad4..1383a5defb 100644 --- a/cli/plans/2026-07-06-graperank-brainstorm-parity.md +++ b/cli/plans/2026-07-06-graperank-brainstorm-parity.md @@ -90,8 +90,16 @@ It is **data**, not math: So the effective scoring input is the same, provided the crawl runs to convergence (our default) rather than a shallow `--max-depth`. 2. **Fringe users / crawl gaps.** Relay timeouts that drop a contact list, or a - `--max-users` cap, remove edges and shift nearby scores. Mitigate with a full - crawl and generous `--timeout`. + `--max-users` cap, remove edges and shift nearby scores. The injector now + mitigates this with a three-tier discovery model mirroring the app's + `pickRelaysToLoadUsers`: each user's kind:10002 **outbox**, then harvested + **relay hints** (from `p`-tag hints in the contact lists we crawl), then the + broad **discovery set** — bootstrap + event-finder + **indexer relays** + (purplepag.es, coracle, …) that serve kind:0/3/10002 for the whole network. + A per-hop **retry pass** re-queries any member whose contact list still didn't + arrive against that indexer + hint set, recovering users the outbox model + alone would miss. Remaining mitigation levers: a full crawl (default) and a + generous `--timeout`. 3. **Convergence precision.** Both stop at delta 0.0001; residual error is < ~0.0001 in influence ⇒ < ~0.01 rank points ⇒ identical integer `rank`. 4. **Seeding.** Their hop-distance seed vs our zero seed — same fixed point, no diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 07f71c7360..ec5136874f 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.amethyst.cli.Context import com.vitorpamplona.amethyst.cli.DataDir import com.vitorpamplona.amethyst.cli.Output import com.vitorpamplona.amethyst.commons.defaults.Constants +import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList import com.vitorpamplona.amethyst.commons.wot.GrapeRank import com.vitorpamplona.amethyst.commons.wot.GrapeRankParams import com.vitorpamplona.amethyst.commons.wot.TrustGraphBuilder @@ -128,24 +129,51 @@ object GrapeRankCommand { } else { val collected = mutableListOf() val discovered = hashSetOf(observer) + // Per-user relay hints harvested from the `p`-tag relay hints in + // the contact lists we crawl (A's follow of B says where B writes). + // A second discovery tier below each user's kind:10002 outbox. + val relayHints = HashMap>() var frontier: Set = setOf(observer) for (hop in 0 until maxDepth) { if (frontier.isEmpty()) break depthReached = hop + 1 - ensureRelayLists(ctx, frontier, timeoutMs) + // 1. Locate each frontier member's kind:10002 write relays. + ensureRelayLists(ctx, frontier, relayHints, timeoutMs) - val filters = routeByOutbox(ctx, frontier, graphKinds) + // 2. Fetch their follows/mutes/reports from those relays. + val filters = routeByOutbox(ctx, frontier, relayHints, graphKinds) collected += ctx.drain(filters, timeoutMs).map { it.second } + // 3. Completeness retry: any member whose contact list still + // didn't arrive (no kind:10002, or its outbox was down) gets + // re-queried against the broad indexer + hint set. Indexers + // like purplepag.es serve kind:3 for the whole network, so + // this recovers users the outbox model alone would miss. + val stillMissing = frontier.filter { ctx.contactsOf(it) == null } + if (stillMissing.isNotEmpty()) { + val retryRelays = discoveryRelays(ctx) + stillMissing.flatMap { relayHints[it].orEmpty() } + val retry = + retryRelays.associateWith { + stillMissing.chunked(AUTHORS_PER_FILTER).map { chunk -> Filter(kinds = graphKinds, authors = chunk) } + } + collected += ctx.drain(retry, timeoutMs).map { it.second } + } + + // 4. Harvest relay hints + expand the follow frontier. val next = hashSetOf() for (pk in frontier) { - ctx.contactsOf(pk)?.verifiedFollowKeySet()?.forEach { followed -> - if (discovered.size < maxUsers && discovered.add(followed)) next += followed + val contacts = ctx.contactsOf(pk) ?: continue + for (tag in contacts.follows()) { + tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } + if (discovered.size < maxUsers && discovered.add(tag.pubKey)) next += tag.pubKey } } - System.err.println("[graperank] hop ${hop + 1}: fetched frontier=${frontier.size}, new=${next.size}, total=${discovered.size}") + val recovered = stillMissing.count { ctx.contactsOf(it) != null } + System.err.println( + "[graperank] hop ${hop + 1}: frontier=${frontier.size}, recovered=$recovered/${stillMissing.size}, new=${next.size}, total=${discovered.size}", + ) if (discovered.size >= maxUsers) { System.err.println("[graperank] reached --max-users=$maxUsers cap; stopping crawl") @@ -394,26 +422,40 @@ object GrapeRankCommand { return providerListOf(ctx, pubKey) } + /** + * The broad, network-wide discovery set: the account's own relays + Amethyst's + * bootstrap defaults + the event-finder relays + the **indexer relays** + * (purplepag.es, coracle, …). Indexers aggregate kind:0 / kind:3 / kind:10002 + * for the whole network, so they are where a stranger's relay list and contact + * list are actually found — the single biggest lever on crawl completeness. + */ + private suspend fun discoveryRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + /** * Fetch kind:10002 relay lists for any frontier member we don't already know, * so [routeByOutbox] can route their content query to their own write relays. - * Uses the broad bootstrap + event-finder relay set as the discovery seed — - * the CLI analog of the app's tiered outbox lookup. + * Queries the broad discovery set (incl. indexers) plus each user's harvested + * relay [hints] — the CLI analog of the app's tiered `pickRelaysToLoadUsers`. */ private suspend fun ensureRelayLists( ctx: Context, pubkeys: Set, + hints: Map>, timeoutMs: Long, ) { val missing = pubkeys.filter { ctx.relaysOf(it) == null } if (missing.isEmpty()) return - val seedRelays = ctx.bootstrapRelays() + Constants.eventFinderRelays - if (seedRelays.isEmpty()) return + val base = discoveryRelays(ctx) + val perRelay = HashMap>() + for (pk in missing) { + for (relay in base + hints[pk].orEmpty()) perRelay.getOrPut(relay) { HashSet() }.add(pk) + } + if (perRelay.isEmpty()) return val filters = - seedRelays.associateWith { - missing.chunked(AUTHORS_PER_FILTER).map { chunk -> + perRelay.mapValues { (_, authors) -> + authors.chunked(AUTHORS_PER_FILTER).map { chunk -> Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) } } @@ -422,21 +464,22 @@ object GrapeRankCommand { /** * Group [pubkeys] by the relays we should query for their events: each user's - * kind:10002 write relays (the outbox model), falling back to the broad - * event-finder set for users with no advertised relay list. Authors are - * chunked per relay to respect relay REQ limits. + * kind:10002 write relays (the outbox model); for users with no advertised + * relay list, their harvested relay [hints] plus the broad discovery set. + * Authors are chunked per relay to respect relay REQ limits. */ private suspend fun routeByOutbox( ctx: Context, pubkeys: Set, + hints: Map>, kinds: List, ): Map> { - val fallback = ctx.bootstrapRelays() + Constants.eventFinderRelays + val fallback = discoveryRelays(ctx) val perRelay = HashMap>() for (pk in pubkeys) { val write = ctx.relaysOf(pk)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() } - val relays = write ?: fallback + val relays = write ?: (hints[pk].orEmpty() + fallback) for (relay in relays) perRelay.getOrPut(relay) { HashSet() }.add(pk) } From dd86b617c1fcd1bd4cccb307f555c9b9ac75f26d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 18:05:17 +0000 Subject: [PATCH 07/58] fix(cli): route graperank content to outboxes, indexers only for 10002 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correct the injector's relay model: indexer relays (purplepag.es, coracle, …) aggregate kind:10002 (and kind:0) for the whole network but do NOT serve kind:3/10000/1984. Those live only on each user's own outbox. - Split the relay sets: `relayListDiscoveryRelays` (bootstrap + event-finder + indexers) is used only to locate kind:10002; `contentFallbackRelays` (bootstrap + event-finder, no indexers) is the best-effort fallback for content when a user's outbox is unknown/down. - Content is fetched from each user's outbox write relays, with harvested relay hints and general relays as fallback — never indexers. Also add progress status (all on stderr, stdout stays the JSON contract): - loading already logs per-hop frontier/recovered/new/total counts; - "graph built: N users, E edges; scoring…" and "scored N users" bracket the calculation; - GrapeRank.compute gains an optional (visited, scored, queued) progress callback, wired to emit a scoring line every 5000 worklist visits so a large graph shows movement instead of hanging silently. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../2026-07-06-graperank-brainstorm-parity.md | 22 ++++--- .../amethyst/cli/commands/GrapeRankCommand.kt | 57 ++++++++++++++----- .../amethyst/commons/wot/GrapeRank.kt | 9 +++ 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/cli/plans/2026-07-06-graperank-brainstorm-parity.md b/cli/plans/2026-07-06-graperank-brainstorm-parity.md index 1383a5defb..3571de22ec 100644 --- a/cli/plans/2026-07-06-graperank-brainstorm-parity.md +++ b/cli/plans/2026-07-06-graperank-brainstorm-parity.md @@ -90,16 +90,20 @@ It is **data**, not math: So the effective scoring input is the same, provided the crawl runs to convergence (our default) rather than a shallow `--max-depth`. 2. **Fringe users / crawl gaps.** Relay timeouts that drop a contact list, or a - `--max-users` cap, remove edges and shift nearby scores. The injector now - mitigates this with a three-tier discovery model mirroring the app's - `pickRelaysToLoadUsers`: each user's kind:10002 **outbox**, then harvested - **relay hints** (from `p`-tag hints in the contact lists we crawl), then the - broad **discovery set** — bootstrap + event-finder + **indexer relays** - (purplepag.es, coracle, …) that serve kind:0/3/10002 for the whole network. + `--max-users` cap, remove edges and shift nearby scores. The injector mitigates + this with a two-stage model mirroring the app's `pickRelaysToLoadUsers`: + - **Relay-list discovery** (kind:10002) queries the account's relays + + bootstrap + event-finder + **indexer relays** (purplepag.es, coracle, …). + Indexers aggregate kind:10002 (and kind:0) for the whole network, so this is + where a stranger's outbox is found — the biggest completeness lever. + - **Content** (kind:3/10000/1984/0) is fetched from each user's **own outbox** + write relays, with harvested **relay hints** (from the `p`-tag hints in + contact lists we crawl) and general-purpose relays as a best-effort fallback + when the outbox is unknown/down. **Indexers are not used for content** — they + don't serve those kinds; kind:3/mutes/reports live only on the user's outbox. A per-hop **retry pass** re-queries any member whose contact list still didn't - arrive against that indexer + hint set, recovering users the outbox model - alone would miss. Remaining mitigation levers: a full crawl (default) and a - generous `--timeout`. + arrive against that hint + general-relay set. Remaining mitigation levers: a + full crawl (default) and a generous `--timeout`. 3. **Convergence precision.** Both stop at delta 0.0001; residual error is < ~0.0001 in influence ⇒ < ~0.01 rank points ⇒ identical integer `rank`. 4. **Seeding.** Their hop-distance seed vs our zero seed — same fixed point, no diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index ec5136874f..4d125e3592 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -79,6 +79,9 @@ object GrapeRankCommand { // Concurrent publishes when writing NIP-85 cards. private const val PUBLISH_CONCURRENCY = 16 + // Emit a scoring-progress line every this many worklist visits. + private const val SCORE_PROGRESS_STEP = 5_000 + suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -148,12 +151,14 @@ object GrapeRankCommand { // 3. Completeness retry: any member whose contact list still // didn't arrive (no kind:10002, or its outbox was down) gets - // re-queried against the broad indexer + hint set. Indexers - // like purplepag.es serve kind:3 for the whole network, so - // this recovers users the outbox model alone would miss. + // re-queried against its relay hints plus the general-purpose + // fallback relays. kind:3/10000/1984 live on the user's own + // outbox — NOT on indexers (those only aggregate kind:10002) — + // so this is best-effort recovery from general relays that may + // hold a copy, not a guaranteed find. val stillMissing = frontier.filter { ctx.contactsOf(it) == null } if (stillMissing.isNotEmpty()) { - val retryRelays = discoveryRelays(ctx) + stillMissing.flatMap { relayHints[it].orEmpty() } + val retryRelays = contentFallbackRelays(ctx) + stillMissing.flatMap { relayHints[it].orEmpty() } val retry = retryRelays.associateWith { stillMissing.chunked(AUTHORS_PER_FILTER).map { chunk -> Filter(kinds = graphKinds, authors = chunk) } @@ -185,7 +190,20 @@ object GrapeRankCommand { } val graph = TrustGraphBuilder.build(events) - val scores = GrapeRank(params).compute(graph, observer) + System.err.println( + "[graperank] graph built: ${graph.users.size} users, ${graph.edgeCount()} edges from ${events.size} events; scoring…", + ) + + // Live scoring progress: the worklist visits each reachable user once + // per relaxation; report every PROGRESS_STEP visits so a large graph + // shows movement instead of hanging silently. + val scores = + GrapeRank(params).compute(graph, observer) { visited, scored, queued -> + if (visited % SCORE_PROGRESS_STEP == 0) { + System.err.println("[graperank] scoring: $visited visited, $scored scored, $queued queued") + } + } + System.err.println("[graperank] scored ${scores.size} users") fun rankOf(score: Double) = (score * 100).roundToInt() @@ -423,19 +441,28 @@ object GrapeRankCommand { } /** - * The broad, network-wide discovery set: the account's own relays + Amethyst's - * bootstrap defaults + the event-finder relays + the **indexer relays** - * (purplepag.es, coracle, …). Indexers aggregate kind:0 / kind:3 / kind:10002 - * for the whole network, so they are where a stranger's relay list and contact - * list are actually found — the single biggest lever on crawl completeness. + * Relays to query for **kind:10002 relay lists** — the account's own relays + + * bootstrap defaults + event-finder relays + the **indexer relays** + * (purplepag.es, coracle, …). Indexers aggregate kind:10002 (and kind:0) for + * the whole network, so this is where a stranger's relay list is found. They + * do NOT hold kind:3/10000/1984 — see [contentFallbackRelays]. */ - private suspend fun discoveryRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + private suspend fun relayListDiscoveryRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + + /** + * Best-effort fallback relays for **content** (kind:3/10000/1984/0) when a + * user's outbox is unknown or unreachable. Content lives on each user's own + * outbox, so this is only general-purpose relays that *might* hold a copy — + * bootstrap + event-finder. **No indexers**: they don't serve these kinds. + */ + private suspend fun contentFallbackRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays /** * Fetch kind:10002 relay lists for any frontier member we don't already know, * so [routeByOutbox] can route their content query to their own write relays. - * Queries the broad discovery set (incl. indexers) plus each user's harvested - * relay [hints] — the CLI analog of the app's tiered `pickRelaysToLoadUsers`. + * Queries the relay-list discovery set (incl. indexers) plus each user's + * harvested relay [hints] — the CLI analog of the app's tiered + * `pickRelaysToLoadUsers`. */ private suspend fun ensureRelayLists( ctx: Context, @@ -446,7 +473,7 @@ object GrapeRankCommand { val missing = pubkeys.filter { ctx.relaysOf(it) == null } if (missing.isEmpty()) return - val base = discoveryRelays(ctx) + val base = relayListDiscoveryRelays(ctx) val perRelay = HashMap>() for (pk in missing) { for (relay in base + hints[pk].orEmpty()) perRelay.getOrPut(relay) { HashSet() }.add(pk) @@ -474,7 +501,7 @@ object GrapeRankCommand { hints: Map>, kinds: List, ): Map> { - val fallback = discoveryRelays(ctx) + val fallback = contentFallbackRelays(ctx) val perRelay = HashMap>() for (pk in pubkeys) { diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt index 848d91cb0c..fd4859cade 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt @@ -86,10 +86,15 @@ class GrapeRank( * Score every user reachable from [observer]. The returned map excludes the * observer itself (its score is a pinned `1.0` and not part of a ranking). * Users with no positive path from the observer are absent (equivalently, 0). + * + * [onProgress] is invoked once per worklist visit with + * `(visited, scored, queued)` running counts, so a caller can report progress + * on a large graph; it defaults to a no-op. */ fun compute( graph: TrustGraph, observer: HexKey, + onProgress: ((visited: Int, scored: Int, queued: Int) -> Unit)? = null, ): Map { val scores = HashMap() scores[observer] = 1.0 @@ -103,6 +108,7 @@ class GrapeRank( graph.outgoing[observer]?.forEach(::enqueue) + var visited = 0 while (queue.isNotEmpty()) { val target = queue.removeFirst() queued.remove(target) @@ -113,6 +119,9 @@ class GrapeRank( if (abs(newScore - oldScore) > params.convergence) { graph.outgoing[target]?.forEach(::enqueue) } + + visited++ + onProgress?.invoke(visited, scores.size, queue.size) } scores.remove(observer) From 4ec3da1e869e5abedcc92eee9bf3f2157f234c7d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 18:23:10 +0000 Subject: [PATCH 08/58] =?UTF-8?q?feat(cli):=20exhaustive=20graperank=20cra?= =?UTF-8?q?wl=20=E2=80=94=20no=20user=20cap,=20check=20every=20outbox?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the depth-limited, user-capped BFS with a completeness loop that runs until every discovered user's kind:10002 outbox has been checked and their latest kind:3/10000/1984 pulled from it: - Delete the --max-users cap entirely. - Crawl round by round until the pending set (discovered minus done) is empty. A user is "done" once we download its contact list, or after --max-attempts (default 3) failed tries of its outbox — so an unreachable outbox can't stall the crawl, and it still terminates on a finite graph. - --max-rounds replaces --max-depth as an (unbounded by default) safety backstop. - Track and report the pool of relays actually contacted (relays_contacted), the "running relays" we connect to as more outboxes are discovered. JSON: `depth_reached` -> `crawl_rounds`, add `relays_contacted`. Per-round and final crawl-summary progress on stderr. Local regression: scores unchanged (rank 26); the crawl retries contact-list-less users then terminates cleanly. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- cli/README.md | 2 +- .../2026-07-06-graperank-brainstorm-parity.md | 23 ++-- .../com/vitorpamplona/amethyst/cli/Main.kt | 18 +-- .../amethyst/cli/commands/GrapeRankCommand.kt | 114 ++++++++++-------- 4 files changed, 90 insertions(+), 67 deletions(-) diff --git a/cli/README.md b/cli/README.md index 88eeeea65d..de4afc3822 100644 --- a/cli/README.md +++ b/cli/README.md @@ -384,7 +384,7 @@ HTTP endpoint. Reuses quartz's `Nip86Client` and the shared `Nip86Retriever` | `amy profile show [USER]` | Print kind:0 metadata. USER accepts npub/nprofile/hex/NIP-05; defaults to self. | | `amy profile edit --name … --about … --picture URL …` | Patch and re-publish your kind:0. | | `amy follow USER` / `amy unfollow USER` | Add/remove USER from your kind:3 contact list (fetches the freshest list first). | -| `amy graperank [OBSERVER] [--max-depth N] [--offline] [--publish]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph, crawled via the outbox model; optionally publish results as NIP-85 kind:30382 cards (unchanged ranks are skipped). | +| `amy graperank [OBSERVER] [--offline] [--publish]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph. Exhaustively crawls each user's kind:10002 outbox for their latest kind:3/10000/1984 until every discovered user is checked (no user cap); optionally publishes results as NIP-85 kind:30382 cards (unchanged ranks are skipped). | | `amy graperank register [PROVIDER] [--service KIND:TAG] [--relay URL]` | Declare a NIP-85 provider in your kind:10040 so clients can discover it (default: self as the `30382:rank` provider). | | `amy graperank providers [USER]` | List a user's declared NIP-85 trusted providers (public + your own private entries). | diff --git a/cli/plans/2026-07-06-graperank-brainstorm-parity.md b/cli/plans/2026-07-06-graperank-brainstorm-parity.md index 3571de22ec..b31e51975f 100644 --- a/cli/plans/2026-07-06-graperank-brainstorm-parity.md +++ b/cli/plans/2026-07-06-graperank-brainstorm-parity.md @@ -87,11 +87,13 @@ It is **data**, not math: the observer's trust graph) contributes **zero**. Only follows/mutes/reports authored by users *inside* the follow graph move a score — and those are exactly the users our crawl discovers and whose kind 3/10000/1984 we fetch. - So the effective scoring input is the same, provided the crawl runs to - convergence (our default) rather than a shallow `--max-depth`. -2. **Fringe users / crawl gaps.** Relay timeouts that drop a contact list, or a - `--max-users` cap, remove edges and shift nearby scores. The injector mitigates - this with a two-stage model mirroring the app's `pickRelaysToLoadUsers`: + So the effective scoring input is the same, as long as the crawl actually + checks every discovered user's outbox — which it now does exhaustively (no + user cap, retrying an unreachable outbox up to `--max-attempts` times). +2. **Fringe users / crawl gaps.** A relay timeout that drops a contact list + removes edges and shifts nearby scores. The injector mitigates this with a + two-stage model mirroring the app's `pickRelaysToLoadUsers`, plus a + completeness loop that retries until every user's outbox has been checked: - **Relay-list discovery** (kind:10002) queries the account's relays + bootstrap + event-finder + **indexer relays** (purplepag.es, coracle, …). Indexers aggregate kind:10002 (and kind:0) for the whole network, so this is @@ -101,9 +103,9 @@ It is **data**, not math: contact lists we crawl) and general-purpose relays as a best-effort fallback when the outbox is unknown/down. **Indexers are not used for content** — they don't serve those kinds; kind:3/mutes/reports live only on the user's outbox. - A per-hop **retry pass** re-queries any member whose contact list still didn't - arrive against that hint + general-relay set. Remaining mitigation levers: a - full crawl (default) and a generous `--timeout`. + The crawl loops round by round, retrying any member whose contact list still + didn't arrive (up to `--max-attempts`), until every discovered user's outbox + has been checked. Remaining mitigation lever: a generous `--timeout`. 3. **Convergence precision.** Both stop at delta 0.0001; residual error is < ~0.0001 in influence ⇒ < ~0.01 rank points ⇒ identical integer `rank`. 4. **Seeding.** Their hop-distance seed vs our zero seed — same fixed point, no @@ -113,8 +115,9 @@ It is **data**, not math: - **Keep the current DEFAULT params** — they are byte-for-byte the Brainstorm DEFAULT preset. No change needed for parity. -- **Crawl to convergence** (the default) rather than a small `--max-depth`; a - shallow crawl is the single biggest source of drift. +- **The crawl is exhaustive by default** (no user cap; every reachable user's + outbox is checked, unreachable outboxes retried up to `--max-attempts`). An + incomplete crawl is the single biggest source of drift, so avoid capping it. - **Optional, for fuller parity (not required for close scores):** - Add `--preset default|permissive|restrictive`. DEFAULT is confirmed; the PERMISSIVE / RESTRICTIVE numbers are DB-seeded in `brainstorm_server` (an diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 08667592ad..92f73a8a3e 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -528,14 +528,16 @@ private fun printUsage() { | |Web of Trust (GrapeRank): | graperank [OBSERVER] compute subjective trust scores (0..1) for every - | [--max-depth N] [--max-users N] user reachable in the follow/mute/report graph, - | [--limit N] [--min-score X] crawled via the outbox model until no new users - | [--rigor X] [--attenuation X] appear (OBSERVER: npub|nprofile|hex|name@domain, - | [--offline] [--timeout SECS] default: active account). --offline scores from - | [--publish] [--min-rank N] the local store only. --publish writes NIP-85 - | [--publish-limit N] [--publish-relay URL] kind:30382 trusted-assertion cards - | (rank = round(score*100)) for each user at or - | above --min-rank (unchanged ranks are skipped). + | [--limit N] [--min-score X] user reachable in the follow/mute/report graph. + | [--rigor X] [--attenuation X] Crawls each user's kind:10002 outbox for their + | [--max-attempts N] [--max-rounds N] latest kind:3/10000/1984 until every discovered + | [--offline] [--timeout SECS] user has been checked (no user cap; --max-attempts + | [--publish] [--min-rank N] bounds retries of an unreachable outbox, default 3). + | [--publish-limit N] [--publish-relay URL] OBSERVER: npub|nprofile|hex|name@domain (default: + | active account). --offline scores from the local + | store only. --publish writes NIP-85 kind:30382 + | cards (rank = round(score*100)) for each user at + | or above --min-rank (unchanged ranks skipped). | graperank register [PROVIDER] declare a NIP-85 provider in your kind:10040 so | [--service KIND:TAG] [--relay URL] clients can discover it (default: self as the | [--private] 30382:rank provider at your first outbox relay). diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 4d125e3592..78eaebb322 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -58,8 +58,10 @@ import kotlin.math.roundToInt * observer has full self-trust). It crawls the follow graph outward using the * outbox model — each user's kind:10002 write relays are located first, then * their kind:3 / kind:10000 / kind:1984 events are fetched from *their own* - * relays — until no new users appear (typically ~8 hops), then runs the scoring - * engine in `commons/wot`. + * relays. The crawl is exhaustive: it keeps going, with no user cap, until every + * discovered user's outbox has been checked and their contact list pulled (an + * unreachable outbox is retried up to `--max-attempts` times), then runs the + * scoring engine in `commons/wot`. * * Prints a ranked list (text, or one JSON object under `--json`). With * `--publish`, results are also published as NIP-85 kind:30382 `ContactCardEvent` @@ -100,8 +102,11 @@ object GrapeRankCommand { ): Int { val args = Args(rest) val observerArg = args.positionalOrNull(0) - val maxDepth = args.intFlag("max-depth", 8) - val maxUsers = args.intFlag("max-users", 50_000) + // Crawl to full convergence by default (every reachable user's outbox + // checked). --max-rounds is only a safety backstop; --max-attempts bounds + // how many times we re-try an unreachable user's outbox before giving up. + val maxRounds = args.intFlag("max-rounds", Int.MAX_VALUE) + val maxAttempts = args.intFlag("max-attempts", 3) val limit = args.intFlag("limit", 100) val minScore = args.flag("min-score")?.toDoubleOrNull() ?: 0.0 val offline = args.bool("offline") @@ -123,7 +128,8 @@ object GrapeRankCommand { val graphKinds = listOf(ContactListEvent.KIND, MuteListEvent.KIND, ReportEvent.KIND) - var depthReached = 0 + var rounds = 0 + var relaysContactedCount = 0 val events: List if (offline) { @@ -132,60 +138,71 @@ object GrapeRankCommand { } else { val collected = mutableListOf() val discovered = hashSetOf(observer) - // Per-user relay hints harvested from the `p`-tag relay hints in - // the contact lists we crawl (A's follow of B says where B writes). - // A second discovery tier below each user's kind:10002 outbox. + // Per-user relay hints harvested from the `p`-tag relay hints in the + // contact lists we crawl (A's follow of B says where B writes) — a + // discovery tier below each user's kind:10002 outbox. val relayHints = HashMap>() - var frontier: Set = setOf(observer) + // Users we're finished with: their outbox was queried and we either + // downloaded their kind:3 or ran out of retry attempts. Growing this + // set toward `discovered` is what drives the crawl to completion. + val done = hashSetOf() + val attempts = HashMap() + // The pool of relays we actually route outbox queries to, grown as + // more users' kind:10002 outboxes are discovered. + val relaysContacted = hashSetOf() - for (hop in 0 until maxDepth) { - if (frontier.isEmpty()) break - depthReached = hop + 1 + // Loop until every discovered user has had their outbox checked and + // their kind:3/10000/1984 pulled from it — no user cap. A user whose + // outbox stays unreachable is dropped after --max-attempts tries so + // the crawl still terminates. + while (rounds < maxRounds) { + val pending = discovered.filterNot { it in done } + if (pending.isEmpty()) break + rounds++ - // 1. Locate each frontier member's kind:10002 write relays. - ensureRelayLists(ctx, frontier, relayHints, timeoutMs) + // 1. Resolve kind:10002 outboxes for pending users missing them. + ensureRelayLists(ctx, pending.toSet(), relayHints, timeoutMs) - // 2. Fetch their follows/mutes/reports from those relays. - val filters = routeByOutbox(ctx, frontier, relayHints, graphKinds) + // 2. Pull kind:3/10000/1984 from each pending user's own outbox + // (hints + general relays only when the outbox is unknown). + val before = collected.size + val filters = routeByOutbox(ctx, pending.toSet(), relayHints, graphKinds) + relaysContacted += filters.keys collected += ctx.drain(filters, timeoutMs).map { it.second } - // 3. Completeness retry: any member whose contact list still - // didn't arrive (no kind:10002, or its outbox was down) gets - // re-queried against its relay hints plus the general-purpose - // fallback relays. kind:3/10000/1984 live on the user's own - // outbox — NOT on indexers (those only aggregate kind:10002) — - // so this is best-effort recovery from general relays that may - // hold a copy, not a guaranteed find. - val stillMissing = frontier.filter { ctx.contactsOf(it) == null } - if (stillMissing.isNotEmpty()) { - val retryRelays = contentFallbackRelays(ctx) + stillMissing.flatMap { relayHints[it].orEmpty() } - val retry = - retryRelays.associateWith { - stillMissing.chunked(AUTHORS_PER_FILTER).map { chunk -> Filter(kinds = graphKinds, authors = chunk) } + // 3. Mark done / retry, harvest hints, expand the follow graph. + var downloaded = 0 + var newUsers = 0 + for (pk in pending) { + val contacts = ctx.contactsOf(pk) + if (contacts != null) { + done += pk + downloaded++ + for (tag in contacts.follows()) { + tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } + if (discovered.add(tag.pubKey)) newUsers++ } - collected += ctx.drain(retry, timeoutMs).map { it.second } - } - - // 4. Harvest relay hints + expand the follow frontier. - val next = hashSetOf() - for (pk in frontier) { - val contacts = ctx.contactsOf(pk) ?: continue - for (tag in contacts.follows()) { - tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } - if (discovered.size < maxUsers && discovered.add(tag.pubKey)) next += tag.pubKey + } else { + val tries = (attempts[pk] ?: 0) + 1 + attempts[pk] = tries + // Give up once we've exhausted retries: either the user has + // no contact list, or their outbox is unreachable. + if (tries >= maxAttempts) done += pk } } - val recovered = stillMissing.count { ctx.contactsOf(it) != null } System.err.println( - "[graperank] hop ${hop + 1}: frontier=${frontier.size}, recovered=$recovered/${stillMissing.size}, new=${next.size}, total=${discovered.size}", + "[graperank] round $rounds: queried=${pending.size}, +events=${collected.size - before}, " + + "downloaded=$downloaded, newUsers=$newUsers, discovered=${discovered.size}, done=${done.size}", ) - - if (discovered.size >= maxUsers) { - System.err.println("[graperank] reached --max-users=$maxUsers cap; stopping crawl") - break - } - frontier = next } + + relaysContactedCount = relaysContacted.size + val unreached = discovered.count { it !in done || ctx.contactsOf(it) == null } + System.err.println( + "[graperank] crawl complete: ${discovered.size} users discovered, " + + "${discovered.size - unreached} contact lists downloaded, $unreached without one, " + + "$relaysContactedCount relays contacted, $rounds rounds", + ) events = collected } @@ -215,7 +232,8 @@ object GrapeRankCommand { val result = linkedMapOf( "observer" to observer, - "depth_reached" to depthReached, + "crawl_rounds" to rounds, + "relays_contacted" to relaysContactedCount, "graph_users" to graph.users.size, "graph_edges" to graph.edgeCount(), "users_scored" to scores.size, From 3f503f0d0473bed85318aa425884ec4373095620 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 18:51:01 +0000 Subject: [PATCH 09/58] refactor(cli): drop graperank --max-attempts, hardcode 3 retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-user outbox retry bound doesn't need to be tunable — replace the --max-attempts flag with a MAX_OUTBOX_ATTEMPTS = 3 constant. Same behaviour, one fewer knob. Updates usage text, README, and the parity doc. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../2026-07-06-graperank-brainstorm-parity.md | 6 +++--- .../com/vitorpamplona/amethyst/cli/Main.kt | 11 +++++------ .../amethyst/cli/commands/GrapeRankCommand.kt | 18 ++++++++++-------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cli/plans/2026-07-06-graperank-brainstorm-parity.md b/cli/plans/2026-07-06-graperank-brainstorm-parity.md index b31e51975f..52f09056d9 100644 --- a/cli/plans/2026-07-06-graperank-brainstorm-parity.md +++ b/cli/plans/2026-07-06-graperank-brainstorm-parity.md @@ -89,7 +89,7 @@ It is **data**, not math: exactly the users our crawl discovers and whose kind 3/10000/1984 we fetch. So the effective scoring input is the same, as long as the crawl actually checks every discovered user's outbox — which it now does exhaustively (no - user cap, retrying an unreachable outbox up to `--max-attempts` times). + user cap, retrying an unreachable outbox a few times). 2. **Fringe users / crawl gaps.** A relay timeout that drops a contact list removes edges and shifts nearby scores. The injector mitigates this with a two-stage model mirroring the app's `pickRelaysToLoadUsers`, plus a @@ -104,7 +104,7 @@ It is **data**, not math: when the outbox is unknown/down. **Indexers are not used for content** — they don't serve those kinds; kind:3/mutes/reports live only on the user's outbox. The crawl loops round by round, retrying any member whose contact list still - didn't arrive (up to `--max-attempts`), until every discovered user's outbox + didn't arrive (a few times), until every discovered user's outbox has been checked. Remaining mitigation lever: a generous `--timeout`. 3. **Convergence precision.** Both stop at delta 0.0001; residual error is < ~0.0001 in influence ⇒ < ~0.01 rank points ⇒ identical integer `rank`. @@ -116,7 +116,7 @@ It is **data**, not math: - **Keep the current DEFAULT params** — they are byte-for-byte the Brainstorm DEFAULT preset. No change needed for parity. - **The crawl is exhaustive by default** (no user cap; every reachable user's - outbox is checked, unreachable outboxes retried up to `--max-attempts`). An + outbox is checked, unreachable outboxes retried a few times). An incomplete crawl is the single biggest source of drift, so avoid capping it. - **Optional, for fuller parity (not required for close scores):** - Add `--preset default|permissive|restrictive`. DEFAULT is confirmed; the diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 92f73a8a3e..38bd66bb2d 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -529,12 +529,11 @@ private fun printUsage() { |Web of Trust (GrapeRank): | graperank [OBSERVER] compute subjective trust scores (0..1) for every | [--limit N] [--min-score X] user reachable in the follow/mute/report graph. - | [--rigor X] [--attenuation X] Crawls each user's kind:10002 outbox for their - | [--max-attempts N] [--max-rounds N] latest kind:3/10000/1984 until every discovered - | [--offline] [--timeout SECS] user has been checked (no user cap; --max-attempts - | [--publish] [--min-rank N] bounds retries of an unreachable outbox, default 3). - | [--publish-limit N] [--publish-relay URL] OBSERVER: npub|nprofile|hex|name@domain (default: - | active account). --offline scores from the local + | [--rigor X] [--attenuation X] Exhaustively crawls each user's kind:10002 outbox + | [--max-rounds N] for their latest kind:3/10000/1984 until every + | [--offline] [--timeout SECS] discovered user has been checked (no user cap). + | [--publish] [--min-rank N] OBSERVER: npub|nprofile|hex|name@domain (default: + | [--publish-limit N] [--publish-relay URL] active account). --offline scores from the local | store only. --publish writes NIP-85 kind:30382 | cards (rank = round(score*100)) for each user at | or above --min-rank (unchanged ranks skipped). diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 78eaebb322..86294079f0 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -60,8 +60,8 @@ import kotlin.math.roundToInt * their kind:3 / kind:10000 / kind:1984 events are fetched from *their own* * relays. The crawl is exhaustive: it keeps going, with no user cap, until every * discovered user's outbox has been checked and their contact list pulled (an - * unreachable outbox is retried up to `--max-attempts` times), then runs the - * scoring engine in `commons/wot`. + * unreachable outbox is retried a few times), then runs the scoring engine in + * `commons/wot`. * * Prints a ranked list (text, or one JSON object under `--json`). With * `--publish`, results are also published as NIP-85 kind:30382 `ContactCardEvent` @@ -84,6 +84,10 @@ object GrapeRankCommand { // Emit a scoring-progress line every this many worklist visits. private const val SCORE_PROGRESS_STEP = 5_000 + // Times we re-query an unreachable user's outbox before giving up on it, so + // the crawl still terminates on a finite graph. + private const val MAX_OUTBOX_ATTEMPTS = 3 + suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -103,10 +107,8 @@ object GrapeRankCommand { val args = Args(rest) val observerArg = args.positionalOrNull(0) // Crawl to full convergence by default (every reachable user's outbox - // checked). --max-rounds is only a safety backstop; --max-attempts bounds - // how many times we re-try an unreachable user's outbox before giving up. + // checked). --max-rounds is only a safety backstop. val maxRounds = args.intFlag("max-rounds", Int.MAX_VALUE) - val maxAttempts = args.intFlag("max-attempts", 3) val limit = args.intFlag("limit", 100) val minScore = args.flag("min-score")?.toDoubleOrNull() ?: 0.0 val offline = args.bool("offline") @@ -153,8 +155,8 @@ object GrapeRankCommand { // Loop until every discovered user has had their outbox checked and // their kind:3/10000/1984 pulled from it — no user cap. A user whose - // outbox stays unreachable is dropped after --max-attempts tries so - // the crawl still terminates. + // outbox stays unreachable is dropped after MAX_OUTBOX_ATTEMPTS tries + // so the crawl still terminates. while (rounds < maxRounds) { val pending = discovered.filterNot { it in done } if (pending.isEmpty()) break @@ -187,7 +189,7 @@ object GrapeRankCommand { attempts[pk] = tries // Give up once we've exhausted retries: either the user has // no contact list, or their outbox is unreachable. - if (tries >= maxAttempts) done += pk + if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk } } System.err.println( From 8f7b9cf59166aeb36196284efca9dfb828ba1248 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 19:40:54 +0000 Subject: [PATCH 10/58] fix(cli): batch graperank outbox fetches so contact lists actually download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live run against Vitor's ~110k-user WoT exposed the crawl's real bottleneck: draining every pending user's outbox in one subscription saturates connections and times out. Round-by-round evidence — 250 users queried downloaded 205 contact lists (82%), but 17,055 queried downloaded only 137 (0.8%). Net: 93k outboxes found but only ~14k contact lists pulled, so most users had no outgoing edges and scores came out far below Brainstorm's. Fix: fetch content in bounded batches (USER_BATCH=256) drained a few at a time (DRAIN_CONCURRENCY=8). Routing (store reads) runs serially; only the drains run concurrently — inserts serialize on the store write lock, so that is safe. kind:10002 discovery stays a bulk indexer query (they aggregate 10002 and handle bulk author filters fine); only the per-user outbox fan-out is batched. Effect on the same graph: round 3 went from 137 → 4,157 contact lists downloaded (+36k events), and users scored after 3 rounds jumped 34,976 → 109,760. Full-run parity verification against the live Brainstorm set is in progress. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 90 +++++++++++-------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 86294079f0..f1f010f81b 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -88,6 +88,14 @@ object GrapeRankCommand { // the crawl still terminates on a finite graph. private const val MAX_OUTBOX_ATTEMPTS = 3 + // Users whose outboxes we fetch in a single drain. Draining thousands of + // distinct outbox relays at once saturates connections and times out + // (empirically ~250 users/drain succeeds, ~17k fails); keep the fan-out small. + private const val USER_BATCH = 256 + + // Concurrent content drains. Bounded so total open connections stay sane. + private const val DRAIN_CONCURRENCY = 8 + suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -162,34 +170,47 @@ object GrapeRankCommand { if (pending.isEmpty()) break rounds++ - // 1. Resolve kind:10002 outboxes for pending users missing them. - ensureRelayLists(ctx, pending.toSet(), relayHints, timeoutMs) + // 1. Resolve kind:10002 outboxes for pending users missing them, + // in bulk from the indexer set (they aggregate kind:10002). + ensureRelayLists(ctx, pending.toSet(), timeoutMs) - // 2. Pull kind:3/10000/1984 from each pending user's own outbox - // (hints + general relays only when the outbox is unknown). val before = collected.size - val filters = routeByOutbox(ctx, pending.toSet(), relayHints, graphKinds) - relaysContacted += filters.keys - collected += ctx.drain(filters, timeoutMs).map { it.second } - - // 3. Mark done / retry, harvest hints, expand the follow graph. var downloaded = 0 var newUsers = 0 - for (pk in pending) { - val contacts = ctx.contactsOf(pk) - if (contacts != null) { - done += pk - downloaded++ - for (tag in contacts.follows()) { - tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } - if (discovered.add(tag.pubKey)) newUsers++ + + // 2. Pull kind:3/10000/1984 from each user's own outbox, in small + // batches drained a few at a time. Routing (store reads) is done + // serially; only the drains run concurrently — inserts serialize + // on the store write lock, so that is safe. Processing each + // batch's results is serial. + for (group in pending.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { + val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, graphKinds) } + val drained = + coroutineScope { + prepared + .map { (batch, filters) -> + async { Triple(batch, filters.keys, ctx.drain(filters, timeoutMs).map { it.second }) } + }.awaitAll() + } + for ((batch, relays, ev) in drained) { + relaysContacted += relays + collected += ev + for (pk in batch) { + val contacts = ctx.contactsOf(pk) + if (contacts != null) { + done += pk + downloaded++ + for (tag in contacts.follows()) { + tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } + if (discovered.add(tag.pubKey)) newUsers++ + } + } else { + val tries = (attempts[pk] ?: 0) + 1 + attempts[pk] = tries + // Give up after retries: no contact list, or outbox unreachable. + if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk + } } - } else { - val tries = (attempts[pk] ?: 0) + 1 - attempts[pk] = tries - // Give up once we've exhausted retries: either the user has - // no contact list, or their outbox is unreachable. - if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk } } System.err.println( @@ -478,31 +499,26 @@ object GrapeRankCommand { private suspend fun contentFallbackRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays /** - * Fetch kind:10002 relay lists for any frontier member we don't already know, - * so [routeByOutbox] can route their content query to their own write relays. - * Queries the relay-list discovery set (incl. indexers) plus each user's - * harvested relay [hints] — the CLI analog of the app's tiered - * `pickRelaysToLoadUsers`. + * Fetch kind:10002 relay lists for any [pubkeys] we don't already know, so + * [routeByOutbox] can route their content query to their own write relays. + * Queries the bounded relay-list discovery set (indexers + general defaults), + * which aggregate kind:10002 for the whole network — reliable in bulk, unlike + * fanning out to thousands of per-user outboxes. */ private suspend fun ensureRelayLists( ctx: Context, pubkeys: Set, - hints: Map>, timeoutMs: Long, ) { val missing = pubkeys.filter { ctx.relaysOf(it) == null } if (missing.isEmpty()) return - val base = relayListDiscoveryRelays(ctx) - val perRelay = HashMap>() - for (pk in missing) { - for (relay in base + hints[pk].orEmpty()) perRelay.getOrPut(relay) { HashSet() }.add(pk) - } - if (perRelay.isEmpty()) return + val relays = relayListDiscoveryRelays(ctx) + if (relays.isEmpty()) return val filters = - perRelay.mapValues { (_, authors) -> - authors.chunked(AUTHORS_PER_FILTER).map { chunk -> + relays.associateWith { + missing.chunked(AUTHORS_PER_FILTER).map { chunk -> Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) } } From e63ca5d637c42b12ac69f0e70097450b2d6e31f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 20:35:51 +0000 Subject: [PATCH 11/58] fix(cli): graperank skips contact lists already in the store Two related fixes so a warm/shared store isn't re-downloaded every run: - Only DOWNLOAD contact lists we don't already have. Each round now splits pending users into "already in the store" (expanded from disk with zero network) vs "need to fetch" (routed to their outbox). Verified on a warm store: round 2 pending=250 -> cached=236, downloaded=7; round 3 pending=19491 -> cached=15827, downloaded=87. - Build the trust graph from the store (kind:3/10000/1984 query) instead of the in-run `collected` list, so cached-and-skipped lists still contribute their edges. Online and offline paths now share the same graph source. No behavioural change on a cold store (nothing cached -> download everything once), and the crawl still runs to full graph depth with no user cap. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 137 +++++++++++------- 1 file changed, 81 insertions(+), 56 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index f1f010f81b..c2e22b322e 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -140,95 +140,120 @@ object GrapeRankCommand { var rounds = 0 var relaysContactedCount = 0 - val events: List + var contactListsHeld = 0 - if (offline) { - events = ctx.store.query(Filter(kinds = graphKinds)) - System.err.println("[graperank] offline: ${events.size} events from local store") - } else { - val collected = mutableListOf() + if (!offline) { val discovered = hashSetOf(observer) // Per-user relay hints harvested from the `p`-tag relay hints in the // contact lists we crawl (A's follow of B says where B writes) — a // discovery tier below each user's kind:10002 outbox. val relayHints = HashMap>() - // Users we're finished with: their outbox was queried and we either - // downloaded their kind:3 or ran out of retry attempts. Growing this - // set toward `discovered` is what drives the crawl to completion. + // Users we're finished with: we hold their kind:3 (cached or freshly + // downloaded), or ran out of retry attempts. val done = hashSetOf() val attempts = HashMap() - // The pool of relays we actually route outbox queries to, grown as - // more users' kind:10002 outboxes are discovered. + // The pool of relays we actually route outbox queries to. val relaysContacted = hashSetOf() - // Loop until every discovered user has had their outbox checked and - // their kind:3/10000/1984 pulled from it — no user cap. A user whose - // outbox stays unreachable is dropped after MAX_OUTBOX_ATTEMPTS tries - // so the crawl still terminates. + // Harvest a contact list: record its relay hints and add its follows to + // the frontier. Returns the count of newly-discovered users. + fun expand(contacts: ContactListEvent): Int { + var fresh = 0 + for (tag in contacts.follows()) { + tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } + if (discovered.add(tag.pubKey)) fresh++ + } + return fresh + } + + // Loop until every discovered user's contact list is in hand — no user + // cap, to full graph depth. We only DOWNLOAD lists we don't already + // have; a list already in the store is expanded from disk with no + // network (so re-runs and a warm shared store are cheap). An + // unreachable outbox is dropped after MAX_OUTBOX_ATTEMPTS tries so the + // crawl still terminates. while (rounds < maxRounds) { val pending = discovered.filterNot { it in done } if (pending.isEmpty()) break rounds++ - // 1. Resolve kind:10002 outboxes for pending users missing them, - // in bulk from the indexer set (they aggregate kind:10002). - ensureRelayLists(ctx, pending.toSet(), timeoutMs) - - val before = collected.size + var cached = 0 var downloaded = 0 var newUsers = 0 - // 2. Pull kind:3/10000/1984 from each user's own outbox, in small - // batches drained a few at a time. Routing (store reads) is done - // serially; only the drains run concurrently — inserts serialize - // on the store write lock, so that is safe. Processing each - // batch's results is serial. - for (group in pending.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { - val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, graphKinds) } - val drained = - coroutineScope { - prepared - .map { (batch, filters) -> - async { Triple(batch, filters.keys, ctx.drain(filters, timeoutMs).map { it.second }) } - }.awaitAll() - } - for ((batch, relays, ev) in drained) { - relaysContacted += relays - collected += ev - for (pk in batch) { - val contacts = ctx.contactsOf(pk) - if (contacts != null) { - done += pk - downloaded++ - for (tag in contacts.follows()) { - tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } - if (discovered.add(tag.pubKey)) newUsers++ + // 1. Users whose kind:3 is already in the store: expand, no network. + val need = ArrayList() + for (pk in pending) { + val contacts = ctx.contactsOf(pk) + if (contacts != null) { + done += pk + contactListsHeld++ + cached++ + newUsers += expand(contacts) + } else { + need += pk + } + } + + // 2. Download the rest from their own outboxes. Resolve kind:10002 + // in bulk (indexers aggregate it), then fetch content in small + // batches drained a few at a time — one giant drain over + // thousands of outbox relays saturates connections and times out. + // Routing (store reads) is serial; only the drains run + // concurrently, which is safe: inserts serialize on the store + // write lock. + if (need.isNotEmpty()) { + ensureRelayLists(ctx, need.toSet(), timeoutMs) + for (group in need.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { + val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, graphKinds) } + val drained = + coroutineScope { + prepared + .map { (batch, filters) -> + async { + ctx.drain(filters, timeoutMs) + batch to filters.keys + } + }.awaitAll() + } + for ((batch, relays) in drained) { + relaysContacted += relays + for (pk in batch) { + val contacts = ctx.contactsOf(pk) + if (contacts != null) { + done += pk + contactListsHeld++ + downloaded++ + newUsers += expand(contacts) + } else { + val tries = (attempts[pk] ?: 0) + 1 + attempts[pk] = tries + // Give up after retries: no contact list, or outbox unreachable. + if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk } - } else { - val tries = (attempts[pk] ?: 0) + 1 - attempts[pk] = tries - // Give up after retries: no contact list, or outbox unreachable. - if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk } } } } + System.err.println( - "[graperank] round $rounds: queried=${pending.size}, +events=${collected.size - before}, " + - "downloaded=$downloaded, newUsers=$newUsers, discovered=${discovered.size}, done=${done.size}", + "[graperank] round $rounds: pending=${pending.size}, cached=$cached, downloaded=$downloaded, " + + "newUsers=$newUsers, discovered=${discovered.size}, done=${done.size}", ) } relaysContactedCount = relaysContacted.size - val unreached = discovered.count { it !in done || ctx.contactsOf(it) == null } System.err.println( - "[graperank] crawl complete: ${discovered.size} users discovered, " + - "${discovered.size - unreached} contact lists downloaded, $unreached without one, " + + "[graperank] crawl complete: ${discovered.size} discovered, $contactListsHeld with a contact list, " + "$relaysContactedCount relays contacted, $rounds rounds", ) - events = collected } + // Build the graph from the store so it includes BOTH freshly-downloaded and + // already-cached contact lists (and, offline, everything on disk). + val events = ctx.store.query(Filter(kinds = graphKinds)) + if (offline) System.err.println("[graperank] offline: ${events.size} events from local store") + val graph = TrustGraphBuilder.build(events) System.err.println( "[graperank] graph built: ${graph.users.size} users, ${graph.edgeCount()} edges from ${events.size} events; scoring…", From d91673bb34e0c5e0b1338c965d483b4f89811ead Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 21:21:48 +0000 Subject: [PATCH 12/58] perf(wot): compact int-CSR trust graph + freshness pass; stream into it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point #1 (freshness): each run now fetches every discovered user's LATEST kind:3/10000/1984 once from their outbox (grouped by write relay in routeByOutbox; empty-tagged relays already count as write), instead of skipping users whose list is already cached. `done` still guarantees once-per-run and that a fresh download isn't repeated. Point #2 (memory): replace the HexKey-keyed, TrustEdge-object graph with a compact representation that scales to the whole network: - commons/wot: pubkeys interned to dense Int ids; edges stored in two CSR IntArray layouts (by target for scoring, by source for the worklist), each incoming entry packing source id + relation into one int. GrapeRank.compute returns a DoubleArray by node id (no boxed map at millions of nodes). TrustGraphBuilder is now stateful/streaming (addFollows/addMutes/addReports). Tests rewritten; the full-sweep cross-check still passes. - cli: contact lists stream straight into the builder as they arrive and the Event is discarded — the crawl never holds millions of kind:3 objects. Mutes and reports (far fewer) are fed from the store. Output de-interns the top-N. Validated: a fresh online run built a 108,961-user / 1.67M-edge graph and scored 83,613 users in a 4 GB heap. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 200 ++++++++--------- .../amethyst/commons/wot/GrapeRank.kt | 156 +++++++------- .../amethyst/commons/wot/TrustGraph.kt | 106 +++++---- .../amethyst/commons/wot/TrustGraphBuilder.kt | 147 ++++++++----- .../amethyst/commons/wot/GrapeRankTest.kt | 204 ++++++++++-------- .../commons/wot/TrustGraphBuilderTest.kt | 163 ++++++-------- 6 files changed, 517 insertions(+), 459 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index c2e22b322e..2d7e19e176 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -138,9 +138,13 @@ object GrapeRankCommand { val graphKinds = listOf(ContactListEvent.KIND, MuteListEvent.KIND, ReportEvent.KIND) + // The graph is built incrementally: contact lists stream straight into a + // compact int-CSR structure and the Event is discarded, so the whole + // network fits in memory without holding millions of kind:3 objects. + val builder = TrustGraphBuilder() var rounds = 0 var relaysContactedCount = 0 - var contactListsHeld = 0 + var contactListsFed = 0 if (!offline) { val discovered = hashSetOf(observer) @@ -148,146 +152,146 @@ object GrapeRankCommand { // contact lists we crawl (A's follow of B says where B writes) — a // discovery tier below each user's kind:10002 outbox. val relayHints = HashMap>() - // Users we're finished with: we hold their kind:3 (cached or freshly - // downloaded), or ran out of retry attempts. + // Users we're finished with this run: we fed their latest kind:3, or + // ran out of retry attempts on an unreachable outbox. val done = hashSetOf() val attempts = HashMap() - // The pool of relays we actually route outbox queries to. val relaysContacted = hashSetOf() - // Harvest a contact list: record its relay hints and add its follows to - // the frontier. Returns the count of newly-discovered users. - fun expand(contacts: ContactListEvent): Int { + // Feed a user's contact list into the graph, harvest relay hints, and + // add its follows to the frontier. Called once per user (guarded by + // `done`). Returns the count of newly-discovered users. + fun ingest( + source: HexKey, + contacts: ContactListEvent, + ): Int { + val follows = ArrayList() var fresh = 0 for (tag in contacts.follows()) { + follows.add(tag.pubKey) tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } if (discovered.add(tag.pubKey)) fresh++ } + builder.addFollows(source, follows) + contactListsFed++ return fresh } - // Loop until every discovered user's contact list is in hand — no user - // cap, to full graph depth. We only DOWNLOAD lists we don't already - // have; a list already in the store is expanded from disk with no - // network (so re-runs and a warm shared store are cheap). An - // unreachable outbox is dropped after MAX_OUTBOX_ATTEMPTS tries so the - // crawl still terminates. + // Crawl to full graph depth (no user cap). Each run fetches every + // discovered user's LATEST kind:3/10000/1984 once from their outbox + // (a freshness pass — grouped by write relay in routeByOutbox), unless + // we already fetched it this run (`done`). An unreachable outbox is + // retried up to MAX_OUTBOX_ATTEMPTS then dropped so the crawl terminates. while (rounds < maxRounds) { val pending = discovered.filterNot { it in done } if (pending.isEmpty()) break rounds++ - var cached = 0 - var downloaded = 0 + var gotList = 0 var newUsers = 0 - // 1. Users whose kind:3 is already in the store: expand, no network. - val need = ArrayList() - for (pk in pending) { - val contacts = ctx.contactsOf(pk) - if (contacts != null) { - done += pk - contactListsHeld++ - cached++ - newUsers += expand(contacts) - } else { - need += pk - } - } - - // 2. Download the rest from their own outboxes. Resolve kind:10002 - // in bulk (indexers aggregate it), then fetch content in small - // batches drained a few at a time — one giant drain over - // thousands of outbox relays saturates connections and times out. - // Routing (store reads) is serial; only the drains run - // concurrently, which is safe: inserts serialize on the store - // write lock. - if (need.isNotEmpty()) { - ensureRelayLists(ctx, need.toSet(), timeoutMs) - for (group in need.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { - val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, graphKinds) } - val drained = - coroutineScope { - prepared - .map { (batch, filters) -> - async { - ctx.drain(filters, timeoutMs) - batch to filters.keys - } - }.awaitAll() - } - for ((batch, relays) in drained) { - relaysContacted += relays - for (pk in batch) { - val contacts = ctx.contactsOf(pk) - if (contacts != null) { - done += pk - contactListsHeld++ - downloaded++ - newUsers += expand(contacts) - } else { - val tries = (attempts[pk] ?: 0) + 1 - attempts[pk] = tries - // Give up after retries: no contact list, or outbox unreachable. - if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk - } + // Resolve kind:10002 outboxes in bulk (indexers aggregate them), + // then fetch content in small batches drained a few at a time — one + // giant drain over thousands of outbox relays saturates connections + // and times out. Routing (store reads) is serial; only the drains + // run concurrently, which is safe: inserts serialize on the store + // write lock. + ensureRelayLists(ctx, pending.toSet(), timeoutMs) + for (group in pending.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { + val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, graphKinds) } + val drained = + coroutineScope { + prepared + .map { (batch, filters) -> + async { + ctx.drain(filters, timeoutMs) + batch to filters.keys + } + }.awaitAll() + } + for ((batch, relays) in drained) { + relaysContacted += relays + for (pk in batch) { + val contacts = ctx.contactsOf(pk) + if (contacts != null) { + done += pk + gotList++ + newUsers += ingest(pk, contacts) + } else { + val tries = (attempts[pk] ?: 0) + 1 + attempts[pk] = tries + if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk } } } } System.err.println( - "[graperank] round $rounds: pending=${pending.size}, cached=$cached, downloaded=$downloaded, " + + "[graperank] round $rounds: fetched=${pending.size}, gotList=$gotList, " + "newUsers=$newUsers, discovered=${discovered.size}, done=${done.size}", ) } relaysContactedCount = relaysContacted.size System.err.println( - "[graperank] crawl complete: ${discovered.size} discovered, $contactListsHeld with a contact list, " + + "[graperank] crawl complete: ${discovered.size} discovered, $contactListsFed contact lists fed, " + "$relaysContactedCount relays contacted, $rounds rounds", ) - } - - // Build the graph from the store so it includes BOTH freshly-downloaded and - // already-cached contact lists (and, offline, everything on disk). - val events = ctx.store.query(Filter(kinds = graphKinds)) - if (offline) System.err.println("[graperank] offline: ${events.size} events from local store") - - val graph = TrustGraphBuilder.build(events) - System.err.println( - "[graperank] graph built: ${graph.users.size} users, ${graph.edgeCount()} edges from ${events.size} events; scoring…", - ) - - // Live scoring progress: the worklist visits each reachable user once - // per relaxation; report every PROGRESS_STEP visits so a large graph - // shows movement instead of hanging silently. - val scores = - GrapeRank(params).compute(graph, observer) { visited, scored, queued -> - if (visited % SCORE_PROGRESS_STEP == 0) { - System.err.println("[graperank] scoring: $visited visited, $scored scored, $queued queued") + } else { + // Offline: stream contact lists from the local store into the graph. + for (event in ctx.store.query(Filter(kinds = listOf(ContactListEvent.KIND)))) { + if (event is ContactListEvent) { + builder.addFollows(event.pubKey, event.verifiedFollowKeySet()) + contactListsFed++ + } + } + System.err.println("[graperank] offline: $contactListsFed contact lists from local store") + } + + // Mutes + reports come from the store (both paths). Far fewer than contact + // lists, so materialising them is cheap. + for (event in ctx.store.query(Filter(kinds = listOf(MuteListEvent.KIND)))) { + if (event is MuteListEvent) builder.addMutes(event.pubKey, event.linkedPubKeys()) + } + for (event in ctx.store.query(Filter(kinds = listOf(ReportEvent.KIND)))) { + if (event is ReportEvent) builder.addReports(event.pubKey, event.reportedAuthor().map { it.pubkey }) + } + + val graph = builder.build() + System.err.println("[graperank] graph built: ${graph.nodeCount} users, ${graph.edgeCount()} edges; scoring…") + + // Live scoring progress: the worklist visits each reachable user once per + // relaxation; report every SCORE_PROGRESS_STEP visits so a large graph shows + // movement instead of hanging silently. + val scores = + GrapeRank(params).compute(graph, observer) { visited, queued -> + if (visited % SCORE_PROGRESS_STEP == 0L) { + System.err.println("[graperank] scoring: $visited visited, $queued queued") } } - System.err.println("[graperank] scored ${scores.size} users") fun rankOf(score: Double) = (score * 100).roundToInt() - val ranked = - scores.entries - .filter { it.value >= minScore } - .sortedByDescending { it.value } + val observerId = graph.idOf(observer) + // Reachable users with positive trust at or above --min-score, high→low. + val rankedIds = ArrayList() + for (id in 0 until graph.nodeCount) { + if (id != observerId && scores[id] > 0.0 && scores[id] >= minScore) rankedIds.add(id) + } + rankedIds.sortByDescending { scores[it] } + System.err.println("[graperank] scored ${rankedIds.size} users") val result = linkedMapOf( "observer" to observer, "crawl_rounds" to rounds, "relays_contacted" to relaysContactedCount, - "graph_users" to graph.users.size, + "graph_users" to graph.nodeCount, "graph_edges" to graph.edgeCount(), - "users_scored" to scores.size, + "users_scored" to rankedIds.size, "scores" to - ranked.take(limit).map { - mapOf("pubkey" to it.key, "score" to it.value, "rank" to rankOf(it.value)) + rankedIds.take(limit).map { + mapOf("pubkey" to graph.pubkeyOf(it), "score" to scores[it], "rank" to rankOf(scores[it])) }, ) @@ -306,9 +310,9 @@ object GrapeRankCommand { val publishedRanks = publishedCardRanks(ctx) val candidates = - ranked - .filter { rankOf(it.value) >= minRank } - .map { it.key to rankOf(it.value) } + rankedIds + .filter { rankOf(scores[it]) >= minRank } + .map { graph.pubkeyOf(it) to rankOf(scores[it]) } val changed = candidates.filter { (target, rank) -> publishedRanks[target] != rank } val toPublish = changed.take(publishLimit) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt index fd4859cade..725ba3d470 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt @@ -28,12 +28,8 @@ import kotlin.math.ln /** * Tunable GrapeRank parameters. Defaults mirror the reference implementation at - * . - * - * A follow from the observer themselves counts far more than a follow from a - * stranger deep in the graph ([directFollowConfidence] vs - * [indirectFollowConfidence]); mutes and reports are trusted more heavily than - * an indirect follow because negative signals are rarer and more deliberate. + * and NosFabrica's Brainstorm + * `DEFAULT` preset. */ @Immutable data class GrapeRankParams( @@ -48,105 +44,121 @@ data class GrapeRankParams( /** * GrapeRank — a subjective, observer-centric web-of-trust score in `[0, 1]` for - * every user reachable from an observer in a [TrustGraph]. The observer has full - * self-trust (`1.0`); trust decays by roughly the attenuation factor each hop, - * so scores fall to ~0 within a handful of hops. + * every user reachable from an observer in a [TrustGraph]. See the algorithm + * notes in `TrustGraph`/`GrapeRankTest`; this is the single-observer worklist + * form, operating on the compact int-CSR graph so it scales to the whole network. * - * This is a faithful single-observer port of the reference `v3TargetedBFS` - * variant. Rather than the reactive per-edge propagation the reference uses (it - * assumes edges stream in one at a time), this recomputes over a graph that is - * already fully loaded, using a worklist that: - * 1. seeds the observer at `1.0` and enqueues the users it attests about, - * 2. dequeues a target, recomputes its score over *all* its incoming edges, - * 3. re-enqueues that target's out-neighbours whenever its score moved by more - * than [GrapeRankParams.convergence]. - * - * Attenuation makes the update a contraction, so the worklist reaches the same - * fixed point a full sweep would — while only ever touching users reachable from - * the observer. See `GrapeRankTest` for the full-sweep cross-check. + * [compute] returns a `DoubleArray` indexed by node id (`graph.idOf(pubkey)`), + * not a map — at millions of nodes a boxed map would dwarf the graph itself. The + * observer's own entry stays pinned at `1.0`; callers rank the others. */ class GrapeRank( val params: GrapeRankParams = GrapeRankParams(), ) { - /** Confidence weight [source]→target contributes, from [observer]'s point of view. */ + private val rigidity = -ln(params.rigor) + + /** Exponential saturation turning accumulated weight into a confidence in `[0, 1)`. */ + private fun weightToConfidence(weight: Double): Double = 1.0 - exp(-weight * rigidity) + private fun confidence( - edge: TrustEdge, - observer: HexKey, + relationCode: Int, + sourceIsObserver: Boolean, ): Double = - when (edge.relation) { - TrustRelation.FOLLOW -> if (edge.source == observer) params.directFollowConfidence else params.indirectFollowConfidence - TrustRelation.MUTE -> params.muteConfidence - TrustRelation.REPORT -> params.reportConfidence + when (relationCode) { + TrustRelation.FOLLOW.code -> if (sourceIsObserver) params.directFollowConfidence else params.indirectFollowConfidence + TrustRelation.MUTE.code -> params.muteConfidence + else -> params.reportConfidence } - /** Exponential saturation curve turning accumulated weight into a confidence in `[0, 1)`. */ - private fun weightToConfidence(weight: Double): Double = 1.0 - exp(-weight * -ln(params.rigor)) + private fun rating(relationCode: Int): Double = + when (relationCode) { + TrustRelation.FOLLOW.code -> TrustRelation.FOLLOW.rating + TrustRelation.MUTE.code -> TrustRelation.MUTE.rating + else -> TrustRelation.REPORT.rating + } /** - * Score every user reachable from [observer]. The returned map excludes the - * observer itself (its score is a pinned `1.0` and not part of a ranking). - * Users with no positive path from the observer are absent (equivalently, 0). - * - * [onProgress] is invoked once per worklist visit with - * `(visited, scored, queued)` running counts, so a caller can report progress - * on a large graph; it defaults to a no-op. + * Score every node reachable from [observer]. Returns scores by node id, or an + * all-zero array if the observer isn't in the graph. [onProgress] fires once + * per worklist visit with `(visited, queued)` running counts. */ fun compute( graph: TrustGraph, observer: HexKey, - onProgress: ((visited: Int, scored: Int, queued: Int) -> Unit)? = null, - ): Map { - val scores = HashMap() - scores[observer] = 1.0 + onProgress: ((visited: Long, queued: Int) -> Unit)? = null, + ): DoubleArray { + val n = graph.nodeCount + val scores = DoubleArray(n) + val observerId = graph.idOf(observer) + if (observerId < 0) return scores - val queue = ArrayDeque() - val queued = HashSet() + scores[observerId] = 1.0 - fun enqueue(user: HexKey) { - if (user != observer && queued.add(user)) queue.addLast(user) + val inQueue = BooleanArray(n) + val queue = IntArrayList(1024) + + fun enqueue(node: Int) { + if (node != observerId && !inQueue[node]) { + inQueue[node] = true + queue.add(node) + } } - graph.outgoing[observer]?.forEach(::enqueue) + enqueueOutNeighbours(graph, observerId, ::enqueue) - var visited = 0 + var visited = 0L while (queue.isNotEmpty()) { - val target = queue.removeFirst() - queued.remove(target) + val target = queue.removeLast() + inQueue[target] = false - val newScore = scoreOf(graph, scores, target, observer) - val oldScore = scores.put(target, newScore) ?: 0.0 + var sumOfWeights = 0.0 + var sumOfWeightedRatings = 0.0 + var i = graph.inOffsets[target] + val end = graph.inOffsets[target + 1] + while (i < end) { + val packed = graph.inPacked[i] + val source = packed and TrustGraph.SOURCE_MASK + val sourceScore = scores[source] + if (sourceScore != 0.0) { + val relationCode = packed ushr TrustGraph.SOURCE_BITS + val weight = confidence(relationCode, source == observerId) * sourceScore * params.attenuation + sumOfWeights += weight + sumOfWeightedRatings += weight * rating(relationCode) + } + i++ + } + val newScore = + if (abs(sumOfWeights) < 0.00001) { + 0.0 + } else { + val s = weightToConfidence(sumOfWeights) * sumOfWeightedRatings / sumOfWeights + if (s > 0.0) s else 0.0 + } + + val oldScore = scores[target] + scores[target] = newScore if (abs(newScore - oldScore) > params.convergence) { - graph.outgoing[target]?.forEach(::enqueue) + enqueueOutNeighbours(graph, target, ::enqueue) } visited++ - onProgress?.invoke(visited, scores.size, queue.size) + onProgress?.invoke(visited, queue.size) } - scores.remove(observer) return scores } - private fun scoreOf( + private inline fun enqueueOutNeighbours( graph: TrustGraph, - scores: Map, - target: HexKey, - observer: HexKey, - ): Double { - var sumOfWeights = 0.0 - var sumOfWeightedRatings = 0.0 - - val edges = graph.incoming[target] ?: return 0.0 - for (edge in edges) { - val sourceScore = scores[edge.source] ?: continue - val weight = confidence(edge, observer) * sourceScore * params.attenuation - sumOfWeights += weight - sumOfWeightedRatings += weight * edge.relation.rating + node: Int, + enqueue: (Int) -> Unit, + ) { + var i = graph.outOffsets[node] + val end = graph.outOffsets[node + 1] + while (i < end) { + enqueue(graph.outTargets[i]) + i++ } - - if (abs(sumOfWeights) < 0.00001) return 0.0 - val score = weightToConfidence(sumOfWeights) * sumOfWeightedRatings / sumOfWeights - return if (score > 0.0) score else 0.0 } } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt index d27e855c03..b84063b87c 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt @@ -20,64 +20,82 @@ */ package com.vitorpamplona.amethyst.commons.wot -import androidx.compose.runtime.Immutable import com.vitorpamplona.quartz.nip01Core.core.HexKey /** - * A single directed trust attestation between two Nostr users, mapped from a - * kind:3 follow / kind:10000 mute / kind:1984 report. Each carries a [rating] - * (how the relationship reflects on the target) that GrapeRank multiplies by an - * observer-relative confidence — see [GrapeRank]. + * A trust relationship kind and the GrapeRank rating it carries. [code] is the + * 2-bit tag packed alongside a source node id in the edge arrays (see + * [TrustGraph]); keep it in `0..3`. */ -@Immutable enum class TrustRelation( val rating: Double, + val code: Int, ) { - FOLLOW(1.0), - MUTE(-0.1), - REPORT(-0.1), + FOLLOW(1.0, 0), + MUTE(-0.1, 1), + REPORT(-0.1, 2), } -/** [source] asserts [relation] about the (implicit) target it is indexed under. */ -@Immutable -data class TrustEdge( - val source: HexKey, - val relation: TrustRelation, -) - /** - * A protocol-agnostic web-of-trust graph keyed by pubkey hex. + * A web-of-trust graph over Nostr pubkeys, stored compactly so it scales to the + * whole network (millions of edges) without a `String`-keyed edge object per + * relationship. * - * [incoming] maps every target user to the attestations pointing *at* it — the - * only view GrapeRank needs to score a node. [outgoing] (source → the set of - * users it attests about) is derived once and used by the propagation worklist - * to know which nodes to re-score when a source's score moves. + * Pubkeys are interned to dense `Int` node ids. Edges live in two + * compressed-sparse-row (CSR) layouts backed by flat `IntArray`s — one indexed + * by target (what [GrapeRank] reads to score a node) and one by source (what the + * propagation worklist follows). Each incoming entry packs the source id in the + * low 29 bits and the [TrustRelation.code] in the top bits, so an edge is a + * single `int`. A 100M-edge graph is then ~0.8 GB of primitive arrays instead of + * tens of GB of objects. * - * Build one with [TrustGraphBuilder.build] from a bag of Nostr events; score it - * with [GrapeRank.compute]. + * Build one with [TrustGraphBuilder], feeding contact lists / mutes / reports in + * as they stream off the relays. */ -class TrustGraph( - val incoming: Map>, +class TrustGraph internal constructor( + val nodeCount: Int, + private val pubkeys: Array, + private val ids: HashMap, + // CSR by target: incoming edges of node t are inPacked[inOffsets[t] until inOffsets[t+1]], + // each packing source id (low 29 bits) + relation code (top bits). + internal val inOffsets: IntArray, + internal val inPacked: IntArray, + // CSR by source: out-neighbour targets of node s are outTargets[outOffsets[s] until outOffsets[s+1]]. + internal val outOffsets: IntArray, + internal val outTargets: IntArray, ) { - /** source pubkey → the targets it has an outgoing edge to. */ - val outgoing: Map> by lazy { - val out = HashMap>() - for ((target, edges) in incoming) { - for (edge in edges) { - out.getOrPut(edge.source) { HashSet() }.add(target) - } - } - out - } + /** Node id for [pubkey], or `-1` if it never appeared in the graph. */ + fun idOf(pubkey: HexKey): Int = ids[pubkey] ?: -1 - /** Every user that appears in the graph, as a target or as an edge source. */ - val users: Set by lazy { - val all = HashSet(incoming.keys) - for (edges in incoming.values) { - for (edge in edges) all.add(edge.source) - } - all - } + /** Pubkey for a node [id]. */ + fun pubkeyOf(id: Int): HexKey = pubkeys[id] - fun edgeCount(): Int = incoming.values.sumOf { it.size } + fun edgeCount(): Int = inPacked.size + + companion object { + const val SOURCE_BITS = 29 + const val SOURCE_MASK = (1 shl SOURCE_BITS) - 1 + const val MAX_NODES = SOURCE_MASK // ids must fit in the low 29 bits + } +} + +/** A minimal growable `int[]` — avoids boxing `Int`s in an `ArrayList` at graph scale. */ +internal class IntArrayList( + initialCapacity: Int = 16, +) { + var data: IntArray = IntArray(initialCapacity.coerceAtLeast(1)) + private set + var size: Int = 0 + private set + + fun add(value: Int) { + if (size == data.size) data = data.copyOf(data.size * 2) + data[size++] = value + } + + fun get(index: Int): Int = data[index] + + fun removeLast(): Int = data[--size] + + fun isNotEmpty(): Boolean = size > 0 } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt index 485ad91eb0..55a223949b 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt @@ -20,79 +20,108 @@ */ package com.vitorpamplona.amethyst.commons.wot -import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey -import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent -import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent -import com.vitorpamplona.quartz.nip56Reports.ReportEvent /** - * Turns a bag of Nostr events into a [TrustGraph]. Pure: no network, no state — - * hand it whatever kind:3 / kind:10000 / kind:1984 events you have collected. + * Builds a [TrustGraph] incrementally so callers never have to hold every contact + * list in memory at once — feed each user's follows / mutes / reports as they + * stream off the relays (or out of the store), then call [build]. * - * - **kind:3** [ContactListEvent] → a [TrustRelation.FOLLOW] edge per followed key. - * - **kind:10000** [MuteListEvent] → a [TrustRelation.MUTE] edge per publicly muted - * key. Private (NIP-44 encrypted) mutes are ignored — they aren't ours to - * decrypt and aren't fetchable from another user's relays anyway. - * - **kind:1984** [ReportEvent] → a [TrustRelation.REPORT] edge per reported author. - * - * kind:3 and kind:10000 are replaceable, so only the newest per author is kept. - * Reports are regular events; every distinct `(reporter → reported)` pair counts - * once. Self-edges are dropped. + * Interns pubkeys to dense ids on the fly and accumulates edges in flat growable + * int arrays. Follows and mutes are replaceable (one list per author, deduped by + * the caller via latest-per-author + set-valued tags); reports are regular events, + * so `(reporter → reported)` report edges are deduped here. Self-edges are dropped. */ -object TrustGraphBuilder { - fun build(events: Collection): TrustGraph { - // Latest replaceable-per-author for kind 3 / 10000. - val latestContacts = HashMap() - val latestMutes = HashMap() - val reports = ArrayList() +class TrustGraphBuilder { + private val ids = HashMap() + private val pubkeys = ArrayList() - for (event in events) { - when (event) { - is ContactListEvent -> { - val prev = latestContacts[event.pubKey] - if (prev == null || event.createdAt > prev.createdAt) latestContacts[event.pubKey] = event - } + // Parallel edge arrays: edge i is source edgeSource[i] --relation--> edgeTarget[i], + // with the relation packed into the top bits of edgeSource[i]. + private val edgeTargets = IntArrayList() + private val edgeSourcesPacked = IntArrayList() - is MuteListEvent -> { - val prev = latestMutes[event.pubKey] - if (prev == null || event.createdAt > prev.createdAt) latestMutes[event.pubKey] = event - } + // Dedup for report edges only (reporters can file many kind:1984 for one target). + private val reportSeen = HashSet() - is ReportEvent -> reports.add(event) - } + private fun intern(pubkey: HexKey): Int = + ids.getOrPut(pubkey) { + val id = pubkeys.size + pubkeys.add(pubkey) + id } - // target -> distinct incoming edges (dedup identical source+relation pairs). - val incoming = HashMap>() + private fun addEdge( + source: HexKey, + target: HexKey, + relation: TrustRelation, + ) { + if (source == target) return + val s = intern(source) + val t = intern(target) + if (relation == TrustRelation.REPORT) { + val key = (s.toLong() shl 32) or (t.toLong() and 0xFFFFFFFFL) + if (!reportSeen.add(key)) return + } + edgeTargets.add(t) + edgeSourcesPacked.add(s or (relation.code shl TrustGraph.SOURCE_BITS)) + } - fun addEdge( - source: HexKey, - target: HexKey, - relation: TrustRelation, - ) { - if (source == target) return - incoming.getOrPut(target) { LinkedHashSet() }.add(TrustEdge(source, relation)) + fun addFollows( + source: HexKey, + follows: Iterable, + ) { + for (target in follows) addEdge(source, target, TrustRelation.FOLLOW) + } + + fun addMutes( + source: HexKey, + muted: Iterable, + ) { + for (target in muted) addEdge(source, target, TrustRelation.MUTE) + } + + fun addReports( + source: HexKey, + reported: Iterable, + ) { + for (target in reported) addEdge(source, target, TrustRelation.REPORT) + } + + fun nodeCount(): Int = pubkeys.size + + fun edgeCount(): Int = edgeTargets.size + + /** Freeze the accumulated edges into the two CSR layouts. */ + fun build(): TrustGraph { + val n = pubkeys.size + val m = edgeTargets.size + + // Incoming CSR (by target). + val inOffsets = IntArray(n + 1) + for (i in 0 until m) inOffsets[edgeTargets.get(i) + 1]++ + for (i in 1..n) inOffsets[i] += inOffsets[i - 1] + val inPacked = IntArray(m) + val inCursor = inOffsets.copyOf() + for (i in 0 until m) { + val t = edgeTargets.get(i) + inPacked[inCursor[t]++] = edgeSourcesPacked.get(i) } - for (contacts in latestContacts.values) { - for (target in contacts.verifiedFollowKeySet()) { - addEdge(contacts.pubKey, target, TrustRelation.FOLLOW) - } + // Outgoing CSR (by source). + val outOffsets = IntArray(n + 1) + for (i in 0 until m) { + val s = edgeSourcesPacked.get(i) and TrustGraph.SOURCE_MASK + outOffsets[s + 1]++ + } + for (i in 1..n) outOffsets[i] += outOffsets[i - 1] + val outTargets = IntArray(m) + val outCursor = outOffsets.copyOf() + for (i in 0 until m) { + val s = edgeSourcesPacked.get(i) and TrustGraph.SOURCE_MASK + outTargets[outCursor[s]++] = edgeTargets.get(i) } - for (mutes in latestMutes.values) { - for (target in mutes.linkedPubKeys()) { - addEdge(mutes.pubKey, target, TrustRelation.MUTE) - } - } - - for (report in reports) { - for (reported in report.reportedAuthor()) { - addEdge(report.pubKey, reported.pubkey, TrustRelation.REPORT) - } - } - - return TrustGraph(incoming.mapValues { (_, edges) -> edges.toList() }) + return TrustGraph(n, pubkeys.toTypedArray(), ids, inOffsets, inPacked, outOffsets, outTargets) } } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt index 11bb658575..d9a3282a4f 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt @@ -28,111 +28,131 @@ import kotlin.math.max import kotlin.random.Random import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertNull import kotlin.test.assertTrue class GrapeRankTest { private val obs = "observer" - private fun graphOf(vararg edges: Triple): TrustGraph { - val incoming = HashMap>() + private fun graphOf(edges: List>): TrustGraph { + val b = TrustGraphBuilder() for ((source, target, relation) in edges) { - incoming.getOrPut(target) { mutableListOf() }.add(TrustEdge(source, relation)) + when (relation) { + TrustRelation.FOLLOW -> b.addFollows(source, listOf(target)) + TrustRelation.MUTE -> b.addMutes(source, listOf(target)) + TrustRelation.REPORT -> b.addReports(source, listOf(target)) + } } - return TrustGraph(incoming) + return b.build() + } + + private fun graphOf(vararg edges: Triple) = graphOf(edges.toList()) + + /** Score for a pubkey (0.0 if absent from the graph). */ + private fun DoubleArray.of( + graph: TrustGraph, + pubkey: HexKey, + ): Double { + val id = graph.idOf(pubkey) + return if (id < 0) 0.0 else this[id] } @Test - fun observerIsExcludedFromRanking() { - val scores = GrapeRank().compute(graphOf(Triple(obs, "a", TrustRelation.FOLLOW)), obs) - assertNull(scores[obs], "observer's pinned self-trust is not part of the ranking") + fun observerIsPinnedAtFullSelfTrust() { + val graph = graphOf(Triple(obs, "a", TrustRelation.FOLLOW)) + val scores = GrapeRank().compute(graph, obs) + assertEquals(1.0, scores.of(graph, obs), 1e-12) } @Test fun directFollowMatchesHandComputedValue() { - val scores = GrapeRank().compute(graphOf(Triple(obs, "a", TrustRelation.FOLLOW)), obs) - // weight = 0.5 * 1.0 * 0.85 = 0.425 ; conf(0.425) = 1 - 2^-0.425 - // score = conf * (0.425 / 0.425) = 0.2551612... - assertEquals(0.25516127, scores.getValue("a"), 1e-6) + val graph = graphOf(Triple(obs, "a", TrustRelation.FOLLOW)) + val scores = GrapeRank().compute(graph, obs) + // weight = 0.5 * 1.0 * 0.85 = 0.425 ; score = conf(0.425) = 0.2551612... + assertEquals(0.25516127, scores.of(graph, "a"), 1e-6) } @Test fun trustDecaysSteeplyAcrossHops() { - val scores = - GrapeRank().compute( - graphOf( - Triple(obs, "a", TrustRelation.FOLLOW), - Triple("a", "b", TrustRelation.FOLLOW), - ), - obs, + val graph = + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("a", "b", TrustRelation.FOLLOW), ) - val a = scores.getValue("a") - val b = scores.getValue("b") - // Indirect follow from a (conf 0.03) two hops out: ~0.0045, an ~56x drop. + val scores = GrapeRank().compute(graph, obs) + val a = scores.of(graph, "a") + val b = scores.of(graph, "b") assertEquals(0.004499, b, 1e-5) assertTrue(b < a / 10.0, "two-hop trust should be far below one-hop trust") } @Test fun aMuteFromAnEndorsedUserLowersTheScore() { - val followOnly = GrapeRank().compute(graphOf(Triple(obs, "b", TrustRelation.FOLLOW)), obs) - val withMute = - GrapeRank().compute( - graphOf( - Triple(obs, "a", TrustRelation.FOLLOW), - Triple(obs, "b", TrustRelation.FOLLOW), - Triple("a", "b", TrustRelation.MUTE), - ), - obs, + val followOnlyGraph = graphOf(Triple(obs, "b", TrustRelation.FOLLOW)) + val followOnly = GrapeRank().compute(followOnlyGraph, obs).of(followOnlyGraph, "b") + + val muteGraph = + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple(obs, "b", TrustRelation.FOLLOW), + Triple("a", "b", TrustRelation.MUTE), ) - assertTrue( - withMute.getValue("b") < followOnly.getValue("b"), - "a mute from a trusted user should pull b's score below the follow-only baseline", - ) + val withMute = GrapeRank().compute(muteGraph, obs).of(muteGraph, "b") + + assertTrue(withMute < followOnly, "a mute from a trusted user should pull b below the follow-only baseline") } @Test fun purelyReportedUserFloorsAtZero() { - val scores = - GrapeRank().compute( - graphOf( - Triple(obs, "a", TrustRelation.FOLLOW), - Triple("a", "d", TrustRelation.REPORT), - ), - obs, + val graph = + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("a", "d", TrustRelation.REPORT), ) - assertEquals(0.0, scores.getValue("d"), 1e-9, "negative-only signals floor at zero") + val scores = GrapeRank().compute(graph, obs) + assertEquals(0.0, scores.of(graph, "d"), 1e-9) } @Test fun unreachableUsersAreNotScored() { - // x -> y exists but neither is reachable from the observer. - val scores = - GrapeRank().compute( - graphOf( - Triple(obs, "a", TrustRelation.FOLLOW), - Triple("x", "y", TrustRelation.FOLLOW), - ), - obs, + val graph = + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("x", "y", TrustRelation.FOLLOW), ) - assertTrue("a" in scores) - assertNull(scores["y"], "a user with no path from the observer is absent from the result") + val scores = GrapeRank().compute(graph, obs) + assertTrue(scores.of(graph, "a") > 0.0) + assertEquals(0.0, scores.of(graph, "y"), 1e-12, "a user with no path from the observer stays 0") } @Test fun cyclesConverge() { - // a<->b mutual follow plus observer->a. Must terminate at a fixed point. - val scores = - GrapeRank().compute( - graphOf( - Triple(obs, "a", TrustRelation.FOLLOW), - Triple("a", "b", TrustRelation.FOLLOW), - Triple("b", "a", TrustRelation.FOLLOW), - ), - obs, + val graph = + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("a", "b", TrustRelation.FOLLOW), + Triple("b", "a", TrustRelation.FOLLOW), ) - assertTrue(scores.getValue("a") > 0.0) - assertTrue(scores.getValue("b") > 0.0) + val scores = GrapeRank().compute(graph, obs) + assertTrue(scores.of(graph, "a") > 0.0) + assertTrue(scores.of(graph, "b") > 0.0) + } + + @Test + fun deduplicatesRepeatedReportEdges() { + // Two report edges a->d collapse to one; the score matches a single report. + val once = graphOf(Triple(obs, "a", TrustRelation.FOLLOW), Triple("a", "d", TrustRelation.REPORT)) + val twice = + graphOf( + Triple(obs, "a", TrustRelation.FOLLOW), + Triple("a", "d", TrustRelation.REPORT), + Triple("a", "d", TrustRelation.REPORT), + ) + assertEquals(2, twice.edgeCount(), "duplicate report edge should be dropped") + assertEquals( + GrapeRank().compute(once, obs).of(once, "d"), + GrapeRank().compute(twice, obs).of(twice, "d"), + 1e-12, + ) } /** @@ -141,9 +161,6 @@ class GrapeRankTest { */ @Test fun worklistMatchesFullSweepOnRandomGraphs() { - // Tight convergence so both methods settle onto essentially the same - // fixed point (attenuation < 1 makes the update a contraction), leaving - // only floating-point slop to compare against. val params = GrapeRankParams(convergence = 1e-10) val engine = GrapeRank(params) repeat(50) { seed -> @@ -165,33 +182,43 @@ class GrapeRankTest { } } } - val graph = graphOf(*edges.toTypedArray()) val observer = nodes.first() + val graph = graphOf(edges) + val scores = engine.compute(graph, observer) + val reference = fullSweep(edges, nodes, observer, params) - val worklist = engine.compute(graph, observer) - val fullSweep = fullSweep(graph, observer, params) - - for (node in graph.users) { - if (node == observer) continue - val a = worklist[node] ?: 0.0 - val b = fullSweep[node] ?: 0.0 + for (node in nodes) { + if (node == observer) continue // observer self-trust is not part of a ranking + val a = scores.of(graph, node) + val b = reference[node] ?: 0.0 assertEquals(b, a, 1e-5, "seed=$seed node=$node worklist=$a fullSweep=$b") } } } - // Reference implementation: blind full sweep over every user until nothing changes. + // Reference: blind full sweep over every user until nothing changes. private fun fullSweep( - graph: TrustGraph, + edges: List>, + nodes: List, observer: HexKey, params: GrapeRankParams, ): Map { - fun confidence(edge: TrustEdge): Double = - when (edge.relation) { - TrustRelation.FOLLOW -> if (edge.source == observer) params.directFollowConfidence else params.indirectFollowConfidence - TrustRelation.MUTE -> params.muteConfidence - TrustRelation.REPORT -> params.reportConfidence - } + // Dedup identical edges (mirrors the builder: report edges dedup; follow/mute + // sets are unique per source anyway). + val incoming = HashMap>>() + for ((s, t, r) in edges) { + if (s == t) continue + incoming.getOrPut(t) { LinkedHashSet() }.add(s to r) + } + + fun confidence( + r: TrustRelation, + source: HexKey, + ) = when (r) { + TrustRelation.FOLLOW -> if (source == observer) params.directFollowConfidence else params.indirectFollowConfidence + TrustRelation.MUTE -> params.muteConfidence + TrustRelation.REPORT -> params.reportConfidence + } fun weightToConfidence(w: Double) = 1.0 - exp(-w * -ln(params.rigor)) @@ -199,22 +226,21 @@ class GrapeRankTest { scores[observer] = 1.0 do { var changed = false - for (target in graph.users) { + for (target in nodes) { if (target == observer) continue var sumW = 0.0 var sumWR = 0.0 - for (edge in graph.incoming[target] ?: emptyList()) { - val s = scores[edge.source] ?: continue - val w = confidence(edge) * s * params.attenuation + for ((source, r) in incoming[target] ?: emptySet()) { + val s = scores[source] ?: continue + val w = confidence(r, source) * s * params.attenuation sumW += w - sumWR += w * edge.relation.rating + sumWR += w * r.rating } val newScore = if (abs(sumW) < 0.00001) 0.0 else max(weightToConfidence(sumW) * sumWR / sumW, 0.0) val old = scores.put(target, newScore) ?: 0.0 changed = changed || abs(newScore - old) > params.convergence } } while (changed) - scores.remove(observer) return scores } } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt index d43ecdc459..01b9fb6cb3 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt @@ -21,118 +21,87 @@ package com.vitorpamplona.amethyst.commons.wot import com.vitorpamplona.quartz.nip01Core.core.HexKey -import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent -import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent -import com.vitorpamplona.quartz.nip56Reports.ReportEvent import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue class TrustGraphBuilderTest { - // Distinct valid 64-hex pubkeys. - private fun pk(n: Int): HexKey = n.toString(16).padStart(64, '0') + private val alice = "alice" + private val bob = "bob" + private val carol = "carol" + private val dave = "dave" - private val alice = pk(0xA1) - private val bob = pk(0xB0) - private val carol = pk(0xC0) - private val dave = pk(0xD0) - - private val dummySig = "0".repeat(128) - - private fun contactList( - author: HexKey, - follows: List, - createdAt: Long = 1000, - ) = ContactListEvent( - id = pk(author.hashCode() xor createdAt.toInt()), - pubKey = author, - createdAt = createdAt, - tags = follows.map { arrayOf("p", it) }.toTypedArray(), - content = "", - sig = dummySig, - ) - - private fun muteList( - author: HexKey, - mutes: List, - createdAt: Long = 1000, - ) = MuteListEvent( - id = pk(author.hashCode() xor createdAt.toInt() xor 0x5555), - pubKey = author, - createdAt = createdAt, - tags = mutes.map { arrayOf("p", it) }.toTypedArray(), - content = "", - sig = dummySig, - ) - - private fun report( - author: HexKey, - reported: HexKey, - createdAt: Long = 1000, - ) = ReportEvent( - id = pk(author.hashCode() xor reported.hashCode() xor createdAt.toInt()), - pubKey = author, - createdAt = createdAt, - tags = arrayOf(arrayOf("p", reported, "spam")), - content = "", - sig = dummySig, - ) + /** Decode a node's incoming edges back to (source, relation) pairs from the CSR. */ + private fun TrustGraph.incomingOf(pubkey: HexKey): Set> { + val t = idOf(pubkey) + if (t < 0) return emptySet() + val out = HashSet>() + var i = inOffsets[t] + val end = inOffsets[t + 1] + while (i < end) { + val packed = inPacked[i] + val source = pubkeyOf(packed and TrustGraph.SOURCE_MASK) + val relation = TrustRelation.entries.first { it.code == (packed ushr TrustGraph.SOURCE_BITS) } + out.add(source to relation) + i++ + } + return out + } @Test fun buildsFollowMuteAndReportEdges() { - val graph = - TrustGraphBuilder.build( - listOf( - contactList(alice, listOf(bob, carol)), - muteList(bob, listOf(dave)), - report(carol, dave), - ), - ) + val b = TrustGraphBuilder() + b.addFollows(alice, listOf(bob, carol)) + b.addMutes(bob, listOf(dave)) + b.addReports(carol, listOf(dave)) + val graph = b.build() + assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(bob)) + assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(carol)) assertEquals( - setOf(TrustEdge(alice, TrustRelation.FOLLOW)), - graph.incoming[bob]?.toSet(), + setOf(bob to TrustRelation.MUTE, carol to TrustRelation.REPORT), + graph.incomingOf(dave), ) - assertEquals( - setOf(TrustEdge(alice, TrustRelation.FOLLOW)), - graph.incoming[carol]?.toSet(), - ) - assertEquals( - setOf(TrustEdge(bob, TrustRelation.MUTE), TrustEdge(carol, TrustRelation.REPORT)), - graph.incoming[dave]?.toSet(), - ) - } - - @Test - fun keepsOnlyLatestReplaceablePerAuthor() { - val graph = - TrustGraphBuilder.build( - listOf( - contactList(alice, listOf(bob), createdAt = 1000), - contactList(alice, listOf(carol), createdAt = 2000), - ), - ) - // The newer list (follows carol) wins; the stale bob follow is gone. - assertTrue(graph.incoming[bob].isNullOrEmpty()) - assertEquals(setOf(TrustEdge(alice, TrustRelation.FOLLOW)), graph.incoming[carol]?.toSet()) - } - - @Test - fun dedupesRepeatedReports() { - val graph = - TrustGraphBuilder.build( - listOf( - report(alice, dave, createdAt = 1000), - report(alice, dave, createdAt = 2000), - ), - ) - assertEquals(listOf(TrustEdge(alice, TrustRelation.REPORT)), graph.incoming[dave]) } @Test fun dropsSelfEdges() { - val graph = TrustGraphBuilder.build(listOf(contactList(alice, listOf(alice, bob)))) - assertTrue(graph.incoming[alice].isNullOrEmpty(), "a self-follow must not become an edge") - assertEquals(setOf(TrustEdge(alice, TrustRelation.FOLLOW)), graph.incoming[bob]?.toSet()) + val b = TrustGraphBuilder() + b.addFollows(alice, listOf(alice, bob)) + val graph = b.build() + assertTrue(graph.incomingOf(alice).isEmpty(), "a self-follow must not become an edge") + assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(bob)) + } + + @Test + fun dedupesRepeatedReports() { + val b = TrustGraphBuilder() + b.addReports(alice, listOf(dave)) + b.addReports(alice, listOf(dave)) + val graph = b.build() + assertEquals(1, graph.edgeCount()) + assertEquals(setOf(alice to TrustRelation.REPORT), graph.incomingOf(dave)) + } + + @Test + fun keepsFollowAndMuteFromSameSourceAsDistinctEdges() { + val b = TrustGraphBuilder() + b.addFollows(alice, listOf(bob)) + b.addMutes(alice, listOf(bob)) + val graph = b.build() + assertEquals( + setOf(alice to TrustRelation.FOLLOW, alice to TrustRelation.MUTE), + graph.incomingOf(bob), + ) + } + + @Test + fun internsEachPubkeyOnce() { + val b = TrustGraphBuilder() + b.addFollows(alice, listOf(bob, carol)) + b.addFollows(bob, listOf(carol)) + val graph = b.build() + assertEquals(3, graph.nodeCount, "alice, bob, carol interned once each") + assertEquals(3, graph.edgeCount()) } } From a2ab9876f44c6170eb104b43bb3458afd6174fee Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 21:40:17 +0000 Subject: [PATCH 13/58] feat(cli): track and report graperank crawl hop distance + --max-hops Stamp each discovered user with its follow-graph hop distance from the observer (observer=0, a user's fresh follows = its hop+1). Report the per-hop histogram on the crawl-complete line and as `max_hop_reached` / `users_by_hop` in --json, and add `--max-hops N` to bound how deep the crawl fetches (deeper users still appear as follow targets). Brainstorm's graph for an observer saturates within ~8 hops, so `--max-hops 8` matches its scope. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../com/vitorpamplona/amethyst/cli/Main.kt | 5 +- .../amethyst/cli/commands/GrapeRankCommand.kt | 49 ++++++++++++++----- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 38bd66bb2d..2c8a1ea3c3 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -530,8 +530,9 @@ private fun printUsage() { | graperank [OBSERVER] compute subjective trust scores (0..1) for every | [--limit N] [--min-score X] user reachable in the follow/mute/report graph. | [--rigor X] [--attenuation X] Exhaustively crawls each user's kind:10002 outbox - | [--max-rounds N] for their latest kind:3/10000/1984 until every - | [--offline] [--timeout SECS] discovered user has been checked (no user cap). + | [--max-rounds N] [--max-hops N] for their latest kind:3/10000/1984 until every + | [--offline] [--timeout SECS] discovered user has been checked (no user cap; + | --max-hops bounds follow distance, e.g. 8). | [--publish] [--min-rank N] OBSERVER: npub|nprofile|hex|name@domain (default: | [--publish-limit N] [--publish-relay URL] active account). --offline scores from the local | store only. --publish writes NIP-85 kind:30382 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 2d7e19e176..64cf6aaa07 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -115,8 +115,10 @@ object GrapeRankCommand { val args = Args(rest) val observerArg = args.positionalOrNull(0) // Crawl to full convergence by default (every reachable user's outbox - // checked). --max-rounds is only a safety backstop. + // checked). --max-rounds is only a safety backstop; --max-hops bounds the + // follow-graph distance from the observer that we crawl (Brainstorm uses 8). val maxRounds = args.intFlag("max-rounds", Int.MAX_VALUE) + val maxHops = args.intFlag("max-hops", Int.MAX_VALUE) val limit = args.intFlag("limit", 100) val minScore = args.flag("min-score")?.toDoubleOrNull() ?: 0.0 val offline = args.bool("offline") @@ -146,8 +148,10 @@ object GrapeRankCommand { var relaysContactedCount = 0 var contactListsFed = 0 + val hopOf = HashMap() if (!offline) { val discovered = hashSetOf(observer) + hopOf[observer] = 0 // Per-user relay hints harvested from the `p`-tag relay hints in the // contact lists we crawl (A's follow of B says where B writes) — a // discovery tier below each user's kind:10002 outbox. @@ -158,32 +162,40 @@ object GrapeRankCommand { val attempts = HashMap() val relaysContacted = hashSetOf() - // Feed a user's contact list into the graph, harvest relay hints, and - // add its follows to the frontier. Called once per user (guarded by - // `done`). Returns the count of newly-discovered users. + // Feed a user's contact list into the graph, harvest relay hints, stamp + // the hop distance of newly-seen follows, and add them to the frontier. + // Called once per user (guarded by `done`). Returns the count of + // newly-discovered users. fun ingest( source: HexKey, contacts: ContactListEvent, ): Int { + val nextHop = (hopOf[source] ?: 0) + 1 val follows = ArrayList() var fresh = 0 for (tag in contacts.follows()) { follows.add(tag.pubKey) tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } - if (discovered.add(tag.pubKey)) fresh++ + if (discovered.add(tag.pubKey)) { + hopOf[tag.pubKey] = nextHop + fresh++ + } } builder.addFollows(source, follows) contactListsFed++ return fresh } - // Crawl to full graph depth (no user cap). Each run fetches every - // discovered user's LATEST kind:3/10000/1984 once from their outbox - // (a freshness pass — grouped by write relay in routeByOutbox), unless - // we already fetched it this run (`done`). An unreachable outbox is - // retried up to MAX_OUTBOX_ATTEMPTS then dropped so the crawl terminates. + // Crawl to full graph depth (no user cap; --max-hops bounds the follow + // distance). Each run fetches every discovered user's LATEST + // kind:3/10000/1984 once from their outbox (a freshness pass — grouped + // by write relay in routeByOutbox), unless we already fetched it this + // run (`done`). An unreachable outbox is retried up to + // MAX_OUTBOX_ATTEMPTS then dropped so the crawl terminates. while (rounds < maxRounds) { - val pending = discovered.filterNot { it in done } + // Only crawl users within the hop budget; deeper users still appear + // in the graph as follow targets, we just don't fetch their lists. + val pending = discovered.filter { it !in done && (hopOf[it] ?: 0) < maxHops } if (pending.isEmpty()) break rounds++ @@ -233,9 +245,15 @@ object GrapeRankCommand { } relaysContactedCount = relaysContacted.size + val perHop = + hopOf.values + .groupingBy { it } + .eachCount() + .toSortedMap() System.err.println( "[graperank] crawl complete: ${discovered.size} discovered, $contactListsFed contact lists fed, " + - "$relaysContactedCount relays contacted, $rounds rounds", + "$relaysContactedCount relays contacted, $rounds rounds; " + + "by hop: " + perHop.entries.joinToString(" ") { "${it.key}=${it.value}" }, ) } else { // Offline: stream contact lists from the local store into the graph. @@ -286,6 +304,13 @@ object GrapeRankCommand { "observer" to observer, "crawl_rounds" to rounds, "relays_contacted" to relaysContactedCount, + "max_hop_reached" to (hopOf.values.maxOrNull() ?: 0), + "users_by_hop" to + hopOf.values + .groupingBy { it } + .eachCount() + .toSortedMap() + .mapKeys { it.key.toString() }, "graph_users" to graph.nodeCount, "graph_edges" to graph.edgeCount(), "users_scored" to rankedIds.size, From 07a9b2d116046b05c207d18617956fbc2d7021f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 21:56:25 +0000 Subject: [PATCH 14/58] feat(cli): widen graperank 10002 discovery + log slow relays on timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the hop-2 (and general) completeness undercount, and makes the cause observable. - Add a broad set of aggregator + big general relays (relay.nostr.band, relay.damus.io, snort, offchain.pub, relayable.org, …) to the kind:10002 discovery set. Effect: for Vitor, hop-2 discovery went from ~16,980 to 19,886 (~83% -> ~98% of Brainstorm's 20,332). - `Context.drain` gains a `diagnoseSlow` flag: on a timeout it logs which relays stalled and why — slow (no EOSE, with the event count they did send) vs cannot-connect (with the failure reason) vs closed. `amy graperank --diagnose` turns it on. This showed the remaining misses are relay-side: users advertise dead/misconfigured write relays (HTTP 404/530/503, "Unexpected response", read/connect timeouts), not anything blocking on our end. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../com/vitorpamplona/amethyst/cli/Context.kt | 71 ++++++++++++++----- .../com/vitorpamplona/amethyst/cli/Main.kt | 3 +- .../amethyst/cli/commands/GrapeRankCommand.kt | 25 +++++-- 3 files changed, 77 insertions(+), 22 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index dffb600206..6cac69446f 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -411,11 +411,15 @@ class Context( suspend fun drain( filters: Map>, timeoutMs: Long = 8_000, + diagnoseSlow: Boolean = false, ): List> { if (filters.isEmpty()) return emptyList() val eventChannel = Channel>(UNLIMITED) - val doneChannel = Channel(UNLIMITED) + // Carries the terminal reason per relay so a timeout can distinguish a slow + // relay (never terminal) from a connect failure / CLOSED. + val doneChannel = Channel>(UNLIMITED) val remaining = filters.keys.toMutableSet() + val doneReasons = HashMap() val subId = newSubId() val listener = object : SubscriptionListener { @@ -432,7 +436,7 @@ class Context( relay: NormalizedRelayUrl, forFilters: List?, ) { - doneChannel.trySend(relay) + doneChannel.trySend(relay to "eose") } override fun onClosed( @@ -440,7 +444,7 @@ class Context( relay: NormalizedRelayUrl, forFilters: List?, ) { - doneChannel.trySend(relay) + doneChannel.trySend(relay to "closed:$message") } override fun onCannotConnect( @@ -448,28 +452,36 @@ class Context( message: String, forFilters: List?, ) { - doneChannel.trySend(relay) + doneChannel.trySend(relay to "cannot:$message") } } val collected = mutableListOf>() try { client.subscribe(subId, filters, listener) - withTimeoutOrNull(timeoutMs) { - while (remaining.isNotEmpty()) { - select { - eventChannel.onReceive { pair -> - if (verifyAndStore(pair.second)) collected.add(pair) + val completed = + withTimeoutOrNull(timeoutMs) { + while (remaining.isNotEmpty()) { + select { + eventChannel.onReceive { pair -> + if (verifyAndStore(pair.second)) collected.add(pair) + } + doneChannel.onReceive { (relay, reason) -> + remaining.remove(relay) + doneReasons[relay] = reason + } } - doneChannel.onReceive { r -> remaining.remove(r) } } + // Drain any events that landed after EOSE but before cancel + while (true) { + val r = eventChannel.tryReceive() + if (!r.isSuccess) break + val pair = r.getOrThrow() + if (verifyAndStore(pair.second)) collected.add(pair) + } + true } - // Drain any events that landed after EOSE but before cancel - while (true) { - val r = eventChannel.tryReceive() - if (!r.isSuccess) break - val pair = r.getOrThrow() - if (verifyAndStore(pair.second)) collected.add(pair) - } + if (diagnoseSlow && completed == null && remaining.isNotEmpty()) { + logSlowDrain(timeoutMs, remaining, doneReasons, collected) } } finally { client.unsubscribe(subId) @@ -479,6 +491,31 @@ class Context( return collected } + /** + * On a [drain] timeout, report which relays stalled and why — a relay that + * never sent EOSE (slow, possibly still streaming) vs one that couldn't be + * reached (CANNOT-CONNECT, which points at our side / the network) vs one + * that CLOSED the sub. Includes how many events each slow relay did send, so + * "relay is slow" and "we never connected" are easy to tell apart. + */ + private fun logSlowDrain( + timeoutMs: Long, + stalled: Set, + doneReasons: Map, + collected: List>, + ) { + val eventsPer = collected.groupingBy { it.first }.eachCount() + val cannot = doneReasons.filterValues { it.startsWith("cannot") } + val closed = doneReasons.filterValues { it.startsWith("closed") } + val slowDetail = stalled.take(12).joinToString(", ") { "${it.url}(${eventsPer[it] ?: 0}ev)" } + val cannotDetail = cannot.entries.take(8).joinToString(", ") { "${it.key.url}=${it.value.removePrefix("cannot:").take(40)}" } + System.err.println( + "[drain] timeout ${timeoutMs}ms: ${stalled.size} slow(no EOSE), ${cannot.size} cannot-connect, ${closed.size} closed" + + (if (slowDetail.isNotEmpty()) " | slow: $slowDetail" else "") + + (if (cannotDetail.isNotEmpty()) " | cannot: $cannotDetail" else ""), + ) + } + /** * Publish [request] to [relays], then wait for the FIRST event matching [responseFilter] * — a live reply that arrives after our own EOSE, which [drain] would miss (it returns at diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 2c8a1ea3c3..4c0a746b31 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -532,7 +532,8 @@ private fun printUsage() { | [--rigor X] [--attenuation X] Exhaustively crawls each user's kind:10002 outbox | [--max-rounds N] [--max-hops N] for their latest kind:3/10000/1984 until every | [--offline] [--timeout SECS] discovered user has been checked (no user cap; - | --max-hops bounds follow distance, e.g. 8). + | [--diagnose] --max-hops bounds follow distance, e.g. 8; + | --diagnose logs slow/failed relays on timeout). | [--publish] [--min-rank N] OBSERVER: npub|nprofile|hex|name@domain (default: | [--publish-limit N] [--publish-relay URL] active account). --offline scores from the local | store only. --publish writes NIP-85 kind:30382 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 64cf6aaa07..0c4bf68677 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -96,6 +96,21 @@ object GrapeRankCommand { // Concurrent content drains. Bounded so total open connections stay sane. private const val DRAIN_CONCURRENCY = 8 + // Broad relays that carry kind:10002 for many users — aggregators + big + // general relays — added to the discovery set to raise the odds of resolving + // a stranger's outbox quickly. + private val EXTRA_DISCOVERY_RELAYS: Set = + listOf( + "wss://relay.nostr.band", + "wss://relay.damus.io", + "wss://relay.snort.social", + "wss://offchain.pub", + "wss://relayable.org", + "wss://nostr.land", + "wss://eden.nostr.land", + "wss://relay.nostr.bg", + ).mapNotNull { RelayUrlNormalizer.normalizeOrNull(it) }.toSet() + suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -122,6 +137,7 @@ object GrapeRankCommand { val limit = args.intFlag("limit", 100) val minScore = args.flag("min-score")?.toDoubleOrNull() ?: 0.0 val offline = args.bool("offline") + val diagnose = args.bool("diagnose") val timeoutMs = args.longFlag("timeout", 10L) * 1000 val doPublish = args.bool("publish") val minRank = args.intFlag("min-rank", 1) @@ -208,7 +224,7 @@ object GrapeRankCommand { // and times out. Routing (store reads) is serial; only the drains // run concurrently, which is safe: inserts serialize on the store // write lock. - ensureRelayLists(ctx, pending.toSet(), timeoutMs) + ensureRelayLists(ctx, pending.toSet(), timeoutMs, diagnose) for (group in pending.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, graphKinds) } val drained = @@ -216,7 +232,7 @@ object GrapeRankCommand { prepared .map { (batch, filters) -> async { - ctx.drain(filters, timeoutMs) + ctx.drain(filters, timeoutMs, diagnose) batch to filters.keys } }.awaitAll() @@ -542,7 +558,7 @@ object GrapeRankCommand { * the whole network, so this is where a stranger's relay list is found. They * do NOT hold kind:3/10000/1984 — see [contentFallbackRelays]. */ - private suspend fun relayListDiscoveryRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + private suspend fun relayListDiscoveryRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + EXTRA_DISCOVERY_RELAYS /** * Best-effort fallback relays for **content** (kind:3/10000/1984/0) when a @@ -563,6 +579,7 @@ object GrapeRankCommand { ctx: Context, pubkeys: Set, timeoutMs: Long, + diagnose: Boolean, ) { val missing = pubkeys.filter { ctx.relaysOf(it) == null } if (missing.isEmpty()) return @@ -576,7 +593,7 @@ object GrapeRankCommand { Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) } } - ctx.drain(filters, timeoutMs) + ctx.drain(filters, timeoutMs, diagnose) } /** From ccb5912e33a0e81a998311ed8320d83bc2a66abd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 22:13:22 +0000 Subject: [PATCH 15/58] feat(cli): learn a known-good relay backbone; retry unreachable users on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers "do we try relays we know work from other people's lists when a user isn't in the first set?" — now yes. - Drop dead relays I'd added unchecked (relay.nostr.band, relayable.org, relay.nostr.bg); keep only ones that reply to a limit:1 query. NIP-11 presence is not used for liveness (it's optional). - Learn a backbone dynamically from the crawl: tally how often each relay appears as someone's kind:10002 write relay, and mark relays that actually delivered events as live. The most-used live relays form the backbone. - Route retried users (whose own outbox already failed) and outbox-less users to outbox + backbone, since popular relays usually hold a copy of their kind:3. Effect for Vitor (--max-hops 3): hop-3 coverage rose from ~112,600 to 146,413 (95% of Brainstorm's 153,409), as round-3 contact-list fetches more than doubled. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 67 ++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 0c4bf68677..8189b641d8 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -96,21 +96,22 @@ object GrapeRankCommand { // Concurrent content drains. Bounded so total open connections stay sane. private const val DRAIN_CONCURRENCY = 8 - // Broad relays that carry kind:10002 for many users — aggregators + big - // general relays — added to the discovery set to raise the odds of resolving - // a stranger's outbox quickly. + // Broad, big general relays that carry kind:10002 for many users, added to the + // discovery set to raise the odds of resolving a stranger's outbox. Every entry + // is NIP-11 liveness-checked — dead relays only add timeouts. private val EXTRA_DISCOVERY_RELAYS: Set = listOf( - "wss://relay.nostr.band", "wss://relay.damus.io", "wss://relay.snort.social", "wss://offchain.pub", - "wss://relayable.org", "wss://nostr.land", "wss://eden.nostr.land", - "wss://relay.nostr.bg", ).mapNotNull { RelayUrlNormalizer.normalizeOrNull(it) }.toSet() + // How many of the most-used write relays (learned from everyone's kind:10002) + // to keep as the known-good backbone for retrying users we couldn't reach. + private const val BACKBONE_SIZE = 30 + suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -177,6 +178,12 @@ object GrapeRankCommand { val done = hashSetOf() val attempts = HashMap() val relaysContacted = hashSetOf() + // Known-good relay pool, learned from the crawl itself: how often each + // relay appears as someone's write relay, and which relays actually + // delivered events (so we know they connect and work). The most-common + // live relays become the `backbone` we retry unreachable users against. + val writeRelayFreq = HashMap() + val liveRelays = hashSetOf() // Feed a user's contact list into the graph, harvest relay hints, stamp // the hop distance of newly-seen follows, and add them to the frontier. @@ -218,6 +225,19 @@ object GrapeRankCommand { var gotList = 0 var newUsers = 0 + // The known-good backbone this round: the most-used write relays + // that have actually delivered events. Retried / outbox-less users + // are also queried here — these are relays we know work, learned + // from everyone else's lists. + val backbone = + writeRelayFreq.entries + .asSequence() + .filter { it.key in liveRelays } + .sortedByDescending { it.value } + .take(BACKBONE_SIZE) + .map { it.key } + .toSet() + // Resolve kind:10002 outboxes in bulk (indexers aggregate them), // then fetch content in small batches drained a few at a time — one // giant drain over thousands of outbox relays saturates connections @@ -226,19 +246,21 @@ object GrapeRankCommand { // write lock. ensureRelayLists(ctx, pending.toSet(), timeoutMs, diagnose) for (group in pending.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { - val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, graphKinds) } + val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, graphKinds) } val drained = coroutineScope { prepared .map { (batch, filters) -> async { - ctx.drain(filters, timeoutMs, diagnose) - batch to filters.keys + val events = ctx.drain(filters, timeoutMs, diagnose) + Triple(batch, filters.keys, events) } }.awaitAll() } - for ((batch, relays) in drained) { + for ((batch, relays, events) in drained) { relaysContacted += relays + // Any relay that gave us an event is proven live + useful. + for ((relay, _) in events) liveRelays.add(relay) for (pk in batch) { val contacts = ctx.contactsOf(pk) if (contacts != null) { @@ -597,15 +619,24 @@ object GrapeRankCommand { } /** - * Group [pubkeys] by the relays we should query for their events: each user's - * kind:10002 write relays (the outbox model); for users with no advertised - * relay list, their harvested relay [hints] plus the broad discovery set. - * Authors are chunked per relay to respect relay REQ limits. + * Group [pubkeys] by the relays we should query for their events: + * - first try: the user's own kind:10002 write relays (the outbox model); + * - a retry (`attempts[pk] > 0`, its outbox already failed): outbox + + * [backbone] — the known-good relays other people write to, which likely + * hold a copy; + * - no outbox at all: harvested [hints] + backbone + the general fallback. + * + * Also tallies each user's write relays into [writeRelayFreq] so the backbone + * can be learned from the crawl. Authors are chunked per relay to respect REQ + * limits. */ private suspend fun routeByOutbox( ctx: Context, pubkeys: Set, hints: Map>, + backbone: Set, + attempts: Map, + writeRelayFreq: MutableMap, kinds: List, ): Map> { val fallback = contentFallbackRelays(ctx) @@ -613,7 +644,13 @@ object GrapeRankCommand { for (pk in pubkeys) { val write = ctx.relaysOf(pk)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() } - val relays = write ?: (hints[pk].orEmpty() + fallback) + write?.forEach { writeRelayFreq.merge(it, 1, Int::plus) } + val relays = + when { + write == null -> hints[pk].orEmpty() + backbone + fallback + (attempts[pk] ?: 0) > 0 -> write + backbone + else -> write + } for (relay in relays) perRelay.getOrPut(relay) { HashSet() }.add(pk) } From 8531c6475de2a831049b19c3fb1f6f533cb7341f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 22:27:23 +0000 Subject: [PATCH 16/58] feat(cli): last-mile relay sweep for graperank crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the outbox crawl gives up on users whose own kind:10002 relays never answered (dead/misconfigured outboxes), take one more pass at every still-missing user within the hop budget against the WHOLE known-good relay pool — the busiest live relays learned from the crawl (which include the big aggregators) plus the discovery set — instead of re-asking each straggler's broken outbox. Recovered contact lists feed the graph and can reveal a few more reachable users, so the sweep repeats up to LAST_MILE_PASSES times until it stops recovering. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 8189b641d8..e613a888d5 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -112,6 +112,15 @@ object GrapeRankCommand { // to keep as the known-good backbone for retrying users we couldn't reach. private const val BACKBONE_SIZE = 30 + // Last-mile sweep: after the outbox crawl gives up on the users whose own + // relays never answered, we take one more run at them against the WHOLE + // known-good relay pool — the busiest live relays we learned from everyone + // else's lists (they include the big aggregators). LAST_MILE_RELAYS caps that + // pool; LAST_MILE_PASSES bounds how many times we re-sweep as recovered lists + // reveal a few more reachable users. + private const val LAST_MILE_RELAYS = 80 + private const val LAST_MILE_PASSES = 2 + suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -282,6 +291,67 @@ object GrapeRankCommand { ) } + // Last-mile sweep. The outbox crawl leaves a tail of users whose own + // relays never answered (dead/misconfigured outboxes). Their contact + // lists very likely still exist — on the big aggregators and busy + // relays everyone else writes to. So instead of asking each straggler's + // broken outbox again, ask the WHOLE known-good pool at once: the + // busiest live relays learned from the crawl, plus the discovery set. + val goodPool = + ( + writeRelayFreq.entries + .asSequence() + .filter { it.key in liveRelays } + .sortedByDescending { it.value } + .take(LAST_MILE_RELAYS) + .map { it.key } + .toSet() + relayListDiscoveryRelays(ctx) + ).toList() + if (goodPool.isNotEmpty()) { + for (pass in 1..LAST_MILE_PASSES) { + val missing = discovered.filter { (hopOf[it] ?: 0) < maxHops && ctx.contactsOf(it) == null } + if (missing.isEmpty()) break + + var recovered = 0 + var newUsers = 0 + for (group in missing.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { + val drained = + coroutineScope { + group + .map { batch -> + val filters = + goodPool.associateWith { + batch.chunked(AUTHORS_PER_FILTER).map { chunk -> + Filter(kinds = graphKinds, authors = chunk) + } + } + async { + val events = ctx.drain(filters, timeoutMs, diagnose) + batch to events + } + }.awaitAll() + } + for ((batch, events) in drained) { + relaysContacted += goodPool + for ((relay, _) in events) liveRelays.add(relay) + for (pk in batch) { + val contacts = ctx.contactsOf(pk) + if (contacts != null) { + recovered++ + done += pk + newUsers += ingest(pk, contacts) + } + } + } + } + System.err.println( + "[graperank] last-mile pass $pass: swept=${missing.size}, recovered=$recovered, " + + "newUsers=$newUsers, discovered=${discovered.size}, still-missing=${discovered.count { (hopOf[it] ?: 0) < maxHops && ctx.contactsOf(it) == null }}", + ) + if (recovered == 0) break + } + } + relaysContactedCount = relaysContacted.size val perHop = hopOf.values From 0d6f7fd186e209e11706391b99ce05ed5fe09de7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 23:24:10 +0000 Subject: [PATCH 17/58] feat(cli): SQLite event-store backend for amy (default), FS opt-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FS event store writes one pretty-printed JSON file per event plus one file per index posting (kind, author, every p-tag value). At crawl scale this explodes: a 96k-event GrapeRank crawl produced 5.6M tiny index files rounding up to 2.8GB on disk — only 457MB of which was actual event data. An 8-hop crawl would blow past available disk. Wire amy's shared store through a new StoreFactory that selects the backend from AMY_STORE (default `sqlite`, opt into the legacy tree with `fs`). Both implement IEventStore, so every command works unchanged. SQLite packs the same postings into shared B-tree pages — several times smaller on disk and the natural fit for large crawls. The two stores live side by side under `/shared/` (events.db vs events-store/) so switching never clobbers the other's data. `amy store` maintenance verbs are now backend-aware: stat reports the total + disk bytes for both (kind histogram/mtime stay fs-only); scrub is a no-op on sqlite (indexes are transactional); compact runs VACUUM on sqlite. Verified end-to-end via the built amy image on both backends: init, notes post round-trip (event persisted + read back), and every store verb. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../com/vitorpamplona/amethyst/cli/Config.kt | 9 + .../com/vitorpamplona/amethyst/cli/Context.kt | 31 +--- .../com/vitorpamplona/amethyst/cli/Main.kt | 15 +- .../amethyst/cli/StoreFactory.kt | 89 +++++++++ .../amethyst/cli/commands/StoreCommands.kt | 171 +++++++++++++----- 5 files changed, 244 insertions(+), 71 deletions(-) create mode 100644 cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/StoreFactory.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt index dd7c0fa984..ef9c5ed655 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt @@ -213,6 +213,15 @@ class DataDir( val groupsDir = File(marmotDir, "groups") val keyPackageBundleFile = File(marmotDir, "keypackages.bundle") + /** + * SQLite event-store DB file, a sibling of [eventsDir] under + * `/shared/`. Used when the store backend is SQLite (the + * default — see [StoreFactory]); the FS backend uses [eventsDir] + * instead. Kept alongside the FS store so switching backends never + * clobbers the other's data. + */ + val eventsDbFile: File = File(eventsDir.parentFile ?: root, "events.db") + init { SecureFileIO.secureMkdirs(root) SecureFileIO.secureMkdirs(groupsDir) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 6cac69446f..2e6b4dcccd 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -39,7 +39,6 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.crypto.verify -import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.publishAndConfirmDetailed @@ -54,7 +53,6 @@ import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.TcpNoDelaySocketF import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip17Dm.settings.ChatMessageRelayListEvent import com.vitorpamplona.quartz.nip46RemoteSigner.signer.NostrSignerRemote @@ -94,9 +92,10 @@ import okhttp3.OkHttpClient * Every Nostr event Amy observes — whether received from a relay * subscription, unwrapped from a NIP-59 gift wrap, or generated locally * before publish — is verified (NIP-01 signature + id check via - * [Event.verify]) and persisted to the file-backed [IEventStore] at - * `/events-store/`. Malformed events are dropped before - * reaching command code. + * [Event.verify]) and persisted to the shared [IEventStore] under + * `/shared/` (a SQLite DB by default, or the FS tree when + * `AMY_STORE=fs` — see [StoreFactory]). Malformed events are dropped + * before reaching command code. * * This makes [store] the authoritative cache of everything Amy has ever * seen: profile metadata, relay lists, contact lists, gift wraps, @@ -159,23 +158,13 @@ class Context( private val messageStore = FileMarmotMessageStore(dataDir.groupsDir) /** - * Filesystem-backed Nostr event store, rooted at [DataDir.eventsDir]. - * Lazy so commands that don't touch persistent event state pay zero - * open cost (no `.lock` file, no seed allocation). Closed by - * [close] when this Context shuts down. - * - * Files are written pretty-printed (not the compact NIP-01 canonical - * form) so `cat`, `jq`, `git diff` are useful out of the box — - * humans inspect these files. Verification always re-canonicalises, - * so the stored bytes never feed back into a signature check. + * Shared Nostr event store for this run, opened via [StoreFactory] + * (SQLite by default, or the FS tree when `AMY_STORE=fs`). Lazy so + * commands that don't touch persistent event state pay zero open cost + * (no DB file / `.lock`, no seed allocation). Closed by [close] when + * this Context shuts down. */ - private val storeDelegate: Lazy = - lazy { - FsEventStore( - root = dataDir.eventsDir.toPath(), - eventToJson = JacksonMapper::toJsonPretty, - ) - } + private val storeDelegate: Lazy = lazy { StoreFactory.open(dataDir) } val store: IEventStore by storeDelegate /** Fully-wired manager. Call [prepare] once before use to load persisted state. */ diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 4c0a746b31..8f4e077eb9 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -321,7 +321,8 @@ private fun printUsage() { | All state lives under ~/.amy/. Per-account directories | ~/.amy// hold identity, cursors, MLS state, and | aliases; every observed Nostr event lands in the shared - | ~/.amy/shared/events-store/. ACCOUNT must match + | store under ~/.amy/shared/ (a SQLite `events.db` by default, or + | the `events-store/` tree when AMY_STORE=fs). ACCOUNT must match | [a-zA-Z0-9_-]{1,64} (no spaces, no slashes). | | Resolution order: @@ -619,11 +620,15 @@ private fun printUsage() { | | marmot reset [--yes] wipe all local MLS/KeyPackage state (destructive) | - |Local event store (`/events-store/`): - | store stat event count, kind histogram, disk usage + |Local event store (shared, under `/shared/`): + | Backend selected by AMY_STORE: sqlite (default; `shared/events.db`) + | or fs (`AMY_STORE=fs`; the `shared/events-store/` tree). SQLite is + | far more compact at scale — the FS tree spends one file per index + | posting, so large crawls balloon on disk. + | store stat event count + disk usage (kind histogram/mtime on fs) | store sweep-expired delete events past their NIP-40 expiration - | store scrub rebuild idx/ from canonical events (after edits / crashes) - | store compact drop dangling idx entries (canonical gone) + | store scrub fs: rebuild idx/ from canonical events; sqlite: no-op + | store compact fs: drop dangling idx entries; sqlite: VACUUM | store reindex-fts rebuild the NIP-50 search index (after a searchable-kinds change) """.trimMargin(), ) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/StoreFactory.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/StoreFactory.kt new file mode 100644 index 0000000000..8c75bbf25f --- /dev/null +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/StoreFactory.kt @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.cli + +import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import kotlin.io.path.Path + +/** On-disk backend for the shared event store. */ +enum class StoreBackend { + /** + * Single SQLite database file at [DataDir.eventsDbFile]. Postings live + * in shared B-tree pages, so an event's kind/author/tag indexes cost a + * handful of rows — not one 4 KB-block file each, the way the FS store + * lays them out. For crawl-scale corpora (hundreds of thousands of + * follow lists) this is several times smaller on disk and the default. + */ + SQLITE, + + /** + * Filesystem tree at [DataDir.eventsDir] — one pretty-printed JSON file + * per event plus one file per index posting. Human-inspectable with + * `cat`/`jq`/`git diff`, but every posting rounds up to a filesystem + * block, so a large corpus balloons. Opt in with `AMY_STORE=fs`. + */ + FS, +} + +/** + * Chooses and opens the event-store backend for `amy`. The backend is + * selected by the `AMY_STORE` environment variable and defaults to + * [StoreBackend.SQLITE]; set `AMY_STORE=fs` for the legacy filesystem + * store. Both backends implement [IEventStore], so every command works + * unchanged regardless of the choice — the only user-visible difference + * is where bytes land ([DataDir.eventsDbFile] vs [DataDir.eventsDir]) and + * how much disk they take. + */ +object StoreFactory { + const val ENV = "AMY_STORE" + + /** Resolve the configured backend. Unrecognised values fall back to the default. */ + fun backend(): StoreBackend = + when (System.getenv(ENV)?.trim()?.lowercase()) { + "fs", "file", "files", "filesystem" -> StoreBackend.FS + else -> StoreBackend.SQLITE + } + + /** + * Open the store for [dataDir] using the configured [backend]. Events + * are written pretty-printed on the FS backend so the on-disk JSON stays + * inspection-friendly; the SQLite backend stores the compact NIP-01 + * form internally. Neither is re-used for signature checks (verification + * always re-canonicalises), so the stored representation is purely an + * implementation detail. Callers own [IEventStore.close]. + */ + fun open(dataDir: DataDir): IEventStore = + when (backend()) { + StoreBackend.SQLITE -> { + // BundledSQLiteDriver won't create parent directories. + dataDir.eventsDbFile.parentFile?.mkdirs() + EventStore(dbName = dataDir.eventsDbFile.absolutePath, relay = null) + } + StoreBackend.FS -> + FsEventStore( + root = Path(dataDir.eventsDir.absolutePath), + eventToJson = JacksonMapper::toJsonPretty, + ) + } +} diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt index bab4c94a55..16a84361d4 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt @@ -22,9 +22,13 @@ package com.vitorpamplona.amethyst.cli.commands import com.vitorpamplona.amethyst.cli.DataDir import com.vitorpamplona.amethyst.cli.Output -import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper +import com.vitorpamplona.amethyst.cli.StoreBackend +import com.vitorpamplona.amethyst.cli.StoreFactory +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import java.io.File import java.io.IOException import java.nio.file.Files import java.nio.file.Path @@ -33,19 +37,24 @@ import kotlin.io.path.exists /** * `amy store ` — direct introspection - * and maintenance of the file-backed event store at - * `/events-store/`. + * and maintenance of the shared event store under `/shared/`. * - * - `stat` total event count, kind histogram, disk bytes, - * mtime range — pure read, no relay traffic. + * The store backend is selected by `AMY_STORE` (SQLite by default, or the + * FS tree with `AMY_STORE=fs` — see [StoreFactory]); each verb adapts to + * whichever is active: + * + * - `stat` total event count, disk bytes, backend, plus (FS only) + * the per-kind histogram and mtime range — pure read, + * no relay traffic. * - `sweep-expired` delete events whose NIP-40 `expiration` tag has * passed (per the store's own sweep logic). Run * from cron / scheduler / `amy` periodically. - * - `scrub` rebuild every `idx/` entry from the canonical - * events. Recovers from partial-write crashes or - * external edits. - * - `compact` drop dangling `idx/` entries whose canonical is - * gone. Cheaper than scrub. + * - `scrub` FS: rebuild every `idx/` entry from the canonical + * events, recovering from partial-write crashes or + * external edits. SQLite: a no-op (indexes are updated + * transactionally and can't drift). + * - `compact` FS: drop dangling `idx/` entries whose canonical is + * gone. SQLite: `VACUUM` the database to reclaim space. * - `reindex-fts` wipe and rebuild only the NIP-50 full-text search * index from the stored events. Run after a quartz * upgrade that changes which kinds are searchable. @@ -68,7 +77,52 @@ object StoreCommands { ), ) - private fun stat(dataDir: DataDir): Int { + private suspend fun stat(dataDir: DataDir): Int = + when (StoreFactory.backend()) { + StoreBackend.SQLITE -> sqliteStat(dataDir) + StoreBackend.FS -> fsStat(dataDir) + } + + /** + * SQLite `stat`: total count via `COUNT(*)` and on-disk bytes from the + * DB file plus its `-wal`/`-shm` sidecars. The per-kind histogram and + * mtime range are FS-store concepts (they read the `idx/kind` tree and + * file mtimes), so they're omitted here. + */ + private suspend fun sqliteStat(dataDir: DataDir): Int { + val dbFile = dataDir.eventsDbFile + if (!dbFile.exists()) { + Output.emit( + mapOf( + "backend" to "sqlite", + "events" to 0, + "disk_bytes" to 0L, + "root" to dbFile.absolutePath, + ), + ) + return 0 + } + val count = + EventStore(dbName = dbFile.absolutePath, relay = null).use { store -> + store.count(Filter()) + } + val diskBytes = + listOf("", "-wal", "-shm").sumOf { suffix -> + val f = File(dbFile.absolutePath + suffix) + if (f.isFile) f.length() else 0L + } + Output.emit( + mapOf( + "backend" to "sqlite", + "events" to count, + "disk_bytes" to diskBytes, + "root" to dbFile.absolutePath, + ), + ) + return 0 + } + + private fun fsStat(dataDir: DataDir): Int { val storeRoot = dataDir.eventsDir.toPath() if (!storeRoot.exists()) { Output.emit( @@ -140,37 +194,65 @@ object StoreCommands { private suspend fun sweepExpired(dataDir: DataDir): Int = withStore(dataDir) { store -> - val expiresAtDir = dataDir.eventsDir.toPath().resolve("idx/expires_at") - val before = countEntries(expiresAtDir) - store.deleteExpiredEvents() - val after = countEntries(expiresAtDir) - Output.emit( - mapOf( - "swept" to (before - after).coerceAtLeast(0L), - "remaining" to after, - ), - ) + if (store is FsEventStore) { + // The FS store exposes its expiration index as a directory, + // so we can report exactly how many entries the sweep cleared. + val expiresAtDir = dataDir.eventsDir.toPath().resolve("idx/expires_at") + val before = countEntries(expiresAtDir) + store.deleteExpiredEvents() + val after = countEntries(expiresAtDir) + Output.emit( + mapOf( + "swept" to (before - after).coerceAtLeast(0L), + "remaining" to after, + ), + ) + } else { + store.deleteExpiredEvents() + Output.emit(mapOf("ok" to true)) + } 0 } - private fun scrub(dataDir: DataDir): Int = + private suspend fun scrub(dataDir: DataDir): Int = withStore(dataDir) { store -> - store.scrub() - Output.emit(mapOf("ok" to true)) + when (store) { + is FsEventStore -> { + store.scrub() + Output.emit(mapOf("ok" to true)) + } + // SQLite indexes are written in the same transaction as the + // event, so they can't drift the way the FS `idx/` tree can — + // there is nothing to rebuild. + else -> + Output.emit( + mapOf( + "ok" to true, + "note" to "scrub is a no-op for the sqlite backend (indexes update transactionally)", + ), + ) + } 0 } - private fun compact(dataDir: DataDir): Int = + private suspend fun compact(dataDir: DataDir): Int = withStore(dataDir) { store -> - store.compact() + when (store) { + // FS: drop dangling idx/ postings. SQLite: VACUUM to rebuild + // the file and hand freed pages back to the OS. + is FsEventStore -> store.compact() + is EventStore -> store.store.vacuum() + else -> Unit + } Output.emit(mapOf("ok" to true)) 0 } private suspend fun reindexFts(dataDir: DataDir): Int = withStore(dataDir) { store -> + val fsBacked = store is FsEventStore val ftsDir = dataDir.eventsDir.toPath().resolve("idx/fts") - val before = countEntries(ftsDir) + val before = if (fsBacked) countEntries(ftsDir) else 0L // Drive the resumable, batched path to completion so a huge // store is processed without holding the writer lock for the // whole pass. A real long-running caller would persist the @@ -184,35 +266,34 @@ object StoreCommands { processed += progress.processedThisBatch batches++ } while (!progress.done) - val after = countEntries(ftsDir) - Output.emit( - mapOf( + val out = + linkedMapOf( "ok" to true, "processed" to processed, "batches" to batches, - "tokens_before" to before, - "tokens_after" to after, - ), - ) + ) + if (fsBacked) { + // Token-file counts are an FS-store notion (idx/fts is a + // directory); the SQLite FTS index doesn't expose one. + out["tokens_before"] = before + out["tokens_after"] = countEntries(ftsDir) + } + Output.emit(out) 0 } /** * Maintenance verbs only need the store — not identity, not relays, - * not the signer. Skip [Context.open] (which throws if no identity - * has been bootstrapped) and construct the [FsEventStore] directly - * from [DataDir.eventsDir]. Pretty formatter matches what the rest - * of the CLI uses for inspection-friendly output. + * not the signer. Skip [Context.open] (which throws if no identity has + * been bootstrapped) and open the configured backend directly via + * [StoreFactory], so `amy store` acts on whichever store the rest of + * the CLI is using. */ - private inline fun withStore( + private suspend fun withStore( dataDir: DataDir, - body: (FsEventStore) -> Int, + body: suspend (IEventStore) -> Int, ): Int { - val store = - FsEventStore( - root = dataDir.eventsDir.toPath(), - eventToJson = JacksonMapper::toJsonPretty, - ) + val store = StoreFactory.open(dataDir) try { return body(store) } finally { From c75e0ff1abd6d59b8829f368cdcccfb361985406 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 00:09:44 +0000 Subject: [PATCH 18/58] fix(cli): don't log benign UNIQUE-constraint dups as store failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SQLite backend raises a catchable UNIQUE-constraint exception when an event is a duplicate id or an older/duplicate replaceable (kind 0/3/10000- 19999) — which the outbox model produces constantly, since each user's replaceable is fetched from several of their write relays. verifyAndStore was logging every one as `[cli] store insert failed`, so a full-network GrapeRank crawl emitted ~294k spurious error lines. The store is behaving correctly (its partial unique index + trigger keep the newest version and reject stale copies); the FS backend simply no-ops on the same duplicates. Suppress UNIQUE-constraint rejections (normal dedup) while still surfacing genuine persistence failures (I/O, full disk, corruption). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../kotlin/com/vitorpamplona/amethyst/cli/Context.kt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 2e6b4dcccd..ba8c0befea 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -559,7 +559,17 @@ class Context( try { store.insert(event) } catch (t: Throwable) { - System.err.println("[cli] store insert failed for ${event.id.take(8)}: ${t.message}") + // A UNIQUE-constraint rejection is normal, not a failure: the + // store already holds this id, or a newer version of a + // replaceable (kind 0/3/10000-19999). The outbox model routinely + // delivers the same event from several of a user's write relays, + // so a crawl produces these by the hundred-thousand. Only surface + // genuine persistence failures (I/O, full disk, corruption). The + // FS backend no-ops on such duplicates; this keeps the SQLite + // backend just as quiet. + if (t.message?.contains("UNIQUE constraint", ignoreCase = true) != true) { + System.err.println("[cli] store insert failed for ${event.id.take(8)}: ${t.message}") + } } return true } From 3c45a3187d0b91dbf93892706162498ed2c9ba5a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 01:03:12 +0000 Subject: [PATCH 19/58] feat(cli): report graperank graph-build and scoring time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measure and surface the two compute phases of `amy graperank`: building the int-CSR trust graph and running GrapeRank to convergence. Both are logged to stderr ("graph built … in N ms", "scored N users in M ms") and added to the --json result as graph_build_ms / scoring_ms, so the pure scoring cost over a given dataset is measurable without eyeballing logs — e.g. an `--offline` pass over a fully-crawled store times score generation with no network in the loop. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index e613a888d5..96a355424b 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -383,12 +383,15 @@ object GrapeRankCommand { if (event is ReportEvent) builder.addReports(event.pubKey, event.reportedAuthor().map { it.pubkey }) } + val buildStart = System.nanoTime() val graph = builder.build() - System.err.println("[graperank] graph built: ${graph.nodeCount} users, ${graph.edgeCount()} edges; scoring…") + val buildMs = (System.nanoTime() - buildStart) / 1_000_000 + System.err.println("[graperank] graph built: ${graph.nodeCount} users, ${graph.edgeCount()} edges in $buildMs ms; scoring…") // Live scoring progress: the worklist visits each reachable user once per // relaxation; report every SCORE_PROGRESS_STEP visits so a large graph shows // movement instead of hanging silently. + val scoreStart = System.nanoTime() val scores = GrapeRank(params).compute(graph, observer) { visited, queued -> if (visited % SCORE_PROGRESS_STEP == 0L) { @@ -405,7 +408,8 @@ object GrapeRankCommand { if (id != observerId && scores[id] > 0.0 && scores[id] >= minScore) rankedIds.add(id) } rankedIds.sortByDescending { scores[it] } - System.err.println("[graperank] scored ${rankedIds.size} users") + val scoringMs = (System.nanoTime() - scoreStart) / 1_000_000 + System.err.println("[graperank] scored ${rankedIds.size} users in $scoringMs ms") val result = linkedMapOf( @@ -422,6 +426,8 @@ object GrapeRankCommand { "graph_users" to graph.nodeCount, "graph_edges" to graph.edgeCount(), "users_scored" to rankedIds.size, + "graph_build_ms" to buildMs, + "scoring_ms" to scoringMs, "scores" to rankedIds.take(limit).map { mapOf("pubkey" to graph.pubkeyOf(it), "score" to scores[it], "rank" to rankOf(scores[it])) From f24e0f92d4f30ee83ce147ba6d5b74f4ac890d66 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 01:23:31 +0000 Subject: [PATCH 20/58] feat(cli): second-tier kind:10002 discovery over the learned relay pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Outbox resolution used only the fixed indexer/aggregator set to find a user's kind:10002. Users the aggregators don't carry got no relay list, so their content couldn't be routed to their own outbox (falling back to hints / the broad last-mile content sweep). Add a tier-2 pass in ensureRelayLists: any pubkey still without a relay list after the indexer sweep is retried for kind:10002 against the known-good backbone — the busiest live relays learned from the `r` tags in everyone else's 10002s. A user publishes their own 10002 to their own write relays, which overlap heavily with that pool, so this recovers relay lists the aggregators miss. Bounded to the backbone (not an unbounded fan-out to every working relay) to avoid connection saturation; early rounds no-op until the backbone is learned. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 96a355424b..30ba9fc090 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -253,7 +253,7 @@ object GrapeRankCommand { // and times out. Routing (store reads) is serial; only the drains // run concurrently, which is safe: inserts serialize on the store // write lock. - ensureRelayLists(ctx, pending.toSet(), timeoutMs, diagnose) + ensureRelayLists(ctx, pending.toSet(), backbone, timeoutMs, diagnose) for (group in pending.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, graphKinds) } val drained = @@ -669,29 +669,51 @@ object GrapeRankCommand { /** * Fetch kind:10002 relay lists for any [pubkeys] we don't already know, so * [routeByOutbox] can route their content query to their own write relays. - * Queries the bounded relay-list discovery set (indexers + general defaults), - * which aggregate kind:10002 for the whole network — reliable in bulk, unlike - * fanning out to thousands of per-user outboxes. + * + * Tier 1 queries the bounded relay-list discovery set (indexers + general + * defaults), which aggregate kind:10002 for the whole network — reliable in + * bulk, unlike fanning out to thousands of per-user outboxes. + * + * Tier 2 is a completeness net for the stragglers the indexers don't cover: + * a user publishes their own kind:10002 to their own write relays, and those + * relays overlap heavily with [fallbackRelays] — the known-good backbone we + * learned from the `r` tags in *everyone else's* 10002s. So after tier 1, + * any pubkey still without a relay list is retried against that learned pool + * (minus the tier-1 relays we already asked). Early rounds skip tier 2 + * harmlessly because the backbone is still empty; it kicks in once the crawl + * has learned which relays actually carry 10002s. */ private suspend fun ensureRelayLists( ctx: Context, pubkeys: Set, + fallbackRelays: Set, timeoutMs: Long, diagnose: Boolean, ) { val missing = pubkeys.filter { ctx.relaysOf(it) == null } if (missing.isEmpty()) return - val relays = relayListDiscoveryRelays(ctx) - if (relays.isEmpty()) return - - val filters = - relays.associateWith { - missing.chunked(AUTHORS_PER_FILTER).map { chunk -> - Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) + suspend fun query( + authors: List, + relays: Set, + ) { + if (relays.isEmpty() || authors.isEmpty()) return + val filters = + relays.associateWith { + authors.chunked(AUTHORS_PER_FILTER).map { chunk -> + Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) + } } - } - ctx.drain(filters, timeoutMs, diagnose) + ctx.drain(filters, timeoutMs, diagnose) + } + + val discovery = relayListDiscoveryRelays(ctx) + query(missing, discovery) + + // Tier 2: whoever the aggregators still don't have, ask the relays the + // rest of the graph actually writes to. + val stillMissing = missing.filter { ctx.relaysOf(it) == null } + query(stillMissing, fallbackRelays - discovery) } /** From d20f3d71e3b5995e9cb153e453c8ce85fd5d54da Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 01:50:07 +0000 Subject: [PATCH 21/58] feat(cli): time the offline store-load separately from graph build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The graph_build_ms timer covered only builder.build() (the in-memory int-CSR pack, a few hundred ms). It excluded the real pre-scoring cost: reading and deserializing every kind:3 contact list out of the store. Add store_load_ms (offline path) so the three phases — store load, CSR build, scoring — are each measured and reported (stderr + JSON), instead of the load hiding behind a misleadingly small build number. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 30ba9fc090..f9775cf3c1 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -173,6 +173,11 @@ object GrapeRankCommand { var rounds = 0 var relaysContactedCount = 0 var contactListsFed = 0 + // Wall time to read + deserialize the contact lists out of the store + // (offline path only; online streams them in during the crawl). This + // is the real pre-scoring cost — the int-CSR build afterwards is a + // cheap in-memory pack. + var storeLoadMs: Long? = null val hopOf = HashMap() if (!offline) { @@ -365,13 +370,15 @@ object GrapeRankCommand { ) } else { // Offline: stream contact lists from the local store into the graph. + val loadStart = System.nanoTime() for (event in ctx.store.query(Filter(kinds = listOf(ContactListEvent.KIND)))) { if (event is ContactListEvent) { builder.addFollows(event.pubKey, event.verifiedFollowKeySet()) contactListsFed++ } } - System.err.println("[graperank] offline: $contactListsFed contact lists from local store") + storeLoadMs = (System.nanoTime() - loadStart) / 1_000_000 + System.err.println("[graperank] offline: $contactListsFed contact lists from local store in $storeLoadMs ms") } // Mutes + reports come from the store (both paths). Far fewer than contact @@ -426,6 +433,7 @@ object GrapeRankCommand { "graph_users" to graph.nodeCount, "graph_edges" to graph.edgeCount(), "users_scored" to rankedIds.size, + "store_load_ms" to storeLoadMs, "graph_build_ms" to buildMs, "scoring_ms" to scoringMs, "scores" to From fd63d3843b26f9e6c9a5a4c7d0e3f0a048cdf777 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 02:11:46 +0000 Subject: [PATCH 22/58] perf(wot): score with Gauss-Seidel sweeps instead of a change-propagating worklist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worklist re-enqueued a node's dependents on every >convergence nudge, so on a dense graph its total work scaled with the in-degree of the churning core — an offline score over the full ~419k-node / 19.3M-edge Vitor graph ran 640M+ node-visits and still had not converged after 32 minutes. Replace it with synchronous Gauss-Seidel sweeps over all nodes (in-place updates, so trust flows outward within a sweep), ending when no node moves more than the convergence delta. Same per-node formula, same 0.0001 threshold, same unique fixed point as NosFabrica's Brainstorm reference (which iterates the same way) — verified byte-identical by the existing GrapeRankTest suite — but each node is touched once per sweep instead of once per churning rater, cutting the work by roughly the average in-degree. Progress now reports per-sweep (node-updates + nodes-still-moving) and the CLI surfaces the sweep count. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 19 ++- .../amethyst/commons/wot/GrapeRank.kt | 124 +++++++++--------- 2 files changed, 73 insertions(+), 70 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index f9775cf3c1..507c22f27c 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -81,9 +81,6 @@ object GrapeRankCommand { // Concurrent publishes when writing NIP-85 cards. private const val PUBLISH_CONCURRENCY = 16 - // Emit a scoring-progress line every this many worklist visits. - private const val SCORE_PROGRESS_STEP = 5_000 - // Times we re-query an unreachable user's outbox before giving up on it, so // the crawl still terminates on a finite graph. private const val MAX_OUTBOX_ATTEMPTS = 3 @@ -395,15 +392,16 @@ object GrapeRankCommand { val buildMs = (System.nanoTime() - buildStart) / 1_000_000 System.err.println("[graperank] graph built: ${graph.nodeCount} users, ${graph.edgeCount()} edges in $buildMs ms; scoring…") - // Live scoring progress: the worklist visits each reachable user once per - // relaxation; report every SCORE_PROGRESS_STEP visits so a large graph shows - // movement instead of hanging silently. + // Live scoring progress: fires once per Gauss-Seidel sweep with the + // running node-update count and how many nodes still moved more than the + // convergence delta this sweep — that second number trends to 0, so a + // large graph shows convergence instead of hanging silently. val scoreStart = System.nanoTime() + var sweeps = 0 val scores = - GrapeRank(params).compute(graph, observer) { visited, queued -> - if (visited % SCORE_PROGRESS_STEP == 0L) { - System.err.println("[graperank] scoring: $visited visited, $queued queued") - } + GrapeRank(params).compute(graph, observer) { visited, stillMoving -> + sweeps++ + System.err.println("[graperank] scoring sweep $sweeps: $visited node-updates, $stillMoving still moving") } fun rankOf(score: Double) = (score * 100).roundToInt() @@ -436,6 +434,7 @@ object GrapeRankCommand { "store_load_ms" to storeLoadMs, "graph_build_ms" to buildMs, "scoring_ms" to scoringMs, + "scoring_sweeps" to sweeps, "scores" to rankedIds.take(limit).map { mapOf("pubkey" to graph.pubkeyOf(it), "score" to scores[it], "rank" to rankOf(scores[it])) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt index 725ba3d470..c5674317c8 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt @@ -45,8 +45,9 @@ data class GrapeRankParams( /** * GrapeRank — a subjective, observer-centric web-of-trust score in `[0, 1]` for * every user reachable from an observer in a [TrustGraph]. See the algorithm - * notes in `TrustGraph`/`GrapeRankTest`; this is the single-observer worklist - * form, operating on the compact int-CSR graph so it scales to the whole network. + * notes in `TrustGraph`/`GrapeRankTest`; this is the single-observer + * Gauss-Seidel form, operating on the compact int-CSR graph so it scales to the + * whole network. * * [compute] returns a `DoubleArray` indexed by node id (`graph.idOf(pubkey)`), * not a map — at millions of nodes a boxed map would dwarf the graph itself. The @@ -80,7 +81,25 @@ class GrapeRank( /** * Score every node reachable from [observer]. Returns scores by node id, or an * all-zero array if the observer isn't in the graph. [onProgress] fires once - * per worklist visit with `(visited, queued)` running counts. + * per sweep with `(totalNodeUpdates, nodesStillMoving)` — the second value is + * how many nodes moved more than [GrapeRankParams.convergence] this sweep, so + * it trends to 0 as the graph settles. + * + * Iterates synchronous **Gauss-Seidel** sweeps over every node, updating scores + * in place so a value computed earlier in a sweep is already visible to nodes + * later in the same sweep (this converges faster than a double-buffered Jacobi + * pass). A sweep that moves no node by more than the convergence delta ends the + * loop — the same per-node threshold and fixed point as NosFabrica's Brainstorm + * reference. On a dense graph this is far less total work than a + * change-propagating worklist: a worklist re-visits a node once per rater whose + * score nudges, so its cost scales with the in-degree of the churning core, + * whereas a sweep touches each node exactly once per iteration. Attenuation < 1 + * makes the update a contraction, so the fixed point is unique regardless of + * sweep order; ids run in roughly BFS order from the observer, which lets + * trust flow outward within a single sweep and keeps the iteration count low. + * + * Nodes unreachable from the observer settle to 0 for free: all of their raters + * stay at 0, so the inner loop's `sourceScore != 0.0` guard skips every edge. */ fun compute( graph: TrustGraph, @@ -94,71 +113,56 @@ class GrapeRank( scores[observerId] = 1.0 - val inQueue = BooleanArray(n) - val queue = IntArrayList(1024) - - fun enqueue(node: Int) { - if (node != observerId && !inQueue[node]) { - inQueue[node] = true - queue.add(node) - } - } - - enqueueOutNeighbours(graph, observerId, ::enqueue) + val attenuation = params.attenuation + val convergence = params.convergence + val inOffsets = graph.inOffsets + val inPacked = graph.inPacked var visited = 0L - while (queue.isNotEmpty()) { - val target = queue.removeLast() - inQueue[target] = false + while (true) { + var stillMoving = 0 + var target = 0 + while (target < n) { + if (target != observerId) { + var sumOfWeights = 0.0 + var sumOfWeightedRatings = 0.0 + var i = inOffsets[target] + val end = inOffsets[target + 1] + while (i < end) { + val packed = inPacked[i] + val source = packed and TrustGraph.SOURCE_MASK + val sourceScore = scores[source] + if (sourceScore != 0.0) { + val relationCode = packed ushr TrustGraph.SOURCE_BITS + val weight = confidence(relationCode, source == observerId) * sourceScore * attenuation + sumOfWeights += weight + sumOfWeightedRatings += weight * rating(relationCode) + } + i++ + } - var sumOfWeights = 0.0 - var sumOfWeightedRatings = 0.0 - var i = graph.inOffsets[target] - val end = graph.inOffsets[target + 1] - while (i < end) { - val packed = graph.inPacked[i] - val source = packed and TrustGraph.SOURCE_MASK - val sourceScore = scores[source] - if (sourceScore != 0.0) { - val relationCode = packed ushr TrustGraph.SOURCE_BITS - val weight = confidence(relationCode, source == observerId) * sourceScore * params.attenuation - sumOfWeights += weight - sumOfWeightedRatings += weight * rating(relationCode) + val newScore = + if (abs(sumOfWeights) < 0.00001) { + 0.0 + } else { + val s = weightToConfidence(sumOfWeights) * sumOfWeightedRatings / sumOfWeights + if (s > 0.0) s else 0.0 + } + + val oldScore = scores[target] + if (newScore != oldScore) { + scores[target] = newScore + if (abs(newScore - oldScore) > convergence) stillMoving++ + } + visited++ } - i++ + target++ } - val newScore = - if (abs(sumOfWeights) < 0.00001) { - 0.0 - } else { - val s = weightToConfidence(sumOfWeights) * sumOfWeightedRatings / sumOfWeights - if (s > 0.0) s else 0.0 - } - - val oldScore = scores[target] - scores[target] = newScore - if (abs(newScore - oldScore) > params.convergence) { - enqueueOutNeighbours(graph, target, ::enqueue) - } - - visited++ - onProgress?.invoke(visited, queue.size) + onProgress?.invoke(visited, stillMoving) + if (stillMoving == 0) break } return scores } - - private inline fun enqueueOutNeighbours( - graph: TrustGraph, - node: Int, - enqueue: (Int) -> Unit, - ) { - var i = graph.outOffsets[node] - val end = graph.outOffsets[node + 1] - while (i < end) { - enqueue(graph.outTargets[i]) - i++ - } - } } From 6a1988976ea3599fa5390673572a166187312ff8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 02:20:42 +0000 Subject: [PATCH 23/58] feat(cli): --bench-sign to time kind:30382 card generation (no publish) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a benchmark path to `amy graperank`: after scoring, build + sign one NIP-85 kind:30382 ContactCardEvent per scored user (rank >= --min-rank) with a throwaway keypair, fanned out across CPU cores, and report bench_signed + bench_sign_ms. The signed events are discarded — this measures the id-hash + Schnorr-sign cost of emitting the full card set without touching any relay or real identity. Complements the existing store_load_ms / graph_build_ms / scoring_ms phase timings so the whole pipeline (load → build → score → sign) is measured end to end. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 507c22f27c..b187116ebc 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -31,9 +31,12 @@ import com.vitorpamplona.amethyst.commons.wot.GrapeRankParams import com.vitorpamplona.amethyst.commons.wot.TrustGraphBuilder import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent import com.vitorpamplona.quartz.nip56Reports.ReportEvent @@ -45,6 +48,7 @@ import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProvider import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceType import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope @@ -150,6 +154,10 @@ object GrapeRankCommand { val minRank = args.intFlag("min-rank", 1) val publishLimit = args.intFlag("publish-limit", 500) val publishRelaysArg = args.flag("publish-relay") + // Benchmark: build + sign one kind:30382 card per scored user (rank >= + // --min-rank) with a throwaway key and time it, WITHOUT publishing. + // Measures the id-hash + Schnorr-sign cost of emitting the full card set. + val benchSign = args.bool("bench-sign") val params = GrapeRankParams( @@ -479,11 +487,60 @@ object GrapeRankCommand { } } + if (benchSign) { + // Throwaway key — these cards are for timing only and never leave + // the process, so no real identity signs them. + val tempSigner = NostrSignerInternal(KeyPair()) + val cards = + rankedIds + .filter { rankOf(scores[it]) >= minRank } + .map { graph.pubkeyOf(it) to rankOf(scores[it]) } + val signStart = System.nanoTime() + val signed = signCards(cards, tempSigner) + val signMs = (System.nanoTime() - signStart) / 1_000_000 + val perSec = if (signMs > 0) signed * 1000L / signMs else 0 + System.err.println("[graperank] signed $signed kind:30382 cards in $signMs ms ($perSec/s, temp key, not published)") + result["bench_signed"] = signed + result["bench_sign_ms"] = signMs + } + Output.emit(result) return 0 } } + /** + * Build + sign one kind:30382 [ContactCardEvent] per (target, rank), fanned + * out across CPU cores (id-hash + Schnorr sign is CPU-bound). The signed + * events are discarded — this only exists to time card generation. Returns + * the number signed. + */ + private suspend fun signCards( + cards: List>, + signer: NostrSigner, + ): Int { + if (cards.isEmpty()) return 0 + val cores = Runtime.getRuntime().availableProcessors().coerceAtLeast(1) + val chunkSize = ((cards.size + cores - 1) / cores).coerceAtLeast(1) + return coroutineScope { + cards + .chunked(chunkSize) + .map { chunk -> + async(Dispatchers.Default) { + for ((target, rank) in chunk) { + ContactCardEvent.create( + targetUser = target, + signer = signer, + publicInitializer = { add(RankTag.assemble(rank)) }, + ) + } + chunk.size + } + }.awaitAll() + .sum() + } + } + /** * `amy graperank register [PROVIDER] [--service KIND:TAG] [--relay URL] [--private]` * From 35078c460def04cbc1a7f945bd4c5a145bfe51ef Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 02:48:49 +0000 Subject: [PATCH 24/58] feat(cli): measure crawl/download time in graperank (download_ms) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The online crawl phase — the network-bound fetch of the whole graph off the relays (rounds + last-mile sweep) — was untimed; only the offline store-load had a phase timer. Add download_ms around the crawl block and fold it into the "crawl complete" log and the JSON, so a from-scratch run reports every phase: download -> graph build -> scoring (and, with --bench-sign, card signing). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index b187116ebc..e8905456a4 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -183,9 +183,15 @@ object GrapeRankCommand { // is the real pre-scoring cost — the int-CSR build afterwards is a // cheap in-memory pack. var storeLoadMs: Long? = null + // Wall time to crawl + download the whole graph off the relays + // (online path only) — rounds + last-mile sweep, i.e. everything up + // to the point the graph is fully fetched. This is network-bound and + // dominates a from-scratch run. + var downloadMs: Long? = null val hopOf = HashMap() if (!offline) { + val crawlStart = System.nanoTime() val discovered = hashSetOf(observer) hopOf[observer] = 0 // Per-user relay hints harvested from the `p`-tag relay hints in the @@ -368,9 +374,10 @@ object GrapeRankCommand { .groupingBy { it } .eachCount() .toSortedMap() + downloadMs = (System.nanoTime() - crawlStart) / 1_000_000 System.err.println( "[graperank] crawl complete: ${discovered.size} discovered, $contactListsFed contact lists fed, " + - "$relaysContactedCount relays contacted, $rounds rounds; " + + "$relaysContactedCount relays contacted, $rounds rounds in $downloadMs ms; " + "by hop: " + perHop.entries.joinToString(" ") { "${it.key}=${it.value}" }, ) } else { @@ -439,6 +446,7 @@ object GrapeRankCommand { "graph_users" to graph.nodeCount, "graph_edges" to graph.edgeCount(), "users_scored" to rankedIds.size, + "download_ms" to downloadMs, "store_load_ms" to storeLoadMs, "graph_build_ms" to buildMs, "scoring_ms" to scoringMs, From 6e51f9c262701cd9c2ec82599cf5d440fc07a3ab Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 03:58:01 +0000 Subject: [PATCH 25/58] =?UTF-8?q?perf(cli):=20faster=20graperank=20crawl?= =?UTF-8?q?=20=E2=80=94=20dead-relay=20pruning,=20higher=20concurrency,=20?= =?UTF-8?q?sharded=20backbone=20sweep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The download phase was ~90% idle, blocked on the per-wave drain timeout waiting on dead/stalled outbox relays. Three changes: 1. Dead-relay pruning. Context.drain now reports relays that failed to CONNECT via a deadOut set; the crawl strikes them (MAX_DEAD_STRIKES=2) into a deadRelays set and excludes them from routeByOutbox and the sweep, so a wave stops re-paying the timeout on the same dead outboxes. 2. DRAIN_CONCURRENCY 8 -> 24, safe now that dead relays are pruned rather than piling up as stalled connections. 3. Sharded backbone sweep (Phase A each round): split the pending authors across the top-SHARD_RELAYS(10) live relays — one shard per relay, no relay gets the same list twice — drain all concurrently, rotate the still-missing onto different relays for up to SHARD_ROTATIONS(6) passes, then broadcast the remainder to all top relays only once it drops below SHARD_BROADCAST_THRESHOLD (2000). Phase B then outbox-routes only whoever the popular relays lacked. The old broadcast-everyone last-mile is removed (the sweep subsumes it). Each concurrent drain uses its own dead-set (no shared-HashSet race); harvest feeds from the drain's returned events instead of re-scanning contactsOf over the whole missing set. Adds download_ms so the crawl phase is timed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../com/vitorpamplona/amethyst/cli/Context.kt | 12 + .../amethyst/cli/commands/GrapeRankCommand.kt | 300 +++++++++++------- 2 files changed, 191 insertions(+), 121 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 5a6984e481..c02a982d03 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -400,11 +400,18 @@ class Context( * Subscribe to the given filters across the given relays, drain all events * until either every relay has sent EOSE or the timeout elapses, and * return them. Used for one-shot catch-up queries — not live subscriptions. + * + * When [deadOut] is provided, every relay that reported it could not be + * connected to (`onCannotConnect`) is added to it, so callers can prune + * proven-dead relays from future routing instead of paying the full + * [timeoutMs] on them again. Slow-but-connected relays are NOT reported — + * only hard connect failures, so a temporarily-busy relay isn't discarded. */ suspend fun drain( filters: Map>, timeoutMs: Long = 8_000, diagnoseSlow: Boolean = false, + deadOut: MutableSet? = null, ): List> { if (filters.isEmpty()) return emptyList() val eventChannel = Channel>(UNLIMITED) @@ -481,6 +488,11 @@ class Context( eventChannel.close() doneChannel.close() } + deadOut?.let { out -> + for ((relay, reason) in doneReasons) { + if (reason.startsWith("cannot")) out.add(relay) + } + } return collected } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index e8905456a4..122994e7b1 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -94,8 +94,26 @@ object GrapeRankCommand { // (empirically ~250 users/drain succeeds, ~17k fails); keep the fan-out small. private const val USER_BATCH = 256 - // Concurrent content drains. Bounded so total open connections stay sane. - private const val DRAIN_CONCURRENCY = 8 + // Concurrent content drains. Higher fan-out is safe now that proven-dead + // relays are pruned from routing (see deadRelays) — most of what a wave + // used to wait on was dead outboxes, so we no longer just pile up stalled + // connections. + private const val DRAIN_CONCURRENCY = 24 + + // Sharded backbone sweep: instead of asking every popular relay for the + // same full author list (N× redundant), split the still-missing authors + // into SHARD_RELAYS lists and send each to ONE of the top relays. Authors a + // relay doesn't have rotate onto a different relay next pass, up to + // SHARD_ROTATIONS times, so over a few passes each author is tried on + // several popular relays. Once the remaining set drops below + // SHARD_BROADCAST_THRESHOLD it's cheap to just ask them all at once. + private const val SHARD_RELAYS = 10 + private const val SHARD_ROTATIONS = 6 + private const val SHARD_BROADCAST_THRESHOLD = 2000 + + // A relay that fails to CONNECT this many times is treated as dead and + // dropped from routing, so we stop paying the drain timeout on it. + private const val MAX_DEAD_STRIKES = 2 // Broad, big general relays that carry kind:10002 for many users, added to the // discovery set to raise the odds of resolving a stranger's outbox. Every entry @@ -113,15 +131,6 @@ object GrapeRankCommand { // to keep as the known-good backbone for retrying users we couldn't reach. private const val BACKBONE_SIZE = 30 - // Last-mile sweep: after the outbox crawl gives up on the users whose own - // relays never answered, we take one more run at them against the WHOLE - // known-good relay pool — the busiest live relays we learned from everyone - // else's lists (they include the big aggregators). LAST_MILE_RELAYS caps that - // pool; LAST_MILE_PASSES bounds how many times we re-sweep as recovered lists - // reveal a few more reachable users. - private const val LAST_MILE_RELAYS = 80 - private const val LAST_MILE_PASSES = 2 - suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -209,6 +218,26 @@ object GrapeRankCommand { // live relays become the `backbone` we retry unreachable users against. val writeRelayFreq = HashMap() val liveRelays = hashSetOf() + // Relays that failed to connect MAX_DEAD_STRIKES times — dropped + // from all routing so a wave stops eating the timeout on them. + val deadRelays = hashSetOf() + val relayStrikes = HashMap() + + fun recordDead(failed: Set) { + for (r in failed) { + if (relayStrikes.merge(r, 1, Int::plus)!! >= MAX_DEAD_STRIKES) deadRelays.add(r) + } + } + + // The busiest live relays we've learned, excluding the dead ones. + fun topLiveRelays(cap: Int): List = + writeRelayFreq.entries + .asSequence() + .filter { it.key in liveRelays && it.key !in deadRelays } + .sortedByDescending { it.value } + .take(cap) + .map { it.key } + .toList() // Feed a user's contact list into the graph, harvest relay hints, stamp // the hop distance of newly-seen follows, and add them to the frontier. @@ -234,6 +263,91 @@ object GrapeRankCommand { return fresh } + // Feed into the graph the contact lists a drain just returned + // (deduped by author; the store's canonical latest wins), marking + // fed authors done. Only the authors we actually received are + // touched — no scan over the whole still-missing set. Returns the + // count newly fed. + suspend fun harvest(events: List>): Int { + var got = 0 + for ((_, ev) in events) { + if (ev !is ContactListEvent) continue + val pk = ev.pubKey + if (pk in done) continue + val contacts = ctx.contactsOf(pk) ?: continue + done += pk + ingest(pk, contacts) + got++ + } + return got + } + + // Sharded backbone sweep (see SHARD_RELAYS). Splits the missing + // authors across the top live relays — one shard per relay, so no + // relay gets the same list twice — drains all shards concurrently, + // then rotates whoever's still missing onto a different relay for up + // to SHARD_ROTATIONS passes. Once the remainder is small it's cheap + // to broadcast it to every top relay at once. Returns lists fed. + suspend fun shardedSweep(authors: Collection): Int { + val top = topLiveRelays(SHARD_RELAYS) + if (top.isEmpty()) return 0 + val n = top.size + var missing = authors.filter { it !in done && ctx.contactsOf(it) == null } + var got = 0 + var rotation = 0 + while (missing.size > SHARD_BROADCAST_THRESHOLD && rotation < SHARD_ROTATIONS) { + val shards = Array(n) { ArrayList() } + for (pk in missing) { + val base = ((pk.hashCode() % n) + n) % n + shards[(base + rotation) % n].add(pk) + } + val results = + coroutineScope { + top + .mapIndexedNotNull { i, relay -> + val shard = shards[i] + if (shard.isEmpty()) { + null + } else { + // Each drain gets its own dead-set — the concurrent + // drains must not share a mutable HashSet. + async { + val dead = hashSetOf() + val filters = + mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = graphKinds, authors = it) }) + ctx.drain(filters, timeoutMs, diagnose, dead) to dead + } + } + }.awaitAll() + } + for ((_, dead) in results) recordDead(dead) + relaysContacted += top + val flat = results.flatMap { it.first } + for ((relay, _) in flat) liveRelays.add(relay) + got += harvest(flat) + missing = missing.filter { it !in done } + rotation++ + } + // Once the remainder is small it's cheap to ask every top relay + // for it at once. If the rotations bailed with a still-large set, + // those authors just aren't on the popular relays — leave them to + // the caller's outbox pass rather than broadcast a huge list. + if (missing.isNotEmpty() && missing.size <= SHARD_BROADCAST_THRESHOLD) { + val live = top.filter { it !in deadRelays } + if (live.isNotEmpty()) { + val dead = hashSetOf() + val filters = + live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = graphKinds, authors = it) } } + val events = ctx.drain(filters, timeoutMs, diagnose, dead) + recordDead(dead) + relaysContacted += live + for ((relay, _) in events) liveRelays.add(relay) + got += harvest(events) + } + } + return got + } + // Crawl to full graph depth (no user cap; --max-hops bounds the follow // distance). Each run fetches every discovered user's LATEST // kind:3/10000/1984 once from their outbox (a freshness pass — grouped @@ -247,126 +361,67 @@ object GrapeRankCommand { if (pending.isEmpty()) break rounds++ - var gotList = 0 - var newUsers = 0 + val discoveredBefore = discovered.size + val fedBefore = contactListsFed - // The known-good backbone this round: the most-used write relays - // that have actually delivered events. Retried / outbox-less users - // are also queried here — these are relays we know work, learned - // from everyone else's lists. - val backbone = - writeRelayFreq.entries - .asSequence() - .filter { it.key in liveRelays } - .sortedByDescending { it.value } - .take(BACKBONE_SIZE) - .map { it.key } - .toSet() + // Phase A — bulk-fetch from the busiest relays via the sharded + // sweep. Most users' kind:3 lives on the big popular relays, so + // this clears the majority cheaply, without asking every relay for + // the same authors (early rounds no-op until a backbone is learned). + shardedSweep(pending) - // Resolve kind:10002 outboxes in bulk (indexers aggregate them), - // then fetch content in small batches drained a few at a time — one - // giant drain over thousands of outbox relays saturates connections - // and times out. Routing (store reads) is serial; only the drains - // run concurrently, which is safe: inserts serialize on the store - // write lock. - ensureRelayLists(ctx, pending.toSet(), backbone, timeoutMs, diagnose) - for (group in pending.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { - val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, graphKinds) } - val drained = - coroutineScope { - prepared - .map { (batch, filters) -> - async { - val events = ctx.drain(filters, timeoutMs, diagnose) - Triple(batch, filters.keys, events) - } - }.awaitAll() - } - for ((batch, relays, events) in drained) { - relaysContacted += relays - // Any relay that gave us an event is proven live + useful. - for ((relay, _) in events) liveRelays.add(relay) - for (pk in batch) { - val contacts = ctx.contactsOf(pk) - if (contacts != null) { - done += pk - gotList++ - newUsers += ingest(pk, contacts) - } else { - val tries = (attempts[pk] ?: 0) + 1 - attempts[pk] = tries - if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk + // Phase B — whoever the popular relays didn't have (niche + // outboxes): resolve their kind:10002, then fetch from their own + // write relays, drained a few at a time and skipping dead relays. + val stragglers = pending.filter { it !in done } + if (stragglers.isNotEmpty()) { + val backbone = topLiveRelays(BACKBONE_SIZE).toSet() + ensureRelayLists(ctx, stragglers.toSet(), backbone, timeoutMs, diagnose) + for (group in stragglers.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { + val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, graphKinds, deadRelays) } + val drained = + coroutineScope { + prepared + .map { (batch, filters) -> + async { + val dead = hashSetOf() + val events = ctx.drain(filters, timeoutMs, diagnose, dead) + recordDead(dead) + Triple(batch, filters.keys, events) + } + }.awaitAll() + } + for ((batch, relays, events) in drained) { + relaysContacted += relays + // Any relay that gave us an event is proven live + useful. + for ((relay, _) in events) liveRelays.add(relay) + for (pk in batch) { + if (pk in done) continue + val contacts = ctx.contactsOf(pk) + if (contacts != null) { + done += pk + ingest(pk, contacts) + } else { + val tries = (attempts[pk] ?: 0) + 1 + attempts[pk] = tries + if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk + } } } } } System.err.println( - "[graperank] round $rounds: fetched=${pending.size}, gotList=$gotList, " + - "newUsers=$newUsers, discovered=${discovered.size}, done=${done.size}", + "[graperank] round $rounds: pending=${pending.size}, " + + "gotList=${contactListsFed - fedBefore}, newUsers=${discovered.size - discoveredBefore}, " + + "discovered=${discovered.size}, done=${done.size}, dead=${deadRelays.size}", ) } - // Last-mile sweep. The outbox crawl leaves a tail of users whose own - // relays never answered (dead/misconfigured outboxes). Their contact - // lists very likely still exist — on the big aggregators and busy - // relays everyone else writes to. So instead of asking each straggler's - // broken outbox again, ask the WHOLE known-good pool at once: the - // busiest live relays learned from the crawl, plus the discovery set. - val goodPool = - ( - writeRelayFreq.entries - .asSequence() - .filter { it.key in liveRelays } - .sortedByDescending { it.value } - .take(LAST_MILE_RELAYS) - .map { it.key } - .toSet() + relayListDiscoveryRelays(ctx) - ).toList() - if (goodPool.isNotEmpty()) { - for (pass in 1..LAST_MILE_PASSES) { - val missing = discovered.filter { (hopOf[it] ?: 0) < maxHops && ctx.contactsOf(it) == null } - if (missing.isEmpty()) break - - var recovered = 0 - var newUsers = 0 - for (group in missing.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { - val drained = - coroutineScope { - group - .map { batch -> - val filters = - goodPool.associateWith { - batch.chunked(AUTHORS_PER_FILTER).map { chunk -> - Filter(kinds = graphKinds, authors = chunk) - } - } - async { - val events = ctx.drain(filters, timeoutMs, diagnose) - batch to events - } - }.awaitAll() - } - for ((batch, events) in drained) { - relaysContacted += goodPool - for ((relay, _) in events) liveRelays.add(relay) - for (pk in batch) { - val contacts = ctx.contactsOf(pk) - if (contacts != null) { - recovered++ - done += pk - newUsers += ingest(pk, contacts) - } - } - } - } - System.err.println( - "[graperank] last-mile pass $pass: swept=${missing.size}, recovered=$recovered, " + - "newUsers=$newUsers, discovered=${discovered.size}, still-missing=${discovered.count { (hopOf[it] ?: 0) < maxHops && ctx.contactsOf(it) == null }}", - ) - if (recovered == 0) break - } - } + // No separate last-mile pass: the per-round sharded sweep already + // broadcasts the small remaining set to every top relay once it drops + // below SHARD_BROADCAST_THRESHOLD, and the round loop only exits when + // every reachable user within the hop budget is done. relaysContactedCount = relaysContacted.size val perHop = @@ -808,6 +863,7 @@ object GrapeRankCommand { attempts: Map, writeRelayFreq: MutableMap, kinds: List, + deadRelays: Set, ): Map> { val fallback = contentFallbackRelays(ctx) val perRelay = HashMap>() @@ -821,7 +877,9 @@ object GrapeRankCommand { (attempts[pk] ?: 0) > 0 -> write + backbone else -> write } - for (relay in relays) perRelay.getOrPut(relay) { HashSet() }.add(pk) + // Skip relays already proven dead — routing to them only burns the + // drain timeout. + for (relay in relays) if (relay !in deadRelays) perRelay.getOrPut(relay) { HashSet() }.add(pk) } return perRelay.mapValues { (_, authors) -> From 3af168eff91bf130e9cbd87750260ff2a76c1eaf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 04:00:50 +0000 Subject: [PATCH 26/58] =?UTF-8?q?perf(cli):=20preserve=20crawl=20recall=20?= =?UTF-8?q?=E2=80=94=20wider=20broadcast=20pool,=20softer=20dead-strike?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the sharded-sweep restructure flagged two recall regressions vs the removed last-mile: - the sweep + broadcast only ever hit the top SHARD_RELAYS (10), while the old last-mile reached busy relays ranked 11-80 where a user's kind:3 is often mirrored. Broadcast the small remainder to BROADCAST_RELAYS (60) top live relays instead of just the rotation's 10, restoring that reach (indexers are intentionally excluded — they don't serve kind:3). - MAX_DEAD_STRIKES was 2 with no recovery, so two transient connect blips evicted a relay for the whole run. Raise to 3 for a safety margin; drain still only counts hard connect failures, not slow relays. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 122994e7b1..8b754f6fa1 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -111,9 +111,16 @@ object GrapeRankCommand { private const val SHARD_ROTATIONS = 6 private const val SHARD_BROADCAST_THRESHOLD = 2000 + // The small-remainder broadcast (once a sweep is under the threshold) goes to + // this many top live relays, not just the SHARD_RELAYS the rotation used — + // a user's kind:3 is often mirrored on a busy relay ranked below the top 10, + // which is where the old last-mile pass found its stragglers. + private const val BROADCAST_RELAYS = 60 + // A relay that fails to CONNECT this many times is treated as dead and - // dropped from routing, so we stop paying the drain timeout on it. - private const val MAX_DEAD_STRIKES = 2 + // dropped from routing, so we stop paying the drain timeout on it. Kept above + // 1 so a single transient connect blip doesn't evict a relay for the run. + private const val MAX_DEAD_STRIKES = 3 // Broad, big general relays that carry kind:10002 for many users, added to the // discovery set to raise the odds of resolving a stranger's outbox. Every entry @@ -333,7 +340,10 @@ object GrapeRankCommand { // those authors just aren't on the popular relays — leave them to // the caller's outbox pass rather than broadcast a huge list. if (missing.isNotEmpty() && missing.size <= SHARD_BROADCAST_THRESHOLD) { - val live = top.filter { it !in deadRelays } + // Broadcast the small remainder to a wider set of busy relays + // than the rotation used — recovers users whose list is only + // on a relay ranked below the top SHARD_RELAYS. + val live = topLiveRelays(BROADCAST_RELAYS) if (live.isNotEmpty()) { val dead = hashSetOf() val filters = From 82f0c3cc6c54adf6dfab733f94dc2655d7829bf1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 04:40:35 +0000 Subject: [PATCH 27/58] feat(cli): NIP-42 auth + relay-feedback diagnostics in the crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two blind spots in the crawl's relay I/O: - No NIP-42 auth. Auth-gated relays sent AUTH, the client never answered, and their sub CLOSed 'auth-required' — so those outboxes served us nothing. Wire a RelayAuthenticator into Context that signs the AUTH challenge with the account key (local signer only; a remote bunker is skipped to avoid a per-relay round-trip storm mid-crawl). Signing with any key still unlocks relays that just want some auth. - No visibility into REQ failures. Add RelayDiagnostics, a connection listener that tallies NOTICE frames, CLOSED reasons by NIP-01 prefix (auth-required / rate-limited / restricted / …), and AUTH challenges. Surfaced in the crawl's stderr summary and as relay_feedback in the JSON, so a failed fetch can be explained instead of guessed at. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../com/vitorpamplona/amethyst/cli/Context.kt | 29 ++++++ .../amethyst/cli/RelayDiagnostics.kt | 96 +++++++++++++++++++ .../amethyst/cli/commands/GrapeRankCommand.kt | 6 +- 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/RelayDiagnostics.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index c02a982d03..6e2bbcf5ea 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -43,6 +43,7 @@ import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPagesFromPool import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.publishAndConfirmDetailed +import com.vitorpamplona.quartz.nip01Core.relay.client.auth.RelayAuthenticator import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener import com.vitorpamplona.quartz.nip01Core.relay.client.single.newSubId import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.CachingEventDecoder @@ -145,6 +146,34 @@ class Context( ) } ?: NostrSignerInternal(identity.keyPair()) + /** + * Client-wide tally of relay feedback — NOTICE frames, CLOSED reasons + * (auth-required / rate-limited / restricted / …), and NIP-42 AUTH + * challenges — so a failed REQ can be explained instead of guessed at. + * Registered on [client] for the life of this run. + */ + val relayDiagnostics: RelayDiagnostics = RelayDiagnostics().also { client.addConnectionListener(it) } + + /** + * NIP-42 responder: answers a relay's AUTH challenge by signing with the + * account key, so auth-gated relays serve our reads instead of CLOSing the + * subscription. Constructing it registers its own listener on [client]. + * Only a local key auto-signs — a remote bunker signer is skipped, since a + * per-relay remote round-trip during a crawl would stall it (and signing an + * auth event with any key still unlocks relays that just want *some* auth). + */ + private val relayAuth: RelayAuthenticator = + RelayAuthenticator( + client = client, + signWithAllLoggedInUsers = { _, template -> + if (signer is NostrSignerInternal) { + runCatching { listOf(signer.sign(template)) }.getOrElse { emptyList() } + } else { + emptyList() + } + }, + ) + /** * NIP-05 resolver for turning `alice@damus.io`-style identifiers into pubkeys. * Uses the same OkHttp instance as the WebSocket client so we share connection diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/RelayDiagnostics.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/RelayDiagnostics.kt new file mode 100644 index 0000000000..be275a71dd --- /dev/null +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/RelayDiagnostics.kt @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.cli + +import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener +import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.AuthMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.ClosedMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NoticeMessage +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong + +/** + * Client-wide tally of the relay feedback the crawl would otherwise never see: + * `NOTICE` frames, `CLOSED` reasons (`auth-required` / `rate-limited` / + * `restricted` / …), and NIP-42 `AUTH` challenges. Registered as a + * [RelayConnectionListener] on the shared client, so every incoming message + * during a run is counted and a REQ failure can be explained instead of + * guessed at. + * + * Callbacks fire on the per-relay socket threads, so all state is concurrent. + */ +class RelayDiagnostics : RelayConnectionListener { + private val closedByReason = ConcurrentHashMap() + private val noticeSamples = ConcurrentHashMap() + private val authChallenges = AtomicLong() + + override fun onIncomingMessage( + relay: IRelayClient, + msgStr: String, + msg: Message, + ) { + when (msg) { + // CLOSED reasons follow the NIP-01 machine-readable "word: text" + // convention, so the prefix categorises the failure. + is ClosedMessage -> bump(closedByReason, prefix(msg.message)) + // NOTICE is free-form; keep the (truncated) text so recurring + // relay complaints ("too many concurrent REQs", …) are visible. + is NoticeMessage -> if (noticeSamples.size < MAX_DISTINCT_NOTICES) bump(noticeSamples, msg.message.trim().take(80)) + is AuthMessage -> authChallenges.incrementAndGet() + else -> Unit + } + } + + private fun bump( + map: ConcurrentHashMap, + key: String, + ) { + map.getOrPut(key) { AtomicLong() }.incrementAndGet() + } + + /** The NIP-01 machine-readable prefix (`word` before `:`), or `other`. */ + private fun prefix(message: String): String { + val head = message.substringBefore(':').trim().lowercase() + return head.ifEmpty { "other" }.take(24) + } + + fun hadFeedback(): Boolean = authChallenges.get() > 0 || closedByReason.isNotEmpty() || noticeSamples.isNotEmpty() + + /** JSON-friendly summary for the command output. */ + fun snapshot(): Map = + mapOf( + "auth_challenges" to authChallenges.get(), + "closed_by_reason" to closedByReason.entries.associate { it.key to it.value.get() }.toSortedMap(), + "notices" to noticeSamples.values.sumOf { it.get() }, + "notice_top" to + noticeSamples.entries + .sortedByDescending { it.value.get() } + .take(TOP_NOTICES) + .map { "${it.key} (${it.value.get()})" }, + ) + + companion object { + private const val MAX_DISTINCT_NOTICES = 500 + private const val TOP_NOTICES = 8 + } +} diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 8b754f6fa1..b76a3dd6a8 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -442,9 +442,12 @@ object GrapeRankCommand { downloadMs = (System.nanoTime() - crawlStart) / 1_000_000 System.err.println( "[graperank] crawl complete: ${discovered.size} discovered, $contactListsFed contact lists fed, " + - "$relaysContactedCount relays contacted, $rounds rounds in $downloadMs ms; " + + "$relaysContactedCount relays contacted, ${deadRelays.size} dead, $rounds rounds in $downloadMs ms; " + "by hop: " + perHop.entries.joinToString(" ") { "${it.key}=${it.value}" }, ) + if (ctx.relayDiagnostics.hadFeedback()) { + System.err.println("[graperank] relay feedback: ${ctx.relayDiagnostics.snapshot()}") + } } else { // Offline: stream contact lists from the local store into the graph. val loadStart = System.nanoTime() @@ -501,6 +504,7 @@ object GrapeRankCommand { "observer" to observer, "crawl_rounds" to rounds, "relays_contacted" to relaysContactedCount, + "relay_feedback" to if (ctx.relayDiagnostics.hadFeedback()) ctx.relayDiagnostics.snapshot() else null, "max_hop_reached" to (hopOf.values.maxOrNull() ?: 0), "users_by_hop" to hopOf.values From 6060c50f79f7074a59efc399ee2702684b245f6f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 04:53:38 +0000 Subject: [PATCH 28/58] perf(cli): keep a warm connection pool to the top relays during the crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The client's relay pool reconciles open sockets to the relays that active subscriptions currently need, so between-round gaps (routing + contactsOf scans > ~300ms) and niche-relay churn dropped connections we reuse every round, then reconnected them — a TCP+TLS+WS handshake each time. Hold a persistent do-nothing subscription (WARM_SUB_ID) open to the busiest WARM_POOL_SIZE(20) live relays, refreshed to the current top set at each round start (same subId → just updates the desired-relay set) and closed when the crawl finishes. Its filter matches an impossible event id, so the relay EOSEs immediately and streams nothing — it only keeps the socket warm. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index b76a3dd6a8..39e2813f76 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -138,6 +138,16 @@ object GrapeRankCommand { // to keep as the known-good backbone for retrying users we couldn't reach. private const val BACKBONE_SIZE = 30 + // Warm pool: hold a persistent, do-nothing subscription open to the busiest + // WARM_POOL_SIZE relays for the whole crawl, so the connections we reuse + // every round survive the between-round routing gaps (and niche-relay churn) + // instead of being dropped ~300ms after a wave ends and reconnected next + // round. The filter matches an impossible event id, so the relay EOSEs + // immediately and streams nothing — it only keeps the socket warm. + private const val WARM_POOL_SIZE = 20 + private const val WARM_SUB_ID = "graperank-warm" + private val WARM_FILTERS = listOf(Filter(ids = listOf("0".repeat(64)))) + suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -371,6 +381,13 @@ object GrapeRankCommand { if (pending.isEmpty()) break rounds++ + // Refresh the warm pool to this round's busiest relays and keep + // that subscription open — reusing the same subId just updates the + // desired-relay set, so these sockets stay up across the round. + topLiveRelays(WARM_POOL_SIZE).takeIf { it.isNotEmpty() }?.let { warm -> + ctx.client.subscribe(WARM_SUB_ID, warm.associateWith { WARM_FILTERS }, null) + } + val discoveredBefore = discovered.size val fedBefore = contactListsFed @@ -428,6 +445,9 @@ object GrapeRankCommand { ) } + // Crawl done — drop the warm pool. + ctx.client.unsubscribe(WARM_SUB_ID) + // No separate last-mile pass: the per-round sharded sweep already // broadcasts the small remaining set to every top relay once it drops // below SHARD_BROADCAST_THRESHOLD, and the round loop only exits when From fb0011129d90c34128ab174e09b657d00d029777 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 11:11:32 +0000 Subject: [PATCH 29/58] perf(cli): cap crawl at ~20 subscriptions per relay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each concurrent content drain opens exactly one subscription per relay it touches, so DRAIN_CONCURRENCY is effectively the per-relay concurrent-sub cap. RelayDiagnostics showed the previous value (24) blew past typical relay limits — rate-limited=1433, "too many concurrent REQs"=1286, "too many subscriptions"=710 — causing dropped fetches and retry churn. Lower it to 18 so the peak (18 drain subs + 1 persistent warm-pool sub) stays ~19, just under the common ~20 cap. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 39e2813f76..deb06d817f 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -94,11 +94,16 @@ object GrapeRankCommand { // (empirically ~250 users/drain succeeds, ~17k fails); keep the fan-out small. private const val USER_BATCH = 256 - // Concurrent content drains. Higher fan-out is safe now that proven-dead - // relays are pruned from routing (see deadRelays) — most of what a wave - // used to wait on was dead outboxes, so we no longer just pile up stalled - // connections. - private const val DRAIN_CONCURRENCY = 24 + // Concurrent content drains. Each drain opens exactly ONE subscription per + // relay it touches (one REQ per relay under the drain's subId), so this IS + // the per-relay concurrency cap: a popular relay shared by many pending users + // receives at most this many concurrent subs from us — while the fan-out + // across *different* relays stays fully parallel (each drain hits ~100 distinct + // outboxes). RelayDiagnostics showed 24 blew past typical relay limits + // (rate-limited=1433, "too many concurrent REQs"=1286, "too many + // subscriptions"=710). Target ~20 subs/relay; 18 leaves room for the 1 + // persistent warm-pool sub so the peak stays ~19, just under the common cap. + private const val DRAIN_CONCURRENCY = 18 // Sharded backbone sweep: instead of asking every popular relay for the // same full author list (N× redundant), split the still-missing authors From de8648f3f10e1bb2c96ad12009be119a97548be9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 11:44:54 +0000 Subject: [PATCH 30/58] perf(cli): adaptive per-relay subscription cap for the crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the blunt global concurrency number with per-relay back-pressure. Every relay starts generous (100 concurrent subscriptions) and is demoted down a ladder (100 -> 20 -> 10) only when it complains about concurrency — a CLOSED rate-limited, or a NOTICE like "too many concurrent REQs" / "too many subscriptions" / "burst exhausted". Well-behaved relays keep the full cap; only the busy hubs that push back get throttled, and only as far as they keep pushing. AdaptiveRelayLimiter registers as a RelayConnectionListener so demotions are driven straight off the same NOTICE/CLOSED frames RelayDiagnostics already observes, keyed by relay.url. Context.drain gains a gatePerRelay path that opens one subscription per relay, each held behind that relay's gate, so our concurrent subs on it never exceed its current cap. The gate is a fair FIFO bounded semaphore whose limit can only be lowered; shrinking below the in-use count admits no new subs until enough finish, so concurrency converges down to the new cap. Because a hot relay can no longer be flooded, the global content-drain fan-out is raised (18 -> 48) to crawl the many well-behaved relays faster. The crawl emits a relay_throttling summary (which relays were capped, and to what) alongside relay_feedback. --- .../amethyst/cli/AdaptiveRelayLimiter.kt | 198 ++++++++++++++++++ .../com/vitorpamplona/amethyst/cli/Context.kt | 115 ++++++++++ .../amethyst/cli/commands/GrapeRankCommand.kt | 33 +-- 3 files changed, 332 insertions(+), 14 deletions(-) create mode 100644 cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt new file mode 100644 index 0000000000..d57e19c034 --- /dev/null +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.cli + +import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener +import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.ClosedMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NoticeMessage +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +/** + * Adaptive per-relay concurrent-subscription cap. + * + * Every relay starts with a generous cap ([startCap], default 100) — we assume a + * relay can take as many concurrent REQs as we throw at it until it tells us + * otherwise. When a relay complains about concurrency (a `CLOSED rate-limited`, + * or a `NOTICE` like "too many concurrent REQs" / "too many subscriptions" / + * "burst exhausted"), we demote *that relay only* down the [ladder] + * (100 → 20 → 10). A well-behaved relay keeps the full cap; only the ones that + * push back get throttled, and only as far as they keep pushing. + * + * This replaces a single blunt global concurrency number with per-relay + * back-pressure: the crawl can fan out widely across the many relays that don't + * mind, while automatically easing off the few busy hubs that do — exactly the + * signals [RelayDiagnostics] already observes, here turned into an actuator. + * + * Registered as a [RelayConnectionListener] on the shared client, so demotions + * are driven straight off the incoming NOTICE/CLOSED frames (which fire on the + * per-relay socket threads — all state here is concurrent). Drains gate through + * [withPermit]; [Context.drain]'s `gatePerRelay` path holds a relay's permit for + * the lifetime of that relay's subscription, so at most `cap` of our + * subscriptions are ever open on it at once. + */ +class AdaptiveRelayLimiter( + private val startCap: Int = 100, + private val ladder: List = listOf(20, 10), +) : RelayConnectionListener { + private val gates = ConcurrentHashMap() + + // How many concurrency complaints we've acted on per relay (== index+1 into + // the ladder). Capped at ladder.size: past the floor we stop demoting. + private val demotions = ConcurrentHashMap() + + private fun gate(relay: NormalizedRelayUrl): Gate = gates.getOrPut(relay) { Gate(startCap) } + + /** Run [block] holding one of [relay]'s permits, respecting its current cap. */ + suspend fun withPermit( + relay: NormalizedRelayUrl, + block: suspend () -> T, + ): T { + val g = gate(relay) + g.acquire() + try { + return block() + } finally { + g.release() + } + } + + override fun onIncomingMessage( + relay: IRelayClient, + msgStr: String, + msg: Message, + ) { + when (msg) { + is ClosedMessage -> if (isConcurrencyComplaint(msg.message)) demote(relay.url) + is NoticeMessage -> if (isConcurrencyComplaint(msg.message)) demote(relay.url) + else -> Unit + } + } + + /** Step [relay] one rung down the cap ladder, unless it's already at the floor. */ + private fun demote(relay: NormalizedRelayUrl) { + // Fast path: relays flood identical NOTICEs, so bail once at the floor + // instead of counting them all (the demotion is monotonic and idempotent). + if ((demotions[relay] ?: 0) >= ladder.size) return + val step = demotions.merge(relay, 1, Int::plus)!! + val cap = ladder[(step - 1).coerceIn(0, ladder.size - 1)] + gate(relay).lower(cap) + if (step <= ladder.size) { + System.err.println("[limiter] ${relay.url} capped at $cap concurrent subs (complaint #$step)") + } + } + + private fun isConcurrencyComplaint(text: String): Boolean { + val t = text.lowercase() + return CONCURRENCY_MARKERS.any { it in t } + } + + /** JSON-friendly view of which relays we throttled and how far. */ + fun snapshot(): Map { + val cappedAt = sortedMapOf() + for ((_, step) in demotions) { + val cap = ladder[(step - 1).coerceIn(0, ladder.size - 1)] + cappedAt.merge(cap, 1, Int::plus) + } + return mapOf( + "start_cap" to startCap, + "ladder" to ladder, + "throttled_relays" to demotions.size, + "capped_at" to cappedAt, + ) + } + + fun hadThrottling(): Boolean = demotions.isNotEmpty() + + /** + * A bounded-concurrency gate whose limit can only ever be *lowered* (relays + * never earn their cap back within a run). Fair FIFO hand-off: a released + * permit goes to the longest-waiting acquirer. Lowering the limit below the + * in-use count doesn't cancel live holders — it just refuses to admit new + * ones until enough release that `inUse < limit` again, so the concurrency + * converges down to the new cap as the excess subscriptions finish. + */ + private class Gate( + initialLimit: Int, + ) { + private val limit = AtomicInteger(initialLimit) + private val mutex = Mutex() + private var inUse = 0 + private val waiters = ArrayDeque>() + + suspend fun acquire() { + val wait = + mutex.withLock { + if (inUse < limit.get()) { + inUse++ + null + } else { + CompletableDeferred().also { waiters.addLast(it) } + } + } + wait?.await() + } + + suspend fun release() { + mutex.withLock { + inUse-- + while (inUse < limit.get() && waiters.isNotEmpty()) { + waiters.removeFirst().complete(Unit) + inUse++ + } + } + } + + /** Monotonically shrink the cap. Safe to call from any thread. */ + fun lower(newLimit: Int) { + limit.updateAndGet { if (newLimit < it) newLimit else it } + } + } + + companion object { + // Substrings (matched case-insensitively) that mean "you're opening too + // many concurrent subscriptions / sending too fast" — the failure modes a + // lower per-relay cap actually fixes. Auth/blocked/restricted/unsupported + // are deliberately excluded: throttling wouldn't help those. + private val CONCURRENCY_MARKERS = + listOf( + "too many concurrent", + "concurrent req", + "too many subscription", + "number of subscriptions", + "subscription limit", + "too many req", + "rate-limit", + "rate limit", + "ratelimit", + "burst exhausted", + "throttl", + "too many messages", + "slow down", + ) + } +} diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 6e2bbcf5ea..b44075ad6a 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -74,10 +74,12 @@ import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.selects.select import kotlinx.coroutines.withTimeoutOrNull import okhttp3.OkHttpClient +import java.util.concurrent.ConcurrentHashMap /** * Per-invocation wiring. Each CLI run constructs a Context, does its work, @@ -154,6 +156,16 @@ class Context( */ val relayDiagnostics: RelayDiagnostics = RelayDiagnostics().also { client.addConnectionListener(it) } + /** + * Adaptive per-relay concurrent-subscription cap. Starts every relay + * generous (100) and demotes only the ones that complain about concurrency + * (100 → 20 → 10), driven straight off the NOTICE/CLOSED frames it observes + * as a connection listener. [drain]'s `gatePerRelay` path holds a relay's + * permit for the life of that relay's subscription, so we never exceed the + * cap the relay itself asked for. Idle for commands that don't opt in. + */ + val relayLimiter: AdaptiveRelayLimiter = AdaptiveRelayLimiter().also { client.addConnectionListener(it) } + /** * NIP-42 responder: answers a relay's AUTH challenge by signing with the * account key, so auth-gated relays serve our reads instead of CLOSing the @@ -441,8 +453,10 @@ class Context( timeoutMs: Long = 8_000, diagnoseSlow: Boolean = false, deadOut: MutableSet? = null, + gatePerRelay: Boolean = false, ): List> { if (filters.isEmpty()) return emptyList() + if (gatePerRelay) return drainGated(filters, timeoutMs, diagnoseSlow, deadOut) val eventChannel = Channel>(UNLIMITED) // Carries the terminal reason per relay so a timeout can distinguish a slow // relay (never terminal) from a connect failure / CLOSED. @@ -525,6 +539,107 @@ class Context( return collected } + /** + * Per-relay-gated variant of [drain] used by the crawl. Instead of one + * subscription spanning every relay, each relay gets its own subscription + * held behind [relayLimiter], so we never exceed the relay's adaptive + * concurrent-subscription cap. A relay whose cap is full simply waits for one + * of our other subscriptions on it to finish before its REQ goes out; relays + * we haven't upset run at the full starting cap and never wait. + * + * Semantics match [drain] otherwise: verify+store on a single consumer + * (so store writes stay serialized), return events tagged by relay, and + * report hard connect failures into [deadOut]. + */ + private suspend fun drainGated( + filters: Map>, + timeoutMs: Long, + diagnoseSlow: Boolean, + deadOut: MutableSet?, + ): List> { + val eventChannel = Channel>(UNLIMITED) + // One relay per subId, so the relay alone identifies which subscription a + // callback is for. First terminal frame wins; a timeout leaves it unset. + val relayDone = ConcurrentHashMap>() + for (r in filters.keys) relayDone[r] = CompletableDeferred() + val doneReasons = ConcurrentHashMap() + val listener = + object : SubscriptionListener { + override fun onEvent( + event: Event, + isLive: Boolean, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + eventChannel.trySend(relay to event) + } + + override fun onEose( + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + relayDone[relay]?.complete("eose") + } + + override fun onClosed( + message: String, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + relayDone[relay]?.complete("closed:$message") + } + + override fun onCannotConnect( + relay: NormalizedRelayUrl, + message: String, + forFilters: List?, + ) { + relayDone[relay]?.complete("cannot:$message") + } + } + val collected = mutableListOf>() + coroutineScope { + // Single consumer: verify+store serially, exactly like drain(). + val consumer = + launch { + for ((relay, event) in eventChannel) { + if (verifyAndStore(event)) collected.add(relay to event) + } + } + // One gated subscription per relay. The permit is held for the whole + // life of the relay's REQ, so concurrent subs on it never exceed its cap. + filters + .map { (relay, relayFilters) -> + launch { + relayLimiter.withPermit(relay) { + val subId = newSubId() + client.subscribe(subId, mapOf(relay to relayFilters), listener) + try { + val reason = withTimeoutOrNull(timeoutMs) { relayDone[relay]!!.await() } + doneReasons[relay] = reason ?: "timeout" + } finally { + client.unsubscribe(subId) + } + } + } + }.joinAll() + // All subscriptions are torn down; no more events can arrive. Close the + // channel so the consumer drains what's buffered and completes. + eventChannel.close() + consumer.join() + } + if (diagnoseSlow) { + val stalled = filters.keys.filter { (doneReasons[it] ?: "timeout") == "timeout" }.toSet() + if (stalled.isNotEmpty()) logSlowDrain(timeoutMs, stalled, doneReasons, collected) + } + deadOut?.let { out -> + for ((relay, reason) in doneReasons) { + if (reason.startsWith("cannot")) out.add(relay) + } + } + return collected + } + /** * On a [drain] timeout, report which relays stalled and why — a relay that * never sent EOSE (slow, possibly still streaming) vs one that couldn't be diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index deb06d817f..dc344d4429 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -94,16 +94,17 @@ object GrapeRankCommand { // (empirically ~250 users/drain succeeds, ~17k fails); keep the fan-out small. private const val USER_BATCH = 256 - // Concurrent content drains. Each drain opens exactly ONE subscription per - // relay it touches (one REQ per relay under the drain's subId), so this IS - // the per-relay concurrency cap: a popular relay shared by many pending users - // receives at most this many concurrent subs from us — while the fan-out - // across *different* relays stays fully parallel (each drain hits ~100 distinct - // outboxes). RelayDiagnostics showed 24 blew past typical relay limits - // (rate-limited=1433, "too many concurrent REQs"=1286, "too many - // subscriptions"=710). Target ~20 subs/relay; 18 leaves room for the 1 - // persistent warm-pool sub so the peak stays ~19, just under the common cap. - private const val DRAIN_CONCURRENCY = 18 + // Global content-drain fan-out — how many outbox batches we drain at once. + // This is now purely a GLOBAL bound (memory / open sockets); the per-relay + // concurrency limit is enforced separately and adaptively by + // [AdaptiveRelayLimiter] (drains run with gatePerRelay=true), which starts + // every relay at 100 concurrent subs and demotes only the ones that complain + // (100 → 20 → 10). Because a hot relay can no longer be flooded regardless of + // this number, we can fan out widely across the many well-behaved relays for + // throughput. RelayDiagnostics previously showed a blunt 24 caused + // rate-limited=1433 / "too many concurrent REQs"=1286; the adaptive cap + // targets exactly those relays instead of throttling everyone uniformly. + private const val DRAIN_CONCURRENCY = 48 // Sharded backbone sweep: instead of asking every popular relay for the // same full author list (N× redundant), split the still-missing authors @@ -337,7 +338,7 @@ object GrapeRankCommand { val dead = hashSetOf() val filters = mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = graphKinds, authors = it) }) - ctx.drain(filters, timeoutMs, diagnose, dead) to dead + ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) to dead } } }.awaitAll() @@ -363,7 +364,7 @@ object GrapeRankCommand { val dead = hashSetOf() val filters = live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = graphKinds, authors = it) } } - val events = ctx.drain(filters, timeoutMs, diagnose, dead) + val events = ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) recordDead(dead) relaysContacted += live for ((relay, _) in events) liveRelays.add(relay) @@ -417,7 +418,7 @@ object GrapeRankCommand { .map { (batch, filters) -> async { val dead = hashSetOf() - val events = ctx.drain(filters, timeoutMs, diagnose, dead) + val events = ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) recordDead(dead) Triple(batch, filters.keys, events) } @@ -473,6 +474,9 @@ object GrapeRankCommand { if (ctx.relayDiagnostics.hadFeedback()) { System.err.println("[graperank] relay feedback: ${ctx.relayDiagnostics.snapshot()}") } + if (ctx.relayLimiter.hadThrottling()) { + System.err.println("[graperank] relay throttling: ${ctx.relayLimiter.snapshot()}") + } } else { // Offline: stream contact lists from the local store into the graph. val loadStart = System.nanoTime() @@ -530,6 +534,7 @@ object GrapeRankCommand { "crawl_rounds" to rounds, "relays_contacted" to relaysContactedCount, "relay_feedback" to if (ctx.relayDiagnostics.hadFeedback()) ctx.relayDiagnostics.snapshot() else null, + "relay_throttling" to if (ctx.relayLimiter.hadThrottling()) ctx.relayLimiter.snapshot() else null, "max_hop_reached" to (hopOf.values.maxOrNull() ?: 0), "users_by_hop" to hopOf.values @@ -870,7 +875,7 @@ object GrapeRankCommand { Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) } } - ctx.drain(filters, timeoutMs, diagnose) + ctx.drain(filters, timeoutMs, diagnose, gatePerRelay = true) } val discovery = relayListDiscoveryRelays(ctx) From 143a63c520dbef15e0128e3c4c1f7d81437ce862 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 12:03:59 +0000 Subject: [PATCH 31/58] refactor(quartz): move GrapeRank web-of-trust algorithm into quartz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GrapeRank engine, TrustGraph (compact int-CSR) and TrustGraphBuilder are pure Nostr-social-graph computation over HexKeys — no UI, no Compose, and no commons-only dependency. They're a utility for implementing the NIP-85 rank assertions quartz already models, so they belong in quartz rather than commons. Move commons/wot -> quartz experimental/graperank (package com.vitorpamplona.quartz.experimental.graperank), including both commonTest suites, and repoint the CLI import. TrustGraphBuilder was already protocol-agnostic (takes HexKey lists; the caller does the event->edge extraction), so nothing had to change but the package. Makes the algorithm reusable by the Android app for spam/trust filtering without pulling in commons. --- .../vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt | 6 +++--- .../quartz/experimental/graperank}/GrapeRank.kt | 2 +- .../quartz/experimental/graperank}/TrustGraph.kt | 2 +- .../quartz/experimental/graperank}/TrustGraphBuilder.kt | 2 +- .../quartz/experimental/graperank}/GrapeRankTest.kt | 2 +- .../quartz/experimental/graperank}/TrustGraphBuilderTest.kt | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) rename {commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot => quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank}/GrapeRank.kt (99%) rename {commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot => quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank}/TrustGraph.kt (98%) rename {commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot => quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank}/TrustGraphBuilder.kt (98%) rename {commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot => quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/graperank}/GrapeRankTest.kt (99%) rename {commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot => quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/graperank}/TrustGraphBuilderTest.kt (98%) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index dc344d4429..0f8cb76c73 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -26,9 +26,9 @@ import com.vitorpamplona.amethyst.cli.DataDir import com.vitorpamplona.amethyst.cli.Output import com.vitorpamplona.amethyst.commons.defaults.Constants import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList -import com.vitorpamplona.amethyst.commons.wot.GrapeRank -import com.vitorpamplona.amethyst.commons.wot.GrapeRankParams -import com.vitorpamplona.amethyst.commons.wot.TrustGraphBuilder +import com.vitorpamplona.quartz.experimental.graperank.GrapeRank +import com.vitorpamplona.quartz.experimental.graperank.GrapeRankParams +import com.vitorpamplona.quartz.experimental.graperank.TrustGraphBuilder import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRank.kt similarity index 99% rename from commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRank.kt index c5674317c8..3744d52b41 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRank.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRank.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.commons.wot +package com.vitorpamplona.quartz.experimental.graperank import androidx.compose.runtime.Immutable import com.vitorpamplona.quartz.nip01Core.core.HexKey diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraph.kt similarity index 98% rename from commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraph.kt index b84063b87c..ad78a6e56a 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraph.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraph.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.commons.wot +package com.vitorpamplona.quartz.experimental.graperank import com.vitorpamplona.quartz.nip01Core.core.HexKey diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraphBuilder.kt similarity index 98% rename from commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraphBuilder.kt index 55a223949b..7b3f12fbe4 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraphBuilder.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.commons.wot +package com.vitorpamplona.quartz.experimental.graperank import com.vitorpamplona.quartz.nip01Core.core.HexKey diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankTest.kt similarity index 99% rename from commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt rename to quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankTest.kt index d9a3282a4f..4219095809 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/GrapeRankTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankTest.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.commons.wot +package com.vitorpamplona.quartz.experimental.graperank import com.vitorpamplona.quartz.nip01Core.core.HexKey import kotlin.math.abs diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraphBuilderTest.kt similarity index 98% rename from commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt rename to quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraphBuilderTest.kt index 01b9fb6cb3..f27e064273 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/wot/TrustGraphBuilderTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/graperank/TrustGraphBuilderTest.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.commons.wot +package com.vitorpamplona.quartz.experimental.graperank import com.vitorpamplona.quartz.nip01Core.core.HexKey import kotlin.test.Test From 0c0a8caaff1a81b65fb61f22af7f862a3cc219f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 12:05:46 +0000 Subject: [PATCH 32/58] perf(cli): dial crawl fan-out back to 24 under the adaptive cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured 48 against the per-relay adaptive limiter (hop-4 A/B): it demoted the right hubs and cut rate-limited CLOSEDs further (1433 -> 501), but the higher fan-out re-floods busy relays faster than demotion catches up — a new dominant complaint ("max concurrent subscription count reached") appeared and download_ms regressed ~11% vs the 24 baseline. Keep the adaptive per-relay cap (it targets the misbehaving relays precisely) but return the global fan-out to 24, where the 20/10 ladder still bites below the global bound and wall-time stays at its best-observed value. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 0f8cb76c73..3eb191710e 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -95,16 +95,18 @@ object GrapeRankCommand { private const val USER_BATCH = 256 // Global content-drain fan-out — how many outbox batches we drain at once. - // This is now purely a GLOBAL bound (memory / open sockets); the per-relay - // concurrency limit is enforced separately and adaptively by - // [AdaptiveRelayLimiter] (drains run with gatePerRelay=true), which starts - // every relay at 100 concurrent subs and demotes only the ones that complain - // (100 → 20 → 10). Because a hot relay can no longer be flooded regardless of - // this number, we can fan out widely across the many well-behaved relays for - // throughput. RelayDiagnostics previously showed a blunt 24 caused - // rate-limited=1433 / "too many concurrent REQs"=1286; the adaptive cap - // targets exactly those relays instead of throttling everyone uniformly. - private const val DRAIN_CONCURRENCY = 48 + // This is a GLOBAL bound (memory / open sockets); the per-relay concurrency + // limit is enforced separately and adaptively by [AdaptiveRelayLimiter] + // (drains run with gatePerRelay=true), which starts every relay at 100 + // concurrent subs and demotes only the ones that complain (100 → 20 → 10). + // The two compose: at fan-out 24 a well-behaved relay runs at up to 24 + // concurrent subs, while a relay that pushes back is cut to 20 then 10 — + // below the global bound, so the ladder actually bites. A higher global + // fan-out (measured at 48) *re-floods* the busy hubs faster than demotion + // catches up ("max concurrent subscription count reached" spikes) and + // regressed wall-time, so keep the global bound moderate and let the + // per-relay cap do the targeting. + private const val DRAIN_CONCURRENCY = 24 // Sharded backbone sweep: instead of asking every popular relay for the // same full author list (N× redundant), split the still-missing authors From df0cf4987610a3544dfa2b0f78091dc5d4ec37cd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 12:13:14 +0000 Subject: [PATCH 33/58] perf(cli): widen OkHttp dispatcher + tighten connect timeout for the crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crawl's client ran on OkHttp defaults: Dispatcher.maxRequests=64 and a 10s connectTimeout. Every relay WS-upgrade handshake is an async call through that shared dispatcher, so 64 caps the connection-ramp width — and a dead relay squats on a slot for the full connectTimeout, starving live relays queued behind it (observed: only ~150 sockets open at once during an active wave touching hundreds of relays). Raise maxRequests to 256 / maxRequestsPerHost to 16 and drop connectTimeout to 5s so unreachable relays release their slot fast. This is orthogonal to REQ concurrency (bounded per-relay by AdaptiveRelayLimiter on already-open sockets), so it can't trip a relay's REQ rate-limit — it only speeds connection setup. The dispatcher's executor pool grows threads on demand, and FD headroom is ample (4096 limit vs ~150 in use), so the wider cap just lets more short-lived handshakes run at once. --- .../com/vitorpamplona/amethyst/cli/Context.kt | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index b44075ad6a..8ce736de36 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -78,8 +78,10 @@ import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.selects.select import kotlinx.coroutines.withTimeoutOrNull +import okhttp3.Dispatcher import okhttp3.OkHttpClient import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit /** * Per-invocation wiring. Each CLI run constructs a Context, does its work, @@ -118,7 +120,28 @@ class Context( val identity: Identity, val state: RunState, ) : AutoCloseable { - private val okhttp = OkHttpClient.Builder().socketFactory(TcpNoDelaySocketFactory).build() + private val okhttp = + OkHttpClient + .Builder() + .socketFactory(TcpNoDelaySocketFactory) + // The crawl opens WebSockets to thousands of relays. Each WS-upgrade + // handshake is an async call through OkHttp's shared Dispatcher, whose + // default cap (maxRequests=64) throttles the connection ramp — worse, + // a dead relay holds a slot for the whole connectTimeout, starving live + // relays queued behind it. Widen the dispatcher so handshakes fan out, + // and tighten connectTimeout so an unreachable relay frees its slot + // fast. This is orthogonal to REQ concurrency (that runs on already-open + // sockets, bounded by AdaptiveRelayLimiter), so it can't trip a relay's + // REQ rate-limit — it only speeds connection setup. The executor thread + // pool is unbounded on demand, so raising maxRequests just lets more of + // those short-lived handshakes proceed at once. + .connectTimeout(5, TimeUnit.SECONDS) + .dispatcher( + Dispatcher().apply { + maxRequests = 256 + maxRequestsPerHost = 16 + }, + ).build() val client: NostrClient = NostrClient( From 9611be4135ff463a6c3a550638219e4c525747e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 12:25:48 +0000 Subject: [PATCH 34/58] perf(cli): stream the crawl through a worker pool, no batch barriers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase B drained DRAIN_CONCURRENCY batches, waited for the SLOWEST (a dead relay's full timeout), ingested, then started the next group — so every batch's long tail idled the whole pool, and connections were torn down and rebuilt between groups. Replace the chunked awaitAll barriers with a continuous producer -> workers -> consumer pipeline: - Producer (1 coroutine) routes each author-batch by outbox and feeds a bounded queue (keeps writeRelayFreq single-writer, backpressured so we don't precompute every filter map at once). - DRAIN_CONCURRENCY workers pull a batch, drain it, and grab the next the instant the drain returns — no worker waits on a slow sibling, and hot relays stay connected because some worker is always subscribed to them. - Consumer (1 coroutine) ingests serially (discovered/done/builder/hopOf stay single-writer), now overlapped with draining instead of blocked behind each batch. The four structures now crossed between producer/worker/consumer (relayHints, attempts, deadRelays, relayStrikes) become concurrent; all graph mutation stays single-writer on the consumer. Removes the dead-relay timeout stalls that were serializing the crawl and cuts reconnect churn. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 113 ++++++++++++------ 1 file changed, 79 insertions(+), 34 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 3eb191710e..ec49da1db8 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -51,7 +51,11 @@ import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import java.util.concurrent.ConcurrentHashMap import kotlin.math.roundToInt /** @@ -230,12 +234,17 @@ object GrapeRankCommand { hopOf[observer] = 0 // Per-user relay hints harvested from the `p`-tag relay hints in the // contact lists we crawl (A's follow of B says where B writes) — a - // discovery tier below each user's kind:10002 outbox. - val relayHints = HashMap>() + // discovery tier below each user's kind:10002 outbox. Concurrent: + // the Phase-B producer reads these while the consumer's ingest writes + // them (see the worker-pool below), so both map and inner sets are + // thread-safe. + val relayHints = ConcurrentHashMap>() // Users we're finished with this run: we fed their latest kind:3, or // ran out of retry attempts on an unreachable outbox. val done = hashSetOf() - val attempts = HashMap() + // Outbox retry counts. Concurrent: the producer reads (to widen a + // retry's routing) while the consumer increments. + val attempts = ConcurrentHashMap() val relaysContacted = hashSetOf() // Known-good relay pool, learned from the crawl itself: how often each // relay appears as someone's write relay, and which relays actually @@ -245,8 +254,10 @@ object GrapeRankCommand { val liveRelays = hashSetOf() // Relays that failed to connect MAX_DEAD_STRIKES times — dropped // from all routing so a wave stops eating the timeout on them. - val deadRelays = hashSetOf() - val relayStrikes = HashMap() + // Concurrent: drain workers strike relays while the producer reads + // deadRelays to prune routing. + val deadRelays = ConcurrentHashMap.newKeySet() + val relayStrikes = ConcurrentHashMap() fun recordDead(failed: Set) { for (r in failed) { @@ -277,7 +288,7 @@ object GrapeRankCommand { var fresh = 0 for (tag in contacts.follows()) { follows.add(tag.pubKey) - tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { HashSet() }.add(it) } + tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { ConcurrentHashMap.newKeySet() }.add(it) } if (discovered.add(tag.pubKey)) { hopOf[tag.pubKey] = nextHop fresh++ @@ -412,37 +423,71 @@ object GrapeRankCommand { if (stragglers.isNotEmpty()) { val backbone = topLiveRelays(BACKBONE_SIZE).toSet() ensureRelayLists(ctx, stragglers.toSet(), backbone, timeoutMs, diagnose) - for (group in stragglers.chunked(USER_BATCH).chunked(DRAIN_CONCURRENCY)) { - val prepared = group.map { batch -> batch to routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, graphKinds, deadRelays) } - val drained = - coroutineScope { - prepared - .map { (batch, filters) -> - async { - val dead = hashSetOf() - val events = ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) - recordDead(dead) - Triple(batch, filters.keys, events) - } - }.awaitAll() + + // Continuous worker pool instead of chunked awaitAll barriers. + // The old shape drained DRAIN_CONCURRENCY batches, waited for the + // SLOWEST (a dead relay's full timeout), ingested, then started + // the next group — so every batch's tail idled the whole pool. + // Here a fixed set of DRAIN_CONCURRENCY workers pulls batches off + // a queue and grabs the next the instant a drain returns, so no + // worker waits on a slow sibling and hot relays stay connected + // (some worker is always subscribed). Shared graph state stays + // single-writer: routeByOutbox runs only on the producer (keeps + // writeRelayFreq serial) and ingest runs only on the consumer + // (keeps discovered/done/builder/hopOf serial), now overlapped + // with draining instead of blocked behind each batch. + val routed = Channel, Map>>>(DRAIN_CONCURRENCY * 2) + val drainedOut = Channel, Set, List>>>(Channel.UNLIMITED) + coroutineScope { + // Producer: route each batch by outbox (serial), backpressured + // by the bounded `routed` channel so we don't precompute every + // filter map at once. + val producer = + launch { + for (batch in stragglers.chunked(USER_BATCH)) { + val filters = routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, graphKinds, deadRelays) + routed.send(batch to filters) + } + routed.close() } - for ((batch, relays, events) in drained) { - relaysContacted += relays - // Any relay that gave us an event is proven live + useful. - for ((relay, _) in events) liveRelays.add(relay) - for (pk in batch) { - if (pk in done) continue - val contacts = ctx.contactsOf(pk) - if (contacts != null) { - done += pk - ingest(pk, contacts) - } else { - val tries = (attempts[pk] ?: 0) + 1 - attempts[pk] = tries - if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk + // Drain workers: pure network, no shared graph-state writes + // except recordDead (concurrent-safe now). + val workers = + List(DRAIN_CONCURRENCY) { + launch { + for ((batch, filters) in routed) { + val dead = hashSetOf() + val events = ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) + recordDead(dead) + drainedOut.send(Triple(batch, filters.keys, events)) + } } } - } + // Consumer: single-writer ingest, overlapped with draining. + val consumer = + launch { + for ((batch, relays, events) in drainedOut) { + relaysContacted += relays + // Any relay that gave us an event is proven live + useful. + for ((relay, _) in events) liveRelays.add(relay) + for (pk in batch) { + if (pk in done) continue + val contacts = ctx.contactsOf(pk) + if (contacts != null) { + done += pk + ingest(pk, contacts) + } else { + val tries = (attempts[pk] ?: 0) + 1 + attempts[pk] = tries + if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk + } + } + } + } + producer.join() + workers.joinAll() + drainedOut.close() + consumer.join() } } From 6d64b7b41c982b99fa1f7e3359c47f2b38a09159 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 13:10:58 +0000 Subject: [PATCH 35/58] feat(quartz): include exception type in relay connect-failure message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BasicRelayClient collapsed a connection failure into a message string built from the throwable's text alone. Message text is localized and inconsistent across platforms, so a listener can't reliably tell a busy relay (a connect timeout) from a dead one (bad domain / TLS misconfig) from it. Always append the exception class name (SocketTimeoutException / UnknownHostException / SSLHandshakeException / ConnectException …), which is stable, so listeners can classify the failure by type. Message text is preserved; the type is added in parentheses. Updated the one test that pinned the old format. --- .../relay/client/single/basic/BasicRelayClient.kt | 14 +++++++++++--- .../client/single/basic/BasicRelayClientTest.kt | 6 ++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt index 5992cecdba..b495d42dbb 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt @@ -136,7 +136,9 @@ open class BasicRelayClient( socket?.connect() } catch (e: Exception) { if (e is CancellationException) throw e - listener.onCannotConnect(this, "Error when trying to connect: ${e.message ?: e::class.simpleName}") + val typeName = e::class.simpleName + val detail = e.message?.let { "$it ($typeName)" } ?: (typeName ?: "unknown error") + listener.onCannotConnect(this, "Error when trying to connect: $detail") listener.onDisconnected(this) dontTryAgainForALongTime() markConnectionAsClosed() @@ -187,9 +189,15 @@ open class BasicRelayClient( } else { socket?.disconnect() - // suppression rules below must match the raw message; displayMsg is for listener output only + // suppression rules below must match the raw message; displayMsg is for listener output only. + // Always include the exception's class name: message text is + // localized and inconsistent across platforms, but the type + // (SocketTimeoutException / UnknownHostException / SSLHandshakeException / + // ConnectException …) is stable and lets listeners classify a failure + // reliably — a busy relay (timeout) vs a dead one (bad domain / TLS). val msg = t.message - val displayMsg = msg ?: t::class.simpleName + val typeName = t::class.simpleName + val displayMsg = if (msg != null) "$msg ($typeName)" else (typeName ?: "unknown error") // checks if this is an actual failure. Closing the socket generates an onFailure as well. // ignore tor errors. diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClientTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClientTest.kt index 065ef0fcaf..7a7caaec58 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClientTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClientTest.kt @@ -72,13 +72,15 @@ class BasicRelayClientTest { } @Test - fun onFailureWithMessageKeepsExistingFormat() { + fun onFailureWithMessageAppendsExceptionClassName() { val (socket, listener) = connectAndCapture() socket.onFailure(Exception("Connection reset"), null, null) + // The exception type is appended so listeners can classify the failure by + // its stable class name rather than by localized message text. assertEquals( - listOf("WebSocket Failure: Connection reset"), + listOf("WebSocket Failure: Connection reset (Exception)"), listener.cannotConnectMessages, ) } From 59d5fc752cc72036908467c8f0074eb6a1b43544 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 13:10:58 +0000 Subject: [PATCH 36/58] perf(cli): split rate-limit from subscription-count limit in the crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A relay pushes back for two different reasons that need two different fixes, and treating them the same mishandles the relay: - a subscription-COUNT cap ("too many subscriptions", "maximum concurrent subscription count") is fixed by fewer CONCURRENT subs — demote the per-relay concurrency cap (100 -> 20 -> 10), as before; - a RATE limit ("rate-limited: too many messages", "burst exhausted") is too many subscription CHANGES per second — fewer concurrent subs don't help; the fix is to SPACE the REQs out in time. AdaptiveRelayLimiter now routes each complaint to its own actuator by matching the notice text, and adds a per-relay rate gate: a growing minimum interval between subscription opens (250ms -> 500ms -> 1s -> 2s), enforced in withPermit before the concurrency permit. A relay can be under both controls at once. The snapshot reports each dimension separately. --- .../amethyst/cli/AdaptiveRelayLimiter.kt | 177 ++++++++++++------ 1 file changed, 122 insertions(+), 55 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt index d57e19c034..b2f39fc534 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt @@ -27,51 +27,72 @@ import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NoticeMessage import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.delay import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong /** - * Adaptive per-relay concurrent-subscription cap. + * Adaptive per-relay back-pressure with TWO independent controls, because relays + * push back for two different reasons that need two different responses: * - * Every relay starts with a generous cap ([startCap], default 100) — we assume a - * relay can take as many concurrent REQs as we throw at it until it tells us - * otherwise. When a relay complains about concurrency (a `CLOSED rate-limited`, - * or a `NOTICE` like "too many concurrent REQs" / "too many subscriptions" / - * "burst exhausted"), we demote *that relay only* down the [ladder] - * (100 → 20 → 10). A well-behaved relay keeps the full cap; only the ones that - * push back get throttled, and only as far as they keep pushing. + * 1. **Subscription-count limit** — a max on how many subscriptions may be OPEN + * at once ("too many subscriptions", "maximum concurrent subscription count", + * "number of subscriptions exceeds limit"). The fix is fewer *concurrent* + * subs, so we demote the relay's concurrency cap down [subLadder] + * (100 → 20 → 10). + * 2. **Rate limit** — too many subscription *changes per second* ("rate-limited: + * too many messages", "burst exhausted", "slow down"). Fewer concurrent subs + * wouldn't help; the fix is to *space the REQs out in time*, so we impose a + * minimum interval between opens to that relay, growing it up [rateLadder] + * (250ms → 500ms → 1s → 2s). * - * This replaces a single blunt global concurrency number with per-relay - * back-pressure: the crawl can fan out widely across the many relays that don't - * mind, while automatically easing off the few busy hubs that do — exactly the - * signals [RelayDiagnostics] already observes, here turned into an actuator. + * Mixing the two mishandles the relay: capping concurrency does nothing for a + * rate limit, and slowing the rate does nothing for a subscription-count cap. So + * each complaint is routed to its own actuator by matching the notice text. * - * Registered as a [RelayConnectionListener] on the shared client, so demotions + * A well-behaved relay starts at [startCap] concurrent subs with no rate delay, + * and only the ones that push back get throttled — each only as far, and in the + * dimension, they keep pushing. + * + * Registered as a [RelayConnectionListener] on the shared client, so both signals * are driven straight off the incoming NOTICE/CLOSED frames (which fire on the * per-relay socket threads — all state here is concurrent). Drains gate through * [withPermit]; [Context.drain]'s `gatePerRelay` path holds a relay's permit for - * the lifetime of that relay's subscription, so at most `cap` of our - * subscriptions are ever open on it at once. + * the lifetime of that relay's subscription, and passes the rate gate before it + * opens, so we respect both limits at once. */ class AdaptiveRelayLimiter( private val startCap: Int = 100, - private val ladder: List = listOf(20, 10), + private val subLadder: List = listOf(20, 10), + private val rateLadder: List = listOf(250L, 500L, 1000L, 2000L), ) : RelayConnectionListener { private val gates = ConcurrentHashMap() - // How many concurrency complaints we've acted on per relay (== index+1 into - // the ladder). Capped at ladder.size: past the floor we stop demoting. - private val demotions = ConcurrentHashMap() + // Concurrency-cap demotions per relay (== index+1 into subLadder). Capped at + // subLadder.size: past the floor we stop demoting. + private val subDemotions = ConcurrentHashMap() + + // Rate-limit state per relay: how far down rateLadder we've stepped, the + // current min interval between opens, and the next epoch-ms an open may fire. + private val rateSteps = ConcurrentHashMap() + private val rateDelayMs = ConcurrentHashMap() + private val nextAllowedAtMs = ConcurrentHashMap() private fun gate(relay: NormalizedRelayUrl): Gate = gates.getOrPut(relay) { Gate(startCap) } - /** Run [block] holding one of [relay]'s permits, respecting its current cap. */ + /** + * Run [block] against [relay] respecting both limits: first wait out any rate + * delay (spacing opens in time), then hold one of the relay's concurrency + * permits for the duration. + */ suspend fun withPermit( relay: NormalizedRelayUrl, block: suspend () -> T, ): T { + rateGate(relay) val g = gate(relay) g.acquire() try { @@ -81,52 +102,89 @@ class AdaptiveRelayLimiter( } } + /** If [relay] is rate-limited, reserve and wait for its next allowed open slot. */ + private suspend fun rateGate(relay: NormalizedRelayUrl) { + val delayMs = rateDelayMs[relay] ?: return + if (delayMs <= 0L) return + val now = System.currentTimeMillis() + // Atomically claim the next slot: my turn is max(prevSlot, now); the next + // caller can't fire until delayMs after me. Serializes opens to this relay + // at one per delayMs, in arrival order. + val slot = nextAllowedAtMs.getOrPut(relay) { AtomicLong(now) } + var myTurn: Long + while (true) { + val prev = slot.get() + myTurn = maxOf(prev, now) + if (slot.compareAndSet(prev, myTurn + delayMs)) break + } + val wait = myTurn - now + if (wait > 0) delay(wait) + } + override fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, ) { - when (msg) { - is ClosedMessage -> if (isConcurrencyComplaint(msg.message)) demote(relay.url) - is NoticeMessage -> if (isConcurrencyComplaint(msg.message)) demote(relay.url) - else -> Unit - } - } - - /** Step [relay] one rung down the cap ladder, unless it's already at the floor. */ - private fun demote(relay: NormalizedRelayUrl) { - // Fast path: relays flood identical NOTICEs, so bail once at the floor - // instead of counting them all (the demotion is monotonic and idempotent). - if ((demotions[relay] ?: 0) >= ladder.size) return - val step = demotions.merge(relay, 1, Int::plus)!! - val cap = ladder[(step - 1).coerceIn(0, ladder.size - 1)] - gate(relay).lower(cap) - if (step <= ladder.size) { - System.err.println("[limiter] ${relay.url} capped at $cap concurrent subs (complaint #$step)") - } - } - - private fun isConcurrencyComplaint(text: String): Boolean { + val text = + when (msg) { + is ClosedMessage -> msg.message + is NoticeMessage -> msg.message + else -> return + } val t = text.lowercase() - return CONCURRENCY_MARKERS.any { it in t } + // Route each complaint to the matching actuator. Not mutually exclusive: + // if a relay somehow reports both, we act on both (they don't conflict). + if (RATE_LIMIT_MARKERS.any { it in t }) throttleRate(relay.url) + if (SUB_LIMIT_MARKERS.any { it in t }) demoteConcurrency(relay.url) } - /** JSON-friendly view of which relays we throttled and how far. */ + /** Step [relay] one rung down the concurrency-cap ladder, unless already at the floor. */ + private fun demoteConcurrency(relay: NormalizedRelayUrl) { + if ((subDemotions[relay] ?: 0) >= subLadder.size) return + val step = subDemotions.merge(relay, 1, Int::plus)!! + val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)] + gate(relay).lower(cap) + if (step <= subLadder.size) { + System.err.println("[limiter] ${relay.url} concurrency capped at $cap subs (sub-limit #$step)") + } + } + + /** Step [relay] one rung down the rate ladder, unless already at the slowest. */ + private fun throttleRate(relay: NormalizedRelayUrl) { + if ((rateSteps[relay] ?: 0) >= rateLadder.size) return + val step = rateSteps.merge(relay, 1, Int::plus)!! + val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)] + rateDelayMs[relay] = d + if (step <= rateLadder.size) { + System.err.println("[limiter] ${relay.url} rate-throttled to 1 REQ / ${d}ms (rate-limit #$step)") + } + } + + /** JSON-friendly view of which relays we throttled, in which dimension, how far. */ fun snapshot(): Map { val cappedAt = sortedMapOf() - for ((_, step) in demotions) { - val cap = ladder[(step - 1).coerceIn(0, ladder.size - 1)] + for ((_, step) in subDemotions) { + val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)] cappedAt.merge(cap, 1, Int::plus) } + val rateAt = sortedMapOf() + for ((_, step) in rateSteps) { + val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)] + rateAt.merge(d, 1, Int::plus) + } return mapOf( "start_cap" to startCap, - "ladder" to ladder, - "throttled_relays" to demotions.size, - "capped_at" to cappedAt, + "sub_ladder" to subLadder, + "rate_ladder_ms" to rateLadder, + "concurrency_capped_relays" to subDemotions.size, + "concurrency_capped_at" to cappedAt, + "rate_limited_relays" to rateSteps.size, + "rate_limited_at_ms" to rateAt, ) } - fun hadThrottling(): Boolean = demotions.isNotEmpty() + fun hadThrottling(): Boolean = subDemotions.isNotEmpty() || rateSteps.isNotEmpty() /** * A bounded-concurrency gate whose limit can only ever be *lowered* (relays @@ -174,24 +232,33 @@ class AdaptiveRelayLimiter( } companion object { - // Substrings (matched case-insensitively) that mean "you're opening too - // many concurrent subscriptions / sending too fast" — the failure modes a - // lower per-relay cap actually fixes. Auth/blocked/restricted/unsupported - // are deliberately excluded: throttling wouldn't help those. - private val CONCURRENCY_MARKERS = + // A cap on how many subscriptions may be OPEN at once. Fix: fewer + // concurrent subs (demote the concurrency cap). + private val SUB_LIMIT_MARKERS = listOf( "too many concurrent", "concurrent req", "too many subscription", "number of subscriptions", + "subscriptions exceeds", "subscription limit", + "subscription count", + "maximum concurrent subscription", + "max subscription", "too many req", + ) + + // Too many subscription CHANGES per second. Fix: space the REQs out in + // time (a per-relay min interval), not fewer concurrent subs. + private val RATE_LIMIT_MARKERS = + listOf( "rate-limit", "rate limit", "ratelimit", + "too many messages", + "too many requests", "burst exhausted", "throttl", - "too many messages", "slow down", ) } From dad703baed4dc3c534b21d82a0e2cc3b77da4a33 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 13:11:12 +0000 Subject: [PATCH 37/58] perf(cli): fresher relay lists, wider discovery, and busy-vs-dead pruning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three crawl fixes: 1. Fetch kind:10002 alongside content. The content query asked only for 3/10000/1984, so a user's freshest relay list — which lives on their own outbox — was never pulled from there; we trusted a possibly-stale indexer copy. Fold 10002 into the same fetch. The store keeps newest-by-created_at, so pulling it from popular relays too can't stale it. 2. Widen ensureRelayLists Tier 2. It only asked the top-30 backbone for a still-missing 10002. A stray relay list can sit on any one relay, so Tier 2 now asks EVERY relay we've seen work — fired fire-and-forget on a background scope so the large fan-out never blocks the round; results enrich routing for later rounds. 3. Classify dead relays instead of striking everything the same. A connect TIMEOUT is a busy relay — retried, never marked dead. A HARD failure (bad domain, TLS misconfig, dead HTTP code — see DrainFailure/classifyDrainFailure, keyed on the exception type now in the failure message) is dropped on the first strike. Transient failures (refused/reset/unreachable, 429/5xx) keep the multi-strike leniency. Connect timeout raised 5s -> 7s so slow-but-alive relays finish the handshake. --- .../com/vitorpamplona/amethyst/cli/Context.kt | 80 ++++++++++++++++--- .../amethyst/cli/commands/GrapeRankCommand.kt | 75 +++++++++++++---- 2 files changed, 129 insertions(+), 26 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 8ce736de36..d5b5326dcd 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -83,6 +83,61 @@ import okhttp3.OkHttpClient import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit +/** + * Why a relay could not be used for a drain — when the reason is worth acting on. + * + * - [HARD]: the relay answered wrong, or cannot exist. A bad HTTP upgrade (not a + * websocket / dead status code), an unresolvable domain, or a TLS misconfig. + * This will not fix itself, so one strike is enough to drop it. + * - [TRANSIENT]: a failure that might clear — connection refused / reset, host + * unreachable, or a temporary 429/5xx on the upgrade. Struck a few times + * before we give up. + * + * A pure connect **timeout** is neither. The relay is most likely just busy, so + * we retry it and never mark it dead — [classifyDrainFailure] returns null for + * it (and for any non-failure terminal reason). + */ +enum class DrainFailure { HARD, TRANSIENT } + +/** + * Classify a [Context.drain] per-relay terminal reason. Returns null when the + * relay should simply be retried (a timeout, or a non-failure like eose/closed). + * The reason shape is `cannot:` for a connect failure (see + * `BasicRelayClient.onCannotConnect`), or `eose` / `closed:…` / `timeout`. + */ +fun classifyDrainFailure(reason: String): DrainFailure? { + if (!reason.startsWith("cannot")) return null + val m = reason.removePrefix("cannot:").lowercase() + // The message now carries the exception class name (see BasicRelayClient), so + // we can key on the stable *type* rather than localized message text. + // Busy, not dead: a connect/read timeout means the handshake just didn't + // finish in time. Retry it — the relay is probably fine, only slow or loaded. + if ("timeout" in m || "timed out" in m) return null // SocketTimeoutException, etc. + // Cannot ever work: unresolvable domain (DNS) or a TLS misconfiguration. + // Dead for good — one strike is enough. + if ("unknownhost" in m || // UnknownHostException + "unable to resolve host" in m || + "no address associated" in m || + "nodename nor servname" in m || + "sslhandshake" in m || // SSLHandshakeException + "sslpeerunverified" in m || + "sslexception" in m || + "certificate" in m || // CertificateException + "trust anchor" in m || + "certpath" in m + ) { + return DrainFailure.HARD + } + // Wrong HTTP upgrade. Usually a misconfigured endpoint (not a relay), but + // 429 / 5xx mean "busy, come back later", so those stay transient. + if ("server misconfigured" in m || "not a websocket" in m || "expected http 101" in m) { + val transientCode = Regex("response: (429|500|502|503|504)").containsMatchIn(m) + return if (transientCode) DrainFailure.TRANSIENT else DrainFailure.HARD + } + // Refused / reset / unreachable / anything else: might clear — retry a few times. + return DrainFailure.TRANSIENT +} + /** * Per-invocation wiring. Each CLI run constructs a Context, does its work, * and then closes it — no daemon. @@ -129,13 +184,16 @@ class Context( // default cap (maxRequests=64) throttles the connection ramp — worse, // a dead relay holds a slot for the whole connectTimeout, starving live // relays queued behind it. Widen the dispatcher so handshakes fan out, - // and tighten connectTimeout so an unreachable relay frees its slot - // fast. This is orthogonal to REQ concurrency (that runs on already-open - // sockets, bounded by AdaptiveRelayLimiter), so it can't trip a relay's - // REQ rate-limit — it only speeds connection setup. The executor thread - // pool is unbounded on demand, so raising maxRequests just lets more of - // those short-lived handshakes proceed at once. - .connectTimeout(5, TimeUnit.SECONDS) + // and keep connectTimeout tight-ish so an unreachable relay frees its + // slot fast. This is orthogonal to REQ concurrency (that runs on + // already-open sockets, bounded by AdaptiveRelayLimiter), so it can't + // trip a relay's REQ rate-limit — it only speeds connection setup. The + // executor thread pool is unbounded on demand, so raising maxRequests + // just lets more of those short-lived handshakes proceed at once. 7s + // (not 5s): a 5s cap struck too many merely-busy relays as connect + // failures — the crawl treats a connect *timeout* as retryable anyway, + // but the extra headroom lets slow-but-alive relays finish the handshake. + .connectTimeout(7, TimeUnit.SECONDS) .dispatcher( Dispatcher().apply { maxRequests = 256 @@ -475,7 +533,7 @@ class Context( filters: Map>, timeoutMs: Long = 8_000, diagnoseSlow: Boolean = false, - deadOut: MutableSet? = null, + deadOut: MutableMap? = null, gatePerRelay: Boolean = false, ): List> { if (filters.isEmpty()) return emptyList() @@ -556,7 +614,7 @@ class Context( } deadOut?.let { out -> for ((relay, reason) in doneReasons) { - if (reason.startsWith("cannot")) out.add(relay) + classifyDrainFailure(reason)?.let { out[relay] = it } } } return collected @@ -578,7 +636,7 @@ class Context( filters: Map>, timeoutMs: Long, diagnoseSlow: Boolean, - deadOut: MutableSet?, + deadOut: MutableMap?, ): List> { val eventChannel = Channel>(UNLIMITED) // One relay per subId, so the relay alone identifies which subscription a @@ -657,7 +715,7 @@ class Context( } deadOut?.let { out -> for ((relay, reason) in doneReasons) { - if (reason.startsWith("cannot")) out.add(relay) + classifyDrainFailure(reason)?.let { out[relay] = it } } } return collected diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index ec49da1db8..45061344a8 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.amethyst.cli.commands import com.vitorpamplona.amethyst.cli.Args import com.vitorpamplona.amethyst.cli.Context import com.vitorpamplona.amethyst.cli.DataDir +import com.vitorpamplona.amethyst.cli.DrainFailure import com.vitorpamplona.amethyst.cli.Output import com.vitorpamplona.amethyst.commons.defaults.Constants import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList @@ -48,14 +49,18 @@ import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProvider import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceType import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import java.util.concurrent.ConcurrentHashMap +import kotlin.coroutines.coroutineContext import kotlin.math.roundToInt /** @@ -208,6 +213,14 @@ object GrapeRankCommand { val observer = observerArg?.let { ctx.requireUserHex(it) } ?: ctx.identity.pubKeyHex val graphKinds = listOf(ContactListEvent.KIND, MuteListEvent.KIND, ReportEvent.KIND) + // Kinds requested from relays during the crawl: the graph edges PLUS the + // user's own kind:10002. A user's outbox holds the freshest copy of their + // relay list, so folding 10002 into the same query we send their outbox + // keeps our routing current instead of trusting a possibly-stale indexer + // copy. Safe to also pull from popular relays in the sweep — the store + // keeps newest-by-created_at for the replaceable 10002, so the freshest + // always wins regardless of which relay delivered it. + val fetchKinds = graphKinds + AdvertisedRelayListEvent.KIND // The graph is built incrementally: contact lists stream straight into a // compact int-CSR structure and the Event is discarded, so the whole @@ -230,6 +243,12 @@ object GrapeRankCommand { val hopOf = HashMap() if (!offline) { val crawlStart = System.nanoTime() + // Scope for fire-and-forget relay-list discovery: the wide Tier-2 + // sweep (ensureRelayLists) casts kind:10002 queries across every relay + // we know, but we don't block the crawl on it — its results just + // enrich routing for later rounds. SupervisorJob so one failing sweep + // never cancels the others; cancelled when the crawl finishes. + val bgScope = CoroutineScope(coroutineContext + SupervisorJob()) val discovered = hashSetOf(observer) hopOf[observer] = 0 // Per-user relay hints harvested from the `p`-tag relay hints in the @@ -259,9 +278,19 @@ object GrapeRankCommand { val deadRelays = ConcurrentHashMap.newKeySet() val relayStrikes = ConcurrentHashMap() - fun recordDead(failed: Set) { - for (r in failed) { - if (relayStrikes.merge(r, 1, Int::plus)!! >= MAX_DEAD_STRIKES) deadRelays.add(r) + // A relay that HARD-failed (bad domain, TLS misconfig, dead HTTP + // code — see DrainFailure) is dropped on the first strike: it will + // not fix itself. A TRANSIENT failure (refused/reset/unreachable, + // or a 429/5xx) might clear, so it takes MAX_DEAD_STRIKES before we + // give up. Pure timeouts never reach here — the drain treats them as + // busy-retry and does not report them dead at all. + fun recordDead(failed: Map) { + for ((r, kind) in failed) { + when (kind) { + DrainFailure.HARD -> deadRelays.add(r) + DrainFailure.TRANSIENT -> + if (relayStrikes.merge(r, 1, Int::plus)!! >= MAX_DEAD_STRIKES) deadRelays.add(r) + } } } @@ -348,9 +377,9 @@ object GrapeRankCommand { // Each drain gets its own dead-set — the concurrent // drains must not share a mutable HashSet. async { - val dead = hashSetOf() + val dead = HashMap() val filters = - mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = graphKinds, authors = it) }) + mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = fetchKinds, authors = it) }) ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) to dead } } @@ -374,9 +403,9 @@ object GrapeRankCommand { // on a relay ranked below the top SHARD_RELAYS. val live = topLiveRelays(BROADCAST_RELAYS) if (live.isNotEmpty()) { - val dead = hashSetOf() + val dead = HashMap() val filters = - live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = graphKinds, authors = it) } } + live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = fetchKinds, authors = it) } } val events = ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) recordDead(dead) relaysContacted += live @@ -422,7 +451,11 @@ object GrapeRankCommand { val stragglers = pending.filter { it !in done } if (stragglers.isNotEmpty()) { val backbone = topLiveRelays(BACKBONE_SIZE).toSet() - ensureRelayLists(ctx, stragglers.toSet(), backbone, timeoutMs, diagnose) + // Snapshot of every relay we've seen work, for the wide Tier-2 + // sweep (taken now, on this single coroutine, before the Phase-B + // workers start mutating liveRelays). + val allLive = (liveRelays - deadRelays).toSet() + ensureRelayLists(ctx, stragglers.toSet(), allLive, bgScope, timeoutMs, diagnose) // Continuous worker pool instead of chunked awaitAll barriers. // The old shape drained DRAIN_CONCURRENCY batches, waited for the @@ -445,7 +478,7 @@ object GrapeRankCommand { val producer = launch { for (batch in stragglers.chunked(USER_BATCH)) { - val filters = routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, graphKinds, deadRelays) + val filters = routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, fetchKinds, deadRelays) routed.send(batch to filters) } routed.close() @@ -456,7 +489,7 @@ object GrapeRankCommand { List(DRAIN_CONCURRENCY) { launch { for ((batch, filters) in routed) { - val dead = hashSetOf() + val dead = HashMap() val events = ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) recordDead(dead) drainedOut.send(Triple(batch, filters.keys, events)) @@ -498,8 +531,10 @@ object GrapeRankCommand { ) } - // Crawl done — drop the warm pool. + // Crawl done — drop the warm pool and stop any background relay-list + // sweeps still in flight (their results are already in the store). ctx.client.unsubscribe(WARM_SUB_ID) + bgScope.cancel() // No separate last-mile pass: the per-round sharded sweep already // broadcasts the small remaining set to every top relay once it drops @@ -904,7 +939,8 @@ object GrapeRankCommand { private suspend fun ensureRelayLists( ctx: Context, pubkeys: Set, - fallbackRelays: Set, + allLiveRelays: Set, + bgScope: CoroutineScope, timeoutMs: Long, diagnose: Boolean, ) { @@ -925,13 +961,22 @@ object GrapeRankCommand { ctx.drain(filters, timeoutMs, diagnose, gatePerRelay = true) } + // Tier 1: the index/discovery aggregators, which carry kind:10002 for most + // of the network. Blocking, because this round's routing needs the result. val discovery = relayListDiscoveryRelays(ctx) query(missing, discovery) - // Tier 2: whoever the aggregators still don't have, ask the relays the - // rest of the graph actually writes to. + // Tier 2: whoever the aggregators still don't have, cast the widest net — + // ask EVERY relay we've seen deliver events, not just the backbone. Fired + // fire-and-forget on [bgScope]: a stray 10002 might sit on any one relay, so + // we don't want to skip any, but we also can't block the crawl on a fan-out + // that large. The results land in the store and improve routing for later + // rounds; anyone still unresolved is handled by fallback routing meanwhile. val stillMissing = missing.filter { ctx.relaysOf(it) == null } - query(stillMissing, fallbackRelays - discovery) + val wide = allLiveRelays - discovery + if (stillMissing.isNotEmpty() && wide.isNotEmpty()) { + bgScope.launch { query(stillMissing, wide) } + } } /** From dd6384b19e32566c3e4973a4d7b998f31adc325f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 13:18:47 +0000 Subject: [PATCH 38/58] feat(cli): only publish a 30382 card when its rank tag string changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate publishing on the exact rank TAG VALUE STRING, not a re-parsed Int. A card carries only a `rank` tag (plus the d-tag target), and RankTag.assemble writes `rank.toString()`, so we diff that string against the one on the newest kind:30382 card the signing key already published (read back from the store). An unchanged score is skipped — no new signature, no new event id — so a client that syncs the provider's cards by id only ever downloads the ranks that actually moved. Replaces the prior Int comparison with a faithful what-would-be-written string diff. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 45061344a8..0a510b5efe 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -647,16 +647,20 @@ object GrapeRankCommand { ?.takeIf { it.isNotEmpty() } ?: ctx.outboxRelays() - // Ranks we've already published (read back from the store, which - // holds our own prior cards) — keyed by target, newest per target. - // Lets us leave an unchanged card alone instead of churning it. - val publishedRanks = publishedCardRanks(ctx) + // The rank tag string already published for each target (read back + // from the store, which holds our own prior cards), newest per target. + val publishedRankValues = publishedRankTagValues(ctx) val candidates = rankedIds .filter { rankOf(scores[it]) >= minRank } .map { graph.pubkeyOf(it) to rankOf(scores[it]) } - val changed = candidates.filter { (target, rank) -> publishedRanks[target] != rank } + // Only sign a new card when the rank TAG VALUE STRING would change. + // `RankTag.assemble(rank)` writes `rank.toString()`, so we diff that + // exact string against the one on the newest stored card. Unchanged + // scores are skipped — no new event id — so clients that sync by id + // only ever download the ranks that actually moved. + val changed = candidates.filter { (target, rank) -> publishedRankValues[target] != rank.toString() } val toPublish = changed.take(publishLimit) result["skipped_unchanged"] = candidates.size - changed.size @@ -1026,12 +1030,18 @@ object GrapeRankCommand { } /** - * The rank we last published for each target, read from the active account's - * own kind:30382 cards in the local store (newest card wins per target). - * `ctx.publish` stores every card it sends, so on repeat runs this reflects - * what's already out there and lets us skip targets whose rank is unchanged. + * The exact `rank` tag VALUE STRING we last published for each target, read + * from the active account's own kind:30382 cards in the local store (newest + * card wins per target). `ctx.publish` stores every card it sends, so on + * repeat runs this reflects what's already out there. + * + * We key on the raw tag string, not a re-parsed Int, because that string is + * exactly what a client diffs: creating a new signature (a new event id) is + * only worth it when the written value actually changes. Our cards carry ONLY + * a `rank` tag (plus the d-tag target), so this single tag's value fully + * decides whether the event would differ — see the publish gate. */ - private suspend fun publishedCardRanks(ctx: Context): Map { + private suspend fun publishedRankTagValues(ctx: Context): Map { val self = ctx.identity.pubKeyHex return ctx.store .query(Filter(kinds = listOf(ContactCardEvent.KIND), authors = listOf(self))) @@ -1039,8 +1049,12 @@ object GrapeRankCommand { .groupBy { it.aboutUser() } .mapNotNull { (target, cards) -> val t = target ?: return@mapNotNull null - val rank = cards.maxByOrNull { it.createdAt }?.rank() ?: return@mapNotNull null - t to rank + val newest = cards.maxByOrNull { it.createdAt } ?: return@mapNotNull null + val rankValue = + newest.tags.firstNotNullOfOrNull { tag -> + if (tag.size > 1 && tag[0] == RankTag.TAG_NAME) tag[1] else null + } ?: return@mapNotNull null + t to rankValue }.toMap() } From 8ee8ebdb00707e39e4938b298fdb0559934b45d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 13:26:53 +0000 Subject: [PATCH 39/58] feat(cli): drop retracted reports via NIP-09 deletions in the graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A report the author has since deleted should not count as a negative trust edge. After the crawl, ask each reporter's outbox for kind:5 deletion requests that cite the reports we gathered — #e-filtered to those report ids, so we pull only the deletions that affect our reports, not every deletion the user ever made. When building the graph, a report is dropped iff a kind:5 in the store cites its id AND is signed by the report's own author (NIP-09: a deletion is authoritative only from the event's author). Reports the reporter never retracted are unaffected. The run reports reports_deleted. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 99 ++++++++++++++++++- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 0a510b5efe..964212cbc2 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -39,6 +39,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent +import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent import com.vitorpamplona.quartz.nip56Reports.ReportEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent @@ -536,6 +537,13 @@ object GrapeRankCommand { ctx.client.unsubscribe(WARM_SUB_ID) bgScope.cancel() + // Reports can be retracted. Ask each reporter's outbox for NIP-09 + // kind:5 deletions that cite the reports we gathered (#e-filtered to + // our report ids — not every deletion the user ever made). A report + // the author has since deleted must not count as a negative edge; + // [materializeReports] drops those below. + fetchReportDeletions(ctx, topLiveRelays(BACKBONE_SIZE).toSet(), deadRelays, timeoutMs, diagnose) + // No separate last-mile pass: the per-round sharded sweep already // broadcasts the small remaining set to every top relay once it drops // below SHARD_BROADCAST_THRESHOLD, and the round loop only exits when @@ -577,9 +585,7 @@ object GrapeRankCommand { for (event in ctx.store.query(Filter(kinds = listOf(MuteListEvent.KIND)))) { if (event is MuteListEvent) builder.addMutes(event.pubKey, event.linkedPubKeys()) } - for (event in ctx.store.query(Filter(kinds = listOf(ReportEvent.KIND)))) { - if (event is ReportEvent) builder.addReports(event.pubKey, event.reportedAuthor().map { it.pubkey }) - } + val reportsDeleted = materializeReports(ctx, builder) val buildStart = System.nanoTime() val graph = builder.build() @@ -626,6 +632,7 @@ object GrapeRankCommand { .mapKeys { it.key.toString() }, "graph_users" to graph.nodeCount, "graph_edges" to graph.edgeCount(), + "reports_deleted" to reportsDeleted, "users_scored" to rankedIds.size, "download_ms" to downloadMs, "store_load_ms" to storeLoadMs, @@ -983,6 +990,92 @@ object GrapeRankCommand { } } + /** + * Fetch NIP-09 kind:5 deletion requests that retract any report we gathered. + * + * A reporter can delete their own kind:1984 report. That deletion is valid + * only if it comes from the reporter's own key, and it's published to the + * reporter's outbox — so we group report ids by their author and ask each + * author's write relays for kind:5 events that cite those ids (`#e`). That + * `#e` filter is the point: we pull only the deletions that touch our reports, + * not every deletion the user has ever made. The events land in the store; + * [materializeReports] decides which reports they actually retract. + */ + private suspend fun fetchReportDeletions( + ctx: Context, + backbone: Set, + deadRelays: Set, + timeoutMs: Long, + diagnose: Boolean, + ) { + val idsByAuthor = HashMap>() + for (ev in ctx.store.query(Filter(kinds = listOf(ReportEvent.KIND)))) { + if (ev is ReportEvent) idsByAuthor.getOrPut(ev.pubKey) { ArrayList() }.add(ev.id) + } + if (idsByAuthor.isEmpty()) return + + // Route each reporter to their own write relays (fallback: backbone). + val perRelayAuthors = HashMap>() + for (author in idsByAuthor.keys) { + val write = ctx.relaysOf(author)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() } ?: backbone + for (relay in write) if (relay !in deadRelays) perRelayAuthors.getOrPut(relay) { HashSet() }.add(author) + } + if (perRelayAuthors.isEmpty()) return + + val filters = + perRelayAuthors.mapValues { (_, authors) -> + buildList { + for (authorChunk in authors.chunked(AUTHORS_PER_FILTER)) { + // Scope #e to this author-chunk's own report ids, chunked to + // respect REQ limits. Any over-match (a filter pairing an + // author with another author's id) is harmless — the + // deleter-must-be-author check in materializeReports rejects it. + val chunkIds = authorChunk.flatMap { idsByAuthor[it].orEmpty() } + for (idChunk in chunkIds.chunked(AUTHORS_PER_FILTER)) { + add(Filter(kinds = listOf(DeletionEvent.KIND), authors = authorChunk, tags = mapOf("e" to idChunk))) + } + } + } + } + ctx.drain(filters, timeoutMs, diagnose, gatePerRelay = true) + } + + /** + * Feed reports into [builder], dropping any that a valid NIP-09 deletion has + * retracted. A report id counts as deleted only when a kind:5 in the store + * cites it AND is signed by the report's own author (NIP-09: a deletion is + * only authoritative from the event's author). Returns how many were dropped. + */ + private suspend fun materializeReports( + ctx: Context, + builder: TrustGraphBuilder, + ): Int { + val reports = ctx.store.query(Filter(kinds = listOf(ReportEvent.KIND))).filterIsInstance() + if (reports.isEmpty()) return 0 + + val authorByReportId = HashMap() + for (r in reports) authorByReportId[r.id] = r.pubKey + + val deletedReportIds = HashSet() + for (ev in ctx.store.query(Filter(kinds = listOf(DeletionEvent.KIND)))) { + val del = ev as? DeletionEvent ?: continue + for (id in del.deleteEventIds()) { + if (authorByReportId[id] == del.pubKey) deletedReportIds.add(id) + } + } + + var dropped = 0 + for (r in reports) { + if (r.id in deletedReportIds) { + dropped++ + continue + } + builder.addReports(r.pubKey, r.reportedAuthor().map { it.pubkey }) + } + if (dropped > 0) System.err.println("[graperank] dropped $dropped retracted reports (NIP-09 deletions)") + return dropped + } + /** * Group [pubkeys] by the relays we should query for their events: * - first try: the user's own kind:10002 write relays (the outbox model); From 98709ef9332f58cb43a942ffdede48009cf6415f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 13:31:04 +0000 Subject: [PATCH 40/58] refactor(cli): use quartz DeletionIndex for retracted-report detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the hand-rolled report-id/author matching with quartz's DeletionIndex — the same NIP-09 indexer the Android app's LocalCache uses. It keys each deletion under the deleter's pubkey, so hasBeenDeleted(report) is authoritative only when the report's own author deleted it, and it also handles created_at ordering (and addressable events, for free). Deletions come from the store, which already verified them, so they're added as pre-verified. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 964212cbc2..07af50192f 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent +import com.vitorpamplona.quartz.nip09Deletions.DeletionIndex import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent import com.vitorpamplona.quartz.nip56Reports.ReportEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent @@ -1042,9 +1043,11 @@ object GrapeRankCommand { /** * Feed reports into [builder], dropping any that a valid NIP-09 deletion has - * retracted. A report id counts as deleted only when a kind:5 in the store - * cites it AND is signed by the report's own author (NIP-09: a deletion is - * only authoritative from the event's author). Returns how many were dropped. + * retracted. Uses quartz's [DeletionIndex] — the same indexer the Android + * app's LocalCache runs — which keys each deletion under the DELETER's pubkey, + * so `hasBeenDeleted(report)` is true only when the report's own author + * deleted it (NIP-09: a deletion is authoritative only from the event's + * author). It also honours created_at ordering. Returns how many were dropped. */ private suspend fun materializeReports( ctx: Context, @@ -1053,20 +1056,16 @@ object GrapeRankCommand { val reports = ctx.store.query(Filter(kinds = listOf(ReportEvent.KIND))).filterIsInstance() if (reports.isEmpty()) return 0 - val authorByReportId = HashMap() - for (r in reports) authorByReportId[r.id] = r.pubKey - - val deletedReportIds = HashSet() + // Everything in the store already passed verifyAndStore, so mark the + // deletions as verified and skip the redundant signature check. + val deletions = DeletionIndex() for (ev in ctx.store.query(Filter(kinds = listOf(DeletionEvent.KIND)))) { - val del = ev as? DeletionEvent ?: continue - for (id in del.deleteEventIds()) { - if (authorByReportId[id] == del.pubKey) deletedReportIds.add(id) - } + if (ev is DeletionEvent) deletions.add(ev, wasVerified = true) } var dropped = 0 for (r in reports) { - if (r.id in deletedReportIds) { + if (deletions.hasBeenDeleted(r)) { dropped++ continue } From 8d4dfedd68b1dcc66ff3af9aa237a7b4e85c17ed Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 13:44:23 +0000 Subject: [PATCH 41/58] perf(cli): skip duplicate events before verify in the gated drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The outbox model — and especially the wide relay-list broadcast — delivers the same event from many relays at once, and the gated drain ran a Schnorr verify (and a store insert) on every copy before the store's UNIQUE constraint dropped it. On a fan-out that asks hundreds of relays for the same kind:10002s, that is hundreds of redundant verifications per event and pegged a core. Add a per-drain SeenIds skip-before-verify to the consumer, mirroring drainAllPages: an id is marked seen only after it verifies, so a forged copy (valid id, bad signature) delivered first can't suppress the genuine one. Cuts the redundant verification across the whole crawl, not just the wide sweep. --- .../com/vitorpamplona/amethyst/cli/Context.kt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index d5b5326dcd..759b8f37b0 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -680,11 +680,22 @@ class Context( } val collected = mutableListOf>() coroutineScope { - // Single consumer: verify+store serially, exactly like drain(). + // Single consumer: verify+store serially, exactly like drain(). One + // writer, so SeenIds' single-writer contract holds. The outbox model + // (and especially the wide relay-list broadcast) delivers the SAME event + // from many relays at once; skip a duplicate BEFORE the expensive + // Schnorr verify+store. An id is marked seen only after it verifies, so a + // forged copy (valid id, bad signature) delivered first can't suppress + // the genuine one that follows. val consumer = launch { + val seen = SeenIds(initialSlotsPow2 = 12) for ((relay, event) in eventChannel) { - if (verifyAndStore(event)) collected.add(relay to event) + if (seen.contains(event.id)) continue + if (verifyAndStore(event)) { + seen.add(event.id) + collected.add(relay to event) + } } } // One gated subscription per relay. The permit is held for the whole From c637d1984d97c535429bcfd81e68154b0ddb3e93 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 14:11:14 +0000 Subject: [PATCH 42/58] fix(cli): cap REQ frame size so relays don't reject oversized subscriptions A REQ carries all of a subscription's filters in one frame, so a popular relay routed thousands of authors produced a multi-MB frame that most relays reject outright ("message too large (2MB > 256KB)"), silently dropping every author in it. The gated drain now splits each relay's filters into REQ-sized groups by total entry count (authors + ids + tag values), MAX_REQ_ENTRIES=2500 (~167KB, under the common 256KB cap), and opens one gated subscription per group. A relay with more authors simply gets several smaller REQs instead of one rejected huge one. Each group carries its own subId/listener/terminal signal; per-relay failure classification takes HARD over TRANSIENT across a relay's groups. --- .../com/vitorpamplona/amethyst/cli/Context.kt | 159 +++++++++++------- 1 file changed, 101 insertions(+), 58 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 759b8f37b0..4214dac422 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -138,6 +138,22 @@ fun classifyDrainFailure(reason: String): DrainFailure? { return DrainFailure.TRANSIENT } +/** + * Max total "entries" (authors + ids + tag values) allowed in a single REQ frame. + * A REQ carries all of a subscription's filters at once, and each entry is a + * ~67-byte JSON hex string, so 2500 entries ≈ 167KB — comfortably under the 256KB + * message cap most relays enforce (and reject a frame over, dropping every author + * in it). [Context.drainGated] groups a relay's filters to stay within this. + */ +private const val MAX_REQ_ENTRIES = 2500 + +/** Count the size-driving entries in a filter: authors, ids, and tag values. */ +fun filterEntries(f: Filter): Int = + (f.authors?.size ?: 0) + + (f.ids?.size ?: 0) + + (f.tags?.values?.sumOf { it.size } ?: 0) + + (f.tagsAll?.values?.sumOf { it.size } ?: 0) + /** * Per-invocation wiring. Each CLI run constructs a Context, does its work, * and then closes it — no daemon. @@ -639,54 +655,43 @@ class Context( deadOut: MutableMap?, ): List> { val eventChannel = Channel>(UNLIMITED) - // One relay per subId, so the relay alone identifies which subscription a - // callback is for. First terminal frame wins; a timeout leaves it unset. - val relayDone = ConcurrentHashMap>() - for (r in filters.keys) relayDone[r] = CompletableDeferred() - val doneReasons = ConcurrentHashMap() - val listener = - object : SubscriptionListener { - override fun onEvent( - event: Event, - isLive: Boolean, - relay: NormalizedRelayUrl, - forFilters: List?, - ) { - eventChannel.trySend(relay to event) - } - override fun onEose( - relay: NormalizedRelayUrl, - forFilters: List?, - ) { - relayDone[relay]?.complete("eose") - } - - override fun onClosed( - message: String, - relay: NormalizedRelayUrl, - forFilters: List?, - ) { - relayDone[relay]?.complete("closed:$message") - } - - override fun onCannotConnect( - relay: NormalizedRelayUrl, - message: String, - forFilters: List?, - ) { - relayDone[relay]?.complete("cannot:$message") + // Split each relay's filters into REQ-sized groups. A REQ frame carries ALL + // its filters at once, so a popular relay routed thousands of authors would + // otherwise produce a multi-MB frame that most relays reject outright + // ("message too large") — silently dropping every author in it. Grouping by + // total entry count keeps each REQ well under the common 256KB cap; a relay + // with more authors just gets several smaller REQs, each its own gated sub. + val units = ArrayList>>() + for ((relay, relayFilters) in filters) { + var group = ArrayList() + var entries = 0 + for (f in relayFilters) { + val fe = filterEntries(f) + if (group.isNotEmpty() && entries + fe > MAX_REQ_ENTRIES) { + units.add(relay to group) + group = ArrayList() + entries = 0 } + group.add(f) + entries += fe } + if (group.isNotEmpty()) units.add(relay to group) + } + + // Per-relay failure classification, HARD winning over TRANSIENT across a + // relay's several REQ-groups; plus which relays stalled to a timeout. + val failures = ConcurrentHashMap() + val timedOut = ConcurrentHashMap.newKeySet() + val collected = mutableListOf>() coroutineScope { - // Single consumer: verify+store serially, exactly like drain(). One - // writer, so SeenIds' single-writer contract holds. The outbox model - // (and especially the wide relay-list broadcast) delivers the SAME event - // from many relays at once; skip a duplicate BEFORE the expensive - // Schnorr verify+store. An id is marked seen only after it verifies, so a - // forged copy (valid id, bad signature) delivered first can't suppress - // the genuine one that follows. + // Single consumer: verify+store serially. One writer, so SeenIds' + // single-writer contract holds. The outbox model (and the wide relay- + // list broadcast) delivers the SAME event from many relays at once; skip + // a duplicate BEFORE the expensive Schnorr verify+store. An id is marked + // seen only after it verifies, so a forged copy (valid id, bad signature) + // delivered first can't suppress the genuine one that follows. val consumer = launch { val seen = SeenIds(initialSlotsPow2 = 12) @@ -698,17 +703,60 @@ class Context( } } } - // One gated subscription per relay. The permit is held for the whole - // life of the relay's REQ, so concurrent subs on it never exceed its cap. - filters - .map { (relay, relayFilters) -> + // One gated subscription per (relay, REQ-group). The permit is held for + // the group's whole life, so concurrent subs on a relay never exceed its + // adaptive cap. Each group carries its own subId, listener, and terminal + // signal (relay + subId together identify a group, but a per-group + // listener is simplest). + units + .map { (relay, groupFilters) -> launch { relayLimiter.withPermit(relay) { val subId = newSubId() - client.subscribe(subId, mapOf(relay to relayFilters), listener) + val done = CompletableDeferred() + val groupListener = + object : SubscriptionListener { + override fun onEvent( + event: Event, + isLive: Boolean, + r: NormalizedRelayUrl, + forFilters: List?, + ) { + eventChannel.trySend(r to event) + } + + override fun onEose( + r: NormalizedRelayUrl, + forFilters: List?, + ) { + done.complete("eose") + } + + override fun onClosed( + message: String, + r: NormalizedRelayUrl, + forFilters: List?, + ) { + done.complete("closed:$message") + } + + override fun onCannotConnect( + r: NormalizedRelayUrl, + message: String, + forFilters: List?, + ) { + done.complete("cannot:$message") + } + } + client.subscribe(subId, mapOf(relay to groupFilters), groupListener) try { - val reason = withTimeoutOrNull(timeoutMs) { relayDone[relay]!!.await() } - doneReasons[relay] = reason ?: "timeout" + val reason = withTimeoutOrNull(timeoutMs) { done.await() } ?: "timeout" + if (reason == "timeout") timedOut.add(relay) + classifyDrainFailure(reason)?.let { kind -> + failures.merge(relay, kind) { a, b -> + if (a == DrainFailure.HARD || b == DrainFailure.HARD) DrainFailure.HARD else DrainFailure.TRANSIENT + } + } } finally { client.unsubscribe(subId) } @@ -720,15 +768,10 @@ class Context( eventChannel.close() consumer.join() } - if (diagnoseSlow) { - val stalled = filters.keys.filter { (doneReasons[it] ?: "timeout") == "timeout" }.toSet() - if (stalled.isNotEmpty()) logSlowDrain(timeoutMs, stalled, doneReasons, collected) - } - deadOut?.let { out -> - for ((relay, reason) in doneReasons) { - classifyDrainFailure(reason)?.let { out[relay] = it } - } + if (diagnoseSlow && timedOut.isNotEmpty()) { + logSlowDrain(timeoutMs, timedOut, emptyMap(), collected) } + deadOut?.putAll(failures) return collected } From 0deae996bbc5b528804df7c9196127d7714bff6a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 14:18:32 +0000 Subject: [PATCH 43/58] feat(cli): operator-key module for GrapeRank provider signing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A machine holds one operator master seed, independent of any amy account, stored under ~/.amy/operator/ via the same SecretStore backend the accounts use. From it OperatorKeys deterministically derives one service key per observer — serviceKey(observer) = sha256(masterPriv || "graperank-provider:" || observerHex) — which will sign that observer's kind:30382 rank cards and their retractions. Deterministic derivation gives a stable per-observer identity (so re-signing a card replaces the addressable prior one instead of orphaning it) and one-secret backup (every service key re-derives from the master alone). The manifest (operator.json) records the master pubkey, operator relay(s), and observer -> provider-pubkey mapping — public data; only the master rides the SecretStore. Exposed via DataDir.operatorKeys(). Wiring into publish comes next. --- .../com/vitorpamplona/amethyst/cli/Config.kt | 7 + .../amethyst/cli/OperatorKeys.kt | 156 ++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/OperatorKeys.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt index ef9c5ed655..d2cc32a51d 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt @@ -222,6 +222,13 @@ class DataDir( */ val eventsDbFile: File = File(eventsDir.parentFile ?: root, "events.db") + /** + * Machine-level operator keys for GrapeRank trusted-assertion publishing, + * rooted at `~/.amy/operator/` (the account root's parent) so a single + * operator master is shared across accounts. See [OperatorKeys]. + */ + fun operatorKeys(): OperatorKeys = OperatorKeys(root.parentFile ?: root, secrets) + init { SecureFileIO.secureMkdirs(root) SecureFileIO.secureMkdirs(groupsDir) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/OperatorKeys.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/OperatorKeys.kt new file mode 100644 index 0000000000..71a7dd6d68 --- /dev/null +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/OperatorKeys.kt @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.cli + +import com.fasterxml.jackson.module.kotlin.readValue +import com.vitorpamplona.amethyst.cli.secrets.IdentitySecret +import com.vitorpamplona.amethyst.cli.secrets.SecretStore +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.utils.sha256.sha256 +import java.io.File + +/** + * Operator-level signing keys for GrapeRank trusted-assertion publishing. + * + * A machine holds ONE operator master seed, **independent of any amy account**, + * stored under `~/.amy/operator/` through the same [SecretStore] backend the + * accounts use (OS keychain / NIP-49 ncryptsec / plaintext). From it we + * deterministically derive ONE service key per observer: + * + * ``` + * serviceKey(observer) = sha256(masterPriv ‖ "graperank-provider:" ‖ observerHex ‖ counter) + * ``` + * + * That service key signs the observer's kind:30382 rank cards (and their kind:5 + * retractions). Deterministic derivation buys two things: + * - **Stable identity** — the same observer always maps to the same key, so + * re-signing a card *replaces* the prior one (kind:30382 is addressable) + * instead of orphaning it and spamming clients with duplicates. + * - **One-secret backup** — back up only the master seed; every service key is + * re-derivable even if the [providers] manifest is lost. + * + * The manifest (`~/.amy/operator/operator.json`) records the master pubkey, the + * configured operator relay(s), and the observer → provider-pubkey mapping. Only + * the master itself is a secret; it rides the [SecretStore] descriptor, so the + * manifest holds public data. + */ +class OperatorKeys( + amyHome: File, + private val secrets: SecretStore, +) { + private val dir = File(amyHome, DIR_NAME) + private val configFile = File(dir, CONFIG_NAME) + + data class ProviderRecord( + val providerPubKey: HexKey = "", + ) + + data class Config( + val masterPubKey: HexKey = "", + val master: IdentitySecret? = null, + val relays: List = emptyList(), + val providers: MutableMap = mutableMapOf(), + ) + + private fun load(): Config? = if (configFile.exists()) Output.mapper.readValue(configFile.readText()) else null + + private fun save(cfg: Config) { + SecureFileIO.secureMkdirs(dir) + configFile.writeText(Output.mapper.writeValueAsString(cfg)) + SecureFileIO.tighten(configFile) + } + + /** True once an operator master exists on this machine. */ + fun exists(): Boolean = load()?.master != null + + /** Load (or, on first use, create + persist) the operator master private key. */ + private fun masterPriv(): ByteArray { + load()?.master?.let { return secrets.resolve(it).hexToByteArray() } + val kp = KeyPair() + val pub = kp.pubKey.toHexKey() + val secret = secrets.store(pub, kp.privKey!!.toHexKey()) + save(Config(masterPubKey = pub, master = secret)) + System.err.println("[operator] created operator master ${pub.take(8)}… at ${configFile.path}") + return kp.privKey!! + } + + /** The operator master pubkey, creating the master on first use. */ + fun masterPubKey(): HexKey { + masterPriv() + return load()!!.masterPubKey + } + + /** + * The deterministic service key for [observerHex], recording the observer → + * provider-pubkey mapping in the manifest. The counter loop only ever runs + * once in practice — it's a guard for the ~2^-128 chance a sha256 output isn't + * a valid secp256k1 scalar. + */ + fun serviceKey(observerHex: HexKey): KeyPair { + val master = masterPriv() + var counter = 0 + while (true) { + val material = master + "$DERIVATION_LABEL$observerHex:$counter".encodeToByteArray() + val kp = runCatching { KeyPair(privKey = sha256(material)) }.getOrNull() + if (kp?.privKey != null) { + recordProvider(observerHex, kp.pubKey.toHexKey()) + return kp + } + counter++ + } + } + + private fun recordProvider( + observerHex: HexKey, + providerPubKey: HexKey, + ) { + val cfg = load() ?: return + if (cfg.providers[observerHex]?.providerPubKey == providerPubKey) return + cfg.providers[observerHex] = ProviderRecord(providerPubKey) + save(cfg) + } + + /** Relays the operator publishes all its 30382 cards + retractions to. */ + fun operatorRelays(): Set = + load() + ?.relays + .orEmpty() + .mapNotNull { RelayUrlNormalizer.normalizeOrNull(it) } + .toSet() + + fun setRelays(urls: List) { + masterPriv() // make sure the config (and master) exists first + save(load()!!.copy(relays = urls)) + } + + fun providers(): Map = load()?.providers.orEmpty() + + companion object { + private const val DIR_NAME = "operator" + private const val CONFIG_NAME = "operator.json" + private const val DERIVATION_LABEL = "graperank-provider:" + } +} From 840f8f3153d546a414c915b92c6acb3b62215ac0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 14:23:26 +0000 Subject: [PATCH 44/58] feat(cli): publish 30382 cards under per-observer service keys, reconciled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewire `graperank --publish` onto the operator-key model: - Sign each observer's kind:30382 cards with the dedicated service key derived for that observer (OperatorKeys), not the account key — a stable per-observer identity so re-signing replaces the addressable prior card. - Publish to the operator's configured relay(s) (new `graperank operator relay ` sub-verb; --publish-relay still overrides). Errors clearly if unset. - Three-way reconciliation against what the provider key already published: upsert cards whose rank tag string changed (or are new), skip unchanged, and RETRACT (kind:5, same service key, addressable `a`-tag, chunked under the message cap) any existing card whose target is no longer publishable — dropped from the graph, or below the cutoff. - Raise the default publish cutoff to rank >= 2 (drops the barely-trusted tail); the retract rule removes any now-sub-cutoff cards. - When we hold the observer's key (observer == active account), publish/refresh their kind:10040 pointing 30382:rank -> providerPubkey at the operator relay, to their outbox — the pointer clients follow to find the cards. Adds `operator [status|relay|providers]` for managing the machine's operator. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 237 +++++++++++++++--- 1 file changed, 199 insertions(+), 38 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 07af50192f..0d3a38af35 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -32,6 +32,7 @@ import com.vitorpamplona.quartz.experimental.graperank.GrapeRankParams import com.vitorpamplona.quartz.experimental.graperank.TrustGraphBuilder import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -96,6 +97,11 @@ object GrapeRankCommand { // Concurrent publishes when writing NIP-85 cards. private const val PUBLISH_CONCURRENCY = 16 + // Addressable coordinates cited per kind:5 retraction. Each `a` tag is + // ~130 bytes (30382:<64hex>:<64hex>), so 500 keeps the deletion frame well + // under the common 256KB relay message cap. + private const val DELETE_PER_EVENT = 500 + // Times we re-query an unreachable user's outbox before giving up on it, so // the crawl still terminates on a finite graph. private const val MAX_OUTBOX_ATTEMPTS = 3 @@ -176,6 +182,7 @@ object GrapeRankCommand { when (tail.firstOrNull()) { "register" -> register(dataDir, tail.drop(1).toTypedArray()) "providers" -> providers(dataDir, tail.drop(1).toTypedArray()) + "operator" -> operator(dataDir, tail.drop(1).toTypedArray()) else -> run(dataDir, tail) } @@ -196,7 +203,10 @@ object GrapeRankCommand { val diagnose = args.bool("diagnose") val timeoutMs = args.longFlag("timeout", 10L) * 1000 val doPublish = args.bool("publish") - val minRank = args.intFlag("min-rank", 1) + // Publish cutoff: only cards with rank >= this are published; existing + // cards for targets below it (or gone from the graph) are retracted. Rank + // is round(score*100), so 2 drops the ~0.015-and-below barely-trusted tail. + val minRank = args.intFlag("min-rank", 2) val publishLimit = args.intFlag("publish-limit", 500) val publishRelaysArg = args.flag("publish-relay") // Benchmark: build + sign one kind:30382 card per scored user (rank >= @@ -647,44 +657,75 @@ object GrapeRankCommand { ) if (doPublish) { + // The cards for THIS observer are signed by a dedicated, stable + // per-observer service key derived from the machine's operator + // master (see OperatorKeys) — not the account key. Same key across + // runs means re-signing a card replaces the addressable prior one. + val opKeys = ctx.dataDir.operatorKeys() + val serviceKey = opKeys.serviceKey(observer) + val serviceSigner = NostrSignerInternal(serviceKey) + val providerPubkey = serviceKey.pubKey.toHexKey() + result["provider_pubkey"] = providerPubkey + + // Cards go to the operator's own relay(s), where the whole + // trusted-assertion set lives; --publish-relay overrides. val relays = publishRelaysArg ?.split(",") ?.mapNotNull { RelayUrlNormalizer.normalizeOrNull(it.trim()) } ?.toSet() ?.takeIf { it.isNotEmpty() } - ?: ctx.outboxRelays() - - // The rank tag string already published for each target (read back - // from the store, which holds our own prior cards), newest per target. - val publishedRankValues = publishedRankTagValues(ctx) - - val candidates = - rankedIds - .filter { rankOf(scores[it]) >= minRank } - .map { graph.pubkeyOf(it) to rankOf(scores[it]) } - // Only sign a new card when the rank TAG VALUE STRING would change. - // `RankTag.assemble(rank)` writes `rank.toString()`, so we diff that - // exact string against the one on the newest stored card. Unchanged - // scores are skipped — no new event id — so clients that sync by id - // only ever download the ranks that actually moved. - val changed = candidates.filter { (target, rank) -> publishedRankValues[target] != rank.toString() } - val toPublish = changed.take(publishLimit) - - result["skipped_unchanged"] = candidates.size - changed.size - if (changed.size > toPublish.size) { - result["publish_truncated"] = changed.size - toPublish.size - } + ?: opKeys.operatorRelays() if (relays.isEmpty()) { result["published"] = 0 - result["publish_error"] = "no publish relays configured" + result["publish_error"] = "no operator relay configured — run `amy graperank operator relay ` or pass --publish-relay" } else { - val (ok, rejected) = publishCards(ctx, toPublish, relays) + // Reconcile what the algorithm says should exist against what + // this provider key has already published (newest card per + // target, read back from the store). + val existing = existingCards(ctx, providerPubkey) + + val publishable = + rankedIds + .filter { rankOf(scores[it]) >= minRank } + .map { graph.pubkeyOf(it) to rankOf(scores[it]) } + val publishableTargets = publishable.mapTo(HashSet()) { it.first } + + // Upsert: publishable targets whose rank tag STRING would change + // (or that have no card yet). RankTag.assemble writes + // rank.toString(), so we diff that exact string — an unchanged + // score is skipped, so clients only sync ranks that moved. + val changed = publishable.filter { (target, rank) -> existing[target]?.let(::rankTagValue) != rank.toString() } + val toUpsert = changed.take(publishLimit) + + // Delete: existing cards whose target is no longer publishable — + // it dropped out of the graph, or fell below the cutoff (e.g. a + // rank-0/1 card we would no longer publish). We won't leave a + // stale assertion standing, so we retract it with a kind:5. + val toDelete = existing.filterKeys { it !in publishableTargets }.values.toList() + + result["skipped_unchanged"] = publishable.size - changed.size + if (changed.size > toUpsert.size) { + result["publish_truncated"] = changed.size - toUpsert.size + } + + val (ok, rejected) = publishCards(ctx, serviceSigner, toUpsert, relays) + val (deleted, deleteRejected) = publishDeletions(ctx, serviceSigner, toDelete, relays) + result["published"] = ok result["publish_rejected"] = rejected + result["deleted"] = deleted + result["delete_rejected"] = deleteRejected result["published_kind"] = ContactCardEvent.KIND result["published_to"] = relays.map { it.url } + + // Help the observer point clients at this provider: publish their + // kind:10040 (30382:rank -> providerPubkey @ operator relay) to + // their outbox — but only when we actually hold their key. + maybePublishObserverProviderList(ctx, observer, providerPubkey, relays.first())?.let { + result["observer_10040"] = it + } } } @@ -742,6 +783,60 @@ object GrapeRankCommand { } } + /** + * `amy graperank operator [status | relay … | providers]` + * + * Manage the machine's operator keys used to sign trusted-assertion cards. + * - `status` (default): master pubkey, configured relay(s), provider count. + * - `relay …`: set the operator relay(s) the cards + retractions publish + * to; creates the operator master on first use. + * - `providers`: the observer -> provider-pubkey mapping learned so far. + */ + private fun operator( + dataDir: DataDir, + rest: Array, + ): Int { + val opKeys = dataDir.operatorKeys() + return when (rest.firstOrNull()) { + "relay" -> { + val urls = rest.drop(1).filter { it.isNotBlank() } + val normalized = urls.mapNotNull { RelayUrlNormalizer.normalizeOrNull(it) } + if (normalized.isEmpty()) return Output.error("bad_args", "usage: amy graperank operator relay [ …]") + opKeys.setRelays(urls) + Output.emit(mapOf("master_pubkey" to opKeys.masterPubKey(), "relays" to normalized.map { it.url })) + 0 + } + + "providers" -> { + Output.emit( + mapOf( + "master_pubkey" to if (opKeys.exists()) opKeys.masterPubKey() else null, + "providers" to opKeys.providers().map { (observer, rec) -> mapOf("observer" to observer, "provider_pubkey" to rec.providerPubKey) }, + ), + ) + 0 + } + + null, "status" -> { + if (!opKeys.exists()) { + Output.emit(mapOf("initialized" to false)) + } else { + Output.emit( + mapOf( + "initialized" to true, + "master_pubkey" to opKeys.masterPubKey(), + "relays" to opKeys.operatorRelays().map { it.url }, + "providers" to opKeys.providers().size, + ), + ) + } + 0 + } + + else -> Output.error("bad_args", "unknown operator subcommand '${rest.first()}' (status | relay | providers)") + } + } + /** * `amy graperank register [PROVIDER] [--service KIND:TAG] [--relay URL] [--private]` * @@ -1133,26 +1228,34 @@ object GrapeRankCommand { * a `rank` tag (plus the d-tag target), so this single tag's value fully * decides whether the event would differ — see the publish gate. */ - private suspend fun publishedRankTagValues(ctx: Context): Map { - val self = ctx.identity.pubKeyHex - return ctx.store - .query(Filter(kinds = listOf(ContactCardEvent.KIND), authors = listOf(self))) + private suspend fun existingCards( + ctx: Context, + providerPubkey: HexKey, + ): Map = + ctx.store + .query(Filter(kinds = listOf(ContactCardEvent.KIND), authors = listOf(providerPubkey))) .filterIsInstance() .groupBy { it.aboutUser() } .mapNotNull { (target, cards) -> val t = target ?: return@mapNotNull null - val newest = cards.maxByOrNull { it.createdAt } ?: return@mapNotNull null - val rankValue = - newest.tags.firstNotNullOfOrNull { tag -> - if (tag.size > 1 && tag[0] == RankTag.TAG_NAME) tag[1] else null - } ?: return@mapNotNull null - t to rankValue + t to (cards.maxByOrNull { it.createdAt } ?: return@mapNotNull null) }.toMap() - } - /** Build + publish one NIP-85 kind:30382 card per user, bounded-concurrently. */ + /** + * The raw `rank` tag value string on a card — what a client diffs. We compare + * this against `rank.toString()` (what RankTag.assemble writes) so an unchanged + * score never produces a new signature. Our cards carry only a `rank` tag (plus + * the d-tag target), so this one value decides whether the event would differ. + */ + private fun rankTagValue(card: ContactCardEvent): String? = + card.tags.firstNotNullOfOrNull { tag -> + if (tag.size > 1 && tag[0] == RankTag.TAG_NAME) tag[1] else null + } + + /** Build + publish one NIP-85 kind:30382 card per user, bounded-concurrently, signed by [signer]. */ private suspend fun publishCards( ctx: Context, + signer: NostrSigner, cards: List>, relays: Set, ): Pair { @@ -1167,7 +1270,7 @@ object GrapeRankCommand { val card = ContactCardEvent.create( targetUser = pubkey, - signer = ctx.signer, + signer = signer, publicInitializer = { add(RankTag.assemble(rank)) }, ) ctx.publish(card, relays) @@ -1180,4 +1283,62 @@ object GrapeRankCommand { } return published to rejected } + + /** + * Retract stale cards with NIP-09 kind:5 deletions signed by [signer] (the same + * service key that signed the cards). Batches several addressable coordinates + * per deletion — chunked so the kind:5 frame stays under the relay message cap — + * and each carries the card's `a` tag (30382:provider:target), so re-publishing + * a newer version later isn't blocked. Returns (deleted, rejected) card counts. + */ + private suspend fun publishDeletions( + ctx: Context, + signer: NostrSigner, + cards: List, + relays: Set, + ): Pair { + if (cards.isEmpty()) return 0 to 0 + var deleted = 0 + var rejected = 0 + for (chunk in cards.chunked(DELETE_PER_EVENT)) { + val event = signer.sign(DeletionEvent.build(chunk)) + val ack = ctx.publish(event, relays) + if (ack.values.any { it }) deleted += chunk.size else rejected += chunk.size + } + return deleted to rejected + } + + /** + * If the active account IS the observer (so we hold their key), publish/refresh + * their kind:10040 declaring `30382:rank` -> [providerPubkey] at [relay], to + * their own outbox relays — the NIP-85 pointer a client follows to find these + * cards. Returns the 10040 event id, or null when we don't hold the key (a + * third-party observer must add the provider to their 10040 out-of-band). + */ + private suspend fun maybePublishObserverProviderList( + ctx: Context, + observer: HexKey, + providerPubkey: HexKey, + relay: NormalizedRelayUrl, + ): String? { + if (observer != ctx.identity.pubKeyHex) return null + val service = ProviderTypes.rank + val outbox = ctx.outboxRelays() + val latest = fetchLatestProviderList(ctx, observer, outbox, 8_000) + val alreadyListed = + latest?.serviceProviders()?.any { + it.service == service && it.pubkey == providerPubkey && it.relayUrl == relay + } ?: false + if (alreadyListed) return latest?.id + + val tag = ServiceProviderTag(service, providerPubkey, relay) + val event = + if (latest == null) { + TrustProviderListEvent.create(tag, isPrivate = false, signer = ctx.signer) + } else { + TrustProviderListEvent.add(latest, tag, isPrivate = false, signer = ctx.signer) + } + ctx.publish(event, outbox) + return event.id + } } From be6ff5456d80c2b48f6f90103e1034186ca505e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 14:39:32 +0000 Subject: [PATCH 45/58] docs(cli): document graperank operator keys + publish reconciliation README + amy usage: add the `graperank operator [status|relay|providers]` sub-verb, update the `graperank --publish` description to the per-observer service-key model (sign with a derived key, publish to the operator relay, reconcile new/changed/skip/retract, cutoff rank>=2, NIP-09-drop retracted reports), and add a 'Publishing GrapeRank scores' section explaining the operator master, deterministic per-observer key derivation, and the kind:10040 discovery wiring. --- cli/README.md | 33 ++++++++++++++++++- .../com/vitorpamplona/amethyst/cli/Main.kt | 12 +++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/cli/README.md b/cli/README.md index de4afc3822..6048f6882b 100644 --- a/cli/README.md +++ b/cli/README.md @@ -384,10 +384,41 @@ HTTP endpoint. Reuses quartz's `Nip86Client` and the shared `Nip86Retriever` | `amy profile show [USER]` | Print kind:0 metadata. USER accepts npub/nprofile/hex/NIP-05; defaults to self. | | `amy profile edit --name … --about … --picture URL …` | Patch and re-publish your kind:0. | | `amy follow USER` / `amy unfollow USER` | Add/remove USER from your kind:3 contact list (fetches the freshest list first). | -| `amy graperank [OBSERVER] [--offline] [--publish]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph. Exhaustively crawls each user's kind:10002 outbox for their latest kind:3/10000/1984 until every discovered user is checked (no user cap); optionally publishes results as NIP-85 kind:30382 cards (unchanged ranks are skipped). | +| `amy graperank [OBSERVER] [--offline] [--publish] [--min-rank N] [--publish-relay URL]` | Compute GrapeRank web-of-trust scores (0..1) over the follow/mute/report graph. Exhaustively crawls each user's kind:10002 outbox for their latest kind:3/10000/1984 until every discovered user is checked (no user cap), dropping reports the author retracted via NIP-09. With `--publish`, reconciles NIP-85 kind:30382 cards signed by a per-observer **service key**: publishes changed/new ranks (cutoff `--min-rank`, default 2), skips unchanged, and **retracts** (kind:5) any card whose target left the graph or fell below the cutoff. | +| `amy graperank operator [status \| relay … \| providers]` | Manage the machine's operator keys (independent of any account, under `~/.amy/operator/`). `relay` sets where cards + retractions publish; `status` shows the master pubkey and relays; `providers` lists the observer → service-pubkey map. | | `amy graperank register [PROVIDER] [--service KIND:TAG] [--relay URL]` | Declare a NIP-85 provider in your kind:10040 so clients can discover it (default: self as the `30382:rank` provider). | | `amy graperank providers [USER]` | List a user's declared NIP-85 trusted providers (public + your own private entries). | +#### Publishing GrapeRank scores (NIP-85) + +Ranks are published as kind:30382 cards, but **not** under your account key. A +machine holds one **operator master** seed (`~/.amy/operator/`, stored via the +same `--secret-backend` as accounts, independent of any account). From it a +distinct, deterministic **service key** is derived per observer: + +``` +serviceKey(observer) = sha256(masterPriv ‖ "graperank-provider:" ‖ observerHex) +``` + +Because kind:30382 is addressable (`pubkey + d-tag`), the stable per-observer key +means re-publishing **replaces** a target's card instead of orphaning it — and +losing everything but the master seed still re-derives every key. Set up once and +publish: + +```bash +amy graperank operator relay wss://relay.example.com # where all cards live +amy graperank --publish # sign with the observer's service key +``` + +Each publish **reconciles** against what the service key already published: new or +changed ranks (≥ `--min-rank`, default 2) are signed and sent; unchanged ranks are +skipped (no new event id); and any card whose target dropped out of the graph or +fell below the cutoff is **retracted** with a kind:5. When the observer is your +own account (we hold the key), Amy also writes their kind:10040 pointing +`30382:rank → serviceKey @ operator relay` to their outbox, so clients can find +the cards. For a third-party observer, `graperank operator providers` prints the +`observer → service-pubkey` mapping to wire their kind:10040 out-of-band. + ### Direct messages (NIP-17) | Command | What it does | diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 8f4e077eb9..d9be83fb3b 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -537,9 +537,15 @@ private fun printUsage() { | --diagnose logs slow/failed relays on timeout). | [--publish] [--min-rank N] OBSERVER: npub|nprofile|hex|name@domain (default: | [--publish-limit N] [--publish-relay URL] active account). --offline scores from the local - | store only. --publish writes NIP-85 kind:30382 - | cards (rank = round(score*100)) for each user at - | or above --min-rank (unchanged ranks skipped). + | store only. --publish reconciles NIP-85 kind:30382 + | cards signed by a per-observer service key: sends + | new/changed ranks >= --min-rank (default 2), skips + | unchanged, and retracts (kind:5) any card whose + | target left the graph or fell below the cutoff. + | graperank operator [status|relay … manage the machine's operator keys (~/.amy/operator/, + | |providers] independent of accounts): relay sets where cards + + | retractions publish; status shows master + relays; + | providers lists observer -> service-pubkey. | graperank register [PROVIDER] declare a NIP-85 provider in your kind:10040 so | [--service KIND:TAG] [--relay URL] clients can discover it (default: self as the | [--private] 30382:rank provider at your first outbox relay). From 6bbc1b7d226b6c03b45abeac06f34b82c556f9ac Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 14:42:35 +0000 Subject: [PATCH 46/58] fix(cli): cap kind:5 retractions at 400 a-tags to stay under 64KB events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each addressable coordinate is ~130 bytes, so 500 pushed the deletion event to ~65KB — over the 64KB event-size cap many relays enforce (stricter than the 256KB message cap). Drop DELETE_PER_EVENT to 400 (~52KB). --- .../amethyst/cli/commands/GrapeRankCommand.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 0d3a38af35..955948b741 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -98,9 +98,10 @@ object GrapeRankCommand { private const val PUBLISH_CONCURRENCY = 16 // Addressable coordinates cited per kind:5 retraction. Each `a` tag is - // ~130 bytes (30382:<64hex>:<64hex>), so 500 keeps the deletion frame well - // under the common 256KB relay message cap. - private const val DELETE_PER_EVENT = 500 + // ~130 bytes (30382:<64hex>:<64hex>), so 400 keeps the whole event ~52KB — + // under the 64KB *event* size many relays cap at (stricter than the 256KB + // message cap). + private const val DELETE_PER_EVENT = 400 // Times we re-query an unreachable user's outbox before giving up on it, so // the crawl still terminates on a finite graph. From 667358a06a40c0f36828b3b96b0652aa28598c2a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 15:41:31 +0000 Subject: [PATCH 47/58] refactor(quartz): extract GrapeRankDataCrawler to commonMain The web-of-trust crawl (~400 lines: outbox routing, sharded backbone sweep, Phase-B worker pool, relay-list discovery, report-deletion fetch, warm pool) was making the CLI's GrapeRankCommand unmaintainably large. Move it into a reusable, KMP-portable GrapeRankDataCrawler in quartz commonMain. The crawler takes a NostrClient + IEventStore + AdaptiveRelayLimiter, injected relay policy (discovery + content-fallback sets, since those defaults live in app code, not the protocol library), and a log callback; it streams contact lists into a TrustGraphBuilder and returns crawl Stats. GrapeRankCommand shrinks to arg-parsing + offline load + scoring + publish + sub-verbs, delegating the online path to the crawler. To reach commonMain (portable to every target, incl. iOS): - Add ConcurrentMap / ConcurrentSet expect classes under utils/concurrent, with jvmAndroid actuals (java.util.concurrent) and native actuals (copy-on-write over kotlin.concurrent.atomics.AtomicReference, mirroring ConcurrentHashCache). commonMain has no ConcurrentHashMap, and the crawl's producer/consumer/drain- worker state needs atomic getOrPut/merge plus a concurrent set. - Move AdaptiveRelayLimiter and DrainFailure/classifyDrainFailure from cli to quartz commonMain (java atomics -> kotlin.concurrent.atomics, ConcurrentHashMap -> ConcurrentMap, System.currentTimeMillis -> TimeUtils.nowMillis, stderr -> Log). - The gated drain (REQ-size splitting, per-relay permits, verify+store) moves into the crawler; Context.drain loses its now-unused gatePerRelay path. Net: cli -1077 lines; the crawler + relay machinery are now reusable by the Android app. Adds ConcurrentCollectionsTest; verified via JVM + commonMain metadata compile, the wot/graperank suites, and a bounded live crawl. --- .../com/vitorpamplona/amethyst/cli/Context.kt | 217 +---- .../amethyst/cli/commands/GrapeRankCommand.kt | 639 +------------- .../graperank/GrapeRankDataCrawler.kt | 813 ++++++++++++++++++ .../accessories}/AdaptiveRelayLimiter.kt | 72 +- .../relay/client/accessories/DrainFailure.kt | 77 ++ .../quartz/utils/concurrent/ConcurrentMap.kt | 68 ++ .../quartz/utils/concurrent/ConcurrentSet.kt | 43 + .../concurrent/ConcurrentCollectionsTest.kt | 108 +++ .../concurrent/ConcurrentMap.jvmAndroid.kt | 51 ++ .../concurrent/ConcurrentSet.jvmAndroid.kt | 35 + .../utils/concurrent/ConcurrentMap.native.kt | 80 ++ .../utils/concurrent/ConcurrentSet.native.kt | 46 + 12 files changed, 1406 insertions(+), 843 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt rename {cli/src/main/kotlin/com/vitorpamplona/amethyst/cli => quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories}/AdaptiveRelayLimiter.kt (80%) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/DrainFailure.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentCollectionsTest.kt create mode 100644 quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt create mode 100644 quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.jvmAndroid.kt create mode 100644 quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.native.kt create mode 100644 quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.native.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 4214dac422..531fcbf6c5 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -41,6 +41,9 @@ import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.crypto.verify import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.AdaptiveRelayLimiter +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.DrainFailure +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.classifyDrainFailure import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPagesFromPool import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.publishAndConfirmDetailed import com.vitorpamplona.quartz.nip01Core.relay.client.auth.RelayAuthenticator @@ -74,86 +77,13 @@ import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.selects.select import kotlinx.coroutines.withTimeoutOrNull import okhttp3.Dispatcher import okhttp3.OkHttpClient -import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit -/** - * Why a relay could not be used for a drain — when the reason is worth acting on. - * - * - [HARD]: the relay answered wrong, or cannot exist. A bad HTTP upgrade (not a - * websocket / dead status code), an unresolvable domain, or a TLS misconfig. - * This will not fix itself, so one strike is enough to drop it. - * - [TRANSIENT]: a failure that might clear — connection refused / reset, host - * unreachable, or a temporary 429/5xx on the upgrade. Struck a few times - * before we give up. - * - * A pure connect **timeout** is neither. The relay is most likely just busy, so - * we retry it and never mark it dead — [classifyDrainFailure] returns null for - * it (and for any non-failure terminal reason). - */ -enum class DrainFailure { HARD, TRANSIENT } - -/** - * Classify a [Context.drain] per-relay terminal reason. Returns null when the - * relay should simply be retried (a timeout, or a non-failure like eose/closed). - * The reason shape is `cannot:` for a connect failure (see - * `BasicRelayClient.onCannotConnect`), or `eose` / `closed:…` / `timeout`. - */ -fun classifyDrainFailure(reason: String): DrainFailure? { - if (!reason.startsWith("cannot")) return null - val m = reason.removePrefix("cannot:").lowercase() - // The message now carries the exception class name (see BasicRelayClient), so - // we can key on the stable *type* rather than localized message text. - // Busy, not dead: a connect/read timeout means the handshake just didn't - // finish in time. Retry it — the relay is probably fine, only slow or loaded. - if ("timeout" in m || "timed out" in m) return null // SocketTimeoutException, etc. - // Cannot ever work: unresolvable domain (DNS) or a TLS misconfiguration. - // Dead for good — one strike is enough. - if ("unknownhost" in m || // UnknownHostException - "unable to resolve host" in m || - "no address associated" in m || - "nodename nor servname" in m || - "sslhandshake" in m || // SSLHandshakeException - "sslpeerunverified" in m || - "sslexception" in m || - "certificate" in m || // CertificateException - "trust anchor" in m || - "certpath" in m - ) { - return DrainFailure.HARD - } - // Wrong HTTP upgrade. Usually a misconfigured endpoint (not a relay), but - // 429 / 5xx mean "busy, come back later", so those stay transient. - if ("server misconfigured" in m || "not a websocket" in m || "expected http 101" in m) { - val transientCode = Regex("response: (429|500|502|503|504)").containsMatchIn(m) - return if (transientCode) DrainFailure.TRANSIENT else DrainFailure.HARD - } - // Refused / reset / unreachable / anything else: might clear — retry a few times. - return DrainFailure.TRANSIENT -} - -/** - * Max total "entries" (authors + ids + tag values) allowed in a single REQ frame. - * A REQ carries all of a subscription's filters at once, and each entry is a - * ~67-byte JSON hex string, so 2500 entries ≈ 167KB — comfortably under the 256KB - * message cap most relays enforce (and reject a frame over, dropping every author - * in it). [Context.drainGated] groups a relay's filters to stay within this. - */ -private const val MAX_REQ_ENTRIES = 2500 - -/** Count the size-driving entries in a filter: authors, ids, and tag values. */ -fun filterEntries(f: Filter): Int = - (f.authors?.size ?: 0) + - (f.ids?.size ?: 0) + - (f.tags?.values?.sumOf { it.size } ?: 0) + - (f.tagsAll?.values?.sumOf { it.size } ?: 0) - /** * Per-invocation wiring. Each CLI run constructs a Context, does its work, * and then closes it — no daemon. @@ -550,10 +480,8 @@ class Context( timeoutMs: Long = 8_000, diagnoseSlow: Boolean = false, deadOut: MutableMap? = null, - gatePerRelay: Boolean = false, ): List> { if (filters.isEmpty()) return emptyList() - if (gatePerRelay) return drainGated(filters, timeoutMs, diagnoseSlow, deadOut) val eventChannel = Channel>(UNLIMITED) // Carries the terminal reason per relay so a timeout can distinguish a slow // relay (never terminal) from a connect failure / CLOSED. @@ -636,145 +564,6 @@ class Context( return collected } - /** - * Per-relay-gated variant of [drain] used by the crawl. Instead of one - * subscription spanning every relay, each relay gets its own subscription - * held behind [relayLimiter], so we never exceed the relay's adaptive - * concurrent-subscription cap. A relay whose cap is full simply waits for one - * of our other subscriptions on it to finish before its REQ goes out; relays - * we haven't upset run at the full starting cap and never wait. - * - * Semantics match [drain] otherwise: verify+store on a single consumer - * (so store writes stay serialized), return events tagged by relay, and - * report hard connect failures into [deadOut]. - */ - private suspend fun drainGated( - filters: Map>, - timeoutMs: Long, - diagnoseSlow: Boolean, - deadOut: MutableMap?, - ): List> { - val eventChannel = Channel>(UNLIMITED) - - // Split each relay's filters into REQ-sized groups. A REQ frame carries ALL - // its filters at once, so a popular relay routed thousands of authors would - // otherwise produce a multi-MB frame that most relays reject outright - // ("message too large") — silently dropping every author in it. Grouping by - // total entry count keeps each REQ well under the common 256KB cap; a relay - // with more authors just gets several smaller REQs, each its own gated sub. - val units = ArrayList>>() - for ((relay, relayFilters) in filters) { - var group = ArrayList() - var entries = 0 - for (f in relayFilters) { - val fe = filterEntries(f) - if (group.isNotEmpty() && entries + fe > MAX_REQ_ENTRIES) { - units.add(relay to group) - group = ArrayList() - entries = 0 - } - group.add(f) - entries += fe - } - if (group.isNotEmpty()) units.add(relay to group) - } - - // Per-relay failure classification, HARD winning over TRANSIENT across a - // relay's several REQ-groups; plus which relays stalled to a timeout. - val failures = ConcurrentHashMap() - val timedOut = ConcurrentHashMap.newKeySet() - - val collected = mutableListOf>() - coroutineScope { - // Single consumer: verify+store serially. One writer, so SeenIds' - // single-writer contract holds. The outbox model (and the wide relay- - // list broadcast) delivers the SAME event from many relays at once; skip - // a duplicate BEFORE the expensive Schnorr verify+store. An id is marked - // seen only after it verifies, so a forged copy (valid id, bad signature) - // delivered first can't suppress the genuine one that follows. - val consumer = - launch { - val seen = SeenIds(initialSlotsPow2 = 12) - for ((relay, event) in eventChannel) { - if (seen.contains(event.id)) continue - if (verifyAndStore(event)) { - seen.add(event.id) - collected.add(relay to event) - } - } - } - // One gated subscription per (relay, REQ-group). The permit is held for - // the group's whole life, so concurrent subs on a relay never exceed its - // adaptive cap. Each group carries its own subId, listener, and terminal - // signal (relay + subId together identify a group, but a per-group - // listener is simplest). - units - .map { (relay, groupFilters) -> - launch { - relayLimiter.withPermit(relay) { - val subId = newSubId() - val done = CompletableDeferred() - val groupListener = - object : SubscriptionListener { - override fun onEvent( - event: Event, - isLive: Boolean, - r: NormalizedRelayUrl, - forFilters: List?, - ) { - eventChannel.trySend(r to event) - } - - override fun onEose( - r: NormalizedRelayUrl, - forFilters: List?, - ) { - done.complete("eose") - } - - override fun onClosed( - message: String, - r: NormalizedRelayUrl, - forFilters: List?, - ) { - done.complete("closed:$message") - } - - override fun onCannotConnect( - r: NormalizedRelayUrl, - message: String, - forFilters: List?, - ) { - done.complete("cannot:$message") - } - } - client.subscribe(subId, mapOf(relay to groupFilters), groupListener) - try { - val reason = withTimeoutOrNull(timeoutMs) { done.await() } ?: "timeout" - if (reason == "timeout") timedOut.add(relay) - classifyDrainFailure(reason)?.let { kind -> - failures.merge(relay, kind) { a, b -> - if (a == DrainFailure.HARD || b == DrainFailure.HARD) DrainFailure.HARD else DrainFailure.TRANSIENT - } - } - } finally { - client.unsubscribe(subId) - } - } - } - }.joinAll() - // All subscriptions are torn down; no more events can arrive. Close the - // channel so the consumer drains what's buffered and completes. - eventChannel.close() - consumer.join() - } - if (diagnoseSlow && timedOut.isNotEmpty()) { - logSlowDrain(timeoutMs, timedOut, emptyMap(), collected) - } - deadOut?.putAll(failures) - return collected - } - /** * On a [drain] timeout, report which relays stalled and why — a relay that * never sent EOSE (slow, possibly still streaming) vs one that couldn't be diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 955948b741..39693a5ade 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -23,11 +23,11 @@ package com.vitorpamplona.amethyst.cli.commands import com.vitorpamplona.amethyst.cli.Args import com.vitorpamplona.amethyst.cli.Context import com.vitorpamplona.amethyst.cli.DataDir -import com.vitorpamplona.amethyst.cli.DrainFailure import com.vitorpamplona.amethyst.cli.Output import com.vitorpamplona.amethyst.commons.defaults.Constants import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList import com.vitorpamplona.quartz.experimental.graperank.GrapeRank +import com.vitorpamplona.quartz.experimental.graperank.GrapeRankDataCrawler import com.vitorpamplona.quartz.experimental.graperank.GrapeRankParams import com.vitorpamplona.quartz.experimental.graperank.TrustGraphBuilder import com.vitorpamplona.quartz.nip01Core.core.Event @@ -44,7 +44,6 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip09Deletions.DeletionIndex import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent import com.vitorpamplona.quartz.nip56Reports.ReportEvent -import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent import com.vitorpamplona.quartz.nip85TrustedAssertions.list.TrustProviderListEvent import com.vitorpamplona.quartz.nip85TrustedAssertions.list.serviceProviders import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ProviderTypes @@ -52,18 +51,10 @@ import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProvider import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceType import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.cancel -import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.joinAll -import kotlinx.coroutines.launch -import java.util.concurrent.ConcurrentHashMap -import kotlin.coroutines.coroutineContext import kotlin.math.roundToInt /** @@ -91,9 +82,6 @@ import kotlin.math.roundToInt * - `amy graperank providers [USER]` — list a user's trusted providers. */ object GrapeRankCommand { - // Authors per REQ filter — keeps individual subscriptions within relay limits. - private const val AUTHORS_PER_FILTER = 300 - // Concurrent publishes when writing NIP-85 cards. private const val PUBLISH_CONCURRENCY = 16 @@ -103,54 +91,8 @@ object GrapeRankCommand { // message cap). private const val DELETE_PER_EVENT = 400 - // Times we re-query an unreachable user's outbox before giving up on it, so - // the crawl still terminates on a finite graph. - private const val MAX_OUTBOX_ATTEMPTS = 3 - - // Users whose outboxes we fetch in a single drain. Draining thousands of - // distinct outbox relays at once saturates connections and times out - // (empirically ~250 users/drain succeeds, ~17k fails); keep the fan-out small. - private const val USER_BATCH = 256 - - // Global content-drain fan-out — how many outbox batches we drain at once. - // This is a GLOBAL bound (memory / open sockets); the per-relay concurrency - // limit is enforced separately and adaptively by [AdaptiveRelayLimiter] - // (drains run with gatePerRelay=true), which starts every relay at 100 - // concurrent subs and demotes only the ones that complain (100 → 20 → 10). - // The two compose: at fan-out 24 a well-behaved relay runs at up to 24 - // concurrent subs, while a relay that pushes back is cut to 20 then 10 — - // below the global bound, so the ladder actually bites. A higher global - // fan-out (measured at 48) *re-floods* the busy hubs faster than demotion - // catches up ("max concurrent subscription count reached" spikes) and - // regressed wall-time, so keep the global bound moderate and let the - // per-relay cap do the targeting. - private const val DRAIN_CONCURRENCY = 24 - - // Sharded backbone sweep: instead of asking every popular relay for the - // same full author list (N× redundant), split the still-missing authors - // into SHARD_RELAYS lists and send each to ONE of the top relays. Authors a - // relay doesn't have rotate onto a different relay next pass, up to - // SHARD_ROTATIONS times, so over a few passes each author is tried on - // several popular relays. Once the remaining set drops below - // SHARD_BROADCAST_THRESHOLD it's cheap to just ask them all at once. - private const val SHARD_RELAYS = 10 - private const val SHARD_ROTATIONS = 6 - private const val SHARD_BROADCAST_THRESHOLD = 2000 - - // The small-remainder broadcast (once a sweep is under the threshold) goes to - // this many top live relays, not just the SHARD_RELAYS the rotation used — - // a user's kind:3 is often mirrored on a busy relay ranked below the top 10, - // which is where the old last-mile pass found its stragglers. - private const val BROADCAST_RELAYS = 60 - - // A relay that fails to CONNECT this many times is treated as dead and - // dropped from routing, so we stop paying the drain timeout on it. Kept above - // 1 so a single transient connect blip doesn't evict a relay for the run. - private const val MAX_DEAD_STRIKES = 3 - // Broad, big general relays that carry kind:10002 for many users, added to the - // discovery set to raise the odds of resolving a stranger's outbox. Every entry - // is NIP-11 liveness-checked — dead relays only add timeouts. + // crawler's discovery set to raise the odds of resolving a stranger's outbox. private val EXTRA_DISCOVERY_RELAYS: Set = listOf( "wss://relay.damus.io", @@ -160,20 +102,6 @@ object GrapeRankCommand { "wss://eden.nostr.land", ).mapNotNull { RelayUrlNormalizer.normalizeOrNull(it) }.toSet() - // How many of the most-used write relays (learned from everyone's kind:10002) - // to keep as the known-good backbone for retrying users we couldn't reach. - private const val BACKBONE_SIZE = 30 - - // Warm pool: hold a persistent, do-nothing subscription open to the busiest - // WARM_POOL_SIZE relays for the whole crawl, so the connections we reuse - // every round survive the between-round routing gaps (and niche-relay churn) - // instead of being dropped ~300ms after a wave ends and reconnected next - // round. The filter matches an impossible event id, so the relay EOSEs - // immediately and streams nothing — it only keeps the socket warm. - private const val WARM_POOL_SIZE = 20 - private const val WARM_SUB_ID = "graperank-warm" - private val WARM_FILTERS = listOf(Filter(ids = listOf("0".repeat(64)))) - suspend fun dispatch( dataDir: DataDir, tail: Array, @@ -225,354 +153,47 @@ object GrapeRankCommand { ctx.prepare() val observer = observerArg?.let { ctx.requireUserHex(it) } ?: ctx.identity.pubKeyHex - val graphKinds = listOf(ContactListEvent.KIND, MuteListEvent.KIND, ReportEvent.KIND) - // Kinds requested from relays during the crawl: the graph edges PLUS the - // user's own kind:10002. A user's outbox holds the freshest copy of their - // relay list, so folding 10002 into the same query we send their outbox - // keeps our routing current instead of trusting a possibly-stale indexer - // copy. Safe to also pull from popular relays in the sweep — the store - // keeps newest-by-created_at for the replaceable 10002, so the freshest - // always wins regardless of which relay delivered it. - val fetchKinds = graphKinds + AdvertisedRelayListEvent.KIND - - // The graph is built incrementally: contact lists stream straight into a - // compact int-CSR structure and the Event is discarded, so the whole - // network fits in memory without holding millions of kind:3 objects. + // Contact lists stream straight into a compact int-CSR structure as the + // crawl finds them and the Event is discarded, so the whole network fits + // in memory without holding millions of kind:3 objects. val builder = TrustGraphBuilder() - var rounds = 0 - var relaysContactedCount = 0 var contactListsFed = 0 // Wall time to read + deserialize the contact lists out of the store - // (offline path only; online streams them in during the crawl). This - // is the real pre-scoring cost — the int-CSR build afterwards is a - // cheap in-memory pack. + // (offline path only; online streams them in during the crawl). var storeLoadMs: Long? = null - // Wall time to crawl + download the whole graph off the relays - // (online path only) — rounds + last-mile sweep, i.e. everything up - // to the point the graph is fully fetched. This is network-bound and - // dominates a from-scratch run. - var downloadMs: Long? = null + // Crawl telemetry (online path only): rounds, relays contacted, the + // per-hop histogram, and the network-bound download time that dominates a + // from-scratch run. Null on the offline path. + var crawlStats: GrapeRankDataCrawler.Stats? = null - val hopOf = HashMap() if (!offline) { - val crawlStart = System.nanoTime() - // Scope for fire-and-forget relay-list discovery: the wide Tier-2 - // sweep (ensureRelayLists) casts kind:10002 queries across every relay - // we know, but we don't block the crawl on it — its results just - // enrich routing for later rounds. SupervisorJob so one failing sweep - // never cancels the others; cancelled when the crawl finishes. - val bgScope = CoroutineScope(coroutineContext + SupervisorJob()) - val discovered = hashSetOf(observer) - hopOf[observer] = 0 - // Per-user relay hints harvested from the `p`-tag relay hints in the - // contact lists we crawl (A's follow of B says where B writes) — a - // discovery tier below each user's kind:10002 outbox. Concurrent: - // the Phase-B producer reads these while the consumer's ingest writes - // them (see the worker-pool below), so both map and inner sets are - // thread-safe. - val relayHints = ConcurrentHashMap>() - // Users we're finished with this run: we fed their latest kind:3, or - // ran out of retry attempts on an unreachable outbox. - val done = hashSetOf() - // Outbox retry counts. Concurrent: the producer reads (to widen a - // retry's routing) while the consumer increments. - val attempts = ConcurrentHashMap() - val relaysContacted = hashSetOf() - // Known-good relay pool, learned from the crawl itself: how often each - // relay appears as someone's write relay, and which relays actually - // delivered events (so we know they connect and work). The most-common - // live relays become the `backbone` we retry unreachable users against. - val writeRelayFreq = HashMap() - val liveRelays = hashSetOf() - // Relays that failed to connect MAX_DEAD_STRIKES times — dropped - // from all routing so a wave stops eating the timeout on them. - // Concurrent: drain workers strike relays while the producer reads - // deadRelays to prune routing. - val deadRelays = ConcurrentHashMap.newKeySet() - val relayStrikes = ConcurrentHashMap() - - // A relay that HARD-failed (bad domain, TLS misconfig, dead HTTP - // code — see DrainFailure) is dropped on the first strike: it will - // not fix itself. A TRANSIENT failure (refused/reset/unreachable, - // or a 429/5xx) might clear, so it takes MAX_DEAD_STRIKES before we - // give up. Pure timeouts never reach here — the drain treats them as - // busy-retry and does not report them dead at all. - fun recordDead(failed: Map) { - for ((r, kind) in failed) { - when (kind) { - DrainFailure.HARD -> deadRelays.add(r) - DrainFailure.TRANSIENT -> - if (relayStrikes.merge(r, 1, Int::plus)!! >= MAX_DEAD_STRIKES) deadRelays.add(r) - } - } - } - - // The busiest live relays we've learned, excluding the dead ones. - fun topLiveRelays(cap: Int): List = - writeRelayFreq.entries - .asSequence() - .filter { it.key in liveRelays && it.key !in deadRelays } - .sortedByDescending { it.value } - .take(cap) - .map { it.key } - .toList() - - // Feed a user's contact list into the graph, harvest relay hints, stamp - // the hop distance of newly-seen follows, and add them to the frontier. - // Called once per user (guarded by `done`). Returns the count of - // newly-discovered users. - fun ingest( - source: HexKey, - contacts: ContactListEvent, - ): Int { - val nextHop = (hopOf[source] ?: 0) + 1 - val follows = ArrayList() - var fresh = 0 - for (tag in contacts.follows()) { - follows.add(tag.pubKey) - tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { ConcurrentHashMap.newKeySet() }.add(it) } - if (discovered.add(tag.pubKey)) { - hopOf[tag.pubKey] = nextHop - fresh++ - } - } - builder.addFollows(source, follows) - contactListsFed++ - return fresh - } - - // Feed into the graph the contact lists a drain just returned - // (deduped by author; the store's canonical latest wins), marking - // fed authors done. Only the authors we actually received are - // touched — no scan over the whole still-missing set. Returns the - // count newly fed. - suspend fun harvest(events: List>): Int { - var got = 0 - for ((_, ev) in events) { - if (ev !is ContactListEvent) continue - val pk = ev.pubKey - if (pk in done) continue - val contacts = ctx.contactsOf(pk) ?: continue - done += pk - ingest(pk, contacts) - got++ - } - return got - } - - // Sharded backbone sweep (see SHARD_RELAYS). Splits the missing - // authors across the top live relays — one shard per relay, so no - // relay gets the same list twice — drains all shards concurrently, - // then rotates whoever's still missing onto a different relay for up - // to SHARD_ROTATIONS passes. Once the remainder is small it's cheap - // to broadcast it to every top relay at once. Returns lists fed. - suspend fun shardedSweep(authors: Collection): Int { - val top = topLiveRelays(SHARD_RELAYS) - if (top.isEmpty()) return 0 - val n = top.size - var missing = authors.filter { it !in done && ctx.contactsOf(it) == null } - var got = 0 - var rotation = 0 - while (missing.size > SHARD_BROADCAST_THRESHOLD && rotation < SHARD_ROTATIONS) { - val shards = Array(n) { ArrayList() } - for (pk in missing) { - val base = ((pk.hashCode() % n) + n) % n - shards[(base + rotation) % n].add(pk) - } - val results = - coroutineScope { - top - .mapIndexedNotNull { i, relay -> - val shard = shards[i] - if (shard.isEmpty()) { - null - } else { - // Each drain gets its own dead-set — the concurrent - // drains must not share a mutable HashSet. - async { - val dead = HashMap() - val filters = - mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = fetchKinds, authors = it) }) - ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) to dead - } - } - }.awaitAll() - } - for ((_, dead) in results) recordDead(dead) - relaysContacted += top - val flat = results.flatMap { it.first } - for ((relay, _) in flat) liveRelays.add(relay) - got += harvest(flat) - missing = missing.filter { it !in done } - rotation++ - } - // Once the remainder is small it's cheap to ask every top relay - // for it at once. If the rotations bailed with a still-large set, - // those authors just aren't on the popular relays — leave them to - // the caller's outbox pass rather than broadcast a huge list. - if (missing.isNotEmpty() && missing.size <= SHARD_BROADCAST_THRESHOLD) { - // Broadcast the small remainder to a wider set of busy relays - // than the rotation used — recovers users whose list is only - // on a relay ranked below the top SHARD_RELAYS. - val live = topLiveRelays(BROADCAST_RELAYS) - if (live.isNotEmpty()) { - val dead = HashMap() - val filters = - live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = fetchKinds, authors = it) } } - val events = ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) - recordDead(dead) - relaysContacted += live - for ((relay, _) in events) liveRelays.add(relay) - got += harvest(events) - } - } - return got - } - - // Crawl to full graph depth (no user cap; --max-hops bounds the follow - // distance). Each run fetches every discovered user's LATEST - // kind:3/10000/1984 once from their outbox (a freshness pass — grouped - // by write relay in routeByOutbox), unless we already fetched it this - // run (`done`). An unreachable outbox is retried up to - // MAX_OUTBOX_ATTEMPTS then dropped so the crawl terminates. - while (rounds < maxRounds) { - // Only crawl users within the hop budget; deeper users still appear - // in the graph as follow targets, we just don't fetch their lists. - val pending = discovered.filter { it !in done && (hopOf[it] ?: 0) < maxHops } - if (pending.isEmpty()) break - rounds++ - - // Refresh the warm pool to this round's busiest relays and keep - // that subscription open — reusing the same subId just updates the - // desired-relay set, so these sockets stay up across the round. - topLiveRelays(WARM_POOL_SIZE).takeIf { it.isNotEmpty() }?.let { warm -> - ctx.client.subscribe(WARM_SUB_ID, warm.associateWith { WARM_FILTERS }, null) - } - - val discoveredBefore = discovered.size - val fedBefore = contactListsFed - - // Phase A — bulk-fetch from the busiest relays via the sharded - // sweep. Most users' kind:3 lives on the big popular relays, so - // this clears the majority cheaply, without asking every relay for - // the same authors (early rounds no-op until a backbone is learned). - shardedSweep(pending) - - // Phase B — whoever the popular relays didn't have (niche - // outboxes): resolve their kind:10002, then fetch from their own - // write relays, drained a few at a time and skipping dead relays. - val stragglers = pending.filter { it !in done } - if (stragglers.isNotEmpty()) { - val backbone = topLiveRelays(BACKBONE_SIZE).toSet() - // Snapshot of every relay we've seen work, for the wide Tier-2 - // sweep (taken now, on this single coroutine, before the Phase-B - // workers start mutating liveRelays). - val allLive = (liveRelays - deadRelays).toSet() - ensureRelayLists(ctx, stragglers.toSet(), allLive, bgScope, timeoutMs, diagnose) - - // Continuous worker pool instead of chunked awaitAll barriers. - // The old shape drained DRAIN_CONCURRENCY batches, waited for the - // SLOWEST (a dead relay's full timeout), ingested, then started - // the next group — so every batch's tail idled the whole pool. - // Here a fixed set of DRAIN_CONCURRENCY workers pulls batches off - // a queue and grabs the next the instant a drain returns, so no - // worker waits on a slow sibling and hot relays stay connected - // (some worker is always subscribed). Shared graph state stays - // single-writer: routeByOutbox runs only on the producer (keeps - // writeRelayFreq serial) and ingest runs only on the consumer - // (keeps discovered/done/builder/hopOf serial), now overlapped - // with draining instead of blocked behind each batch. - val routed = Channel, Map>>>(DRAIN_CONCURRENCY * 2) - val drainedOut = Channel, Set, List>>>(Channel.UNLIMITED) - coroutineScope { - // Producer: route each batch by outbox (serial), backpressured - // by the bounded `routed` channel so we don't precompute every - // filter map at once. - val producer = - launch { - for (batch in stragglers.chunked(USER_BATCH)) { - val filters = routeByOutbox(ctx, batch.toSet(), relayHints, backbone, attempts, writeRelayFreq, fetchKinds, deadRelays) - routed.send(batch to filters) - } - routed.close() - } - // Drain workers: pure network, no shared graph-state writes - // except recordDead (concurrent-safe now). - val workers = - List(DRAIN_CONCURRENCY) { - launch { - for ((batch, filters) in routed) { - val dead = HashMap() - val events = ctx.drain(filters, timeoutMs, diagnose, dead, gatePerRelay = true) - recordDead(dead) - drainedOut.send(Triple(batch, filters.keys, events)) - } - } - } - // Consumer: single-writer ingest, overlapped with draining. - val consumer = - launch { - for ((batch, relays, events) in drainedOut) { - relaysContacted += relays - // Any relay that gave us an event is proven live + useful. - for ((relay, _) in events) liveRelays.add(relay) - for (pk in batch) { - if (pk in done) continue - val contacts = ctx.contactsOf(pk) - if (contacts != null) { - done += pk - ingest(pk, contacts) - } else { - val tries = (attempts[pk] ?: 0) + 1 - attempts[pk] = tries - if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk - } - } - } - } - producer.join() - workers.joinAll() - drainedOut.close() - consumer.join() - } - } - - System.err.println( - "[graperank] round $rounds: pending=${pending.size}, " + - "gotList=${contactListsFed - fedBefore}, newUsers=${discovered.size - discoveredBefore}, " + - "discovered=${discovered.size}, done=${done.size}, dead=${deadRelays.size}", + // Relay policy for the crawler — where a stranger's kind:10002 is + // found (index/discovery aggregators + general defaults that carry + // kind:10002 for most of the network) and the best-effort general + // relays that might hold content when an outbox is unknown. These + // defaults live in app code, so the quartz crawler takes them injected. + val discoveryRelays = + ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + EXTRA_DISCOVERY_RELAYS + val contentFallback = ctx.bootstrapRelays() + Constants.eventFinderRelays + val crawler = + GrapeRankDataCrawler( + client = ctx.client, + store = ctx.store, + limiter = ctx.relayLimiter, + config = + GrapeRankDataCrawler.Config( + relayListDiscoveryRelays = discoveryRelays, + contentFallbackRelays = contentFallback, + maxRounds = maxRounds, + maxHops = maxHops, + timeoutMs = timeoutMs, + diagnose = diagnose, + ), + log = { System.err.println(it) }, ) - } - - // Crawl done — drop the warm pool and stop any background relay-list - // sweeps still in flight (their results are already in the store). - ctx.client.unsubscribe(WARM_SUB_ID) - bgScope.cancel() - - // Reports can be retracted. Ask each reporter's outbox for NIP-09 - // kind:5 deletions that cite the reports we gathered (#e-filtered to - // our report ids — not every deletion the user ever made). A report - // the author has since deleted must not count as a negative edge; - // [materializeReports] drops those below. - fetchReportDeletions(ctx, topLiveRelays(BACKBONE_SIZE).toSet(), deadRelays, timeoutMs, diagnose) - - // No separate last-mile pass: the per-round sharded sweep already - // broadcasts the small remaining set to every top relay once it drops - // below SHARD_BROADCAST_THRESHOLD, and the round loop only exits when - // every reachable user within the hop budget is done. - - relaysContactedCount = relaysContacted.size - val perHop = - hopOf.values - .groupingBy { it } - .eachCount() - .toSortedMap() - downloadMs = (System.nanoTime() - crawlStart) / 1_000_000 - System.err.println( - "[graperank] crawl complete: ${discovered.size} discovered, $contactListsFed contact lists fed, " + - "$relaysContactedCount relays contacted, ${deadRelays.size} dead, $rounds rounds in $downloadMs ms; " + - "by hop: " + perHop.entries.joinToString(" ") { "${it.key}=${it.value}" }, - ) + val stats = crawler.crawl(observer, builder) + crawlStats = stats + contactListsFed = stats.contactListsFed if (ctx.relayDiagnostics.hadFeedback()) { System.err.println("[graperank] relay feedback: ${ctx.relayDiagnostics.snapshot()}") } @@ -631,22 +252,17 @@ object GrapeRankCommand { val result = linkedMapOf( "observer" to observer, - "crawl_rounds" to rounds, - "relays_contacted" to relaysContactedCount, + "crawl_rounds" to (crawlStats?.rounds ?: 0), + "relays_contacted" to (crawlStats?.relaysContacted ?: 0), "relay_feedback" to if (ctx.relayDiagnostics.hadFeedback()) ctx.relayDiagnostics.snapshot() else null, "relay_throttling" to if (ctx.relayLimiter.hadThrottling()) ctx.relayLimiter.snapshot() else null, - "max_hop_reached" to (hopOf.values.maxOrNull() ?: 0), - "users_by_hop" to - hopOf.values - .groupingBy { it } - .eachCount() - .toSortedMap() - .mapKeys { it.key.toString() }, + "max_hop_reached" to (crawlStats?.hopHistogram?.keys?.maxOrNull() ?: 0), + "users_by_hop" to (crawlStats?.hopHistogram?.mapKeys { it.key.toString() } ?: emptyMap()), "graph_users" to graph.nodeCount, "graph_edges" to graph.edgeCount(), "reports_deleted" to reportsDeleted, "users_scored" to rankedIds.size, - "download_ms" to downloadMs, + "download_ms" to crawlStats?.downloadMs, "store_load_ms" to storeLoadMs, "graph_build_ms" to buildMs, "scoring_ms" to scoringMs, @@ -1010,133 +626,6 @@ object GrapeRankCommand { return providerListOf(ctx, pubKey) } - /** - * Relays to query for **kind:10002 relay lists** — the account's own relays + - * bootstrap defaults + event-finder relays + the **indexer relays** - * (purplepag.es, coracle, …). Indexers aggregate kind:10002 (and kind:0) for - * the whole network, so this is where a stranger's relay list is found. They - * do NOT hold kind:3/10000/1984 — see [contentFallbackRelays]. - */ - private suspend fun relayListDiscoveryRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + EXTRA_DISCOVERY_RELAYS - - /** - * Best-effort fallback relays for **content** (kind:3/10000/1984/0) when a - * user's outbox is unknown or unreachable. Content lives on each user's own - * outbox, so this is only general-purpose relays that *might* hold a copy — - * bootstrap + event-finder. **No indexers**: they don't serve these kinds. - */ - private suspend fun contentFallbackRelays(ctx: Context): Set = ctx.bootstrapRelays() + Constants.eventFinderRelays - - /** - * Fetch kind:10002 relay lists for any [pubkeys] we don't already know, so - * [routeByOutbox] can route their content query to their own write relays. - * - * Tier 1 queries the bounded relay-list discovery set (indexers + general - * defaults), which aggregate kind:10002 for the whole network — reliable in - * bulk, unlike fanning out to thousands of per-user outboxes. - * - * Tier 2 is a completeness net for the stragglers the indexers don't cover: - * a user publishes their own kind:10002 to their own write relays, and those - * relays overlap heavily with [fallbackRelays] — the known-good backbone we - * learned from the `r` tags in *everyone else's* 10002s. So after tier 1, - * any pubkey still without a relay list is retried against that learned pool - * (minus the tier-1 relays we already asked). Early rounds skip tier 2 - * harmlessly because the backbone is still empty; it kicks in once the crawl - * has learned which relays actually carry 10002s. - */ - private suspend fun ensureRelayLists( - ctx: Context, - pubkeys: Set, - allLiveRelays: Set, - bgScope: CoroutineScope, - timeoutMs: Long, - diagnose: Boolean, - ) { - val missing = pubkeys.filter { ctx.relaysOf(it) == null } - if (missing.isEmpty()) return - - suspend fun query( - authors: List, - relays: Set, - ) { - if (relays.isEmpty() || authors.isEmpty()) return - val filters = - relays.associateWith { - authors.chunked(AUTHORS_PER_FILTER).map { chunk -> - Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) - } - } - ctx.drain(filters, timeoutMs, diagnose, gatePerRelay = true) - } - - // Tier 1: the index/discovery aggregators, which carry kind:10002 for most - // of the network. Blocking, because this round's routing needs the result. - val discovery = relayListDiscoveryRelays(ctx) - query(missing, discovery) - - // Tier 2: whoever the aggregators still don't have, cast the widest net — - // ask EVERY relay we've seen deliver events, not just the backbone. Fired - // fire-and-forget on [bgScope]: a stray 10002 might sit on any one relay, so - // we don't want to skip any, but we also can't block the crawl on a fan-out - // that large. The results land in the store and improve routing for later - // rounds; anyone still unresolved is handled by fallback routing meanwhile. - val stillMissing = missing.filter { ctx.relaysOf(it) == null } - val wide = allLiveRelays - discovery - if (stillMissing.isNotEmpty() && wide.isNotEmpty()) { - bgScope.launch { query(stillMissing, wide) } - } - } - - /** - * Fetch NIP-09 kind:5 deletion requests that retract any report we gathered. - * - * A reporter can delete their own kind:1984 report. That deletion is valid - * only if it comes from the reporter's own key, and it's published to the - * reporter's outbox — so we group report ids by their author and ask each - * author's write relays for kind:5 events that cite those ids (`#e`). That - * `#e` filter is the point: we pull only the deletions that touch our reports, - * not every deletion the user has ever made. The events land in the store; - * [materializeReports] decides which reports they actually retract. - */ - private suspend fun fetchReportDeletions( - ctx: Context, - backbone: Set, - deadRelays: Set, - timeoutMs: Long, - diagnose: Boolean, - ) { - val idsByAuthor = HashMap>() - for (ev in ctx.store.query(Filter(kinds = listOf(ReportEvent.KIND)))) { - if (ev is ReportEvent) idsByAuthor.getOrPut(ev.pubKey) { ArrayList() }.add(ev.id) - } - if (idsByAuthor.isEmpty()) return - - // Route each reporter to their own write relays (fallback: backbone). - val perRelayAuthors = HashMap>() - for (author in idsByAuthor.keys) { - val write = ctx.relaysOf(author)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() } ?: backbone - for (relay in write) if (relay !in deadRelays) perRelayAuthors.getOrPut(relay) { HashSet() }.add(author) - } - if (perRelayAuthors.isEmpty()) return - - val filters = - perRelayAuthors.mapValues { (_, authors) -> - buildList { - for (authorChunk in authors.chunked(AUTHORS_PER_FILTER)) { - // Scope #e to this author-chunk's own report ids, chunked to - // respect REQ limits. Any over-match (a filter pairing an - // author with another author's id) is harmless — the - // deleter-must-be-author check in materializeReports rejects it. - val chunkIds = authorChunk.flatMap { idsByAuthor[it].orEmpty() } - for (idChunk in chunkIds.chunked(AUTHORS_PER_FILTER)) { - add(Filter(kinds = listOf(DeletionEvent.KIND), authors = authorChunk, tags = mapOf("e" to idChunk))) - } - } - } - } - ctx.drain(filters, timeoutMs, diagnose, gatePerRelay = true) - } - /** * Feed reports into [builder], dropping any that a valid NIP-09 deletion has * retracted. Uses quartz's [DeletionIndex] — the same indexer the Android @@ -1171,52 +660,6 @@ object GrapeRankCommand { return dropped } - /** - * Group [pubkeys] by the relays we should query for their events: - * - first try: the user's own kind:10002 write relays (the outbox model); - * - a retry (`attempts[pk] > 0`, its outbox already failed): outbox + - * [backbone] — the known-good relays other people write to, which likely - * hold a copy; - * - no outbox at all: harvested [hints] + backbone + the general fallback. - * - * Also tallies each user's write relays into [writeRelayFreq] so the backbone - * can be learned from the crawl. Authors are chunked per relay to respect REQ - * limits. - */ - private suspend fun routeByOutbox( - ctx: Context, - pubkeys: Set, - hints: Map>, - backbone: Set, - attempts: Map, - writeRelayFreq: MutableMap, - kinds: List, - deadRelays: Set, - ): Map> { - val fallback = contentFallbackRelays(ctx) - val perRelay = HashMap>() - - for (pk in pubkeys) { - val write = ctx.relaysOf(pk)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() } - write?.forEach { writeRelayFreq.merge(it, 1, Int::plus) } - val relays = - when { - write == null -> hints[pk].orEmpty() + backbone + fallback - (attempts[pk] ?: 0) > 0 -> write + backbone - else -> write - } - // Skip relays already proven dead — routing to them only burns the - // drain timeout. - for (relay in relays) if (relay !in deadRelays) perRelay.getOrPut(relay) { HashSet() }.add(pk) - } - - return perRelay.mapValues { (_, authors) -> - authors.chunked(AUTHORS_PER_FILTER).map { chunk -> - Filter(kinds = kinds, authors = chunk) - } - } - } - /** * The exact `rank` tag VALUE STRING we last published for each target, read * from the active account's own kind:30382 cards in the local store (newest diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt new file mode 100644 index 0000000000..f25c177010 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt @@ -0,0 +1,813 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.experimental.graperank + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.crypto.verify +import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.AdaptiveRelayLimiter +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.DrainFailure +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.classifyDrainFailure +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener +import com.vitorpamplona.quartz.nip01Core.relay.client.single.newSubId +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent +import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent +import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent +import com.vitorpamplona.quartz.nip56Reports.ReportEvent +import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import com.vitorpamplona.quartz.utils.Log +import com.vitorpamplona.quartz.utils.SeenIds +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentSet +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull +import kotlin.coroutines.coroutineContext +import kotlin.time.TimeSource + +/** + * Crawls the Nostr follow/mute/report graph outward from an observer and streams + * the contact lists it finds into a [TrustGraphBuilder], so [GrapeRank] can score + * the whole reachable network from that observer's point of view. + * + * It uses the outbox model: each user's kind:10002 write relays are located + * first, then their kind:3 / kind:10000 / kind:1984 events are fetched from + * *their own* relays. The crawl is exhaustive — no user cap; it keeps going until + * every discovered user's outbox has been checked and their contact list pulled + * (an unreachable outbox is retried a few times), bounded only by [Config.maxHops] + * (follow-graph distance) and the [Config.maxRounds] safety backstop. + * + * Every event it fetches (contact lists, mute lists, reports, relay lists, and + * the report deletions it looks up) is verified and persisted to [store], so the + * caller can materialize mutes + reports (honouring NIP-09 retractions) from the + * store afterwards. Only the contact lists are streamed into the [TrustGraphBuilder] + * during the crawl — the compact int-CSR structure keeps the whole network in + * memory without holding millions of kind:3 objects. + * + * The crawler is transport-agnostic within quartz: it takes a [NostrClient], an + * [IEventStore], and the shared [AdaptiveRelayLimiter] (which must already be + * registered as a connection listener on the client so its ladders react to + * NOTICE/CLOSED frames). Relay *policy* — which aggregators know kind:10002, which + * general relays might hold content — is injected via [Config], because those + * defaults live in application code, not the protocol library. Operator progress + * is emitted through [log]; a headless caller routes it to stderr, a UI ignores it. + */ +class GrapeRankDataCrawler( + private val client: NostrClient, + private val store: IEventStore, + private val limiter: AdaptiveRelayLimiter, + private val config: Config, + private val log: (String) -> Unit = {}, +) { + /** + * Relay policy + crawl bounds. The relay sets come from the caller because the + * aggregator/bootstrap defaults live outside quartz. + * + * @param relayListDiscoveryRelays where to look up a stranger's kind:10002 — + * the index/discovery aggregators (purplepag.es, coracle, …) plus general + * defaults that carry kind:10002 for most of the network. + * @param contentFallbackRelays best-effort general relays that *might* hold a + * user's kind:3/10000/1984 when their outbox is unknown or unreachable. + * @param maxRounds safety backstop on freshness passes (default: run to convergence). + * @param maxHops follow-graph distance from the observer to crawl (Brainstorm uses 8). + * @param timeoutMs per-drain timeout. + * @param diagnose log a breakdown of slow/unreachable relays on each drain timeout. + */ + class Config( + val relayListDiscoveryRelays: Set, + val contentFallbackRelays: Set, + val maxRounds: Int = Int.MAX_VALUE, + val maxHops: Int = Int.MAX_VALUE, + val timeoutMs: Long = 10_000, + val diagnose: Boolean = false, + ) + + /** What the crawl fetched — the counters the caller reports and the graph is built from. */ + class Stats( + val rounds: Int, + val discovered: Int, + val contactListsFed: Int, + val relaysContacted: Int, + val deadRelays: Int, + /** Users bucketed by follow-graph distance from the observer (hop -> count), ascending. */ + val hopHistogram: Map, + val downloadMs: Long, + ) + + /** + * Crawl from [observer], streaming discovered contact lists into [builder] + * (follows only — mutes/reports land in the store for the caller to + * materialize). Returns the crawl [Stats]. + */ + suspend fun crawl( + observer: HexKey, + builder: TrustGraphBuilder, + ): Stats = CrawlRun(observer, builder).run() + + /** + * Holds all per-crawl mutable state. Graph state (discovered/done/hopOf/ + * builder/writeRelayFreq/liveRelays/relaysContacted) is single-writer by + * construction — Phase A and the Phase-B consumer never run concurrently, and + * routeByOutbox (the only Phase-B producer write, to writeRelayFreq) touches a + * disjoint field — so those stay plain collections. Only the state genuinely + * shared across the producer / consumer / drain-worker coroutines is concurrent: + * relayHints, attempts, deadRelays, relayStrikes. + */ + private inner class CrawlRun( + val observer: HexKey, + val builder: TrustGraphBuilder, + ) { + val hopOf = HashMap() + val discovered = hashSetOf(observer) + val done = hashSetOf() + val relaysContacted = hashSetOf() + val writeRelayFreq = HashMap() + val liveRelays = hashSetOf() + + // Concurrent: touched by more than one of producer/consumer/drain-workers. + val relayHints = ConcurrentMap>() + val attempts = ConcurrentMap() + val deadRelays = ConcurrentSet() + val relayStrikes = ConcurrentMap() + + var rounds = 0 + var contactListsFed = 0 + + /** + * A relay that HARD-failed (bad domain, TLS misconfig, dead HTTP code) is + * dropped on the first strike: it will not fix itself. A TRANSIENT failure + * (refused/reset/unreachable, or a 429/5xx) might clear, so it takes + * MAX_DEAD_STRIKES before we give up. Pure timeouts never reach here — the + * drain treats them as busy-retry and does not report them dead at all. + */ + fun recordDead(failed: Map) { + for ((r, kind) in failed) { + when (kind) { + DrainFailure.HARD -> deadRelays.add(r) + DrainFailure.TRANSIENT -> + if (relayStrikes.merge(r, 1) { a, b -> a + b } >= MAX_DEAD_STRIKES) deadRelays.add(r) + } + } + } + + /** The busiest live relays we've learned, excluding the dead ones. */ + fun topLiveRelays(cap: Int): List = + writeRelayFreq.entries + .asSequence() + .filter { it.key in liveRelays && it.key !in deadRelays } + .sortedByDescending { it.value } + .take(cap) + .map { it.key } + .toList() + + /** + * Feed a user's contact list into the graph, harvest relay hints, stamp + * the hop distance of newly-seen follows, and add them to the frontier. + * Called once per user (guarded by `done`). Returns the count of + * newly-discovered users. + */ + fun ingest( + source: HexKey, + contacts: ContactListEvent, + ): Int { + val nextHop = (hopOf[source] ?: 0) + 1 + val follows = ArrayList() + var fresh = 0 + for (tag in contacts.follows()) { + follows.add(tag.pubKey) + tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { ConcurrentSet() }.add(it) } + if (discovered.add(tag.pubKey)) { + hopOf[tag.pubKey] = nextHop + fresh++ + } + } + builder.addFollows(source, follows) + contactListsFed++ + return fresh + } + + /** + * Feed into the graph the contact lists a drain just returned (deduped by + * author; the store's canonical latest wins), marking fed authors done. + * Only the authors we actually received are touched — no scan over the + * whole still-missing set. Returns the count newly fed. + */ + suspend fun harvest(events: List>): Int { + var got = 0 + for ((_, ev) in events) { + if (ev !is ContactListEvent) continue + val pk = ev.pubKey + if (pk in done) continue + val contacts = contactsOf(pk) ?: continue + done += pk + ingest(pk, contacts) + got++ + } + return got + } + + /** + * Sharded backbone sweep (see SHARD_RELAYS). Splits the missing authors + * across the top live relays — one shard per relay, so no relay gets the + * same list twice — drains all shards concurrently, then rotates whoever's + * still missing onto a different relay for up to SHARD_ROTATIONS passes. + * Once the remainder is small it's cheap to broadcast it to every top relay + * at once. Returns lists fed. + */ + suspend fun shardedSweep(authors: Collection): Int { + val top = topLiveRelays(SHARD_RELAYS) + if (top.isEmpty()) return 0 + val n = top.size + var missing = authors.filter { it !in done && contactsOf(it) == null } + var got = 0 + var rotation = 0 + while (missing.size > SHARD_BROADCAST_THRESHOLD && rotation < SHARD_ROTATIONS) { + val shards = Array(n) { ArrayList() } + for (pk in missing) { + val base = ((pk.hashCode() % n) + n) % n + shards[(base + rotation) % n].add(pk) + } + val results = + coroutineScope { + top + .mapIndexedNotNull { i, relay -> + val shard = shards[i] + if (shard.isEmpty()) { + null + } else { + // Each drain gets its own dead-set — the concurrent + // drains must not share a mutable HashMap. + async { + val dead = HashMap() + val filters = + mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = FETCH_KINDS, authors = it) }) + drainGated(filters, dead) to dead + } + } + }.awaitAll() + } + for ((_, dead) in results) recordDead(dead) + relaysContacted += top + val flat = results.flatMap { it.first } + for ((relay, _) in flat) liveRelays.add(relay) + got += harvest(flat) + missing = missing.filter { it !in done } + rotation++ + } + // Once the remainder is small it's cheap to ask every top relay for it + // at once. If the rotations bailed with a still-large set, those authors + // just aren't on the popular relays — leave them to the caller's outbox + // pass rather than broadcast a huge list. + if (missing.isNotEmpty() && missing.size <= SHARD_BROADCAST_THRESHOLD) { + // Broadcast the small remainder to a wider set of busy relays than + // the rotation used — recovers users whose list is only on a relay + // ranked below the top SHARD_RELAYS. + val live = topLiveRelays(BROADCAST_RELAYS) + if (live.isNotEmpty()) { + val dead = HashMap() + val filters = + live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = FETCH_KINDS, authors = it) } } + val events = drainGated(filters, dead) + recordDead(dead) + relaysContacted += live + for ((relay, _) in events) liveRelays.add(relay) + got += harvest(events) + } + } + return got + } + + /** + * Fetch kind:10002 relay lists for any [pubkeys] we don't already know, so + * [routeByOutbox] can route their content query to their own write relays. + * + * Tier 1 queries the bounded relay-list discovery set (indexers + general + * defaults), which aggregate kind:10002 for the whole network. Blocking, + * because this round's routing needs the result. + * + * Tier 2 is a completeness net for the stragglers the indexers don't cover: + * cast the widest net — every relay we've seen deliver events. Fired + * fire-and-forget on [bgScope]: a stray 10002 might sit on any one relay, so + * we don't skip any, but we can't block the crawl on a fan-out that large. + * The results land in the store and improve routing for later rounds. + */ + suspend fun ensureRelayLists( + pubkeys: Set, + allLiveRelays: Set, + bgScope: CoroutineScope, + ) { + val missing = pubkeys.filter { relaysOf(it) == null } + if (missing.isEmpty()) return + + suspend fun query( + authors: List, + relays: Set, + ) { + if (relays.isEmpty() || authors.isEmpty()) return + val filters = + relays.associateWith { + authors.chunked(AUTHORS_PER_FILTER).map { chunk -> + Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) + } + } + drainGated(filters, null) + } + + val discovery = config.relayListDiscoveryRelays + query(missing, discovery) + + val stillMissing = missing.filter { relaysOf(it) == null } + val wide = allLiveRelays - discovery + if (stillMissing.isNotEmpty() && wide.isNotEmpty()) { + bgScope.launch { query(stillMissing, wide) } + } + } + + /** + * Fetch NIP-09 kind:5 deletion requests that retract any report we gathered. + * A reporter can delete their own kind:1984 report — a deletion valid only + * from the reporter's own key, published to the reporter's outbox. So we + * group report ids by their author and ask each author's write relays for + * kind:5 events that cite those ids (`#e`), pulling only the deletions that + * touch our reports. The events land in the store for the caller to apply. + */ + suspend fun fetchReportDeletions(backbone: Set) { + val idsByAuthor = HashMap>() + for (ev in store.query(Filter(kinds = listOf(ReportEvent.KIND)))) { + if (ev is ReportEvent) idsByAuthor.getOrPut(ev.pubKey) { ArrayList() }.add(ev.id) + } + if (idsByAuthor.isEmpty()) return + + // Route each reporter to their own write relays (fallback: backbone). + val perRelayAuthors = HashMap>() + for (author in idsByAuthor.keys) { + val write = relaysOf(author)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() } ?: backbone + for (relay in write) if (relay !in deadRelays) perRelayAuthors.getOrPut(relay) { HashSet() }.add(author) + } + if (perRelayAuthors.isEmpty()) return + + val filters = + perRelayAuthors.mapValues { (_, authors) -> + buildList { + for (authorChunk in authors.chunked(AUTHORS_PER_FILTER)) { + // Scope #e to this author-chunk's own report ids, chunked to + // respect REQ limits. Any over-match (a filter pairing an + // author with another author's id) is harmless — the + // deleter-must-be-author check the caller runs rejects it. + val chunkIds = authorChunk.flatMap { idsByAuthor[it].orEmpty() } + for (idChunk in chunkIds.chunked(AUTHORS_PER_FILTER)) { + add(Filter(kinds = listOf(DeletionEvent.KIND), authors = authorChunk, tags = mapOf("e" to idChunk))) + } + } + } + } + drainGated(filters, null) + } + + /** + * Group [pubkeys] by the relays we should query for their events: + * - first try: the user's own kind:10002 write relays (the outbox model); + * - a retry (`attempts[pk] > 0`, its outbox already failed): outbox + + * [backbone] — the known-good relays other people write to; + * - no outbox at all: harvested hints + backbone + the general fallback. + * + * Also tallies each user's write relays into [writeRelayFreq] so the + * backbone can be learned from the crawl. Authors are chunked per relay. + */ + suspend fun routeByOutbox( + pubkeys: Set, + backbone: Set, + ): Map> { + val fallback = config.contentFallbackRelays + val perRelay = HashMap>() + + for (pk in pubkeys) { + val write = relaysOf(pk)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() } + write?.forEach { writeRelayFreq[it] = (writeRelayFreq[it] ?: 0) + 1 } + val relays = + when { + write == null -> relayHints[pk]?.snapshot().orEmpty() + backbone + fallback + (attempts[pk] ?: 0) > 0 -> write + backbone + else -> write + } + // Skip relays already proven dead — routing to them only burns the + // drain timeout. + for (relay in relays) if (relay !in deadRelays) perRelay.getOrPut(relay) { HashSet() }.add(pk) + } + + return perRelay.mapValues { (_, authors) -> + authors.chunked(AUTHORS_PER_FILTER).map { chunk -> + Filter(kinds = FETCH_KINDS, authors = chunk) + } + } + } + + suspend fun run(): Stats { + val crawlMark = TimeSource.Monotonic.markNow() + // Scope for fire-and-forget relay-list discovery (see ensureRelayLists + // Tier 2). SupervisorJob so one failing sweep never cancels the others; + // cancelled when the crawl finishes. + val bgScope = CoroutineScope(coroutineContext + SupervisorJob()) + hopOf[observer] = 0 + + while (rounds < config.maxRounds) { + // Only crawl users within the hop budget; deeper users still appear + // in the graph as follow targets, we just don't fetch their lists. + val pending = discovered.filter { it !in done && (hopOf[it] ?: 0) < config.maxHops } + if (pending.isEmpty()) break + rounds++ + + // Refresh the warm pool to this round's busiest relays and keep that + // subscription open — reusing the same subId just updates the + // desired-relay set, so these sockets stay up across the round. + topLiveRelays(WARM_POOL_SIZE).takeIf { it.isNotEmpty() }?.let { warm -> + client.subscribe(WARM_SUB_ID, warm.associateWith { WARM_FILTERS }, null) + } + + val discoveredBefore = discovered.size + val fedBefore = contactListsFed + + // Phase A — bulk-fetch from the busiest relays via the sharded sweep. + // Most users' kind:3 lives on the big popular relays, so this clears + // the majority cheaply (early rounds no-op until a backbone is learned). + shardedSweep(pending) + + // Phase B — whoever the popular relays didn't have (niche outboxes): + // resolve their kind:10002, then fetch from their own write relays, + // drained a few at a time and skipping dead relays. + val stragglers = pending.filter { it !in done } + if (stragglers.isNotEmpty()) { + val backbone = topLiveRelays(BACKBONE_SIZE).toSet() + // Snapshot of every relay we've seen work, for the wide Tier-2 + // sweep (taken now, before the Phase-B workers mutate liveRelays). + val allLive = liveRelays.filterTo(HashSet()) { it !in deadRelays } + ensureRelayLists(stragglers.toSet(), allLive, bgScope) + + // Continuous worker pool instead of chunked awaitAll barriers, so + // no worker waits on a slow sibling and hot relays stay connected. + // Shared graph state stays single-writer: routeByOutbox runs only + // on the producer (keeps writeRelayFreq serial) and ingest runs + // only on the consumer (keeps discovered/done/builder/hopOf serial), + // now overlapped with draining instead of blocked behind each batch. + val routed = Channel, Map>>>(DRAIN_CONCURRENCY * 2) + val drainedOut = Channel, Set, List>>>(Channel.UNLIMITED) + coroutineScope { + // Producer: route each batch by outbox (serial), backpressured + // by the bounded `routed` channel. + val producer = + launch { + for (batch in stragglers.chunked(USER_BATCH)) { + val filters = routeByOutbox(batch.toSet(), backbone) + routed.send(batch to filters) + } + routed.close() + } + // Drain workers: pure network, no shared graph-state writes + // except recordDead (concurrent-safe). + val workers = + List(DRAIN_CONCURRENCY) { + launch { + for ((batch, filters) in routed) { + val dead = HashMap() + val events = drainGated(filters, dead) + recordDead(dead) + drainedOut.send(Triple(batch, filters.keys, events)) + } + } + } + // Consumer: single-writer ingest, overlapped with draining. + val consumer = + launch { + for ((batch, relays, events) in drainedOut) { + relaysContacted += relays + // Any relay that gave us an event is proven live + useful. + for ((relay, _) in events) liveRelays.add(relay) + for (pk in batch) { + if (pk in done) continue + val contacts = contactsOf(pk) + if (contacts != null) { + done += pk + ingest(pk, contacts) + } else { + val tries = (attempts[pk] ?: 0) + 1 + attempts[pk] = tries + if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk + } + } + } + } + producer.join() + workers.joinAll() + drainedOut.close() + consumer.join() + } + } + + log( + "[graperank] round $rounds: pending=${pending.size}, " + + "gotList=${contactListsFed - fedBefore}, newUsers=${discovered.size - discoveredBefore}, " + + "discovered=${discovered.size}, done=${done.size}, dead=${deadRelays.size()}", + ) + } + + // Crawl done — drop the warm pool and stop any background relay-list + // sweeps still in flight (their results are already in the store). + client.unsubscribe(WARM_SUB_ID) + bgScope.cancel() + + // Reports can be retracted. Ask each reporter's outbox for NIP-09 kind:5 + // deletions that cite the reports we gathered (#e-filtered to our report + // ids). The events land in the store; the caller decides which reports + // they actually retract. + fetchReportDeletions(topLiveRelays(BACKBONE_SIZE).toSet()) + + val hopHistogram = + hopOf.values + .groupingBy { it } + .eachCount() + .toList() + .sortedBy { it.first } + .toMap() + val downloadMs = crawlMark.elapsedNow().inWholeMilliseconds + log( + "[graperank] crawl complete: ${discovered.size} discovered, $contactListsFed contact lists fed, " + + "${relaysContacted.size} relays contacted, ${deadRelays.size()} dead, $rounds rounds in $downloadMs ms; " + + "by hop: " + hopHistogram.entries.joinToString(" ") { "${it.key}=${it.value}" }, + ) + return Stats( + rounds = rounds, + discovered = discovered.size, + contactListsFed = contactListsFed, + relaysContacted = relaysContacted.size, + deadRelays = deadRelays.size(), + hopHistogram = hopHistogram, + downloadMs = downloadMs, + ) + } + } + + /** + * Subscribe each relay to its filters behind [limiter], drain until every + * relay's subscription is terminal or the timeout elapses, verify+store the + * events, and return them tagged by relay. Each relay gets its own gated + * subscription so we never exceed its adaptive concurrent-subscription cap; a + * relay's filters are split into REQ-sized groups so a popular relay routed + * thousands of authors doesn't produce a multi-MB frame that most relays + * reject outright. Hard connect failures are reported into [deadOut]. + */ + private suspend fun drainGated( + filters: Map>, + deadOut: MutableMap?, + ): List> { + if (filters.isEmpty()) return emptyList() + val eventChannel = Channel>(Channel.UNLIMITED) + + // Split each relay's filters into REQ-sized groups. A REQ frame carries ALL + // its filters at once, so a popular relay routed thousands of authors would + // otherwise produce a multi-MB frame that most relays reject ("message too + // large") — silently dropping every author in it. Grouping by total entry + // count keeps each REQ well under the common 256KB cap. + val units = ArrayList>>() + for ((relay, relayFilters) in filters) { + var group = ArrayList() + var entries = 0 + for (f in relayFilters) { + val fe = filterEntries(f) + if (group.isNotEmpty() && entries + fe > MAX_REQ_ENTRIES) { + units.add(relay to group) + group = ArrayList() + entries = 0 + } + group.add(f) + entries += fe + } + if (group.isNotEmpty()) units.add(relay to group) + } + + // Per-relay failure classification, HARD winning over TRANSIENT across a + // relay's several REQ-groups; plus which relays stalled to a timeout. + val failures = ConcurrentMap() + val timedOut = ConcurrentSet() + + val collected = mutableListOf>() + coroutineScope { + // Single consumer: verify+store serially. One writer, so SeenIds' + // single-writer contract holds. The outbox model delivers the SAME event + // from many relays at once; skip a duplicate BEFORE the expensive Schnorr + // verify+store. An id is marked seen only after it verifies, so a forged + // copy (valid id, bad signature) delivered first can't suppress the + // genuine one that follows. + val consumer = + launch { + val seen = SeenIds(initialSlotsPow2 = 12) + for ((relay, event) in eventChannel) { + if (seen.contains(event.id)) continue + if (verifyAndStore(event)) { + seen.add(event.id) + collected.add(relay to event) + } + } + } + // One gated subscription per (relay, REQ-group). The permit is held for + // the group's whole life, so concurrent subs on a relay never exceed its + // adaptive cap. + units + .map { (subRelay, groupFilters) -> + launch { + limiter.withPermit(subRelay) { + val subId = newSubId() + val done = CompletableDeferred() + val groupListener = + object : SubscriptionListener { + override fun onEvent( + event: Event, + isLive: Boolean, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + eventChannel.trySend(relay to event) + } + + override fun onEose( + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + done.complete("eose") + } + + override fun onClosed( + message: String, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + done.complete("closed:$message") + } + + override fun onCannotConnect( + relay: NormalizedRelayUrl, + message: String, + forFilters: List?, + ) { + done.complete("cannot:$message") + } + } + client.subscribe(subId, mapOf(subRelay to groupFilters), groupListener) + try { + val reason = withTimeoutOrNull(config.timeoutMs) { done.await() } ?: "timeout" + if (reason == "timeout") timedOut.add(subRelay) + classifyDrainFailure(reason)?.let { kind -> + failures.merge(subRelay, kind) { a, b -> + if (a == DrainFailure.HARD || b == DrainFailure.HARD) DrainFailure.HARD else DrainFailure.TRANSIENT + } + } + } finally { + client.unsubscribe(subId) + } + } + } + }.joinAll() + // All subscriptions are torn down; no more events can arrive. Close the + // channel so the consumer drains what's buffered and completes. + eventChannel.close() + consumer.join() + } + if (config.diagnose && timedOut.size() > 0) { + val stalled = timedOut.snapshot() + val eventsPer = collected.groupingBy { it.first }.eachCount() + val detail = stalled.take(12).joinToString(", ") { "${it.url}(${eventsPer[it] ?: 0}ev)" } + log("[drain] timeout ${config.timeoutMs}ms: ${stalled.size} slow(no EOSE)" + (if (detail.isNotEmpty()) " | slow: $detail" else "")) + } + deadOut?.putAll(failures.snapshot()) + return collected + } + + /** + * Verify [event]'s NIP-01 id+signature and, if valid, persist it to [store]. + * Returns true when the event was accepted. A UNIQUE-constraint rejection is + * normal (the store already holds this id, or a newer replaceable) — the outbox + * model delivers the same event from several relays, so a crawl produces these + * by the hundred-thousand — so only genuine persistence failures are logged. + */ + private suspend fun verifyAndStore(event: Event): Boolean { + if (!event.verify()) { + Log.w("GrapeRankDataCrawler") { "dropped event ${event.id.take(8)} kind=${event.kind} — bad signature" } + return false + } + try { + store.insert(event) + } catch (t: Throwable) { + if (t.message?.contains("UNIQUE constraint", ignoreCase = true) != true) { + Log.w("GrapeRankDataCrawler") { "store insert failed for ${event.id.take(8)}: ${t.message}" } + } + } + return true + } + + /** Latest known kind:3 contact list for [pubKey] from the local store, or null. */ + private suspend fun contactsOf(pubKey: HexKey): ContactListEvent? = + store + .query(Filter(authors = listOf(pubKey), kinds = listOf(ContactListEvent.KIND), limit = 1)) + .firstOrNull() as? ContactListEvent + + /** Latest known kind:10002 advertised relay list for [pubKey] from the store, or null. */ + private suspend fun relaysOf(pubKey: HexKey): AdvertisedRelayListEvent? = + store + .query(Filter(authors = listOf(pubKey), kinds = listOf(AdvertisedRelayListEvent.KIND), limit = 1)) + .firstOrNull() as? AdvertisedRelayListEvent + + companion object { + // Authors per REQ filter — keeps individual subscriptions within relay limits. + private const val AUTHORS_PER_FILTER = 300 + + // Max total "entries" (authors + ids + tag values) in a single REQ frame. + // Each entry is a ~67-byte hex string, so 2500 ≈ 167KB — under the 256KB + // message cap most relays enforce. drainGated groups filters to stay within. + private const val MAX_REQ_ENTRIES = 2500 + + // Times we re-query an unreachable user's outbox before giving up, so the + // crawl still terminates on a finite graph. + private const val MAX_OUTBOX_ATTEMPTS = 3 + + // Users whose outboxes we fetch in a single drain. Draining thousands of + // distinct outbox relays at once saturates connections and times out + // (~250/drain succeeds, ~17k fails); keep the fan-out small. + private const val USER_BATCH = 256 + + // Global content-drain fan-out — how many outbox batches we drain at once. A + // GLOBAL bound (memory / open sockets); the per-relay concurrency limit is + // enforced separately by AdaptiveRelayLimiter. A higher global fan-out + // re-floods busy hubs faster than demotion catches up, so keep it moderate. + private const val DRAIN_CONCURRENCY = 24 + + // Sharded backbone sweep: split the still-missing authors into SHARD_RELAYS + // lists, one per top relay, rotating up to SHARD_ROTATIONS times; once the + // remainder drops below SHARD_BROADCAST_THRESHOLD, broadcast it at once. + private const val SHARD_RELAYS = 10 + private const val SHARD_ROTATIONS = 6 + private const val SHARD_BROADCAST_THRESHOLD = 2000 + + // The small-remainder broadcast goes to this many top live relays — a user's + // kind:3 is often mirrored on a busy relay ranked below the top 10. + private const val BROADCAST_RELAYS = 60 + + // A relay that fails to CONNECT this many times is treated as dead. Kept + // above 1 so a single transient connect blip doesn't evict a relay. + private const val MAX_DEAD_STRIKES = 3 + + // Most-used write relays kept as the known-good backbone for retrying users. + private const val BACKBONE_SIZE = 30 + + // Warm pool: hold a do-nothing subscription open to the busiest relays for + // the whole crawl, so the connections we reuse every round survive the + // between-round routing gaps. The filter matches an impossible event id, so + // the relay EOSEs immediately and streams nothing — it only keeps sockets warm. + private const val WARM_POOL_SIZE = 20 + private const val WARM_SUB_ID = "graperank-warm" + private val WARM_FILTERS = listOf(Filter(ids = listOf("0".repeat(64)))) + + // Kinds requested from relays during the crawl: the graph edges (contact + // lists, mute lists, reports) PLUS the user's own kind:10002. A user's outbox + // holds the freshest copy of their relay list, so folding 10002 into the same + // query keeps routing current. The store keeps newest-by-created_at for the + // replaceable 10002, so the freshest always wins regardless of source relay. + private val FETCH_KINDS = + listOf(ContactListEvent.KIND, MuteListEvent.KIND, ReportEvent.KIND, AdvertisedRelayListEvent.KIND) + + /** Count the size-driving entries in a filter: authors, ids, and tag values. */ + private fun filterEntries(f: Filter): Int = + (f.authors?.size ?: 0) + + (f.ids?.size ?: 0) + + (f.tags?.values?.sumOf { it.size } ?: 0) + + (f.tagsAll?.values?.sumOf { it.size } ?: 0) + } +} diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt similarity index 80% rename from cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt index b2f39fc534..c66134480f 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/AdaptiveRelayLimiter.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.cli +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient @@ -26,13 +26,16 @@ import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.ClosedMessage import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NoticeMessage import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.Log +import com.vitorpamplona.quartz.utils.TimeUtils +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.delay import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicLong +import kotlin.concurrent.atomics.AtomicInt +import kotlin.concurrent.atomics.AtomicLong +import kotlin.concurrent.atomics.ExperimentalAtomicApi /** * Adaptive per-relay back-pressure with TWO independent controls, because relays @@ -60,26 +63,27 @@ import java.util.concurrent.atomic.AtomicLong * Registered as a [RelayConnectionListener] on the shared client, so both signals * are driven straight off the incoming NOTICE/CLOSED frames (which fire on the * per-relay socket threads — all state here is concurrent). Drains gate through - * [withPermit]; [Context.drain]'s `gatePerRelay` path holds a relay's permit for - * the lifetime of that relay's subscription, and passes the rate gate before it - * opens, so we respect both limits at once. + * [withPermit]: the gated-drain path holds a relay's permit for the lifetime of + * that relay's subscription, and passes the rate gate before it opens, so we + * respect both limits at once. */ +@OptIn(ExperimentalAtomicApi::class) class AdaptiveRelayLimiter( private val startCap: Int = 100, private val subLadder: List = listOf(20, 10), private val rateLadder: List = listOf(250L, 500L, 1000L, 2000L), ) : RelayConnectionListener { - private val gates = ConcurrentHashMap() + private val gates = ConcurrentMap() // Concurrency-cap demotions per relay (== index+1 into subLadder). Capped at // subLadder.size: past the floor we stop demoting. - private val subDemotions = ConcurrentHashMap() + private val subDemotions = ConcurrentMap() // Rate-limit state per relay: how far down rateLadder we've stepped, the // current min interval between opens, and the next epoch-ms an open may fire. - private val rateSteps = ConcurrentHashMap() - private val rateDelayMs = ConcurrentHashMap() - private val nextAllowedAtMs = ConcurrentHashMap() + private val rateSteps = ConcurrentMap() + private val rateDelayMs = ConcurrentMap() + private val nextAllowedAtMs = ConcurrentMap() private fun gate(relay: NormalizedRelayUrl): Gate = gates.getOrPut(relay) { Gate(startCap) } @@ -106,14 +110,14 @@ class AdaptiveRelayLimiter( private suspend fun rateGate(relay: NormalizedRelayUrl) { val delayMs = rateDelayMs[relay] ?: return if (delayMs <= 0L) return - val now = System.currentTimeMillis() + val now = TimeUtils.nowMillis() // Atomically claim the next slot: my turn is max(prevSlot, now); the next // caller can't fire until delayMs after me. Serializes opens to this relay // at one per delayMs, in arrival order. val slot = nextAllowedAtMs.getOrPut(relay) { AtomicLong(now) } var myTurn: Long while (true) { - val prev = slot.get() + val prev = slot.load() myTurn = maxOf(prev, now) if (slot.compareAndSet(prev, myTurn + delayMs)) break } @@ -142,49 +146,51 @@ class AdaptiveRelayLimiter( /** Step [relay] one rung down the concurrency-cap ladder, unless already at the floor. */ private fun demoteConcurrency(relay: NormalizedRelayUrl) { if ((subDemotions[relay] ?: 0) >= subLadder.size) return - val step = subDemotions.merge(relay, 1, Int::plus)!! + val step = subDemotions.merge(relay, 1) { a, b -> a + b } val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)] gate(relay).lower(cap) if (step <= subLadder.size) { - System.err.println("[limiter] ${relay.url} concurrency capped at $cap subs (sub-limit #$step)") + Log.w("AdaptiveRelayLimiter") { "${relay.url} concurrency capped at $cap subs (sub-limit #$step)" } } } /** Step [relay] one rung down the rate ladder, unless already at the slowest. */ private fun throttleRate(relay: NormalizedRelayUrl) { if ((rateSteps[relay] ?: 0) >= rateLadder.size) return - val step = rateSteps.merge(relay, 1, Int::plus)!! + val step = rateSteps.merge(relay, 1) { a, b -> a + b } val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)] rateDelayMs[relay] = d if (step <= rateLadder.size) { - System.err.println("[limiter] ${relay.url} rate-throttled to 1 REQ / ${d}ms (rate-limit #$step)") + Log.w("AdaptiveRelayLimiter") { "${relay.url} rate-throttled to 1 REQ / ${d}ms (rate-limit #$step)" } } } /** JSON-friendly view of which relays we throttled, in which dimension, how far. */ fun snapshot(): Map { - val cappedAt = sortedMapOf() - for ((_, step) in subDemotions) { + val capCounts = HashMap() + for ((_, step) in subDemotions.snapshot()) { val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)] - cappedAt.merge(cap, 1, Int::plus) + capCounts[cap] = (capCounts[cap] ?: 0) + 1 } - val rateAt = sortedMapOf() - for ((_, step) in rateSteps) { + val cappedAt = capCounts.toList().sortedBy { it.first }.toMap() + val rateCounts = HashMap() + for ((_, step) in rateSteps.snapshot()) { val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)] - rateAt.merge(d, 1, Int::plus) + rateCounts[d] = (rateCounts[d] ?: 0) + 1 } + val rateAt = rateCounts.toList().sortedBy { it.first }.toMap() return mapOf( "start_cap" to startCap, "sub_ladder" to subLadder, "rate_ladder_ms" to rateLadder, - "concurrency_capped_relays" to subDemotions.size, + "concurrency_capped_relays" to subDemotions.size(), "concurrency_capped_at" to cappedAt, - "rate_limited_relays" to rateSteps.size, + "rate_limited_relays" to rateSteps.size(), "rate_limited_at_ms" to rateAt, ) } - fun hadThrottling(): Boolean = subDemotions.isNotEmpty() || rateSteps.isNotEmpty() + fun hadThrottling(): Boolean = subDemotions.size() > 0 || rateSteps.size() > 0 /** * A bounded-concurrency gate whose limit can only ever be *lowered* (relays @@ -197,7 +203,7 @@ class AdaptiveRelayLimiter( private class Gate( initialLimit: Int, ) { - private val limit = AtomicInteger(initialLimit) + private val limit = AtomicInt(initialLimit) private val mutex = Mutex() private var inUse = 0 private val waiters = ArrayDeque>() @@ -205,7 +211,7 @@ class AdaptiveRelayLimiter( suspend fun acquire() { val wait = mutex.withLock { - if (inUse < limit.get()) { + if (inUse < limit.load()) { inUse++ null } else { @@ -218,7 +224,7 @@ class AdaptiveRelayLimiter( suspend fun release() { mutex.withLock { inUse-- - while (inUse < limit.get() && waiters.isNotEmpty()) { + while (inUse < limit.load() && waiters.isNotEmpty()) { waiters.removeFirst().complete(Unit) inUse++ } @@ -227,7 +233,11 @@ class AdaptiveRelayLimiter( /** Monotonically shrink the cap. Safe to call from any thread. */ fun lower(newLimit: Int) { - limit.updateAndGet { if (newLimit < it) newLimit else it } + while (true) { + val cur = limit.load() + if (newLimit >= cur) return + if (limit.compareAndSet(cur, newLimit)) return + } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/DrainFailure.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/DrainFailure.kt new file mode 100644 index 0000000000..b2d605d044 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/DrainFailure.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories + +/** + * Why a relay could not be used for a one-shot drain — when the reason is worth + * acting on (dropping the relay from further routing). + * + * - [HARD]: the relay answered wrong, or cannot exist. A bad HTTP upgrade (not a + * websocket / dead status code), an unresolvable domain, or a TLS misconfig. + * This will not fix itself, so one strike is enough to drop it. + * - [TRANSIENT]: a failure that might clear — connection refused / reset, host + * unreachable, or a temporary 429/5xx on the upgrade. Struck a few times + * before we give up. + * + * A pure connect **timeout** is neither. The relay is most likely just busy, so + * we retry it and never mark it dead — [classifyDrainFailure] returns null for + * it (and for any non-failure terminal reason). + */ +enum class DrainFailure { HARD, TRANSIENT } + +/** + * Classify a drain per-relay terminal reason. Returns null when the relay should + * simply be retried (a timeout, or a non-failure like eose/closed). The reason + * shape is `cannot:` for a connect failure (see + * `BasicRelayClient.onCannotConnect`), or `eose` / `closed:…` / `timeout`. + */ +fun classifyDrainFailure(reason: String): DrainFailure? { + if (!reason.startsWith("cannot")) return null + val m = reason.removePrefix("cannot:").lowercase() + // The message now carries the exception class name (see BasicRelayClient), so + // we can key on the stable *type* rather than localized message text. + // Busy, not dead: a connect/read timeout means the handshake just didn't + // finish in time. Retry it — the relay is probably fine, only slow or loaded. + if ("timeout" in m || "timed out" in m) return null // SocketTimeoutException, etc. + // Cannot ever work: unresolvable domain (DNS) or a TLS misconfiguration. + // Dead for good — one strike is enough. + if ("unknownhost" in m || // UnknownHostException + "unable to resolve host" in m || + "no address associated" in m || + "nodename nor servname" in m || + "sslhandshake" in m || // SSLHandshakeException + "sslpeerunverified" in m || + "sslexception" in m || + "certificate" in m || // CertificateException + "trust anchor" in m || + "certpath" in m + ) { + return DrainFailure.HARD + } + // Wrong HTTP upgrade. Usually a misconfigured endpoint (not a relay), but + // 429 / 5xx mean "busy, come back later", so those stay transient. + if ("server misconfigured" in m || "not a websocket" in m || "expected http 101" in m) { + val transientCode = Regex("response: (429|500|502|503|504)").containsMatchIn(m) + return if (transientCode) DrainFailure.TRANSIENT else DrainFailure.HARD + } + // Refused / reset / unreachable / anything else: might clear — retry a few times. + return DrainFailure.TRANSIENT +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.kt new file mode 100644 index 0000000000..47d3233c9b --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.kt @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +/** + * A thread-safe hash map whose compound operations — [getOrPut] and [merge] — + * apply their update **atomically**, not merely one-lock-per-primitive-op. This + * is the contract a concurrent producer/consumer pipeline needs: two coroutines + * racing `getOrPut` on the same key must agree on a single value, and racing + * `merge` must not lose an increment. + * + * commonMain has no `java.util.concurrent.ConcurrentHashMap`, so this is + * expect/actual, matching the split already used by [com.vitorpamplona.quartz.utils.cache.ConcurrentHashCache]: + * - JVM / Android → `ConcurrentHashMap` (lock-free, true atomic `computeIfAbsent` / `merge`). + * - Native (Apple + Linux) → copy-on-write over an atomic reference, with a + * CAS retry loop giving the same atomicity. Correct but O(n)-per-write; the + * native targets never run the heavy crawl this backs, they only compile it. + * + * Only the operations the crawl actually uses are exposed — no full [MutableMap] + * surface — so the native copy-on-write actual stays small and obviously correct. + */ +expect class ConcurrentMap() { + operator fun get(key: K): V? + + operator fun set( + key: K, + value: V, + ) + + /** Atomically return the value for [key], computing and inserting [defaultValue] once if absent. */ + fun getOrPut( + key: K, + defaultValue: () -> V, + ): V + + /** + * Atomically insert [value] if [key] is absent, else replace the existing + * value with `remap(existing, value)`. Returns the value now stored. + */ + fun merge( + key: K, + value: V, + remap: (old: V, new: V) -> V, + ): V + + fun size(): Int + + /** A point-in-time copy of the entries — safe to iterate without holding a lock. */ + fun snapshot(): Map +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.kt new file mode 100644 index 0000000000..d00545acf6 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +/** + * A thread-safe hash set for the crawl's cross-coroutine membership tracking + * (dead relays struck by drain workers while the router reads them, relay hints + * written by the ingest consumer while the producer reads them). + * + * commonMain has no `java.util.concurrent.ConcurrentHashMap.newKeySet()`, so this + * is expect/actual with the same JVM-vs-native split as [ConcurrentMap]: + * - JVM / Android → `ConcurrentHashMap.newKeySet()`. + * - Native → copy-on-write over an atomic reference (compile-only, never the hot path). + */ +expect class ConcurrentSet() { + /** Add [element]; returns true if it was not already present. */ + fun add(element: E): Boolean + + operator fun contains(element: E): Boolean + + fun size(): Int + + /** A point-in-time copy — safe to iterate or diff against without a lock. */ + fun snapshot(): Set +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentCollectionsTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentCollectionsTest.kt new file mode 100644 index 0000000000..208054f818 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentCollectionsTest.kt @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class ConcurrentCollectionsTest { + @Test + fun mapGetSet() { + val m = ConcurrentMap() + assertNull(m["a"]) + m["a"] = 1 + assertEquals(1, m["a"]) + m["a"] = 2 + assertEquals(2, m["a"]) + assertEquals(1, m.size()) + } + + @Test + fun mapGetOrPutComputesOnce() { + val m = ConcurrentMap() + var calls = 0 + assertEquals( + 7, + m.getOrPut("k") { + calls++ + 7 + }, + ) + // Present now: the default must NOT be recomputed. + assertEquals( + 7, + m.getOrPut("k") { + calls++ + 99 + }, + ) + assertEquals(1, calls) + assertEquals(7, m["k"]) + } + + @Test + fun mapMergeInsertsThenCombines() { + val m = ConcurrentMap() + // Absent -> inserts the value verbatim, remap not applied. + assertEquals(1, m.merge("k", 1) { a, b -> a + b }) + // Present -> remap(existing, value). + assertEquals(4, m.merge("k", 3) { a, b -> a + b }) + assertEquals(4, m["k"]) + } + + @Test + fun mapSnapshotIsDetached() { + val m = ConcurrentMap() + m["a"] = 1 + m["b"] = 2 + val snap = m.snapshot() + assertEquals(mapOf("a" to 1, "b" to 2), snap) + // Mutating the map after the snapshot must not change the snapshot. + m["c"] = 3 + assertEquals(2, snap.size) + assertEquals(3, m.size()) + } + + @Test + fun setAddContainsSize() { + val s = ConcurrentSet() + assertFalse("x" in s) + assertTrue(s.add("x")) + // Re-adding is a no-op and reports it. + assertFalse(s.add("x")) + assertTrue("x" in s) + assertTrue(s.add("y")) + assertEquals(2, s.size()) + } + + @Test + fun setSnapshotIsDetached() { + val s = ConcurrentSet() + s.add("a") + val snap = s.snapshot() + s.add("b") + assertEquals(setOf("a"), snap) + assertEquals(2, s.size()) + } +} diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt new file mode 100644 index 0000000000..65a734f8d2 --- /dev/null +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +import java.util.concurrent.ConcurrentHashMap + +actual class ConcurrentMap { + private val map = ConcurrentHashMap() + + actual operator fun get(key: K): V? = map[key] + + actual operator fun set( + key: K, + value: V, + ) { + map[key] = value + } + + actual fun getOrPut( + key: K, + defaultValue: () -> V, + ): V = map.computeIfAbsent(key) { defaultValue() } + + actual fun merge( + key: K, + value: V, + remap: (old: V, new: V) -> V, + ): V = map.merge(key, value) { old, new -> remap(old, new) }!! + + actual fun size(): Int = map.size + + actual fun snapshot(): Map = HashMap(map) +} diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.jvmAndroid.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.jvmAndroid.kt new file mode 100644 index 0000000000..94a0754c11 --- /dev/null +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.jvmAndroid.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +import java.util.concurrent.ConcurrentHashMap + +actual class ConcurrentSet { + private val set: MutableSet = ConcurrentHashMap.newKeySet() + + actual fun add(element: E): Boolean = set.add(element) + + actual operator fun contains(element: E): Boolean = set.contains(element) + + actual fun size(): Int = set.size + + actual fun snapshot(): Set = HashSet(set) +} diff --git a/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.native.kt b/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.native.kt new file mode 100644 index 0000000000..de4ee540d8 --- /dev/null +++ b/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.native.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +import kotlin.concurrent.atomics.AtomicReference +import kotlin.concurrent.atomics.ExperimentalAtomicApi + +// Copy-on-write, mirroring ConcurrentHashCache.linux: correct and simple. The +// native targets never run the crawl this backs (it is JVM/Android-only work); +// they only compile it, so the O(n)-per-write cost is irrelevant. A CAS retry +// loop gives getOrPut/merge the same atomicity the JVM actual gets for free. +@OptIn(ExperimentalAtomicApi::class) +actual class ConcurrentMap { + private val ref = AtomicReference(HashMap()) + + actual operator fun get(key: K): V? = ref.load()[key] + + actual operator fun set( + key: K, + value: V, + ) { + while (true) { + val cur = ref.load() + val copy = HashMap(cur) + copy[key] = value + if (ref.compareAndSet(cur, copy)) return + } + } + + actual fun getOrPut( + key: K, + defaultValue: () -> V, + ): V { + while (true) { + val cur = ref.load() + cur[key]?.let { return it } + val value = defaultValue() + val copy = HashMap(cur) + copy[key] = value + if (ref.compareAndSet(cur, copy)) return value + } + } + + actual fun merge( + key: K, + value: V, + remap: (old: V, new: V) -> V, + ): V { + while (true) { + val cur = ref.load() + val old = cur[key] + val merged = if (old == null) value else remap(old, value) + val copy = HashMap(cur) + copy[key] = merged + if (ref.compareAndSet(cur, copy)) return merged + } + } + + actual fun size(): Int = ref.load().size + + actual fun snapshot(): Map = HashMap(ref.load()) +} diff --git a/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.native.kt b/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.native.kt new file mode 100644 index 0000000000..70bf86f133 --- /dev/null +++ b/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentSet.native.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +import kotlin.concurrent.atomics.AtomicReference +import kotlin.concurrent.atomics.ExperimentalAtomicApi + +// Copy-on-write native actual — see ConcurrentMap.native for the rationale. +@OptIn(ExperimentalAtomicApi::class) +actual class ConcurrentSet { + private val ref = AtomicReference(HashSet()) + + actual fun add(element: E): Boolean { + while (true) { + val cur = ref.load() + if (element in cur) return false + val copy = HashSet(cur) + copy.add(element) + if (ref.compareAndSet(cur, copy)) return true + } + } + + actual operator fun contains(element: E): Boolean = element in ref.load() + + actual fun size(): Int = ref.load().size + + actual fun snapshot(): Set = HashSet(ref.load()) +} From 41e88695e06fa30e6161a13b69297f20b17f053e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 16:00:39 +0000 Subject: [PATCH 48/58] refactor(quartz): simplify GrapeRankDataCrawler after extraction review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanups from a reuse/simplification/efficiency/altitude review of the crawler extraction: - Collapse the redundant `discovered` set into `hopOf` — a user is discovered iff it has a hop stamp, so the two always held the same key set. The frontier is now `hopOf.keys`; one fewer collection to keep in sync. - Drop the unused `Stats.discovered` / `Stats.deadRelays` fields (no reader — the CLI reports rounds / relaysContacted / hopHistogram / downloadMs). - Extract a single shared verify-then-store sink, `IEventStore.verifyAndInsert`, and route both the crawler and `Context.verifyAndStore` through it instead of each carrying its own verify + insert + UNIQUE-swallow copy. - Fast-path the present-key hit in `ConcurrentMap.getOrPut` (jvmAndroid) so the crawl's hot relay-hint accumulation stops allocating a mapping-function closure on every call. - Hoist the repeated `crawlStats?.hopHistogram` null-plumbing in GrapeRankCommand. --- .../com/vitorpamplona/amethyst/cli/Context.kt | 39 +++-------- .../amethyst/cli/commands/GrapeRankCommand.kt | 5 +- .../graperank/GrapeRankDataCrawler.kt | 68 ++++++------------- .../quartz/nip01Core/store/VerifyAndInsert.kt | 55 +++++++++++++++ .../concurrent/ConcurrentMap.jvmAndroid.kt | 6 +- 5 files changed, 94 insertions(+), 79 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/VerifyAndInsert.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 531fcbf6c5..e75e8459df 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -38,7 +38,6 @@ import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray -import com.vitorpamplona.quartz.nip01Core.crypto.verify import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.AdaptiveRelayLimiter @@ -58,6 +57,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.TcpNoDelaySocketF import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.verifyAndInsert import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip17Dm.settings.ChatMessageRelayListEvent import com.vitorpamplona.quartz.nip46RemoteSigner.signer.NostrSignerRemote @@ -688,36 +688,17 @@ class Context( } /** - * Verify [event]'s NIP-01 id+signature and, if valid, persist it - * to [store]. Returns `true` when the event was accepted (and - * therefore should be surfaced to callers). Persistence failures - * (I/O errors, full disk) are logged but do not propagate. + * Verify [event]'s NIP-01 id+signature and, if valid, persist it to [store]. + * Returns `true` when the event was accepted (and therefore should be surfaced + * to callers). Persistence failures (I/O errors, full disk) are logged but do + * not propagate; a UNIQUE-constraint rejection is normal and swallowed quietly. * - * Every event-arrival path in the CLI funnels through this method - * so that [store] is the authoritative cache of what Amy has seen. + * Every event-arrival path in the CLI funnels through this so that [store] is + * the authoritative cache of what Amy has seen. Delegates to the shared quartz + * [verifyAndInsert] sink so the CLI and the GrapeRank crawler apply the exact + * same verify-then-store policy. */ - suspend fun verifyAndStore(event: Event): Boolean { - if (!event.verify()) { - System.err.println("[cli] dropped event ${event.id.take(8)} kind=${event.kind} — bad signature") - return false - } - try { - store.insert(event) - } catch (t: Throwable) { - // A UNIQUE-constraint rejection is normal, not a failure: the - // store already holds this id, or a newer version of a - // replaceable (kind 0/3/10000-19999). The outbox model routinely - // delivers the same event from several of a user's write relays, - // so a crawl produces these by the hundred-thousand. Only surface - // genuine persistence failures (I/O, full disk, corruption). The - // FS backend no-ops on such duplicates; this keeps the SQLite - // backend just as quiet. - if (t.message?.contains("UNIQUE constraint", ignoreCase = true) != true) { - System.err.println("[cli] store insert failed for ${event.id.take(8)}: ${t.message}") - } - } - return true - } + suspend fun verifyAndStore(event: Event): Boolean = store.verifyAndInsert(event) // ------------------------------------------------------------------ // Cache-first reads from [store] diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 39693a5ade..85cc25fc8c 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -249,6 +249,7 @@ object GrapeRankCommand { val scoringMs = (System.nanoTime() - scoreStart) / 1_000_000 System.err.println("[graperank] scored ${rankedIds.size} users in $scoringMs ms") + val hopHistogram = crawlStats?.hopHistogram.orEmpty() val result = linkedMapOf( "observer" to observer, @@ -256,8 +257,8 @@ object GrapeRankCommand { "relays_contacted" to (crawlStats?.relaysContacted ?: 0), "relay_feedback" to if (ctx.relayDiagnostics.hadFeedback()) ctx.relayDiagnostics.snapshot() else null, "relay_throttling" to if (ctx.relayLimiter.hadThrottling()) ctx.relayLimiter.snapshot() else null, - "max_hop_reached" to (crawlStats?.hopHistogram?.keys?.maxOrNull() ?: 0), - "users_by_hop" to (crawlStats?.hopHistogram?.mapKeys { it.key.toString() } ?: emptyMap()), + "max_hop_reached" to (hopHistogram.keys.maxOrNull() ?: 0), + "users_by_hop" to hopHistogram.mapKeys { it.key.toString() }, "graph_users" to graph.nodeCount, "graph_edges" to graph.edgeCount(), "reports_deleted" to reportsDeleted, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt index f25c177010..cc83639fa8 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt @@ -22,7 +22,6 @@ package com.vitorpamplona.quartz.experimental.graperank import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey -import com.vitorpamplona.quartz.nip01Core.crypto.verify import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.AdaptiveRelayLimiter import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.DrainFailure @@ -32,12 +31,12 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.single.newSubId import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.verifyAndInsert import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent import com.vitorpamplona.quartz.nip56Reports.ReportEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent -import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quartz.utils.SeenIds import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap import com.vitorpamplona.quartz.utils.concurrent.ConcurrentSet @@ -115,10 +114,8 @@ class GrapeRankDataCrawler( /** What the crawl fetched — the counters the caller reports and the graph is built from. */ class Stats( val rounds: Int, - val discovered: Int, val contactListsFed: Int, val relaysContacted: Int, - val deadRelays: Int, /** Users bucketed by follow-graph distance from the observer (hop -> count), ascending. */ val hopHistogram: Map, val downloadMs: Long, @@ -135,20 +132,22 @@ class GrapeRankDataCrawler( ): Stats = CrawlRun(observer, builder).run() /** - * Holds all per-crawl mutable state. Graph state (discovered/done/hopOf/ - * builder/writeRelayFreq/liveRelays/relaysContacted) is single-writer by - * construction — Phase A and the Phase-B consumer never run concurrently, and - * routeByOutbox (the only Phase-B producer write, to writeRelayFreq) touches a - * disjoint field — so those stay plain collections. Only the state genuinely - * shared across the producer / consumer / drain-worker coroutines is concurrent: - * relayHints, attempts, deadRelays, relayStrikes. + * Holds all per-crawl mutable state. Graph state (done/hopOf/builder/ + * writeRelayFreq/liveRelays/relaysContacted) is single-writer by construction + * — Phase A and the Phase-B consumer never run concurrently, and routeByOutbox + * (the only Phase-B producer write, to writeRelayFreq) touches a disjoint field + * — so those stay plain collections. The frontier IS [hopOf]'s key set: a user + * is "discovered" iff it has a hop stamp. Only the state genuinely shared across + * the producer / consumer / drain-worker coroutines is concurrent: relayHints, + * attempts, deadRelays, relayStrikes. */ private inner class CrawlRun( val observer: HexKey, val builder: TrustGraphBuilder, ) { - val hopOf = HashMap() - val discovered = hashSetOf(observer) + // hop distance per discovered user; the observer seeds it at 0. Its key set + // is the discovered frontier — no separate `discovered` set to keep in sync. + val hopOf = hashMapOf(observer to 0) val done = hashSetOf() val relaysContacted = hashSetOf() val writeRelayFreq = HashMap() @@ -206,7 +205,7 @@ class GrapeRankDataCrawler( for (tag in contacts.follows()) { follows.add(tag.pubKey) tag.relayUri?.let { relayHints.getOrPut(tag.pubKey) { ConcurrentSet() }.add(it) } - if (discovered.add(tag.pubKey)) { + if (tag.pubKey !in hopOf) { hopOf[tag.pubKey] = nextHop fresh++ } @@ -438,12 +437,11 @@ class GrapeRankDataCrawler( // Tier 2). SupervisorJob so one failing sweep never cancels the others; // cancelled when the crawl finishes. val bgScope = CoroutineScope(coroutineContext + SupervisorJob()) - hopOf[observer] = 0 while (rounds < config.maxRounds) { // Only crawl users within the hop budget; deeper users still appear // in the graph as follow targets, we just don't fetch their lists. - val pending = discovered.filter { it !in done && (hopOf[it] ?: 0) < config.maxHops } + val pending = hopOf.keys.filter { it !in done && (hopOf[it] ?: 0) < config.maxHops } if (pending.isEmpty()) break rounds++ @@ -454,7 +452,7 @@ class GrapeRankDataCrawler( client.subscribe(WARM_SUB_ID, warm.associateWith { WARM_FILTERS }, null) } - val discoveredBefore = discovered.size + val discoveredBefore = hopOf.size val fedBefore = contactListsFed // Phase A — bulk-fetch from the busiest relays via the sharded sweep. @@ -477,8 +475,8 @@ class GrapeRankDataCrawler( // no worker waits on a slow sibling and hot relays stay connected. // Shared graph state stays single-writer: routeByOutbox runs only // on the producer (keeps writeRelayFreq serial) and ingest runs - // only on the consumer (keeps discovered/done/builder/hopOf serial), - // now overlapped with draining instead of blocked behind each batch. + // only on the consumer (keeps done/builder/hopOf serial), now + // overlapped with draining instead of blocked behind each batch. val routed = Channel, Map>>>(DRAIN_CONCURRENCY * 2) val drainedOut = Channel, Set, List>>>(Channel.UNLIMITED) coroutineScope { @@ -535,8 +533,8 @@ class GrapeRankDataCrawler( log( "[graperank] round $rounds: pending=${pending.size}, " + - "gotList=${contactListsFed - fedBefore}, newUsers=${discovered.size - discoveredBefore}, " + - "discovered=${discovered.size}, done=${done.size}, dead=${deadRelays.size()}", + "gotList=${contactListsFed - fedBefore}, newUsers=${hopOf.size - discoveredBefore}, " + + "discovered=${hopOf.size}, done=${done.size}, dead=${deadRelays.size()}", ) } @@ -560,16 +558,14 @@ class GrapeRankDataCrawler( .toMap() val downloadMs = crawlMark.elapsedNow().inWholeMilliseconds log( - "[graperank] crawl complete: ${discovered.size} discovered, $contactListsFed contact lists fed, " + + "[graperank] crawl complete: ${hopOf.size} discovered, $contactListsFed contact lists fed, " + "${relaysContacted.size} relays contacted, ${deadRelays.size()} dead, $rounds rounds in $downloadMs ms; " + "by hop: " + hopHistogram.entries.joinToString(" ") { "${it.key}=${it.value}" }, ) return Stats( rounds = rounds, - discovered = discovered.size, contactListsFed = contactListsFed, relaysContacted = relaysContacted.size, - deadRelays = deadRelays.size(), hopHistogram = hopHistogram, downloadMs = downloadMs, ) @@ -632,7 +628,7 @@ class GrapeRankDataCrawler( val seen = SeenIds(initialSlotsPow2 = 12) for ((relay, event) in eventChannel) { if (seen.contains(event.id)) continue - if (verifyAndStore(event)) { + if (store.verifyAndInsert(event)) { seen.add(event.id) collected.add(relay to event) } @@ -711,28 +707,6 @@ class GrapeRankDataCrawler( return collected } - /** - * Verify [event]'s NIP-01 id+signature and, if valid, persist it to [store]. - * Returns true when the event was accepted. A UNIQUE-constraint rejection is - * normal (the store already holds this id, or a newer replaceable) — the outbox - * model delivers the same event from several relays, so a crawl produces these - * by the hundred-thousand — so only genuine persistence failures are logged. - */ - private suspend fun verifyAndStore(event: Event): Boolean { - if (!event.verify()) { - Log.w("GrapeRankDataCrawler") { "dropped event ${event.id.take(8)} kind=${event.kind} — bad signature" } - return false - } - try { - store.insert(event) - } catch (t: Throwable) { - if (t.message?.contains("UNIQUE constraint", ignoreCase = true) != true) { - Log.w("GrapeRankDataCrawler") { "store insert failed for ${event.id.take(8)}: ${t.message}" } - } - } - return true - } - /** Latest known kind:3 contact list for [pubKey] from the local store, or null. */ private suspend fun contactsOf(pubKey: HexKey): ContactListEvent? = store diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/VerifyAndInsert.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/VerifyAndInsert.kt new file mode 100644 index 0000000000..c2f1ff5193 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/VerifyAndInsert.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.crypto.verify +import com.vitorpamplona.quartz.utils.Log + +/** + * Verify [event]'s NIP-01 id + signature and, if valid, persist it to this store. + * Returns `true` when the event was accepted (verified) — even if the insert was a + * no-op — so callers can gate "surface this event" on the return. + * + * A UNIQUE-constraint rejection is normal, not a failure: the store already holds + * this id, or a newer version of a replaceable (kind 0/3/10000-19999). The outbox + * model routinely delivers the same event from several of a user's write relays, so + * a crawl produces these by the hundred-thousand — so only genuine persistence + * failures (I/O, full disk, corruption) are logged. Persistence is best-effort: an + * insert error is swallowed, not propagated, so it can't break a live subscription. + * + * This is the single verify-then-store sink every event-arrival path should funnel + * through, so the store stays the authoritative cache of what has been seen. + */ +suspend fun IEventStore.verifyAndInsert(event: Event): Boolean { + if (!event.verify()) { + Log.w("EventStore") { "dropped event ${event.id.take(8)} kind=${event.kind} — bad signature" } + return false + } + try { + insert(event) + } catch (t: Throwable) { + if (t.message?.contains("UNIQUE constraint", ignoreCase = true) != true) { + Log.w("EventStore") { "store insert failed for ${event.id.take(8)}: ${t.message}" } + } + } + return true +} diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt index 65a734f8d2..aaf40fdcb7 100644 --- a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt @@ -37,7 +37,11 @@ actual class ConcurrentMap { actual fun getOrPut( key: K, defaultValue: () -> V, - ): V = map.computeIfAbsent(key) { defaultValue() } + ): V = + // Fast-path the present-key hit (the common case in the crawl's hot + // relay-hint accumulation) so it never allocates the mapping-function + // closure; only an absent key pays for the atomic computeIfAbsent. + map[key] ?: map.computeIfAbsent(key) { defaultValue() } actual fun merge( key: K, From f7c3aef6fc7fd9a274e72d2060d3fc63eb98a3ba Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 17:51:27 +0000 Subject: [PATCH 49/58] perf(quartz): batch inserts + crawl-wide dedup in GrapeRankDataCrawler The crawl re-verified and re-inserted the same event many times: the outbox model mirrors each event (especially kind:10002 relay lists) across relays, indexers, and rounds, but dedup lived in a per-drain SeenIds, so only the copies within one drain were caught. Add a crawl-wide seen-set (thread-safe ConcurrentSet of event ids, shared across all 24 concurrent drains and every round), checked before verify and added only after verify so a forged copy can't suppress the genuine one. Group-commit the store writes via IEventStore.batchInsert instead of one transaction per event. Measured on a from-scratch --max-hops 3 crawl: events actually verified+stored dropped ~34% (112k -> 74k) and verify time fell in lockstep. The write path now also reports verify/insert timing + events_stored in Stats, exposed as verify_ms/ insert_ms/events_stored on the CLI, and takes an --insert-batch knob. Finding: with the work reduced, inserts serialize on SQLite's single writer mutex rather than transaction count, and the crawl's wall-clock ceiling is the drain-timeout retry tail on dead outboxes, not the disk. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 9 ++ .../graperank/GrapeRankDataCrawler.kt | 117 ++++++++++++++---- 2 files changed, 104 insertions(+), 22 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index 85cc25fc8c..f00adfe61e 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -131,6 +131,10 @@ object GrapeRankCommand { val offline = args.bool("offline") val diagnose = args.bool("diagnose") val timeoutMs = args.longFlag("timeout", 10L) * 1000 + // How many verified events the crawler group-commits per store write. 1 + // forces the per-event insert path (baseline); higher amortizes the SQLite + // transaction + writer-mutex cost across the batch. + val insertBatch = args.intFlag("insert-batch", 500) val doPublish = args.bool("publish") // Publish cutoff: only cards with rank >= this are published; existing // cards for targets below it (or gone from the graph) are retracted. Rank @@ -188,6 +192,7 @@ object GrapeRankCommand { maxHops = maxHops, timeoutMs = timeoutMs, diagnose = diagnose, + insertBatchSize = insertBatch, ), log = { System.err.println(it) }, ) @@ -264,6 +269,10 @@ object GrapeRankCommand { "reports_deleted" to reportsDeleted, "users_scored" to rankedIds.size, "download_ms" to crawlStats?.downloadMs, + "verify_ms" to crawlStats?.verifyMs, + "insert_ms" to crawlStats?.insertMs, + "events_stored" to crawlStats?.eventsStored, + "insert_batch" to insertBatch, "store_load_ms" to storeLoadMs, "graph_build_ms" to buildMs, "scoring_ms" to scoringMs, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt index cc83639fa8..132690ee81 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.experimental.graperank import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.crypto.verify import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.AdaptiveRelayLimiter import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.DrainFailure @@ -31,13 +32,12 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.single.newSubId import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import com.vitorpamplona.quartz.nip01Core.store.verifyAndInsert import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent import com.vitorpamplona.quartz.nip56Reports.ReportEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent -import com.vitorpamplona.quartz.utils.SeenIds +import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap import com.vitorpamplona.quartz.utils.concurrent.ConcurrentSet import kotlinx.coroutines.CompletableDeferred @@ -51,6 +51,8 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeoutOrNull +import kotlin.concurrent.atomics.AtomicLong +import kotlin.concurrent.atomics.ExperimentalAtomicApi import kotlin.coroutines.coroutineContext import kotlin.time.TimeSource @@ -81,6 +83,7 @@ import kotlin.time.TimeSource * defaults live in application code, not the protocol library. Operator progress * is emitted through [log]; a headless caller routes it to stderr, a UI ignores it. */ +@OptIn(ExperimentalAtomicApi::class) class GrapeRankDataCrawler( private val client: NostrClient, private val store: IEventStore, @@ -88,6 +91,15 @@ class GrapeRankDataCrawler( private val config: Config, private val log: (String) -> Unit = {}, ) { + // Crawl-wide timing, accumulated across every drainGated consumer (24 run at + // once). Nanoseconds spent verifying signatures vs. spent in the store write, + // plus how many verified events reached the store. Surfaced in [Stats] so a + // caller can see whether a from-scratch crawl is verify-, write-, or (by + // subtraction from wall time) network-bound. Reset at the top of each [crawl]. + private val verifyNanos = AtomicLong(0) + private val insertNanos = AtomicLong(0) + private val eventsStored = AtomicLong(0) + /** * Relay policy + crawl bounds. The relay sets come from the caller because the * aggregator/bootstrap defaults live outside quartz. @@ -101,6 +113,10 @@ class GrapeRankDataCrawler( * @param maxHops follow-graph distance from the observer to crawl (Brainstorm uses 8). * @param timeoutMs per-drain timeout. * @param diagnose log a breakdown of slow/unreachable relays on each drain timeout. + * @param insertBatchSize how many verified events to group-commit per + * [IEventStore.batchInsert]. The outbox model streams the same events from + * many relays through a single SQLite writer, so batching amortizes the + * per-transaction + writer-mutex cost across the batch (coerced to `>= 1`). */ class Config( val relayListDiscoveryRelays: Set, @@ -109,6 +125,7 @@ class GrapeRankDataCrawler( val maxHops: Int = Int.MAX_VALUE, val timeoutMs: Long = 10_000, val diagnose: Boolean = false, + val insertBatchSize: Int = 500, ) /** What the crawl fetched — the counters the caller reports and the graph is built from. */ @@ -119,6 +136,12 @@ class GrapeRankDataCrawler( /** Users bucketed by follow-graph distance from the observer (hop -> count), ascending. */ val hopHistogram: Map, val downloadMs: Long, + /** Wall time verifying signatures, summed across the concurrent consumers. */ + val verifyMs: Long, + /** Wall time in the store write path, summed across the concurrent consumers. */ + val insertMs: Long, + /** Verified events handed to the store (duplicates included — the write path dedups). */ + val eventsStored: Long, ) /** @@ -129,7 +152,12 @@ class GrapeRankDataCrawler( suspend fun crawl( observer: HexKey, builder: TrustGraphBuilder, - ): Stats = CrawlRun(observer, builder).run() + ): Stats { + verifyNanos.store(0) + insertNanos.store(0) + eventsStored.store(0) + return CrawlRun(observer, builder).run() + } /** * Holds all per-crawl mutable state. Graph state (done/hopOf/builder/ @@ -159,6 +187,15 @@ class GrapeRankDataCrawler( val deadRelays = ConcurrentSet() val relayStrikes = ConcurrentMap() + // Crawl-wide dedup of event ids, shared across all concurrent drains and + // every round. The outbox model mirrors the SAME event (especially kind:10002 + // relay lists) across many relays, indexers, and rounds; a per-drain set only + // catches the copies within one drain, so without this the majority of events + // would be re-verified + re-inserted (hitting the store's UNIQUE constraint) + // in a later drain. An id is added only AFTER it verifies, so a forged copy + // (valid id, bad signature) delivered first can't suppress the genuine one. + val seenIds = ConcurrentSet() + var rounds = 0 var contactListsFed = 0 @@ -270,7 +307,7 @@ class GrapeRankDataCrawler( val dead = HashMap() val filters = mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = FETCH_KINDS, authors = it) }) - drainGated(filters, dead) to dead + drainGated(filters, dead, seenIds) to dead } } }.awaitAll() @@ -296,7 +333,7 @@ class GrapeRankDataCrawler( val dead = HashMap() val filters = live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = FETCH_KINDS, authors = it) } } - val events = drainGated(filters, dead) + val events = drainGated(filters, dead, seenIds) recordDead(dead) relaysContacted += live for ((relay, _) in events) liveRelays.add(relay) @@ -339,7 +376,7 @@ class GrapeRankDataCrawler( Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) } } - drainGated(filters, null) + drainGated(filters, null, seenIds) } val discovery = config.relayListDiscoveryRelays @@ -390,7 +427,7 @@ class GrapeRankDataCrawler( } } } - drainGated(filters, null) + drainGated(filters, null, seenIds) } /** @@ -497,7 +534,7 @@ class GrapeRankDataCrawler( launch { for ((batch, filters) in routed) { val dead = HashMap() - val events = drainGated(filters, dead) + val events = drainGated(filters, dead, seenIds) recordDead(dead) drainedOut.send(Triple(batch, filters.keys, events)) } @@ -557,17 +594,27 @@ class GrapeRankDataCrawler( .sortedBy { it.first } .toMap() val downloadMs = crawlMark.elapsedNow().inWholeMilliseconds + val verifyMs = verifyNanos.load() / 1_000_000 + val insertMs = insertNanos.load() / 1_000_000 + val stored = eventsStored.load() log( "[graperank] crawl complete: ${hopOf.size} discovered, $contactListsFed contact lists fed, " + "${relaysContacted.size} relays contacted, ${deadRelays.size()} dead, $rounds rounds in $downloadMs ms; " + "by hop: " + hopHistogram.entries.joinToString(" ") { "${it.key}=${it.value}" }, ) + log( + "[graperank] write path: $stored events stored, verify ${verifyMs}ms + insert ${insertMs}ms " + + "(summed across ${DRAIN_CONCURRENCY} consumers, batch=${config.insertBatchSize})", + ) return Stats( rounds = rounds, contactListsFed = contactListsFed, relaysContacted = relaysContacted.size, hopHistogram = hopHistogram, downloadMs = downloadMs, + verifyMs = verifyMs, + insertMs = insertMs, + eventsStored = stored, ) } } @@ -579,11 +626,14 @@ class GrapeRankDataCrawler( * subscription so we never exceed its adaptive concurrent-subscription cap; a * relay's filters are split into REQ-sized groups so a popular relay routed * thousands of authors doesn't produce a multi-MB frame that most relays - * reject outright. Hard connect failures are reported into [deadOut]. + * reject outright. Hard connect failures are reported into [deadOut]. Events + * whose id is already in the crawl-wide [seen] set are dropped before the + * expensive verify+store; verified ids are added to it so later drains skip them. */ private suspend fun drainGated( filters: Map>, deadOut: MutableMap?, + seen: ConcurrentSet, ): List> { if (filters.isEmpty()) return emptyList() val eventChannel = Channel>(Channel.UNLIMITED) @@ -617,22 +667,45 @@ class GrapeRankDataCrawler( val collected = mutableListOf>() coroutineScope { - // Single consumer: verify+store serially. One writer, so SeenIds' - // single-writer contract holds. The outbox model delivers the SAME event - // from many relays at once; skip a duplicate BEFORE the expensive Schnorr - // verify+store. An id is marked seen only after it verifies, so a forged - // copy (valid id, bad signature) delivered first can't suppress the - // genuine one that follows. + // Single consumer per drain: dedup against the crawl-wide [seen] set, + // verify, and group-commit to the store. Duplicates (the same event from + // another relay, drain, or round) are skipped BEFORE the expensive Schnorr + // verify + store write. An id is added to [seen] only after it verifies, so + // a forged copy (valid id, bad signature) delivered first can't suppress + // the genuine one. Verified events are buffered and flushed via batchInsert + // so the per-transaction + writer-mutex cost is paid once per + // [insertBatchSize], not once per event. (A relay whose every event was + // already seen won't be credited into `liveRelays` by this drain — that's + // fine: it's a redundant mirror that added nothing new.) val consumer = launch { - val seen = SeenIds(initialSlotsPow2 = 12) - for ((relay, event) in eventChannel) { - if (seen.contains(event.id)) continue - if (store.verifyAndInsert(event)) { - seen.add(event.id) - collected.add(relay to event) - } + val flushAt = config.insertBatchSize.coerceAtLeast(1) + val buffer = ArrayList(flushAt) + + suspend fun flush() { + if (buffer.isEmpty()) return + val mark = TimeSource.Monotonic.markNow() + store.batchInsert(buffer) + insertNanos.addAndFetch(mark.elapsedNow().inWholeNanoseconds) + eventsStored.addAndFetch(buffer.size.toLong()) + buffer.clear() } + + for ((relay, event) in eventChannel) { + if (event.id in seen) continue + val vMark = TimeSource.Monotonic.markNow() + val ok = event.verify() + verifyNanos.addAndFetch(vMark.elapsedNow().inWholeNanoseconds) + if (!ok) { + Log.w("GrapeRankDataCrawler") { "dropped event ${event.id.take(8)} kind=${event.kind} — bad signature" } + continue + } + seen.add(event.id) + collected.add(relay to event) + buffer.add(event) + if (buffer.size >= flushAt) flush() + } + flush() } // One gated subscription per (relay, REQ-group). The permit is held for // the group's whole life, so concurrent subs on a relay never exceed its From 9da382b6988e8b409fc7756181318088de5bfe7c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 17:56:06 +0000 Subject: [PATCH 50/58] refactor(quartz): extract GrapeRankPublisher from the CLI command Mirror the crawler extraction on the emit side: the NIP-85 kind:30382 card reconcile + publish logic (existingCards read-back, rank-diff upsert, stale-card kind:5 retraction batched under the 64KB event cap) moves out of GrapeRankCommand into a reusable GrapeRankPublisher in quartz experimental/graperank. It takes an IEventStore for the prior-card read-back and an injected publish function (event + relays -> per-relay ack), so the store/relay wiring stays in the app while the reconcile logic is reusable (e.g. by the Android app). GrapeRankCommand is now a thin orchestrator: crawl (GrapeRankDataCrawler) -> score (GrapeRank) -> publish (GrapeRankPublisher). The account-specific bits stay in the CLI: operator-key derivation, the observer's kind:10040 discovery pointer, and the operator/register/providers sub-verbs. --- .../amethyst/cli/commands/GrapeRankCommand.kt | 151 ++----------- .../graperank/GrapeRankPublisher.kt | 207 ++++++++++++++++++ 2 files changed, 227 insertions(+), 131 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankPublisher.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index f00adfe61e..e9fa3e8f84 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList import com.vitorpamplona.quartz.experimental.graperank.GrapeRank import com.vitorpamplona.quartz.experimental.graperank.GrapeRankDataCrawler import com.vitorpamplona.quartz.experimental.graperank.GrapeRankParams +import com.vitorpamplona.quartz.experimental.graperank.GrapeRankPublisher import com.vitorpamplona.quartz.experimental.graperank.TrustGraphBuilder import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey @@ -82,15 +83,6 @@ import kotlin.math.roundToInt * - `amy graperank providers [USER]` — list a user's trusted providers. */ object GrapeRankCommand { - // Concurrent publishes when writing NIP-85 cards. - private const val PUBLISH_CONCURRENCY = 16 - - // Addressable coordinates cited per kind:5 retraction. Each `a` tag is - // ~130 bytes (30382:<64hex>:<64hex>), so 400 keeps the whole event ~52KB — - // under the 64KB *event* size many relays cap at (stricter than the 256KB - // message cap). - private const val DELETE_PER_EVENT = 400 - // Broad, big general relays that carry kind:10002 for many users, added to the // crawler's discovery set to raise the odds of resolving a stranger's outbox. private val EXTRA_DISCOVERY_RELAYS: Set = @@ -308,42 +300,31 @@ object GrapeRankCommand { result["published"] = 0 result["publish_error"] = "no operator relay configured — run `amy graperank operator relay ` or pass --publish-relay" } else { - // Reconcile what the algorithm says should exist against what - // this provider key has already published (newest card per - // target, read back from the store). - val existing = existingCards(ctx, providerPubkey) - + // The scorer's desired card set: every user at or above the rank + // cutoff, as (target, rank). GrapeRankPublisher reconciles this + // against what this provider key already published and upserts / + // retracts the difference. val publishable = rankedIds .filter { rankOf(scores[it]) >= minRank } .map { graph.pubkeyOf(it) to rankOf(scores[it]) } - val publishableTargets = publishable.mapTo(HashSet()) { it.first } - // Upsert: publishable targets whose rank tag STRING would change - // (or that have no card yet). RankTag.assemble writes - // rank.toString(), so we diff that exact string — an unchanged - // score is skipped, so clients only sync ranks that moved. - val changed = publishable.filter { (target, rank) -> existing[target]?.let(::rankTagValue) != rank.toString() } - val toUpsert = changed.take(publishLimit) + val publisher = GrapeRankPublisher(ctx.store) { event, to -> ctx.publish(event, to) } + val pub = + publisher.reconcileAndPublish( + providerSigner = serviceSigner, + providerPubkey = providerPubkey, + scored = publishable, + relays = relays, + publishLimit = publishLimit, + ) - // Delete: existing cards whose target is no longer publishable — - // it dropped out of the graph, or fell below the cutoff (e.g. a - // rank-0/1 card we would no longer publish). We won't leave a - // stale assertion standing, so we retract it with a kind:5. - val toDelete = existing.filterKeys { it !in publishableTargets }.values.toList() - - result["skipped_unchanged"] = publishable.size - changed.size - if (changed.size > toUpsert.size) { - result["publish_truncated"] = changed.size - toUpsert.size - } - - val (ok, rejected) = publishCards(ctx, serviceSigner, toUpsert, relays) - val (deleted, deleteRejected) = publishDeletions(ctx, serviceSigner, toDelete, relays) - - result["published"] = ok - result["publish_rejected"] = rejected - result["deleted"] = deleted - result["delete_rejected"] = deleteRejected + result["skipped_unchanged"] = pub.skippedUnchanged + if (pub.truncated > 0) result["publish_truncated"] = pub.truncated + result["published"] = pub.published + result["publish_rejected"] = pub.publishRejected + result["deleted"] = pub.deleted + result["delete_rejected"] = pub.deleteRejected result["published_kind"] = ContactCardEvent.KIND result["published_to"] = relays.map { it.url } @@ -670,98 +651,6 @@ object GrapeRankCommand { return dropped } - /** - * The exact `rank` tag VALUE STRING we last published for each target, read - * from the active account's own kind:30382 cards in the local store (newest - * card wins per target). `ctx.publish` stores every card it sends, so on - * repeat runs this reflects what's already out there. - * - * We key on the raw tag string, not a re-parsed Int, because that string is - * exactly what a client diffs: creating a new signature (a new event id) is - * only worth it when the written value actually changes. Our cards carry ONLY - * a `rank` tag (plus the d-tag target), so this single tag's value fully - * decides whether the event would differ — see the publish gate. - */ - private suspend fun existingCards( - ctx: Context, - providerPubkey: HexKey, - ): Map = - ctx.store - .query(Filter(kinds = listOf(ContactCardEvent.KIND), authors = listOf(providerPubkey))) - .filterIsInstance() - .groupBy { it.aboutUser() } - .mapNotNull { (target, cards) -> - val t = target ?: return@mapNotNull null - t to (cards.maxByOrNull { it.createdAt } ?: return@mapNotNull null) - }.toMap() - - /** - * The raw `rank` tag value string on a card — what a client diffs. We compare - * this against `rank.toString()` (what RankTag.assemble writes) so an unchanged - * score never produces a new signature. Our cards carry only a `rank` tag (plus - * the d-tag target), so this one value decides whether the event would differ. - */ - private fun rankTagValue(card: ContactCardEvent): String? = - card.tags.firstNotNullOfOrNull { tag -> - if (tag.size > 1 && tag[0] == RankTag.TAG_NAME) tag[1] else null - } - - /** Build + publish one NIP-85 kind:30382 card per user, bounded-concurrently, signed by [signer]. */ - private suspend fun publishCards( - ctx: Context, - signer: NostrSigner, - cards: List>, - relays: Set, - ): Pair { - var published = 0 - var rejected = 0 - for (batch in cards.chunked(PUBLISH_CONCURRENCY)) { - val acks = - coroutineScope { - batch - .map { (pubkey, rank) -> - async { - val card = - ContactCardEvent.create( - targetUser = pubkey, - signer = signer, - publicInitializer = { add(RankTag.assemble(rank)) }, - ) - ctx.publish(card, relays) - } - }.awaitAll() - } - for (ack in acks) { - if (ack.values.any { it }) published++ else rejected++ - } - } - return published to rejected - } - - /** - * Retract stale cards with NIP-09 kind:5 deletions signed by [signer] (the same - * service key that signed the cards). Batches several addressable coordinates - * per deletion — chunked so the kind:5 frame stays under the relay message cap — - * and each carries the card's `a` tag (30382:provider:target), so re-publishing - * a newer version later isn't blocked. Returns (deleted, rejected) card counts. - */ - private suspend fun publishDeletions( - ctx: Context, - signer: NostrSigner, - cards: List, - relays: Set, - ): Pair { - if (cards.isEmpty()) return 0 to 0 - var deleted = 0 - var rejected = 0 - for (chunk in cards.chunked(DELETE_PER_EVENT)) { - val event = signer.sign(DeletionEvent.build(chunk)) - val ack = ctx.publish(event, relays) - if (ack.values.any { it }) deleted += chunk.size else rejected += chunk.size - } - return deleted to rejected - } - /** * If the active account IS the observer (so we hold their key), publish/refresh * their kind:10040 declaring `30382:rank` -> [providerPubkey] at [relay], to diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankPublisher.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankPublisher.kt new file mode 100644 index 0000000000..2735d7e08c --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankPublisher.kt @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.experimental.graperank + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent +import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent +import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope + +/** + * Publishes a set of GrapeRank scores as NIP-85 kind:30382 [ContactCardEvent] + * trusted assertions (one `rank` card per scored user), reconciled against what + * this provider key has already published so a repeat run only writes what moved. + * + * Reconciliation, given the desired `(target, rank)` set the scorer produced: + * - **skip** a target whose stored card already carries the same rank string — + * re-signing an unchanged card would churn a new event id for no client benefit; + * - **upsert** a target whose rank changed (or that has no card yet), up to a + * publish limit; + * - **retract** every stored card whose target is no longer in the desired set + * (it fell below the caller's cutoff, or dropped out of the graph) with a NIP-09 + * kind:5 deletion, batched so the frame stays under the ~64KB event cap. + * + * Transport-agnostic like [GrapeRankDataCrawler]: it reads prior cards from an + * [IEventStore] and emits through an injected [publish] function (event + relays → + * per-relay ack), so the store/relay wiring stays in the application while the + * reconcile + card-construction logic is reusable (e.g. by the Android app). + */ +class GrapeRankPublisher( + private val store: IEventStore, + private val publish: suspend (Event, Set) -> Map, +) { + /** Outcome counts for one reconcile: what was written, retracted, and skipped. */ + class Result( + val published: Int, + val publishRejected: Int, + val deleted: Int, + val deleteRejected: Int, + val skippedUnchanged: Int, + /** Changed cards beyond [publishLimit] that were not upserted this run. */ + val truncated: Int, + ) + + /** + * Reconcile the desired [scored] `(target, rank)` set (the caller has already + * applied any rank cutoff) against the cards [providerPubkey] previously + * published, then upsert the changes and retract the stale cards, all signed by + * [providerSigner]. At most [publishLimit] changed cards are upserted per run. + */ + suspend fun reconcileAndPublish( + providerSigner: NostrSigner, + providerPubkey: HexKey, + scored: List>, + relays: Set, + publishLimit: Int, + publishConcurrency: Int = PUBLISH_CONCURRENCY, + ): Result { + // Newest card per target this provider already published (read back from + // the store, which every published card was persisted to). + val existing = existingCards(providerPubkey) + val publishableTargets = scored.mapTo(HashSet()) { it.first } + + // Upsert publishable targets whose rank tag STRING would change (or that + // have no card yet). RankTag.assemble writes rank.toString(), so we diff + // that exact string — an unchanged score is skipped so clients only sync + // ranks that moved. + val changed = scored.filter { (target, rank) -> existing[target]?.let(::rankTagValue) != rank.toString() } + val toUpsert = changed.take(publishLimit) + + // Retract existing cards whose target is no longer publishable — it dropped + // out of the graph, or fell below the caller's cutoff. We won't leave a + // stale assertion standing. + val toDelete = existing.filterKeys { it !in publishableTargets }.values.toList() + + val (ok, rejected) = publishCards(providerSigner, toUpsert, relays, publishConcurrency) + val (deleted, deleteRejected) = publishDeletions(providerSigner, toDelete, relays) + + return Result( + published = ok, + publishRejected = rejected, + deleted = deleted, + deleteRejected = deleteRejected, + skippedUnchanged = scored.size - changed.size, + truncated = (changed.size - toUpsert.size).coerceAtLeast(0), + ) + } + + /** + * The newest kind:30382 card [providerPubkey] published per target, read from + * the store (every card [publish] sends is persisted first, so on repeat runs + * this reflects what is already out there). + */ + private suspend fun existingCards(providerPubkey: HexKey): Map = + store + .query(Filter(kinds = listOf(ContactCardEvent.KIND), authors = listOf(providerPubkey))) + .filterIsInstance() + .groupBy { it.aboutUser() } + .mapNotNull { (target, cards) -> + val t = target ?: return@mapNotNull null + t to (cards.maxByOrNull { it.createdAt } ?: return@mapNotNull null) + }.toMap() + + /** + * The raw `rank` tag value string on a card — exactly what a client diffs, so an + * unchanged score never produces a new signature. Our cards carry only a `rank` + * tag (plus the d-tag target), so this one value decides whether a re-publish + * would differ. + */ + private fun rankTagValue(card: ContactCardEvent): String? = + card.tags.firstNotNullOfOrNull { tag -> + if (tag.size > 1 && tag[0] == RankTag.TAG_NAME) tag[1] else null + } + + /** Build + publish one kind:30382 card per (target, rank), bounded-concurrently. */ + private suspend fun publishCards( + signer: NostrSigner, + cards: List>, + relays: Set, + concurrency: Int, + ): Pair { + var published = 0 + var rejected = 0 + for (batch in cards.chunked(concurrency)) { + val acks = + coroutineScope { + batch + .map { (pubkey, rank) -> + async { + val card = + ContactCardEvent.create( + targetUser = pubkey, + signer = signer, + publicInitializer = { add(RankTag.assemble(rank)) }, + ) + publish(card, relays) + } + }.awaitAll() + } + for (ack in acks) { + if (ack.values.any { it }) published++ else rejected++ + } + } + return published to rejected + } + + /** + * Retract stale cards with NIP-09 kind:5 deletions signed by [signer] (the same + * key that signed the cards). Batches [DELETE_PER_EVENT] addressable coordinates + * per deletion so the kind:5 frame stays under the ~64KB event cap; each carries + * the card's `a` tag (30382:provider:target), so re-publishing a newer version + * later isn't blocked. Returns (deleted, rejected) card counts. + */ + private suspend fun publishDeletions( + signer: NostrSigner, + cards: List, + relays: Set, + ): Pair { + if (cards.isEmpty()) return 0 to 0 + var deleted = 0 + var rejected = 0 + for (chunk in cards.chunked(DELETE_PER_EVENT)) { + val event = signer.sign(DeletionEvent.build(chunk)) + val ack = publish(event, relays) + if (ack.values.any { it }) deleted += chunk.size else rejected += chunk.size + } + return deleted to rejected + } + + companion object { + /** Concurrent card publishes when upserting. */ + const val PUBLISH_CONCURRENCY = 16 + + /** + * Addressable coordinates cited per kind:5 retraction. Each `a` tag is + * ~130 bytes (30382:<64hex>:<64hex>), so 400 keeps the whole event ~52KB — + * under the 64KB event-size cap many relays enforce (stricter than the + * 256KB message cap). + */ + const val DELETE_PER_EVENT = 400 + } +} From 9eec6323c8567ab9c9ec1e146a5f25f38dcf3535 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 18:21:52 +0000 Subject: [PATCH 51/58] perf(quartz): don't re-query a relay that EOSE'd without a user's list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-user retry counter was blunt: it bumped attempts whether an outbox was dead, timed out, or cleanly EOSE'd with no event — so a straggler kept being re-queried against a live relay that had already definitively answered it lacks their kind:3. Distinguish the cases: drainGated now reports the relays that fully EOSE'd (answeredOut); the consumer records, per user, the relays that answered but did not return their contact list (askedEmpty); routeByOutbox excludes those from the user's candidate relays. A timed-out relay is never added (it might just be slow — still worth a retry), only a clean-EOSE-empty one; dead relays stay pruned as before. Measured on --max-hops 3: redundant fetching dropped ~8% (74k -> 68k events stored). It does NOT move the wall-clock tail, though — that tail is dominated by timeout/dead outboxes (the retryable case), not EOSE-empty relays. The wall-clock lever remains the timeout retry budget (MAX_OUTBOX_ATTEMPTS / drain timeout). --- .../graperank/GrapeRankDataCrawler.kt | 73 ++++++++++++++++--- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt index 132690ee81..02c791ebfa 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt @@ -196,6 +196,13 @@ class GrapeRankDataCrawler( // (valid id, bad signature) delivered first can't suppress the genuine one. val seenIds = ConcurrentSet() + // Per-user relays that answered (EOSE'd) without holding this user's kind:3, + // so re-querying them for this user is guaranteed-empty waste. routeByOutbox + // subtracts these from a user's candidate relays, so a straggler is retried + // only against relays that could plausibly still have it (never-asked, or + // ones that timed out — which unlike a clean EOSE might just be slow). + val askedEmpty = ConcurrentMap>() + var rounds = 0 var contactListsFed = 0 @@ -456,9 +463,15 @@ class GrapeRankDataCrawler( (attempts[pk] ?: 0) > 0 -> write + backbone else -> write } - // Skip relays already proven dead — routing to them only burns the - // drain timeout. - for (relay in relays) if (relay !in deadRelays) perRelay.getOrPut(relay) { HashSet() }.add(pk) + // Skip relays proven dead (routing to them only burns the drain + // timeout) and relays that already EOSE'd without this user's list + // (re-querying them for this user is guaranteed-empty waste). + val emptied = askedEmpty[pk] + for (relay in relays) { + if (relay in deadRelays) continue + if (emptied != null && relay in emptied) continue + perRelay.getOrPut(relay) { HashSet() }.add(pk) + } } return perRelay.mapValues { (_, authors) -> @@ -515,7 +528,7 @@ class GrapeRankDataCrawler( // only on the consumer (keeps done/builder/hopOf serial), now // overlapped with draining instead of blocked behind each batch. val routed = Channel, Map>>>(DRAIN_CONCURRENCY * 2) - val drainedOut = Channel, Set, List>>>(Channel.UNLIMITED) + val drainedOut = Channel(Channel.UNLIMITED) coroutineScope { // Producer: route each batch by outbox (serial), backpressured // by the bounded `routed` channel. @@ -528,26 +541,44 @@ class GrapeRankDataCrawler( routed.close() } // Drain workers: pure network, no shared graph-state writes - // except recordDead (concurrent-safe). + // except recordDead (concurrent-safe). Each captures the relays + // that cleanly EOSE'd, so the consumer can tell "answered empty" + // from "timed out" per user. val workers = List(DRAIN_CONCURRENCY) { launch { for ((batch, filters) in routed) { val dead = HashMap() - val events = drainGated(filters, dead, seenIds) + val answered = HashSet() + val events = drainGated(filters, dead, seenIds, answered) recordDead(dead) - drainedOut.send(Triple(batch, filters.keys, events)) + drainedOut.send(DrainedBatch(batch, filters, answered, events)) } } } // Consumer: single-writer ingest, overlapped with draining. val consumer = launch { - for ((batch, relays, events) in drainedOut) { - relaysContacted += relays + for (d in drainedOut) { + relaysContacted += d.filters.keys // Any relay that gave us an event is proven live + useful. - for ((relay, _) in events) liveRelays.add(relay) - for (pk in batch) { + for ((relay, _) in d.events) liveRelays.add(relay) + + // Per user, record relays that answered (EOSE'd) but did + // not return their kind:3, so they aren't re-queried there. + val returnedByRelay = HashMap>() + for ((relay, ev) in d.events) { + if (ev is ContactListEvent) returnedByRelay.getOrPut(relay) { HashSet() }.add(ev.pubKey) + } + for (relay in d.answered) { + val asked = d.filters[relay]?.flatMapTo(HashSet()) { it.authors.orEmpty() } ?: continue + val returned = returnedByRelay[relay].orEmpty() + for (pk in asked) { + if (pk !in returned) askedEmpty.getOrPut(pk) { ConcurrentSet() }.add(relay) + } + } + + for (pk in d.batch) { if (pk in done) continue val contacts = contactsOf(pk) if (contacts != null) { @@ -619,6 +650,19 @@ class GrapeRankDataCrawler( } } + /** + * One Phase-B batch after draining: the users asked for, the relay->filters map + * they were routed through, the relays that cleanly EOSE'd ([answered]), and the + * fresh events. Carries enough for the consumer to attribute "answered but + * empty" per user without re-deriving the routing. + */ + private class DrainedBatch( + val batch: List, + val filters: Map>, + val answered: Set, + val events: List>, + ) + /** * Subscribe each relay to its filters behind [limiter], drain until every * relay's subscription is terminal or the timeout elapses, verify+store the @@ -634,6 +678,7 @@ class GrapeRankDataCrawler( filters: Map>, deadOut: MutableMap?, seen: ConcurrentSet, + answeredOut: MutableSet? = null, ): List> { if (filters.isEmpty()) return emptyList() val eventChannel = Channel>(Channel.UNLIMITED) @@ -664,6 +709,10 @@ class GrapeRankDataCrawler( // relay's several REQ-groups; plus which relays stalled to a timeout. val failures = ConcurrentMap() val timedOut = ConcurrentSet() + // Relays that did NOT cleanly EOSE every group (timed out, closed, or + // couldn't connect). A relay absent from this set answered definitively — + // so an author it was asked for but didn't return is one it simply lacks. + val notAnswered = ConcurrentSet() val collected = mutableListOf>() coroutineScope { @@ -754,6 +803,7 @@ class GrapeRankDataCrawler( try { val reason = withTimeoutOrNull(config.timeoutMs) { done.await() } ?: "timeout" if (reason == "timeout") timedOut.add(subRelay) + if (reason != "eose") notAnswered.add(subRelay) classifyDrainFailure(reason)?.let { kind -> failures.merge(subRelay, kind) { a, b -> if (a == DrainFailure.HARD || b == DrainFailure.HARD) DrainFailure.HARD else DrainFailure.TRANSIENT @@ -777,6 +827,7 @@ class GrapeRankDataCrawler( log("[drain] timeout ${config.timeoutMs}ms: ${stalled.size} slow(no EOSE)" + (if (detail.isNotEmpty()) " | slow: $detail" else "")) } deadOut?.putAll(failures.snapshot()) + answeredOut?.addAll(filters.keys.filter { it !in notAnswered }) return collected } From c9ee79beba8ec6978b06ce63c26e34d020bd309d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 19:10:33 +0000 Subject: [PATCH 52/58] feat(graperank): log slow/timed-out relays with their query under --diagnose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a --diagnose slow-relay log: every content drain that reaches its terminal (EOSE or timeout) slower than SLOW_DRAIN_LOG_MS, or times out entirely, is recorded with the offending relay URL, the failure/EOSE reason, elapsed ms, and the exact filter shape (kinds + author count + first authors). This lets a human replay that precise REQ later to understand why the relay lags. Gated on --diagnose so there is no per-group timing/collection overhead otherwise. Make the content-drain fan-out configurable via a new --drain-concurrency flag (Config.drainConcurrency), replacing the DRAIN_CONCURRENCY constant. Default stays at the validated 24: an A/B at 64 ran ~2x slower with more dead relays (a higher global fan-out re-floods busy hubs faster than the per-relay demotion catches up), so the flag is a probe knob, not a speedup. Client WebSocket pings were also tried and reverted — busy-but-alive relays don't reliably pong while their query handler runs, so pinging just cut them as dead. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 6 +++ .../graperank/GrapeRankDataCrawler.kt | 40 ++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index e9fa3e8f84..d6f483b0ef 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -127,6 +127,11 @@ object GrapeRankCommand { // forces the per-event insert path (baseline); higher amortizes the SQLite // transaction + writer-mutex cost across the batch. val insertBatch = args.intFlag("insert-batch", 500) + // How many outbox batches drain in parallel (the worker-pool size). 24 is the + // validated default; higher fan-out re-floods busy hubs faster than the + // per-relay demotion catches up (an A/B at 64 was ~2x slower with MORE dead + // relays), so raise it only to probe specific slow relays. + val drainConcurrency = args.intFlag("drain-concurrency", 24) val doPublish = args.bool("publish") // Publish cutoff: only cards with rank >= this are published; existing // cards for targets below it (or gone from the graph) are retracted. Rank @@ -185,6 +190,7 @@ object GrapeRankCommand { timeoutMs = timeoutMs, diagnose = diagnose, insertBatchSize = insertBatch, + drainConcurrency = drainConcurrency, ), log = { System.err.println(it) }, ) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt index 02c791ebfa..e15634abd3 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt @@ -117,6 +117,12 @@ class GrapeRankDataCrawler( * [IEventStore.batchInsert]. The outbox model streams the same events from * many relays through a single SQLite writer, so batching amortizes the * per-transaction + writer-mutex cost across the batch (coerced to `>= 1`). + * @param drainConcurrency how many outbox batches drain at once (the worker + * pool size). A GLOBAL bound (memory / open sockets); the per-relay + * concurrent-sub cap is enforced separately by [AdaptiveRelayLimiter]. Keep it + * moderate: a higher global fan-out re-floods busy hubs faster than demotion + * catches up (an A/B at 64 ran ~2x slower with more dead relays), so 24 is the + * validated default and raising it is a probe, not a speedup. */ class Config( val relayListDiscoveryRelays: Set, @@ -126,6 +132,7 @@ class GrapeRankDataCrawler( val timeoutMs: Long = 10_000, val diagnose: Boolean = false, val insertBatchSize: Int = 500, + val drainConcurrency: Int = 24, ) /** What the crawl fetched — the counters the caller reports and the graph is built from. */ @@ -527,7 +534,7 @@ class GrapeRankDataCrawler( // on the producer (keeps writeRelayFreq serial) and ingest runs // only on the consumer (keeps done/builder/hopOf serial), now // overlapped with draining instead of blocked behind each batch. - val routed = Channel, Map>>>(DRAIN_CONCURRENCY * 2) + val routed = Channel, Map>>>(config.drainConcurrency * 2) val drainedOut = Channel(Channel.UNLIMITED) coroutineScope { // Producer: route each batch by outbox (serial), backpressured @@ -545,7 +552,7 @@ class GrapeRankDataCrawler( // that cleanly EOSE'd, so the consumer can tell "answered empty" // from "timed out" per user. val workers = - List(DRAIN_CONCURRENCY) { + List(config.drainConcurrency) { launch { for ((batch, filters) in routed) { val dead = HashMap() @@ -635,7 +642,7 @@ class GrapeRankDataCrawler( ) log( "[graperank] write path: $stored events stored, verify ${verifyMs}ms + insert ${insertMs}ms " + - "(summed across ${DRAIN_CONCURRENCY} consumers, batch=${config.insertBatchSize})", + "(summed across ${config.drainConcurrency} consumers, batch=${config.insertBatchSize})", ) return Stats( rounds = rounds, @@ -713,6 +720,10 @@ class GrapeRankDataCrawler( // couldn't connect). A relay absent from this set answered definitively — // so an author it was asked for but didn't return is one it simply lacks. val notAnswered = ConcurrentSet() + // --diagnose: which relays were slow (or timed out) and on which query, so a + // human can replay that exact REQ later to understand the slowness. Null when + // diagnosis is off (no per-group timing/collection overhead). + val slowDrains = if (config.diagnose) ConcurrentSet() else null val collected = mutableListOf>() coroutineScope { @@ -801,7 +812,9 @@ class GrapeRankDataCrawler( } client.subscribe(subId, mapOf(subRelay to groupFilters), groupListener) try { + val gMark = TimeSource.Monotonic.markNow() val reason = withTimeoutOrNull(config.timeoutMs) { done.await() } ?: "timeout" + val elapsedMs = gMark.elapsedNow().inWholeMilliseconds if (reason == "timeout") timedOut.add(subRelay) if (reason != "eose") notAnswered.add(subRelay) classifyDrainFailure(reason)?.let { kind -> @@ -809,6 +822,16 @@ class GrapeRankDataCrawler( if (a == DrainFailure.HARD || b == DrainFailure.HARD) DrainFailure.HARD else DrainFailure.TRANSIENT } } + // Record slow/timed-out REQs with their exact query so a + // human can replay them later and see why the relay lags. + if (slowDrains != null && (reason == "timeout" || elapsedMs > SLOW_DRAIN_LOG_MS)) { + val authors = groupFilters.flatMap { it.authors.orEmpty() } + val kinds = groupFilters.flatMap { it.kinds.orEmpty() }.distinct() + slowDrains.add( + "[slow-relay] ${subRelay.url} $reason in ${elapsedMs}ms | kinds=$kinds authors=${authors.size}: " + + authors.take(30).joinToString(",") + (if (authors.size > 30) ",…" else ""), + ) + } } finally { client.unsubscribe(subId) } @@ -826,6 +849,7 @@ class GrapeRankDataCrawler( val detail = stalled.take(12).joinToString(", ") { "${it.url}(${eventsPer[it] ?: 0}ev)" } log("[drain] timeout ${config.timeoutMs}ms: ${stalled.size} slow(no EOSE)" + (if (detail.isNotEmpty()) " | slow: $detail" else "")) } + slowDrains?.snapshot()?.forEach { log(it) } deadOut?.putAll(failures.snapshot()) answeredOut?.addAll(filters.keys.filter { it !in notAnswered }) return collected @@ -852,6 +876,10 @@ class GrapeRankDataCrawler( // message cap most relays enforce. drainGated groups filters to stay within. private const val MAX_REQ_ENTRIES = 2500 + // --diagnose: a REQ that takes longer than this to reach a terminal (EOSE or + // timeout) is logged with its relay + filter, so slow relays can be replayed. + private const val SLOW_DRAIN_LOG_MS = 4000L + // Times we re-query an unreachable user's outbox before giving up, so the // crawl still terminates on a finite graph. private const val MAX_OUTBOX_ATTEMPTS = 3 @@ -861,12 +889,6 @@ class GrapeRankDataCrawler( // (~250/drain succeeds, ~17k fails); keep the fan-out small. private const val USER_BATCH = 256 - // Global content-drain fan-out — how many outbox batches we drain at once. A - // GLOBAL bound (memory / open sockets); the per-relay concurrency limit is - // enforced separately by AdaptiveRelayLimiter. A higher global fan-out - // re-floods busy hubs faster than demotion catches up, so keep it moderate. - private const val DRAIN_CONCURRENCY = 24 - // Sharded backbone sweep: split the still-missing authors into SHARD_RELAYS // lists, one per top relay, rotating up to SHARD_ROTATIONS times; once the // remainder drops below SHARD_BROADCAST_THRESHOLD, broadcast it at once. From 6b957e1950ab660a91922b0f13d1b59666b54e75 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 20:13:21 +0000 Subject: [PATCH 53/58] perf(graperank): park slow relays in the background instead of blocking rounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crawl was round-synchronised: each hop drained all its relays and only started the next hop after the slowest one reached EOSE or the timeout. That made waiting for slow-but-alive relays expensive — every hop paid its slow tail before the next hop's fast relays could begin — so a long timeout for completeness cost ~2x wall-clock (measured), and a short one dropped the slow relays' data. Diagnostics on a ~190k-user crawl showed the genuinely-slow set is a stable ~30 relays that DO reach EOSE, just in 5-25s. So decouple the two concerns: - drainGated now drains on the FAST `timeoutMs` that sets the round cadence. A relay still streaming when it elapses is not cut but PARKED: it hands its open subscription to a background scope (releasing its AdaptiveRelayLimiter permit so the round moves on), keeps receiving for up to the new `parkTimeoutMs`, and its late events are persisted + its late contact lists pushed to a crawl-wide lateHarvest channel. - The round loop folds late harvest into the graph between rounds and won't converge until the frontier is empty AND no relay is still parked — so the crawl waits for slow relays for completeness without paying that wait in each round's wall-clock. Graph state stays single-writer: parked coroutines only touch the store, seenIds, and the channel — never hopOf/done/builder. Persistence moved from a single per-drain consumer to a shared `persist()` that fast and parked units both call; crawl-wide dedup is now race-safe via ConcurrentSet.add's atomic test-and-set (an id is added only after a good signature, so no duplicate reaches the store's UNIQUE constraint and a forged copy can't suppress the genuine one). Also carries the --diagnose slow-relay logging (relay + filter + elapsed for every slow/parked REQ, so a human can replay it) and keeps --drain-concurrency at the validated default of 24 (an A/B at 64 was ~2x slower with more dead relays). New --park-timeout flag (default 40s; set <= --timeout to disable). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 8 + .../graperank/GrapeRankDataCrawler.kt | 521 +++++++++++------- 2 files changed, 327 insertions(+), 202 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index d6f483b0ef..c4afffab32 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -123,6 +123,12 @@ object GrapeRankCommand { val offline = args.bool("offline") val diagnose = args.bool("diagnose") val timeoutMs = args.longFlag("timeout", 10L) * 1000 + // A relay still streaming when --timeout elapses is PARKED, not cut: it keeps + // delivering for up to --park-timeout more while the round moves on, and its + // late contact lists fold into a later round. This is how the crawl waits for + // slow-but-alive relays for completeness without paying that wait per round. + // Set <= --timeout to disable parking (old cut-at-timeout behaviour). + val parkTimeoutMs = args.longFlag("park-timeout", 40L) * 1000 // How many verified events the crawler group-commits per store write. 1 // forces the per-event insert path (baseline); higher amortizes the SQLite // transaction + writer-mutex cost across the batch. @@ -188,6 +194,7 @@ object GrapeRankCommand { maxRounds = maxRounds, maxHops = maxHops, timeoutMs = timeoutMs, + parkTimeoutMs = parkTimeoutMs, diagnose = diagnose, insertBatchSize = insertBatch, drainConcurrency = drainConcurrency, @@ -271,6 +278,7 @@ object GrapeRankCommand { "insert_ms" to crawlStats?.insertMs, "events_stored" to crawlStats?.eventsStored, "insert_batch" to insertBatch, + "park_timeout_ms" to parkTimeoutMs, "store_load_ms" to storeLoadMs, "graph_build_ms" to buildMs, "scoring_ms" to scoringMs, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt index e15634abd3..e61c7a9170 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt @@ -111,7 +111,16 @@ class GrapeRankDataCrawler( * user's kind:3/10000/1984 when their outbox is unknown or unreachable. * @param maxRounds safety backstop on freshness passes (default: run to convergence). * @param maxHops follow-graph distance from the observer to crawl (Brainstorm uses 8). - * @param timeoutMs per-drain timeout. + * @param timeoutMs the FAST per-drain timeout that gates a round's progression. + * A relay that reaches EOSE/CLOSED inside it resolves its authors this round; + * one still streaming is not cut but PARKED (see [parkTimeoutMs]) so the round + * moves on without waiting for it. Keep this short — it is the round cadence. + * @param parkTimeoutMs how long a parked (slow-but-alive) relay is allowed to + * keep delivering after it blew [timeoutMs]. Its late events are persisted and + * its late contact lists folded into the graph in a later round, so the crawl + * waits for slow relays for completeness WITHOUT paying that wait in the + * round's wall-clock. Parked sockets are bounded by the slow-relay population, + * not the whole fan-out. Set `<= timeoutMs` to disable parking. * @param diagnose log a breakdown of slow/unreachable relays on each drain timeout. * @param insertBatchSize how many verified events to group-commit per * [IEventStore.batchInsert]. The outbox model streams the same events from @@ -130,6 +139,7 @@ class GrapeRankDataCrawler( val maxRounds: Int = Int.MAX_VALUE, val maxHops: Int = Int.MAX_VALUE, val timeoutMs: Long = 10_000, + val parkTimeoutMs: Long = 40_000, val diagnose: Boolean = false, val insertBatchSize: Int = 500, val drainConcurrency: Int = 24, @@ -210,6 +220,24 @@ class GrapeRankDataCrawler( // ones that timed out — which unlike a clean EOSE might just be slow). val askedEmpty = ConcurrentMap>() + // Contact lists delivered LATE by parked (slow-but-alive) relays. A parked + // unit persists its events, then pushes any kind:3 it found here; the round + // loop (the single graph-writer) folds these into hopOf/done/builder between + // rounds, so a slow relay's follows still expand the frontier — just a round + // or two later than the fast ones. Unbounded: parked delivery must never + // block on the round loop draining it. + val lateHarvest = Channel>(Channel.UNLIMITED) + + // Parked units still streaming. The crawl isn't done until this hits 0 (and + // the frontier is empty), so we wait for slow relays' completeness without + // gating each round on them. Incremented when a unit parks, decremented when + // it finishes (or its park window elapses). + val parkedInFlight = AtomicLong(0) + + // Background scope owning the parked subscriptions (and Tier-2 relay-list + // sweeps). Set in [run]; cancelled once the crawl converges. + var bgScope: CoroutineScope? = null + var rounds = 0 var contactListsFed = 0 @@ -321,7 +349,7 @@ class GrapeRankDataCrawler( val dead = HashMap() val filters = mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = FETCH_KINDS, authors = it) }) - drainGated(filters, dead, seenIds) to dead + drainGated(filters, dead) to dead } } }.awaitAll() @@ -347,7 +375,7 @@ class GrapeRankDataCrawler( val dead = HashMap() val filters = live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = FETCH_KINDS, authors = it) } } - val events = drainGated(filters, dead, seenIds) + val events = drainGated(filters, dead) recordDead(dead) relaysContacted += live for ((relay, _) in events) liveRelays.add(relay) @@ -390,7 +418,7 @@ class GrapeRankDataCrawler( Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), authors = chunk) } } - drainGated(filters, null, seenIds) + drainGated(filters, null) } val discovery = config.relayListDiscoveryRelays @@ -441,7 +469,7 @@ class GrapeRankDataCrawler( } } } - drainGated(filters, null, seenIds) + drainGated(filters, null) } /** @@ -488,18 +516,285 @@ class GrapeRankDataCrawler( } } + /** + * Dedup (crawl-wide [seenIds]), verify, and group-commit a unit's events, + * returning the newly-stored ones tagged by relay. Safe to call concurrently + * from many fast drain units AND parked coroutines: an id is added to + * [seenIds] only AFTER a good signature (so a forged copy delivered first + * can't suppress the genuine one), and [ConcurrentSet.add] is an atomic + * test-and-set — two relays mirroring the same event race on it and only the + * winner stores it, so a duplicate never reaches the store's UNIQUE constraint. + * The store serializes the actual writes behind its own single-writer mutex. + */ + private suspend fun persist(events: List>): List> { + if (events.isEmpty()) return emptyList() + val flushAt = config.insertBatchSize.coerceAtLeast(1) + val fresh = ArrayList>() + val buffer = ArrayList(flushAt) + + suspend fun flush() { + if (buffer.isEmpty()) return + val mark = TimeSource.Monotonic.markNow() + store.batchInsert(buffer) + insertNanos.addAndFetch(mark.elapsedNow().inWholeNanoseconds) + eventsStored.addAndFetch(buffer.size.toLong()) + buffer.clear() + } + + for ((relay, event) in events) { + if (event.id in seenIds) continue + val vMark = TimeSource.Monotonic.markNow() + val ok = event.verify() + verifyNanos.addAndFetch(vMark.elapsedNow().inWholeNanoseconds) + if (!ok) { + Log.w("GrapeRankDataCrawler") { "dropped event ${event.id.take(8)} kind=${event.kind} — bad signature" } + continue + } + if (!seenIds.add(event.id)) continue // lost the race to a mirror; it stores it + fresh.add(relay to event) + buffer.add(event) + if (buffer.size >= flushAt) flush() + } + flush() + return fresh + } + + /** + * Fold one late-delivered event from a parked relay into the graph. Only the + * round loop calls this (directly or via [foldLateHarvest]), so graph state + * stays single-writer. Returns true if it fed a new contact list. + */ + private suspend fun ingestLate( + relay: NormalizedRelayUrl, + ev: Event, + ): Boolean { + liveRelays.add(relay) + if (ev !is ContactListEvent) return false + val pk = ev.pubKey + // Only authors we actually crawled (in hopOf) and haven't fed yet. A late + // list for an unknown author would get a wrong hop stamp from ingest. + if (pk in done || pk !in hopOf) return false + val contacts = contactsOf(pk) ?: return false + done += pk + ingest(pk, contacts) + return true + } + + /** Drain whatever parked relays have delivered so far. Returns lists fed. */ + private suspend fun foldLateHarvest(): Int { + var got = 0 + while (true) { + val (relay, ev) = lateHarvest.tryReceive().getOrNull() ?: break + if (ingestLate(relay, ev)) got++ + } + return got + } + + /** + * Subscribe each relay to its filters behind [limiter] and drain them. A relay + * that reaches a terminal (EOSE/CLOSED/cannot-connect) within the FAST + * [Config.timeoutMs] has its events persisted and returned so this round can + * resolve the authors it was asked for. A relay still streaming when the fast + * timeout elapses is not cut but PARKED: it hands its open subscription to + * [bgScope] (releasing its limiter permit so the fast pool moves on) and keeps + * receiving for up to [Config.parkTimeoutMs] more; whatever it eventually + * delivers is persisted and its contact lists pushed to [lateHarvest] for the + * round loop to fold in — so slow relays add completeness without holding up + * the round. Each relay's filters are split into REQ-sized groups so a popular + * relay routed thousands of authors doesn't emit a frame most relays reject. + * Hard connect failures (fast into [deadOut], parked straight to [recordDead]) + * are marked dead. Returns only the FAST events, tagged by relay. + */ + private suspend fun drainGated( + filters: Map>, + deadOut: MutableMap?, + answeredOut: MutableSet? = null, + ): List> { + if (filters.isEmpty()) return emptyList() + + // Split each relay's filters into REQ-sized groups. A REQ frame carries ALL + // its filters at once, so a popular relay routed thousands of authors would + // otherwise produce a multi-MB frame that most relays reject ("message too + // large"). Grouping by total entry count keeps each REQ under the 256KB cap. + val units = ArrayList>>() + for ((relay, relayFilters) in filters) { + var group = ArrayList() + var entries = 0 + for (f in relayFilters) { + val fe = filterEntries(f) + if (group.isNotEmpty() && entries + fe > MAX_REQ_ENTRIES) { + units.add(relay to group) + group = ArrayList() + entries = 0 + } + group.add(f) + entries += fe + } + if (group.isNotEmpty()) units.add(relay to group) + } + + // Per-relay failure classification (HARD wins over TRANSIENT); which relays + // stalled past the fast window; and which did NOT cleanly EOSE (timed out, + // parked, closed, or couldn't connect) — a relay absent from that set + // answered definitively, so an author it didn't return is one it lacks. + val failures = ConcurrentMap() + val timedOut = ConcurrentSet() + val notAnswered = ConcurrentSet() + + fun classify( + reason: String, + relay: NormalizedRelayUrl, + into: ConcurrentMap, + ) { + classifyDrainFailure(reason)?.let { kind -> + into.merge(relay, kind) { a, b -> + if (a == DrainFailure.HARD || b == DrainFailure.HARD) DrainFailure.HARD else DrainFailure.TRANSIENT + } + } + } + + fun logSlow( + relay: NormalizedRelayUrl, + reason: String, + elapsedMs: Long, + groupFilters: List, + ) { + if (!config.diagnose) return + val authors = groupFilters.flatMap { it.authors.orEmpty() } + val kinds = groupFilters.flatMap { it.kinds.orEmpty() }.distinct() + log( + "[slow-relay] ${relay.url} $reason in ${elapsedMs}ms | kinds=$kinds authors=${authors.size}: " + + authors.take(30).joinToString(",") + (if (authors.size > 30) ",…" else ""), + ) + } + + val fast = + coroutineScope { + units + .map { (subRelay, groupFilters) -> + async { + limiter.withPermit(subRelay) { + val subId = newSubId() + val done = CompletableDeferred() + val unitEvents = Channel>(Channel.UNLIMITED) + val listener = + object : SubscriptionListener { + override fun onEvent( + event: Event, + isLive: Boolean, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + unitEvents.trySend(relay to event) + } + + override fun onEose( + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + done.complete("eose") + } + + override fun onClosed( + message: String, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + done.complete("closed:$message") + } + + override fun onCannotConnect( + relay: NormalizedRelayUrl, + message: String, + forFilters: List?, + ) { + done.complete("cannot:$message") + } + } + client.subscribe(subId, mapOf(subRelay to groupFilters), listener) + val mark = TimeSource.Monotonic.markNow() + val reason = withTimeoutOrNull(config.timeoutMs) { done.await() } + if (reason != null) { + // Terminal within the fast window — resolve this round. + val elapsedMs = mark.elapsedNow().inWholeMilliseconds + if (reason != "eose") notAnswered.add(subRelay) + classify(reason, subRelay, failures) + if (elapsedMs > SLOW_DRAIN_LOG_MS) logSlow(subRelay, reason, elapsedMs, groupFilters) + unitEvents.close() + client.unsubscribe(subId) + persist(buildList { for (e in unitEvents) add(e) }) + } else { + // Still streaming — hand off and let the round move on. + notAnswered.add(subRelay) + timedOut.add(subRelay) + val scope = bgScope + if (scope != null && config.parkTimeoutMs > config.timeoutMs) { + parkedInFlight.addAndFetch(1) + scope.launch { + try { + val late = withTimeoutOrNull(config.parkTimeoutMs) { done.await() } ?: "timeout" + logSlow(subRelay, "parked→$late", mark.elapsedNow().inWholeMilliseconds, groupFilters) + // A parked relay that ends in a hard/transient failure (not a + // clean EOSE) is reported dead the same way a fast one would be. + val lateDead = ConcurrentMap() + classify(late, subRelay, lateDead) + recordDead(lateDead.snapshot()) + unitEvents.close() + for (pair in persist(buildList { for (e in unitEvents) add(e) })) lateHarvest.trySend(pair) + } finally { + client.unsubscribe(subId) + parkedInFlight.addAndFetch(-1) + } + } + } else { + logSlow(subRelay, "timeout", mark.elapsedNow().inWholeMilliseconds, groupFilters) + unitEvents.close() + client.unsubscribe(subId) + } + emptyList() + } + } + } + }.awaitAll() + .flatten() + } + + if (config.diagnose && timedOut.size() > 0) { + log("[drain] parked ${timedOut.size()} slow relay(s) past ${config.timeoutMs}ms") + } + deadOut?.putAll(failures.snapshot()) + answeredOut?.addAll(filters.keys.filter { it !in notAnswered }) + return fast + } + suspend fun run(): Stats { val crawlMark = TimeSource.Monotonic.markNow() - // Scope for fire-and-forget relay-list discovery (see ensureRelayLists - // Tier 2). SupervisorJob so one failing sweep never cancels the others; - // cancelled when the crawl finishes. - val bgScope = CoroutineScope(coroutineContext + SupervisorJob()) + // Scope owning parked (slow-relay) subscriptions and the fire-and-forget + // Tier-2 relay-list sweeps. SupervisorJob so one failure never cancels the + // others; cancelled once the crawl converges. Published to [bgScope] so + // drainGated can hand slow subs to it. + val scope = CoroutineScope(coroutineContext + SupervisorJob()) + bgScope = scope while (rounds < config.maxRounds) { + // Fold in whatever the parked (slow-but-alive) relays have delivered + // since the last round — their late contact lists expand the frontier + // a round or two behind the fast ones (single-writer: only here). + foldLateHarvest() + // Only crawl users within the hop budget; deeper users still appear // in the graph as follow targets, we just don't fetch their lists. val pending = hopOf.keys.filter { it !in done && (hopOf[it] ?: 0) < config.maxHops } - if (pending.isEmpty()) break + if (pending.isEmpty()) { + // Frontier drained. If no slow relay is still streaming, a final + // fold catches any last-moment delivery and we're done; otherwise + // wait for a parked relay to deliver (completeness) and loop. + if (parkedInFlight.load() == 0L) { + if (foldLateHarvest() == 0) break else continue + } + withTimeoutOrNull(PARK_POLL_MS) { lateHarvest.receive() }?.let { ingestLate(it.first, it.second) } + continue + } rounds++ // Refresh the warm pool to this round's busiest relays and keep that @@ -526,7 +821,7 @@ class GrapeRankDataCrawler( // Snapshot of every relay we've seen work, for the wide Tier-2 // sweep (taken now, before the Phase-B workers mutate liveRelays). val allLive = liveRelays.filterTo(HashSet()) { it !in deadRelays } - ensureRelayLists(stragglers.toSet(), allLive, bgScope) + ensureRelayLists(stragglers.toSet(), allLive, scope) // Continuous worker pool instead of chunked awaitAll barriers, so // no worker waits on a slow sibling and hot relays stay connected. @@ -557,7 +852,7 @@ class GrapeRankDataCrawler( for ((batch, filters) in routed) { val dead = HashMap() val answered = HashSet() - val events = drainGated(filters, dead, seenIds, answered) + val events = drainGated(filters, dead, answered) recordDead(dead) drainedOut.send(DrainedBatch(batch, filters, answered, events)) } @@ -613,17 +908,20 @@ class GrapeRankDataCrawler( ) } - // Crawl done — drop the warm pool and stop any background relay-list - // sweeps still in flight (their results are already in the store). + // Crawl done — drop the warm pool. client.unsubscribe(WARM_SUB_ID) - bgScope.cancel() // Reports can be retracted. Ask each reporter's outbox for NIP-09 kind:5 // deletions that cite the reports we gathered (#e-filtered to our report // ids). The events land in the store; the caller decides which reports - // they actually retract. + // they actually retract. Run before cancelling [scope] so it can still + // park slow relays. fetchReportDeletions(topLiveRelays(BACKBONE_SIZE).toSet()) + // Stop any parked subscriptions + Tier-2 relay-list sweeps still in flight + // (whatever they fetched already landed in the store). + scope.cancel() + val hopHistogram = hopOf.values .groupingBy { it } @@ -642,7 +940,7 @@ class GrapeRankDataCrawler( ) log( "[graperank] write path: $stored events stored, verify ${verifyMs}ms + insert ${insertMs}ms " + - "(summed across ${config.drainConcurrency} consumers, batch=${config.insertBatchSize})", + "(summed across all drains, batch=${config.insertBatchSize})", ) return Stats( rounds = rounds, @@ -670,191 +968,6 @@ class GrapeRankDataCrawler( val events: List>, ) - /** - * Subscribe each relay to its filters behind [limiter], drain until every - * relay's subscription is terminal or the timeout elapses, verify+store the - * events, and return them tagged by relay. Each relay gets its own gated - * subscription so we never exceed its adaptive concurrent-subscription cap; a - * relay's filters are split into REQ-sized groups so a popular relay routed - * thousands of authors doesn't produce a multi-MB frame that most relays - * reject outright. Hard connect failures are reported into [deadOut]. Events - * whose id is already in the crawl-wide [seen] set are dropped before the - * expensive verify+store; verified ids are added to it so later drains skip them. - */ - private suspend fun drainGated( - filters: Map>, - deadOut: MutableMap?, - seen: ConcurrentSet, - answeredOut: MutableSet? = null, - ): List> { - if (filters.isEmpty()) return emptyList() - val eventChannel = Channel>(Channel.UNLIMITED) - - // Split each relay's filters into REQ-sized groups. A REQ frame carries ALL - // its filters at once, so a popular relay routed thousands of authors would - // otherwise produce a multi-MB frame that most relays reject ("message too - // large") — silently dropping every author in it. Grouping by total entry - // count keeps each REQ well under the common 256KB cap. - val units = ArrayList>>() - for ((relay, relayFilters) in filters) { - var group = ArrayList() - var entries = 0 - for (f in relayFilters) { - val fe = filterEntries(f) - if (group.isNotEmpty() && entries + fe > MAX_REQ_ENTRIES) { - units.add(relay to group) - group = ArrayList() - entries = 0 - } - group.add(f) - entries += fe - } - if (group.isNotEmpty()) units.add(relay to group) - } - - // Per-relay failure classification, HARD winning over TRANSIENT across a - // relay's several REQ-groups; plus which relays stalled to a timeout. - val failures = ConcurrentMap() - val timedOut = ConcurrentSet() - // Relays that did NOT cleanly EOSE every group (timed out, closed, or - // couldn't connect). A relay absent from this set answered definitively — - // so an author it was asked for but didn't return is one it simply lacks. - val notAnswered = ConcurrentSet() - // --diagnose: which relays were slow (or timed out) and on which query, so a - // human can replay that exact REQ later to understand the slowness. Null when - // diagnosis is off (no per-group timing/collection overhead). - val slowDrains = if (config.diagnose) ConcurrentSet() else null - - val collected = mutableListOf>() - coroutineScope { - // Single consumer per drain: dedup against the crawl-wide [seen] set, - // verify, and group-commit to the store. Duplicates (the same event from - // another relay, drain, or round) are skipped BEFORE the expensive Schnorr - // verify + store write. An id is added to [seen] only after it verifies, so - // a forged copy (valid id, bad signature) delivered first can't suppress - // the genuine one. Verified events are buffered and flushed via batchInsert - // so the per-transaction + writer-mutex cost is paid once per - // [insertBatchSize], not once per event. (A relay whose every event was - // already seen won't be credited into `liveRelays` by this drain — that's - // fine: it's a redundant mirror that added nothing new.) - val consumer = - launch { - val flushAt = config.insertBatchSize.coerceAtLeast(1) - val buffer = ArrayList(flushAt) - - suspend fun flush() { - if (buffer.isEmpty()) return - val mark = TimeSource.Monotonic.markNow() - store.batchInsert(buffer) - insertNanos.addAndFetch(mark.elapsedNow().inWholeNanoseconds) - eventsStored.addAndFetch(buffer.size.toLong()) - buffer.clear() - } - - for ((relay, event) in eventChannel) { - if (event.id in seen) continue - val vMark = TimeSource.Monotonic.markNow() - val ok = event.verify() - verifyNanos.addAndFetch(vMark.elapsedNow().inWholeNanoseconds) - if (!ok) { - Log.w("GrapeRankDataCrawler") { "dropped event ${event.id.take(8)} kind=${event.kind} — bad signature" } - continue - } - seen.add(event.id) - collected.add(relay to event) - buffer.add(event) - if (buffer.size >= flushAt) flush() - } - flush() - } - // One gated subscription per (relay, REQ-group). The permit is held for - // the group's whole life, so concurrent subs on a relay never exceed its - // adaptive cap. - units - .map { (subRelay, groupFilters) -> - launch { - limiter.withPermit(subRelay) { - val subId = newSubId() - val done = CompletableDeferred() - val groupListener = - object : SubscriptionListener { - override fun onEvent( - event: Event, - isLive: Boolean, - relay: NormalizedRelayUrl, - forFilters: List?, - ) { - eventChannel.trySend(relay to event) - } - - override fun onEose( - relay: NormalizedRelayUrl, - forFilters: List?, - ) { - done.complete("eose") - } - - override fun onClosed( - message: String, - relay: NormalizedRelayUrl, - forFilters: List?, - ) { - done.complete("closed:$message") - } - - override fun onCannotConnect( - relay: NormalizedRelayUrl, - message: String, - forFilters: List?, - ) { - done.complete("cannot:$message") - } - } - client.subscribe(subId, mapOf(subRelay to groupFilters), groupListener) - try { - val gMark = TimeSource.Monotonic.markNow() - val reason = withTimeoutOrNull(config.timeoutMs) { done.await() } ?: "timeout" - val elapsedMs = gMark.elapsedNow().inWholeMilliseconds - if (reason == "timeout") timedOut.add(subRelay) - if (reason != "eose") notAnswered.add(subRelay) - classifyDrainFailure(reason)?.let { kind -> - failures.merge(subRelay, kind) { a, b -> - if (a == DrainFailure.HARD || b == DrainFailure.HARD) DrainFailure.HARD else DrainFailure.TRANSIENT - } - } - // Record slow/timed-out REQs with their exact query so a - // human can replay them later and see why the relay lags. - if (slowDrains != null && (reason == "timeout" || elapsedMs > SLOW_DRAIN_LOG_MS)) { - val authors = groupFilters.flatMap { it.authors.orEmpty() } - val kinds = groupFilters.flatMap { it.kinds.orEmpty() }.distinct() - slowDrains.add( - "[slow-relay] ${subRelay.url} $reason in ${elapsedMs}ms | kinds=$kinds authors=${authors.size}: " + - authors.take(30).joinToString(",") + (if (authors.size > 30) ",…" else ""), - ) - } - } finally { - client.unsubscribe(subId) - } - } - } - }.joinAll() - // All subscriptions are torn down; no more events can arrive. Close the - // channel so the consumer drains what's buffered and completes. - eventChannel.close() - consumer.join() - } - if (config.diagnose && timedOut.size() > 0) { - val stalled = timedOut.snapshot() - val eventsPer = collected.groupingBy { it.first }.eachCount() - val detail = stalled.take(12).joinToString(", ") { "${it.url}(${eventsPer[it] ?: 0}ev)" } - log("[drain] timeout ${config.timeoutMs}ms: ${stalled.size} slow(no EOSE)" + (if (detail.isNotEmpty()) " | slow: $detail" else "")) - } - slowDrains?.snapshot()?.forEach { log(it) } - deadOut?.putAll(failures.snapshot()) - answeredOut?.addAll(filters.keys.filter { it !in notAnswered }) - return collected - } - /** Latest known kind:3 contact list for [pubKey] from the local store, or null. */ private suspend fun contactsOf(pubKey: HexKey): ContactListEvent? = store @@ -880,6 +993,10 @@ class GrapeRankDataCrawler( // timeout) is logged with its relay + filter, so slow relays can be replayed. private const val SLOW_DRAIN_LOG_MS = 4000L + // Once the frontier is empty but parked relays are still streaming, how long + // to block waiting for one of them to deliver before re-checking convergence. + private const val PARK_POLL_MS = 2000L + // Times we re-query an unreachable user's outbox before giving up, so the // crawl still terminates on a finite graph. private const val MAX_OUTBOX_ATTEMPTS = 3 From 660bd86dc7ae2c758150136e232192ae57822378 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 22:11:19 +0000 Subject: [PATCH 54/58] perf(graperank): idle-based park timeout so streaming relays aren't cut mid-flight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The park window's timeout was absolute from subscription open, so a relay still actively streaming a large result set once it passed parkTimeoutMs was unsubscribed and its untransmitted tail lost. Reset the window on every incoming event (a conflated activity signal drives a select against the terminal deferred), so a parked subscription is closed only after parkTimeoutMs of actual silence — never while events are still arriving. The fast window stays absolute: it only decides when to hand a slow relay to the background park lane, which loses nothing. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../graperank/GrapeRankDataCrawler.kt | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt index e61c7a9170..8ff595e545 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt @@ -50,6 +50,7 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch +import kotlinx.coroutines.selects.select import kotlinx.coroutines.withTimeoutOrNull import kotlin.concurrent.atomics.AtomicLong import kotlin.concurrent.atomics.ExperimentalAtomicApi @@ -559,6 +560,34 @@ class GrapeRankDataCrawler( return fresh } + /** + * Wait for a subscription's terminal ([done]: EOSE/CLOSED/cannot), resetting + * the [idleMs] window every time an event pings [activity]. So the wait ends + * with "timeout" only after [idleMs] of actual SILENCE — a relay that keeps + * streaming (however long its result set) is never cut mid-flight; only a + * genuinely stalled one is. Used for the patient park window. + */ + private suspend fun awaitTerminalOrIdle( + done: CompletableDeferred, + activity: Channel, + idleMs: Long, + ): String { + while (true) { + val r = + withTimeoutOrNull(idleMs) { + select { + done.onAwait { it } + activity.onReceive { ACTIVITY } + } + } + when (r) { + null -> return "timeout" // idleMs elapsed with no event and no terminal + ACTIVITY -> Unit // an event arrived — reset the idle window and keep waiting + else -> return r // terminal reason + } + } + } + /** * Fold one late-delivered event from a parked relay into the graph. Only the * round loop calls this (directly or via [foldLateHarvest]), so graph state @@ -677,6 +706,10 @@ class GrapeRankDataCrawler( val subId = newSubId() val done = CompletableDeferred() val unitEvents = Channel>(Channel.UNLIMITED) + // Liveness signal for the parked idle timeout: every event pings + // this (conflated, so bursts collapse to one) and resets the park + // window, so a relay actively streaming is never cut mid-flight. + val activity = Channel(Channel.CONFLATED) val listener = object : SubscriptionListener { override fun onEvent( @@ -686,6 +719,7 @@ class GrapeRankDataCrawler( forFilters: List?, ) { unitEvents.trySend(relay to event) + activity.trySend(Unit) } override fun onEose( @@ -732,7 +766,10 @@ class GrapeRankDataCrawler( parkedInFlight.addAndFetch(1) scope.launch { try { - val late = withTimeoutOrNull(config.parkTimeoutMs) { done.await() } ?: "timeout" + // Idle timeout, not absolute: only cut after parkTimeoutMs + // of SILENCE (no event, no terminal), so a relay still + // streaming a large result set is never chopped mid-flight. + val late = awaitTerminalOrIdle(done, activity, config.parkTimeoutMs) logSlow(subRelay, "parked→$late", mark.elapsedNow().inWholeMilliseconds, groupFilters) // A parked relay that ends in a hard/transient failure (not a // clean EOSE) is reported dead the same way a fast one would be. @@ -997,6 +1034,11 @@ class GrapeRankDataCrawler( // to block waiting for one of them to deliver before re-checking convergence. private const val PARK_POLL_MS = 2000L + // Sentinel returned by the park idle-wait's select when an event arrived + // (resets the window). A control string that can't collide with a relay's + // CLOSED/cannot message, which are the only other select results. + private const val ACTIVITY = "activity" + // Times we re-query an unreachable user's outbox before giving up, so the // crawl still terminates on a finite graph. private const val MAX_OUTBOX_ATTEMPTS = 3 From f95df91d6d76d4199f0063079437c793845fc271 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 22:15:29 +0000 Subject: [PATCH 55/58] chore(quartz): drop AdaptiveRelayLimiter + relay-URL rejection logs to debug The per-relay concurrency/rate throttle notices and the "Rejected " normalizer messages fire constantly during a large crawl (thousands of rejected/throttled relays) and are operational detail, not warnings. Move them from Log.w to Log.d so they stay available under debug logging without flooding a normal run. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../relay/client/accessories/AdaptiveRelayLimiter.kt | 4 ++-- .../quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt index c66134480f..977767fade 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt @@ -150,7 +150,7 @@ class AdaptiveRelayLimiter( val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)] gate(relay).lower(cap) if (step <= subLadder.size) { - Log.w("AdaptiveRelayLimiter") { "${relay.url} concurrency capped at $cap subs (sub-limit #$step)" } + Log.d("AdaptiveRelayLimiter") { "${relay.url} concurrency capped at $cap subs (sub-limit #$step)" } } } @@ -161,7 +161,7 @@ class AdaptiveRelayLimiter( val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)] rateDelayMs[relay] = d if (step <= rateLadder.size) { - Log.w("AdaptiveRelayLimiter") { "${relay.url} rate-throttled to 1 REQ / ${d}ms (rate-limit #$step)" } + Log.d("AdaptiveRelayLimiter") { "${relay.url} rate-throttled to 1 REQ / ${d}ms (rate-limit #$step)" } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt index 81b08a9666..04f8c9cc06 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt @@ -156,7 +156,7 @@ class RelayUrlNormalizer { if (trimmed.contains("://")) { // some other scheme we cannot connect to. - Log.w("RelayUrlNormalizer") { "Rejected $url" } + Log.d("RelayUrlNormalizer") { "Rejected $url" } return null } @@ -189,14 +189,14 @@ class RelayUrlNormalizer { normalizedUrls.put(url, NormalizationResult.Success(normalized)) normalized } else { - Log.w("NormalizedRelayUrl") { "Rejected $url" } + Log.d("NormalizedRelayUrl") { "Rejected $url" } normalizedUrls.put(url, NormalizationResult.Error) null } } catch (e: Exception) { if (e is CancellationException) throw e normalizedUrls.put(url, NormalizationResult.Error) - Log.w("NormalizedRelayUrl") { "Rejected $url" } + Log.d("NormalizedRelayUrl") { "Rejected $url" } null } } From a013bcd9d9f258a4e1b9ef0758f3754523f9dfcd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 22:27:19 +0000 Subject: [PATCH 56/58] feat(cli): quiet quartz DEBUG logging by default, add --verbose/-v The CLI ran at the library's default Log.minLevel = DEBUG, so quartz internal chatter (relay-auth init, MLS restore, URL-rejection, throttle notices) leaked onto stderr around every command's real output. Set Log.minLevel = WARN at startup, before dispatch, so a normal run shows only warnings/errors plus the command's own progress. A new global --verbose / -v flag restores full DEBUG; it's parsed with the other global flags so subcommands never see it. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../kotlin/com/vitorpamplona/amethyst/cli/Main.kt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index d9be83fb3b..b1efba3e7f 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -71,6 +71,8 @@ import com.vitorpamplona.amethyst.cli.commands.cashu.CashuCommands import com.vitorpamplona.amethyst.cli.commands.cashu.CashuMintCommands import com.vitorpamplona.amethyst.cli.commands.route import com.vitorpamplona.amethyst.cli.secrets.SecretStore +import com.vitorpamplona.quartz.utils.Log +import com.vitorpamplona.quartz.utils.LogLevel import kotlinx.coroutines.runBlocking import kotlin.system.exitProcess @@ -104,6 +106,12 @@ fun main(argv: Array) { // braces guard for invocations that bypass the launcher scripts. System.setProperty("java.awt.headless", "true") + // Quiet quartz's internal DEBUG chatter (relay auth, MLS restore, URL + // rejection, throttle notices) by default so it doesn't drown a command's + // own output; --verbose / -v restores full DEBUG. Set before dispatch so + // even startup logging is gated. + Log.minLevel = if (argv.any { it == "--verbose" || it == "-v" }) LogLevel.DEBUG else LogLevel.WARN + // Set output mode before dispatch so even argument-parsing errors // honour --json. if (argv.any { it == "--json" || it == "--json=true" }) { @@ -150,6 +158,7 @@ private suspend fun dispatch(argv: Array): Int { GlobalFlag.SECRET_BACKEND -> secretBackendFlag = consumed.value GlobalFlag.PASSPHRASE_FILE -> passphraseFileFlag = consumed.value GlobalFlag.JSON -> Output.mode = Output.Mode.JSON + GlobalFlag.VERBOSE -> Unit // level already applied in main(); just strip it here null -> filteredArgs.add(a) } i += consumed.tokensConsumed @@ -267,11 +276,13 @@ private suspend fun marmotDispatch( private enum class GlobalFlag( val long: String, val takesValue: Boolean = true, + val short: String? = null, ) { ACCOUNT("--account"), SECRET_BACKEND("--secret-backend"), PASSPHRASE_FILE("--passphrase-file"), JSON("--json", takesValue = false), + VERBOSE("--verbose", takesValue = false, short = "-v"), } private data class ConsumedFlag( @@ -290,7 +301,7 @@ private fun extractGlobalFlag( idx: Int, ): Pair { for (flag in GlobalFlag.values()) { - if (token == flag.long) { + if (token == flag.long || token == flag.short) { return if (flag.takesValue) { flag to ConsumedFlag(argv.getOrNull(idx + 1), 2) } else { @@ -315,6 +326,7 @@ private fun printUsage() { | [--secret-backend auto|keychain|ncryptsec|plaintext] | [--passphrase-file PATH] | [--json] + | [--verbose|-v] | [args...] | |Account selection: From db39536bde51c7f50e407c6ffbc3be465e37deb2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 23:09:09 +0000 Subject: [PATCH 57/58] feat(graperank): live progress heartbeat + optional builder for persist-only sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crawler only logged once per round, so a deep hop (140k users, minutes of work) went silent between lines. Add a heartbeat ticker on the background scope that emits every few seconds with the current round's completion (a real X/Y % against the round's known pending target), a rolling fetch rate + rough ETA for it, and live counts (events stored, relays parked/dead) — and a "finishing" line while draining the parked tail. Scoring stays sub-second, so it keeps its per-sweep lines and needs no ticker. Also make crawl()'s builder nullable: null runs a persist-only pass (every event still lands in the store, the frontier still expands off each contact list) with no in-memory graph — the basis for a `sync` that loads data without scoring. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../graperank/GrapeRankDataCrawler.kt | 85 ++++++++++++++++++- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt index 8ff595e545..682c633066 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankDataCrawler.kt @@ -48,6 +48,7 @@ import kotlinx.coroutines.awaitAll import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.selects.select @@ -165,11 +166,13 @@ class GrapeRankDataCrawler( /** * Crawl from [observer], streaming discovered contact lists into [builder] * (follows only — mutes/reports land in the store for the caller to - * materialize). Returns the crawl [Stats]. + * materialize). Pass `null` for a persist-only *sync*: every event still lands + * in the store, the frontier still expands off each contact list, but no graph + * is assembled in memory (the caller scores later from the store). Returns [Stats]. */ suspend fun crawl( observer: HexKey, - builder: TrustGraphBuilder, + builder: TrustGraphBuilder?, ): Stats { verifyNanos.store(0) insertNanos.store(0) @@ -189,7 +192,7 @@ class GrapeRankDataCrawler( */ private inner class CrawlRun( val observer: HexKey, - val builder: TrustGraphBuilder, + val builder: TrustGraphBuilder?, ) { // hop distance per discovered user; the observer seeds it at 0. Its key set // is the discovered frontier — no separate `discovered` set to keep in sync. @@ -242,6 +245,15 @@ class GrapeRankDataCrawler( var rounds = 0 var contactListsFed = 0 + // Live-progress context the heartbeat ticker reads (plain vars set only by the + // single round-loop coroutine; the ticker's reads are benign racy int/bool + // reads — a stale value just shows in one progress line). progTarget/progBase + // frame the CURRENT round so the ticker can show a real "X of Y (Z%)" for it. + var progRound = 0 + var progTarget = 0 + var progBaseDone = 0 + var progConverging = false + /** * A relay that HARD-failed (bad domain, TLS misconfig, dead HTTP code) is * dropped on the first strike: it will not fix itself. A TRANSIENT failure @@ -290,7 +302,7 @@ class GrapeRankDataCrawler( fresh++ } } - builder.addFollows(source, follows) + builder?.addFollows(source, follows) contactListsFed++ return fresh } @@ -619,6 +631,46 @@ class GrapeRankDataCrawler( return got } + /** + * Heartbeat so a long round never goes silent: every [PROGRESS_INTERVAL_MS] + * emit a one-liner with the CURRENT round's completion (a real X/Y % — the + * round's pending set is a known target), a rolling fetch rate + rough ETA for + * it, and live counts (events stored, slow relays parked, live/dead relays). + * Runs for the whole crawl on the background scope; cancelled when it ends. + */ + private suspend fun progressTicker() { + var lastFed = 0 + var lastMark = TimeSource.Monotonic.markNow() + while (true) { + delay(PROGRESS_INTERVAL_MS) + val nowMark = TimeSource.Monotonic.markNow() + val dtMs = (nowMark - lastMark).inWholeMilliseconds.coerceAtLeast(1) + lastMark = nowMark + val fed = contactListsFed + val rate = (fed - lastFed) * 1000L / dtMs // lists/sec over this interval + lastFed = fed + val events = eventsStored.load() + val parked = parkedInFlight.load() + when { + progConverging -> + log( + "[graperank] finishing · ${human(fed.toLong())} lists · ${human(events)} events" + + (if (parked > 0) " · $parked slow relay(s) still delivering" else " · draining"), + ) + progTarget > 0 -> { + val roundDone = (done.size - progBaseDone).coerceAtLeast(0) + val pct = (100L * roundDone / progTarget).coerceIn(0, 100) + val remaining = (progTarget - roundDone).coerceAtLeast(0) + val eta = if (rate > 0) etaFmt(remaining / rate) else "…" + log( + "[graperank] round $progRound · ${human(roundDone.toLong())}/${human(progTarget.toLong())} ($pct%)" + + " · $rate/s · ~$eta · ${human(events)} ev · $parked slow · ${deadRelays.size()} dead", + ) + } + } + } + } + /** * Subscribe each relay to its filters behind [limiter] and drain them. A relay * that reaches a terminal (EOSE/CLOSED/cannot-connect) within the FAST @@ -813,6 +865,10 @@ class GrapeRankDataCrawler( val scope = CoroutineScope(coroutineContext + SupervisorJob()) bgScope = scope + // Heartbeat: keeps a long, silent round feeling alive with live % + ETA. + // Runs on [scope], so scope.cancel() at crawl end stops it. + scope.launch { progressTicker() } + while (rounds < config.maxRounds) { // Fold in whatever the parked (slow-but-alive) relays have delivered // since the last round — their late contact lists expand the frontier @@ -826,6 +882,7 @@ class GrapeRankDataCrawler( // Frontier drained. If no slow relay is still streaming, a final // fold catches any last-moment delivery and we're done; otherwise // wait for a parked relay to deliver (completeness) and loop. + progConverging = true if (parkedInFlight.load() == 0L) { if (foldLateHarvest() == 0) break else continue } @@ -833,6 +890,12 @@ class GrapeRankDataCrawler( continue } rounds++ + // Frame this round for the heartbeat ticker: its target is the pending + // set, its baseline is how many users were already done going in. + progRound = rounds + progTarget = pending.size + progBaseDone = done.size + progConverging = false // Refresh the warm pool to this round's busiest relays and keep that // subscription open — reusing the same subId just updates the @@ -1034,6 +1097,20 @@ class GrapeRankDataCrawler( // to block waiting for one of them to deliver before re-checking convergence. private const val PARK_POLL_MS = 2000L + // How often the heartbeat ticker emits a live-progress line. + private const val PROGRESS_INTERVAL_MS = 3000L + + /** Compact human count: 1234 -> "1.2k", 1_500_000 -> "1.5M". */ + private fun human(n: Long): String = + when { + n >= 1_000_000 -> "${n / 1_000_000}.${(n % 1_000_000) / 100_000}M" + n >= 1_000 -> "${n / 1_000}.${(n % 1_000) / 100}k" + else -> n.toString() + } + + /** Seconds as "45s" or "3m20s". */ + private fun etaFmt(secs: Long): String = if (secs >= 60) "${secs / 60}m${secs % 60}s" else "${secs}s" + // Sentinel returned by the park idle-wait's select when an event arrived // (resets the window). A control string that can't collide with a relay's // CLOSED/cannot message, which are the only other select results. From c6f90b20a05e49504ddbe022b94fff7285207913 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 23:09:09 +0000 Subject: [PATCH 58/58] feat(cli): split graperank into sync (load) and score (compute) sub-verbs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crawl persists everything to the store and the score is a pure function over it, so separate them: `amy graperank sync` crawls the reachable graph into the store (idempotent + cumulative — run it a few times to be sure it's loaded) and reports what it loaded without scoring; `amy graperank score` builds from the store and scores instantly, repeatable with different params and no re-crawl (same as bare `--offline`). Bare `amy graperank` stays the sync+score combo. Extract the shared crawler wiring into newCrawler(); score() just forces the offline path, and sync() runs a persist-only crawl (null builder). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../amethyst/cli/commands/GrapeRankCommand.kt | 158 ++++++++++++------ 1 file changed, 106 insertions(+), 52 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index c4afffab32..f3f5b21e68 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -75,6 +75,16 @@ import kotlin.math.roundToInt * `--publish`, results are also published as NIP-85 kind:30382 `ContactCardEvent` * trusted assertions (one per scored user, `rank = round(score*100)`). * + * The crawl and the computation are separable, because the crawl persists every + * event it fetches to the store and the score is a pure function over it: + * - `amy graperank sync [OBSERVER]` — network only: crawl the reachable graph's + * kind 3/10000/1984/10002 into the local store. Idempotent and cumulative, so + * run it a few times to make sure everything is loaded. Scores nothing. + * - `amy graperank score [OBSERVER]` — local only: build the graph from the store + * and score (same as bare `--offline`). Instant and param-tunable; repeat with + * different `--rigor`/`--attenuation`/cutoffs without re-crawling. + * - bare `amy graperank [OBSERVER]` — the convenience combo: sync then score. + * * Sub-verbs complete the NIP-85 provider experience — the discovery layer that * lets clients find and consume those assertions: * - `amy graperank register` — advertise a `30382:rank` provider in the @@ -104,40 +114,31 @@ object GrapeRankCommand { "register" -> register(dataDir, tail.drop(1).toTypedArray()) "providers" -> providers(dataDir, tail.drop(1).toTypedArray()) "operator" -> operator(dataDir, tail.drop(1).toTypedArray()) + "sync" -> sync(dataDir, tail.drop(1).toTypedArray()) + "score" -> run(dataDir, tail.drop(1).toTypedArray(), forceOffline = true) else -> run(dataDir, tail) } suspend fun run( dataDir: DataDir, rest: Array, + forceOffline: Boolean = false, ): Int { val args = Args(rest) val observerArg = args.positionalOrNull(0) // Crawl to full convergence by default (every reachable user's outbox // checked). --max-rounds is only a safety backstop; --max-hops bounds the // follow-graph distance from the observer that we crawl (Brainstorm uses 8). - val maxRounds = args.intFlag("max-rounds", Int.MAX_VALUE) - val maxHops = args.intFlag("max-hops", Int.MAX_VALUE) val limit = args.intFlag("limit", 100) val minScore = args.flag("min-score")?.toDoubleOrNull() ?: 0.0 - val offline = args.bool("offline") - val diagnose = args.bool("diagnose") - val timeoutMs = args.longFlag("timeout", 10L) * 1000 - // A relay still streaming when --timeout elapses is PARKED, not cut: it keeps - // delivering for up to --park-timeout more while the round moves on, and its - // late contact lists fold into a later round. This is how the crawl waits for - // slow-but-alive relays for completeness without paying that wait per round. - // Set <= --timeout to disable parking (old cut-at-timeout behaviour). + // `graperank score` forces the local (no-network) path; `--offline` does the + // same on the bare command. Either way we build + score from the store only. + val offline = forceOffline || args.bool("offline") + // Crawl tuning (--max-rounds/--max-hops/--timeout/--diagnose/--drain-concurrency) + // is read straight from args by [newCrawler]; only these two are surfaced in + // the result JSON, so keep local copies for that. val parkTimeoutMs = args.longFlag("park-timeout", 40L) * 1000 - // How many verified events the crawler group-commits per store write. 1 - // forces the per-event insert path (baseline); higher amortizes the SQLite - // transaction + writer-mutex cost across the batch. val insertBatch = args.intFlag("insert-batch", 500) - // How many outbox batches drain in parallel (the worker-pool size). 24 is the - // validated default; higher fan-out re-floods busy hubs faster than the - // per-relay demotion catches up (an A/B at 64 was ~2x slower with MORE dead - // relays), so raise it only to probe specific slow relays. - val drainConcurrency = args.intFlag("drain-concurrency", 24) val doPublish = args.bool("publish") // Publish cutoff: only cards with rank >= this are published; existing // cards for targets below it (or gone from the graph) are retracted. Rank @@ -174,42 +175,10 @@ object GrapeRankCommand { var crawlStats: GrapeRankDataCrawler.Stats? = null if (!offline) { - // Relay policy for the crawler — where a stranger's kind:10002 is - // found (index/discovery aggregators + general defaults that carry - // kind:10002 for most of the network) and the best-effort general - // relays that might hold content when an outbox is unknown. These - // defaults live in app code, so the quartz crawler takes them injected. - val discoveryRelays = - ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + EXTRA_DISCOVERY_RELAYS - val contentFallback = ctx.bootstrapRelays() + Constants.eventFinderRelays - val crawler = - GrapeRankDataCrawler( - client = ctx.client, - store = ctx.store, - limiter = ctx.relayLimiter, - config = - GrapeRankDataCrawler.Config( - relayListDiscoveryRelays = discoveryRelays, - contentFallbackRelays = contentFallback, - maxRounds = maxRounds, - maxHops = maxHops, - timeoutMs = timeoutMs, - parkTimeoutMs = parkTimeoutMs, - diagnose = diagnose, - insertBatchSize = insertBatch, - drainConcurrency = drainConcurrency, - ), - log = { System.err.println(it) }, - ) - val stats = crawler.crawl(observer, builder) + val stats = newCrawler(ctx, args).crawl(observer, builder) crawlStats = stats contactListsFed = stats.contactListsFed - if (ctx.relayDiagnostics.hadFeedback()) { - System.err.println("[graperank] relay feedback: ${ctx.relayDiagnostics.snapshot()}") - } - if (ctx.relayLimiter.hadThrottling()) { - System.err.println("[graperank] relay throttling: ${ctx.relayLimiter.snapshot()}") - } + reportRelayFeedback(ctx) } else { // Offline: stream contact lists from the local store into the graph. val loadStart = System.nanoTime() @@ -373,6 +342,91 @@ object GrapeRankCommand { } } + /** + * Configure the outbox-model crawler from the crawl flags on [args] plus the + * account's relay policy. Shared by the bare command and `graperank sync`. + * Relay policy — where a stranger's kind:10002 is found (index/discovery + * aggregators + general defaults) and best-effort general relays that might + * hold content when an outbox is unknown — lives in app code, so the quartz + * crawler takes it injected. + */ + private suspend fun newCrawler( + ctx: Context, + args: Args, + ): GrapeRankDataCrawler { + val discoveryRelays = + ctx.bootstrapRelays() + Constants.eventFinderRelays + DefaultIndexerRelayList + EXTRA_DISCOVERY_RELAYS + val contentFallback = ctx.bootstrapRelays() + Constants.eventFinderRelays + return GrapeRankDataCrawler( + client = ctx.client, + store = ctx.store, + limiter = ctx.relayLimiter, + config = + GrapeRankDataCrawler.Config( + relayListDiscoveryRelays = discoveryRelays, + contentFallbackRelays = contentFallback, + maxRounds = args.intFlag("max-rounds", Int.MAX_VALUE), + maxHops = args.intFlag("max-hops", Int.MAX_VALUE), + timeoutMs = args.longFlag("timeout", 10L) * 1000, + parkTimeoutMs = args.longFlag("park-timeout", 40L) * 1000, + diagnose = args.bool("diagnose"), + insertBatchSize = args.intFlag("insert-batch", 500), + drainConcurrency = args.intFlag("drain-concurrency", 24), + ), + log = { System.err.println(it) }, + ) + } + + /** Echo any relay NOTICE/CLOSED feedback + adaptive throttling the crawl saw. */ + private fun reportRelayFeedback(ctx: Context) { + if (ctx.relayDiagnostics.hadFeedback()) { + System.err.println("[graperank] relay feedback: ${ctx.relayDiagnostics.snapshot()}") + } + if (ctx.relayLimiter.hadThrottling()) { + System.err.println("[graperank] relay throttling: ${ctx.relayLimiter.snapshot()}") + } + } + + /** + * `amy graperank sync [OBSERVER]` — network-only WoT data sync. Crawls the + * reachable follow/mute/report graph into the local store (kind 3/10000/1984/ + * 10002) and reports what it loaded, WITHOUT scoring. Idempotent + cumulative: + * run it a few times to make sure everything is loaded, then `graperank score`. + */ + private suspend fun sync( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val observerArg = args.positionalOrNull(0) + Context.open(dataDir).use { ctx -> + ctx.prepare() + val observer = observerArg?.let { ctx.requireUserHex(it) } ?: ctx.identity.pubKeyHex + // Persist-only crawl: no in-memory graph (null builder); every event + // still lands in the store for a later `score`. + val stats = newCrawler(ctx, args).crawl(observer, null) + reportRelayFeedback(ctx) + Output.emit( + linkedMapOf( + "observer" to observer, + "crawl_rounds" to stats.rounds, + "relays_contacted" to stats.relaysContacted, + "relay_feedback" to if (ctx.relayDiagnostics.hadFeedback()) ctx.relayDiagnostics.snapshot() else null, + "relay_throttling" to if (ctx.relayLimiter.hadThrottling()) ctx.relayLimiter.snapshot() else null, + "max_hop_reached" to (stats.hopHistogram.keys.maxOrNull() ?: 0), + "users_by_hop" to stats.hopHistogram.mapKeys { it.key.toString() }, + "users_discovered" to stats.hopHistogram.values.sum(), + "contact_lists_fed" to stats.contactListsFed, + "download_ms" to stats.downloadMs, + "verify_ms" to stats.verifyMs, + "insert_ms" to stats.insertMs, + "events_stored" to stats.eventsStored, + ), + ) + } + return 0 + } + /** * Build + sign one kind:30382 [ContactCardEvent] per (target, rank), fanned * out across CPU cores (id-hash + Schnorr sign is CPU-bound). The signed