Merge remote-tracking branch 'origin/main' into claude/armada-nip29-integration-lwqard

# Conflicts:
#	cli/tests/.gitignore
This commit is contained in:
Claude
2026-07-09 21:48:04 +00:00
226 changed files with 21723 additions and 1072 deletions
@@ -24,8 +24,7 @@ package com.vitorpamplona.quartz.utils.secp256k1
internal actual class ScratchLocal<T> actual constructor(
initializer: () -> T,
) {
private val tl = ThreadLocal.withInitial(initializer)
private val tl: ThreadLocal<T> = ThreadLocal.withInitial(initializer)
@Suppress("NULLABILITY_MISMATCH_BASED_ON_JAVA_ANNOTATIONS")
actual fun get(): T = tl.get()
actual fun get(): T = tl.get()!!
}
@@ -0,0 +1,168 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.experimental.graperank
import androidx.compose.runtime.Immutable
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import kotlin.math.abs
import kotlin.math.exp
import kotlin.math.ln
/**
* Tunable GrapeRank parameters. Defaults mirror the reference implementation at
* <https://github.com/vitorpamplona/graperank> and NosFabrica's Brainstorm
* `DEFAULT` preset.
*/
@Immutable
data class GrapeRankParams(
val attenuation: Double = 0.85,
val rigor: Double = 0.5,
val directFollowConfidence: Double = 0.5,
val indirectFollowConfidence: Double = 0.03,
val muteConfidence: Double = 0.5,
val reportConfidence: Double = 0.5,
val convergence: Double = 0.0001,
)
/**
* GrapeRank — a subjective, observer-centric web-of-trust score in `[0, 1]` for
* every user reachable from an observer in a [TrustGraph]. See the algorithm
* notes in `TrustGraph`/`GrapeRankTest`; this is the single-observer
* Gauss-Seidel form, operating on the compact int-CSR graph so it scales to the
* whole network.
*
* [compute] returns a `DoubleArray` indexed by node id (`graph.idOf(pubkey)`),
* not a map — at millions of nodes a boxed map would dwarf the graph itself. The
* observer's own entry stays pinned at `1.0`; callers rank the others.
*/
class GrapeRank(
val params: GrapeRankParams = GrapeRankParams(),
) {
private val rigidity = -ln(params.rigor)
/** Exponential saturation turning accumulated weight into a confidence in `[0, 1)`. */
private fun weightToConfidence(weight: Double): Double = 1.0 - exp(-weight * rigidity)
private fun confidence(
relationCode: Int,
sourceIsObserver: Boolean,
): Double =
when (relationCode) {
TrustRelation.FOLLOW.code -> if (sourceIsObserver) params.directFollowConfidence else params.indirectFollowConfidence
TrustRelation.MUTE.code -> params.muteConfidence
else -> params.reportConfidence
}
private fun rating(relationCode: Int): Double =
when (relationCode) {
TrustRelation.FOLLOW.code -> TrustRelation.FOLLOW.rating
TrustRelation.MUTE.code -> TrustRelation.MUTE.rating
else -> TrustRelation.REPORT.rating
}
/**
* Score every node reachable from [observer]. Returns scores by node id, or an
* all-zero array if the observer isn't in the graph. [onProgress] fires once
* per sweep with `(totalNodeUpdates, nodesStillMoving)` — the second value is
* how many nodes moved more than [GrapeRankParams.convergence] this sweep, so
* it trends to 0 as the graph settles.
*
* Iterates synchronous **Gauss-Seidel** sweeps over every node, updating scores
* in place so a value computed earlier in a sweep is already visible to nodes
* later in the same sweep (this converges faster than a double-buffered Jacobi
* pass). A sweep that moves no node by more than the convergence delta ends the
* loop — the same per-node threshold and fixed point as NosFabrica's Brainstorm
* reference. On a dense graph this is far less total work than a
* change-propagating worklist: a worklist re-visits a node once per rater whose
* score nudges, so its cost scales with the in-degree of the churning core,
* whereas a sweep touches each node exactly once per iteration. Attenuation < 1
* makes the update a contraction, so the fixed point is unique regardless of
* sweep order; ids run in roughly BFS order from the observer, which lets
* trust flow outward within a single sweep and keeps the iteration count low.
*
* Nodes unreachable from the observer settle to 0 for free: all of their raters
* stay at 0, so the inner loop's `sourceScore != 0.0` guard skips every edge.
*/
fun compute(
graph: TrustGraph,
observer: HexKey,
onProgress: ((visited: Long, queued: Int) -> Unit)? = null,
): DoubleArray {
val n = graph.nodeCount
val scores = DoubleArray(n)
val observerId = graph.idOf(observer)
if (observerId < 0) return scores
scores[observerId] = 1.0
val attenuation = params.attenuation
val convergence = params.convergence
val inOffsets = graph.inOffsets
val inPacked = graph.inPacked
var visited = 0L
while (true) {
var stillMoving = 0
var target = 0
while (target < n) {
if (target != observerId) {
var sumOfWeights = 0.0
var sumOfWeightedRatings = 0.0
var i = inOffsets[target]
val end = inOffsets[target + 1]
while (i < end) {
val packed = inPacked[i]
val source = packed and TrustGraph.SOURCE_MASK
val sourceScore = scores[source]
if (sourceScore != 0.0) {
val relationCode = packed ushr TrustGraph.SOURCE_BITS
val weight = confidence(relationCode, source == observerId) * sourceScore * attenuation
sumOfWeights += weight
sumOfWeightedRatings += weight * rating(relationCode)
}
i++
}
val newScore =
if (abs(sumOfWeights) < 0.00001) {
0.0
} else {
val s = weightToConfidence(sumOfWeights) * sumOfWeightedRatings / sumOfWeights
if (s > 0.0) s else 0.0
}
val oldScore = scores[target]
if (newScore != oldScore) {
scores[target] = newScore
if (abs(newScore - oldScore) > convergence) stillMoving++
}
visited++
}
target++
}
onProgress?.invoke(visited, stillMoving)
if (stillMoving == 0) break
}
return scores
}
}
@@ -0,0 +1,207 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.experimental.graperank
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent
import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
/**
* Publishes a set of GrapeRank scores as NIP-85 kind:30382 [ContactCardEvent]
* trusted assertions (one `rank` card per scored user), reconciled against what
* this provider key has already published so a repeat run only writes what moved.
*
* Reconciliation, given the desired `(target, rank)` set the scorer produced:
* - **skip** a target whose stored card already carries the same rank string —
* re-signing an unchanged card would churn a new event id for no client benefit;
* - **upsert** a target whose rank changed (or that has no card yet), up to a
* publish limit;
* - **retract** every stored card whose target is no longer in the desired set
* (it fell below the caller's cutoff, or dropped out of the graph) with a NIP-09
* kind:5 deletion, batched so the frame stays under the ~64KB event cap.
*
* Transport-agnostic like [GrapeRankCrawler]: it reads prior cards from an
* [IEventStore] and emits through an injected [publish] function (event + relays →
* per-relay ack), so the store/relay wiring stays in the application while the
* reconcile + card-construction logic is reusable (e.g. by the Android app).
*/
class GrapeRankPublisher(
private val store: IEventStore,
private val publish: suspend (Event, Set<NormalizedRelayUrl>) -> Map<NormalizedRelayUrl, Boolean>,
) {
/** Outcome counts for one reconcile: what was written, retracted, and skipped. */
class Result(
val published: Int,
val publishRejected: Int,
val deleted: Int,
val deleteRejected: Int,
val skippedUnchanged: Int,
/** Changed cards beyond [publishLimit] that were not upserted this run. */
val truncated: Int,
)
/**
* Reconcile the desired [scored] `(target, rank)` set (the caller has already
* applied any rank cutoff) against the cards [providerPubkey] previously
* published, then upsert the changes and retract the stale cards, all signed by
* [providerSigner]. At most [publishLimit] changed cards are upserted per run.
*/
suspend fun reconcileAndPublish(
providerSigner: NostrSigner,
providerPubkey: HexKey,
scored: List<Pair<HexKey, Int>>,
relays: Set<NormalizedRelayUrl>,
publishLimit: Int,
publishConcurrency: Int = PUBLISH_CONCURRENCY,
): Result {
// Newest card per target this provider already published (read back from
// the store, which every published card was persisted to).
val existing = existingCards(providerPubkey)
val publishableTargets = scored.mapTo(HashSet()) { it.first }
// Upsert publishable targets whose rank tag STRING would change (or that
// have no card yet). RankTag.assemble writes rank.toString(), so we diff
// that exact string — an unchanged score is skipped so clients only sync
// ranks that moved.
val changed = scored.filter { (target, rank) -> existing[target]?.let(::rankTagValue) != rank.toString() }
val toUpsert = changed.take(publishLimit)
// Retract existing cards whose target is no longer publishable — it dropped
// out of the graph, or fell below the caller's cutoff. We won't leave a
// stale assertion standing.
val toDelete = existing.filterKeys { it !in publishableTargets }.values.toList()
val (ok, rejected) = publishCards(providerSigner, toUpsert, relays, publishConcurrency)
val (deleted, deleteRejected) = publishDeletions(providerSigner, toDelete, relays)
return Result(
published = ok,
publishRejected = rejected,
deleted = deleted,
deleteRejected = deleteRejected,
skippedUnchanged = scored.size - changed.size,
truncated = (changed.size - toUpsert.size).coerceAtLeast(0),
)
}
/**
* The newest kind:30382 card [providerPubkey] published per target, read from
* the store (every card [publish] sends is persisted first, so on repeat runs
* this reflects what is already out there).
*/
private suspend fun existingCards(providerPubkey: HexKey): Map<HexKey, ContactCardEvent> =
store
.query<Event>(Filter(kinds = listOf(ContactCardEvent.KIND), authors = listOf(providerPubkey)))
.filterIsInstance<ContactCardEvent>()
.groupBy { it.aboutUser() }
.mapNotNull { (target, cards) ->
val t = target.ifBlank { return@mapNotNull null }
t to (cards.maxByOrNull { it.createdAt } ?: return@mapNotNull null)
}.toMap()
/**
* The raw `rank` tag value string on a card — exactly what a client diffs, so an
* unchanged score never produces a new signature. Our cards carry only a `rank`
* tag (plus the d-tag target), so this one value decides whether a re-publish
* would differ.
*/
private fun rankTagValue(card: ContactCardEvent): String? =
card.tags.firstNotNullOfOrNull { tag ->
if (tag.size > 1 && tag[0] == RankTag.TAG_NAME) tag[1] else null
}
/** Build + publish one kind:30382 card per (target, rank), bounded-concurrently. */
private suspend fun publishCards(
signer: NostrSigner,
cards: List<Pair<HexKey, Int>>,
relays: Set<NormalizedRelayUrl>,
concurrency: Int,
): Pair<Int, Int> {
var published = 0
var rejected = 0
for (batch in cards.chunked(concurrency)) {
val acks =
coroutineScope {
batch
.map { (pubkey, rank) ->
async {
val card =
ContactCardEvent.create(
targetUser = pubkey,
signer = signer,
publicInitializer = { add(RankTag.assemble(rank)) },
)
publish(card, relays)
}
}.awaitAll()
}
for (ack in acks) {
if (ack.values.any { it }) published++ else rejected++
}
}
return published to rejected
}
/**
* Retract stale cards with NIP-09 kind:5 deletions signed by [signer] (the same
* key that signed the cards). Batches [DELETE_PER_EVENT] addressable coordinates
* per deletion so the kind:5 frame stays under the ~64KB event cap; each carries
* the card's `a` tag (30382:provider:target), so re-publishing a newer version
* later isn't blocked. Returns (deleted, rejected) card counts.
*/
private suspend fun publishDeletions(
signer: NostrSigner,
cards: List<ContactCardEvent>,
relays: Set<NormalizedRelayUrl>,
): Pair<Int, Int> {
if (cards.isEmpty()) return 0 to 0
var deleted = 0
var rejected = 0
for (chunk in cards.chunked(DELETE_PER_EVENT)) {
val event = signer.sign(DeletionEvent.build(chunk))
val ack = publish(event, relays)
if (ack.values.any { it }) deleted += chunk.size else rejected += chunk.size
}
return deleted to rejected
}
companion object {
/** Concurrent card publishes when upserting. */
const val PUBLISH_CONCURRENCY = 16
/**
* Addressable coordinates cited per kind:5 retraction. Each `a` tag is
* ~130 bytes (30382:<64hex>:<64hex>), so 400 keeps the whole event ~52KB —
* under the 64KB event-size cap many relays enforce (stricter than the
* 256KB message cap).
*/
const val DELETE_PER_EVENT = 400
}
}
@@ -0,0 +1,259 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.experimental.graperank
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent
import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient
import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.NegentropyStoreSync
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent
import com.vitorpamplona.quartz.nip56Reports.ReportEvent
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
/**
* Store-driven, outbox-model refresh of the record kinds a [GrapeRank] score is a
* function of — the profiles (kind:0), follows (kind:3), outbox relay lists
* (kind:10002), and reports (kind:1984) of every author already known to the
* local [store].
*
* Where [GrapeRankCrawler] discovers the graph by walking follows outward from
* an observer, this refreshes what is *already* known: it reads every kind:10002 in
* the store, inverts them into a `write-relay -> authors` map (the outbox model — an
* author's events live on the relays they write to), fans those into one filter per
* `(write relay, author chunk)`, and hands them to [NegentropyStoreSync] — the generic
* two-pass sync engine. So each relay is asked only for the authors it hosts, and each
* author is reconciled only against their own relays. Run it periodically to keep a
* scored network current without paying a full from-scratch crawl.
*
* The engine does the work per `(relay, filter)` group: a bidirectional NIP-77
* reconcile against [store] ([Config.down] / [Config.up]), a deletion settle over the
* residual ([Config.syncDeletions] — its applyDown direction downloads the relay's
* kind:5 when an uploaded record was rejected because the author retracted it), and a
* paged-download fallback when a relay can't reconcile ([Config.pageFallback]). This
* class only builds the outbox filter set and folds the engine's per-group results back
* up per relay.
*
* Transport-agnostic within quartz: it takes an [INostrClient] and an [IEventStore].
* Progress is emitted through [log]; a headless caller routes it to stderr, a UI ignores it.
*/
class GrapeRankUpdater(
private val client: INostrClient,
private val store: IEventStore,
private val config: Config = Config(),
private val log: (String) -> Unit = {},
) {
/**
* @param kinds the record kinds refreshed per author (default: the WoT set
* 0 / 3 / 10002 / 1984).
* @param down download records the relay has that the store lacks.
* @param up upload records the store has that the relay lacks (also arms the
* deletion **applyDown** path — a rejected upload pulls the relay's kind:5 down).
* @param syncDeletions run the deletion settle over the reconcile residual.
* @param pageFallback page the filter when negentropy can't reconcile a relay.
* @param idChunk ids per reconcile chunk and per by-id fetch.
* @param downloadWorkers concurrent by-id download fetches per group.
* @param reconcileConcurrency overlapped `created_at`-window reconciles after an over-cap split.
* @param maxDeletionRounds hard cap on deletion-settle rounds (converges in 12).
* @param relayConcurrency write relays synced at once.
* @param authorChunk authors per reconcile filter (a relay with more is split into several).
* @param minAuthors skip relays hosting fewer than this many of the store's authors.
* @param idleTimeoutMs idle watchdog for reconciles / fetches / pages.
* @param publishTimeoutSecs OK-confirmation wait per uploaded event.
*/
class Config(
val kinds: List<Int> = DEFAULT_KINDS,
val down: Boolean = true,
val up: Boolean = true,
val syncDeletions: Boolean = true,
val pageFallback: Boolean = true,
val idChunk: Int = 500,
val downloadWorkers: Int = 4,
val reconcileConcurrency: Int = 2,
val maxDeletionRounds: Int = 4,
val relayConcurrency: Int = 4,
val authorChunk: Int = 500,
val minAuthors: Int = 1,
val idleTimeoutMs: Long = 30_000L,
val publishTimeoutSecs: Long = 15,
/**
* Relays proven unreachable within the reachability cache's TTL (kind:30166).
* Skipped from the reconcile plan so we don't burn a connect timeout per dead
* relay — the crawl already found them dead, and a dead relay cannot serve its
* authors anyway. TTL'd, so a recovered relay is retried once the record ages
* out; this never drops a *live* author-advertised relay. See RelayReachabilityStore.
*/
val knownDead: Set<NormalizedRelayUrl> = emptySet(),
) {
/** Project the shared engine knobs onto a [NegentropyStoreSync.Config]. */
internal fun toEngineConfig() =
NegentropyStoreSync.Config(
down = down,
up = up,
syncDeletions = syncDeletions,
pageFallback = pageFallback,
idChunk = idChunk,
downloadWorkers = downloadWorkers,
reconcileConcurrency = reconcileConcurrency,
maxDeletionRounds = maxDeletionRounds,
concurrency = relayConcurrency,
idleTimeoutMs = idleTimeoutMs,
publishTimeoutSecs = publishTimeoutSecs,
)
}
/** Per-write-relay outcome of an [update] (folded from the engine's group results). */
class RelayResult(
val relay: NormalizedRelayUrl,
val authors: Int,
val need: Int,
val have: Int,
val downloaded: Int,
val uploaded: Int,
val deletionsSentUp: Int,
val deletionsAppliedDown: Int,
val pagedFallback: Boolean,
/** null when every chunk of this relay succeeded; the first failure otherwise. */
val error: String?,
)
/** Aggregate outcome of an [update], plus the per-relay breakdown. */
class Result(
val relayListsInStore: Int,
val authorsWithOutbox: Int,
val relays: Int,
val relaysOk: Int,
val relaysFailed: Int,
val relaysPagedFallback: Int,
val downloaded: Int,
val uploaded: Int,
val deletionsSentUp: Int,
val deletionsAppliedDown: Int,
val perRelay: List<RelayResult>,
)
/**
* Group the store's authors by their kind:10002 write relays (the outbox model).
* The latest kind:10002 per author wins; an author with no write-marked relays
* contributes nothing (there is nowhere to reconcile them). Public so callers can
* inspect the plan (relay count, largest groups) before running [update].
*/
suspend fun writeRelayGroups(): Map<NormalizedRelayUrl, Set<HexKey>> = groupByWriteRelay(loadLatestRelayLists())
/**
* Run the full outbox-model refresh: [writeRelayGroups] then hand every group
* hosting at least [Config.minAuthors] authors (largest first, chunked to
* [Config.authorChunk]) to [NegentropyStoreSync], folding its per-group results back
* up per relay. Best-effort — a relay that fails is recorded in [RelayResult.error]
* and never aborts the run.
*/
suspend fun update(): Result {
val latest = loadLatestRelayLists()
val groups = groupByWriteRelay(latest)
val authorsWithOutbox = groups.values.flatMapTo(HashSet()) { it }.size
// Plan: relays with enough authors, largest first so the heaviest groups start
// while engine permits are free.
val plan =
groups.entries
.filter { it.value.size >= config.minAuthors }
.filterNot { it.key in config.knownDead }
.sortedByDescending { it.value.size }
.map { it.key to it.value }
// Fan each relay's authors into one filter per authorChunk-sized slice.
val authorChunk = config.authorChunk.coerceAtLeast(1)
val filtersByRelay =
plan.associate { (relay, authors) ->
relay to authors.toList().chunked(authorChunk).map { Filter(kinds = config.kinds, authors = it) }
}
val groupResults = NegentropyStoreSync(client, store, config.toEngineConfig(), log).sync(filtersByRelay)
val byRelay = groupResults.groupBy { it.relay }
// Fold each relay's chunk results back into one RelayResult (plan order preserved).
val perRelay =
plan.map { (relay, authors) ->
val chunks = byRelay[relay].orEmpty()
RelayResult(
relay = relay,
authors = authors.size,
need = chunks.sumOf { it.need },
have = chunks.sumOf { it.have },
downloaded = chunks.sumOf { it.downloaded },
uploaded = chunks.sumOf { it.uploaded },
deletionsSentUp = chunks.sumOf { it.deletionsSentUp },
deletionsAppliedDown = chunks.sumOf { it.deletionsAppliedDown },
pagedFallback = chunks.any { it.pagedFallback },
error = chunks.firstNotNullOfOrNull { it.error },
)
}
return Result(
relayListsInStore = latest.size,
authorsWithOutbox = authorsWithOutbox,
relays = perRelay.size,
relaysOk = perRelay.count { it.error == null },
relaysFailed = perRelay.count { it.error != null },
relaysPagedFallback = perRelay.count { it.pagedFallback },
downloaded = perRelay.sumOf { it.downloaded },
uploaded = perRelay.sumOf { it.uploaded },
deletionsSentUp = perRelay.sumOf { it.deletionsSentUp },
deletionsAppliedDown = perRelay.sumOf { it.deletionsAppliedDown },
perRelay = perRelay,
)
}
/** Latest kind:10002 per author from the store (replaceable — newest createdAt wins). */
private suspend fun loadLatestRelayLists(): Map<HexKey, AdvertisedRelayListEvent> {
val latest = HashMap<HexKey, AdvertisedRelayListEvent>()
for (event in store.query<Event>(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND)))) {
if (event !is AdvertisedRelayListEvent) continue
val prev = latest[event.pubKey]
if (prev == null || event.createdAt > prev.createdAt) latest[event.pubKey] = event
}
return latest
}
/** Invert the per-author relay lists into `write-relay -> authors`. */
private fun groupByWriteRelay(latest: Map<HexKey, AdvertisedRelayListEvent>): Map<NormalizedRelayUrl, Set<HexKey>> {
val relayToAuthors = HashMap<NormalizedRelayUrl, MutableSet<HexKey>>()
for ((author, list) in latest) {
val writes = list.writeRelaysNorm() ?: continue
for (relay in writes) relayToAuthors.getOrPut(relay) { HashSet() }.add(author)
}
return relayToAuthors
}
companion object {
/** The record kinds a GrapeRank score is a function of. */
val DEFAULT_KINDS =
listOf(
MetadataEvent.KIND, // 0 — profiles
ContactListEvent.KIND, // 3 — follows
AdvertisedRelayListEvent.KIND, // 10002 — outbox relay lists
ReportEvent.KIND, // 1984 — reports
)
}
}
@@ -0,0 +1,101 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.experimental.graperank
import com.vitorpamplona.quartz.nip01Core.core.HexKey
/**
* A trust relationship kind and the GrapeRank rating it carries. [code] is the
* 2-bit tag packed alongside a source node id in the edge arrays (see
* [TrustGraph]); keep it in `0..3`.
*/
enum class TrustRelation(
val rating: Double,
val code: Int,
) {
FOLLOW(1.0, 0),
MUTE(-0.1, 1),
REPORT(-0.1, 2),
}
/**
* A web-of-trust graph over Nostr pubkeys, stored compactly so it scales to the
* whole network (millions of edges) without a `String`-keyed edge object per
* relationship.
*
* Pubkeys are interned to dense `Int` node ids. Edges live in two
* compressed-sparse-row (CSR) layouts backed by flat `IntArray`s — one indexed
* by target (what [GrapeRank] reads to score a node) and one by source (what the
* propagation worklist follows). Each incoming entry packs the source id in the
* low 29 bits and the [TrustRelation.code] in the top bits, so an edge is a
* single `int`. A 100M-edge graph is then ~0.8 GB of primitive arrays instead of
* tens of GB of objects.
*
* Build one with [TrustGraphBuilder], feeding contact lists / mutes / reports in
* as they stream off the relays.
*/
class TrustGraph internal constructor(
val nodeCount: Int,
private val pubkeys: Array<HexKey>,
private val ids: HashMap<HexKey, Int>,
// CSR by target: incoming edges of node t are inPacked[inOffsets[t] until inOffsets[t+1]],
// each packing source id (low 29 bits) + relation code (top bits).
internal val inOffsets: IntArray,
internal val inPacked: IntArray,
// CSR by source: out-neighbour targets of node s are outTargets[outOffsets[s] until outOffsets[s+1]].
internal val outOffsets: IntArray,
internal val outTargets: IntArray,
) {
/** Node id for [pubkey], or `-1` if it never appeared in the graph. */
fun idOf(pubkey: HexKey): Int = ids[pubkey] ?: -1
/** Pubkey for a node [id]. */
fun pubkeyOf(id: Int): HexKey = pubkeys[id]
fun edgeCount(): Int = inPacked.size
companion object {
const val SOURCE_BITS = 29
const val SOURCE_MASK = (1 shl SOURCE_BITS) - 1
const val MAX_NODES = SOURCE_MASK // ids must fit in the low 29 bits
}
}
/** A minimal growable `int[]` — avoids boxing `Int`s in an `ArrayList` at graph scale. */
internal class IntArrayList(
initialCapacity: Int = 16,
) {
var data: IntArray = IntArray(initialCapacity.coerceAtLeast(1))
private set
var size: Int = 0
private set
fun add(value: Int) {
if (size == data.size) data = data.copyOf(data.size * 2)
data[size++] = value
}
fun get(index: Int): Int = data[index]
fun removeLast(): Int = data[--size]
fun isNotEmpty(): Boolean = size > 0
}
@@ -0,0 +1,127 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.experimental.graperank
import com.vitorpamplona.quartz.nip01Core.core.HexKey
/**
* Builds a [TrustGraph] incrementally so callers never have to hold every contact
* list in memory at once — feed each user's follows / mutes / reports as they
* stream off the relays (or out of the store), then call [build].
*
* Interns pubkeys to dense ids on the fly and accumulates edges in flat growable
* int arrays. Follows and mutes are replaceable (one list per author, deduped by
* the caller via latest-per-author + set-valued tags); reports are regular events,
* so `(reporter → reported)` report edges are deduped here. Self-edges are dropped.
*/
class TrustGraphBuilder {
private val ids = HashMap<HexKey, Int>()
private val pubkeys = ArrayList<HexKey>()
// Parallel edge arrays: edge i is source edgeSource[i] --relation--> edgeTarget[i],
// with the relation packed into the top bits of edgeSource[i].
private val edgeTargets = IntArrayList()
private val edgeSourcesPacked = IntArrayList()
// Dedup for report edges only (reporters can file many kind:1984 for one target).
private val reportSeen = HashSet<Long>()
private fun intern(pubkey: HexKey): Int =
ids.getOrPut(pubkey) {
val id = pubkeys.size
pubkeys.add(pubkey)
id
}
private fun addEdge(
source: HexKey,
target: HexKey,
relation: TrustRelation,
) {
if (source == target) return
val s = intern(source)
val t = intern(target)
if (relation == TrustRelation.REPORT) {
val key = (s.toLong() shl 32) or (t.toLong() and 0xFFFFFFFFL)
if (!reportSeen.add(key)) return
}
edgeTargets.add(t)
edgeSourcesPacked.add(s or (relation.code shl TrustGraph.SOURCE_BITS))
}
fun addFollows(
source: HexKey,
follows: Iterable<HexKey>,
) {
for (target in follows) addEdge(source, target, TrustRelation.FOLLOW)
}
fun addMutes(
source: HexKey,
muted: Iterable<HexKey>,
) {
for (target in muted) addEdge(source, target, TrustRelation.MUTE)
}
fun addReports(
source: HexKey,
reported: Iterable<HexKey>,
) {
for (target in reported) addEdge(source, target, TrustRelation.REPORT)
}
fun nodeCount(): Int = pubkeys.size
fun edgeCount(): Int = edgeTargets.size
/** Freeze the accumulated edges into the two CSR layouts. */
fun build(): TrustGraph {
val n = pubkeys.size
val m = edgeTargets.size
// Incoming CSR (by target).
val inOffsets = IntArray(n + 1)
for (i in 0 until m) inOffsets[edgeTargets.get(i) + 1]++
for (i in 1..n) inOffsets[i] += inOffsets[i - 1]
val inPacked = IntArray(m)
val inCursor = inOffsets.copyOf()
for (i in 0 until m) {
val t = edgeTargets.get(i)
inPacked[inCursor[t]++] = edgeSourcesPacked.get(i)
}
// Outgoing CSR (by source).
val outOffsets = IntArray(n + 1)
for (i in 0 until m) {
val s = edgeSourcesPacked.get(i) and TrustGraph.SOURCE_MASK
outOffsets[s + 1]++
}
for (i in 1..n) outOffsets[i] += outOffsets[i - 1]
val outTargets = IntArray(m)
val outCursor = outOffsets.copyOf()
for (i in 0 until m) {
val s = edgeSourcesPacked.get(i) and TrustGraph.SOURCE_MASK
outTargets[outCursor[s]++] = edgeTargets.get(i)
}
return TrustGraph(n, pubkeys.toTypedArray(), ids, inOffsets, inPacked, outOffsets, outTargets)
}
}
@@ -327,6 +327,7 @@ data class KindName(
* platform concern layered on top, never a fork of this data.
*/
object KindNames {
@Suppress("DEPRECATION") // registry intentionally names deprecated kinds (GitReply, TorrentComment) for display
val names: Map<Int, KindName> =
mapOf(
AcceptedBadgeSetEvent.KIND to KindName("Accepted Badge Set", "58"),
@@ -215,6 +215,10 @@ class MlsGroup private constructor(
encryptionPrivateKey = encryptionPrivateKey,
interimTranscriptHash = interimTranscriptHash,
encryptionSecret = epochSecrets.encryptionSecret,
// Preserve the SecretTree ratchet positions so a restore doesn't
// rewind our own generation counter to 0 and reuse an AEAD
// key+nonce within this epoch (RFC 9420 §9).
senderRatchetStates = secretTree.exportSenderStates(),
)
}
@@ -3501,15 +3505,23 @@ class MlsGroup private constructor(
/**
* Restore a group from a previously saved [MlsGroupState].
*
* The SecretTree is reconstructed from the stored encryption_secret.
* Note: SecretTree ratchet state (per-sender generation counters) is
* NOT preserved — messages sent/received before the save point cannot
* be re-decrypted, which is acceptable because they would already
* have been processed.
* The SecretTree is reconstructed from the stored encryption_secret,
* then seeded with the persisted per-sender ratchet positions
* ([MlsGroupState.senderRatchetStates]). Seeding is what keeps the
* local member's generation counter monotonic across a restart — a
* fresh SecretTree would restart every sender at generation 0, so our
* next send would reuse generation 0's AEAD key+nonce within the same
* epoch and be rejected by strict receivers (openmls / MDK /
* Whitenoise) that forbid generation reuse.
*
* Receive-only ratchets that weren't persisted (STATE_VERSION 1 blobs,
* or senders we never decrypted) simply re-derive from generation 0 on
* first use — safe, because those messages were already processed.
*/
fun restore(state: MlsGroupState): MlsGroup {
val tree = RatchetTree.decodeTls(TlsReader(state.treeBytes))
val secretTree = SecretTree(state.encryptionSecret, tree.leafCount)
secretTree.importSenderStates(state.senderRatchetStates)
return MlsGroup(
groupContext = state.groupContext,
@@ -348,13 +348,23 @@ class MlsGroupManager(
/**
* Encrypt an application message.
* Synchronized to prevent nonce reuse from concurrent encryption.
*
* The group state is persisted after every send. Encrypting advances the
* SecretTree ratchet (RFC 9420 §9) but does not change the epoch, so
* without this save a restart between two messages would reload the
* pre-send ratchet position and re-emit an already-used generation —
* reusing the AEAD key+nonce and getting rejected by strict receivers.
* State was previously persisted only at commits, which left every
* inter-commit send unprotected.
*/
suspend fun encrypt(
nostrGroupId: HexKey,
plaintext: ByteArray,
): ByteArray =
mutex.withLock {
requireGroup(nostrGroupId).encrypt(plaintext)
val ciphertext = requireGroup(nostrGroupId).encrypt(plaintext)
persistGroup(nostrGroupId)
ciphertext
}
/**
@@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.marmot.mls.codec.TlsReader
import com.vitorpamplona.quartz.marmot.mls.codec.TlsWriter
import com.vitorpamplona.quartz.marmot.mls.messages.GroupContext
import com.vitorpamplona.quartz.marmot.mls.schedule.EpochSecrets
import com.vitorpamplona.quartz.marmot.mls.schedule.SenderRatchetState
/**
* Serializable snapshot of an MLS group's complete state.
@@ -40,6 +41,13 @@ import com.vitorpamplona.quartz.marmot.mls.schedule.EpochSecrets
*
* Security: This blob contains secret key material (signing key, encryption key,
* epoch secrets). It MUST be stored in encrypted local storage.
*
* [senderRatchetStates] carries each sender's live SecretTree ratchet position
* (RFC 9420 §9). Preserving it is what stops the restored local member from
* re-emitting an already-used generation within the same epoch see
* [com.vitorpamplona.quartz.marmot.mls.schedule.SecretTree.exportSenderStates].
* It is optional (empty for STATE_VERSION 1 blobs) so older persisted state
* still decodes.
*/
data class MlsGroupState(
val groupContext: GroupContext,
@@ -51,6 +59,7 @@ data class MlsGroupState(
val encryptionPrivateKey: ByteArray,
val interimTranscriptHash: ByteArray,
val encryptionSecret: ByteArray,
val senderRatchetStates: Map<Int, SenderRatchetState> = emptyMap(),
) {
fun encodeTls(): ByteArray {
val writer = TlsWriter()
@@ -94,6 +103,18 @@ data class MlsGroupState(
// Encryption secret for SecretTree reconstruction
writer.putOpaqueVarInt(encryptionSecret)
// Per-sender SecretTree ratchet positions (STATE_VERSION 2+).
// Preserving the local sender's generation counter is what prevents
// AEAD key+nonce reuse (and strict-receiver rejection) after a restore.
writer.putUint32(senderRatchetStates.size.toLong())
for ((leafIndex, ratchet) in senderRatchetStates) {
writer.putUint32(leafIndex.toLong())
writer.putOpaqueVarInt(ratchet.handshakeSecret)
writer.putUint32(ratchet.handshakeGeneration.toLong())
writer.putOpaqueVarInt(ratchet.applicationSecret)
writer.putUint32(ratchet.applicationGeneration.toLong())
}
return writer.toByteArray()
}
@@ -110,13 +131,18 @@ data class MlsGroupState(
}
companion object {
private const val STATE_VERSION = 1
/**
* v1: original layout (no SecretTree ratchet positions).
* v2: appends [senderRatchetStates] so restores don't reset the
* ratchet to generation 0. v1 blobs still decode (empty map).
*/
private const val STATE_VERSION = 2
fun decodeTls(data: ByteArray): MlsGroupState {
val reader = TlsReader(data)
val version = reader.readUint16()
require(version == STATE_VERSION) { "Unsupported state version: $version" }
require(version in 1..STATE_VERSION) { "Unsupported state version: $version" }
val groupContext = GroupContext.decodeTls(reader)
val treeBytes = reader.readOpaqueVarInt()
@@ -144,6 +170,33 @@ data class MlsGroupState(
val interimTranscriptHash = reader.readOpaqueVarInt()
val encryptionSecret = reader.readOpaqueVarInt()
// v2+: per-sender SecretTree ratchet positions. Absent (or an
// empty count) for v1 blobs, which restore at generation 0.
val senderRatchetStates =
if (version >= 2 && reader.hasRemaining) {
val count = reader.readUint32().toInt()
buildMap {
repeat(count) {
val leafIndex = reader.readUint32().toInt()
val handshakeSecret = reader.readOpaqueVarInt()
val handshakeGeneration = reader.readUint32().toInt()
val applicationSecret = reader.readOpaqueVarInt()
val applicationGeneration = reader.readUint32().toInt()
put(
leafIndex,
SenderRatchetState(
handshakeSecret = handshakeSecret,
handshakeGeneration = handshakeGeneration,
applicationSecret = applicationSecret,
applicationGeneration = applicationGeneration,
),
)
}
}
} else {
emptyMap()
}
return MlsGroupState(
groupContext = groupContext,
treeBytes = treeBytes,
@@ -154,6 +207,7 @@ data class MlsGroupState(
encryptionPrivateKey = encryptionPrivateKey,
interimTranscriptHash = interimTranscriptHash,
encryptionSecret = encryptionSecret,
senderRatchetStates = senderRatchetStates,
)
}
}
@@ -389,6 +389,32 @@ class SecretTree(
return currentSecret
}
/**
* Snapshot every sender's current ratchet position so the enclosing
* group state can be persisted (RFC 9420 §9).
*
* Without this, a restore rebuilds the tree at generation 0 for every
* sender, and the LOCAL member then re-emits generation 0 within the
* same epoch on its next send reusing the AEAD key+nonce (a
* confidentiality break) and getting rejected by strict receivers
* (openmls / MDK / Whitenoise) that forbid generation reuse.
*
* Only the live ratchet position (secret + generation) per sender is
* captured. The replay-detection and skipped-key caches are runtime-only
* and deliberately excluded they are safe to drop across a restart.
*/
fun exportSenderStates(): Map<Int, SenderRatchetState> = senderState.toMap()
/**
* Seed per-sender ratchet positions from an [exportSenderStates]
* snapshot. Called by `MlsGroup.restore`. Any sender absent from
* [states] simply re-derives from generation 0 on first use, which is
* correct for receive-only ratchets.
*/
fun importSenderStates(states: Map<Int, SenderRatchetState>) {
senderState.putAll(states)
}
}
data class SenderRatchetState(
@@ -0,0 +1,287 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener
import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient
import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.ClosedMessage
import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message
import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NoticeMessage
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.utils.Log
import com.vitorpamplona.quartz.utils.TimeUtils
import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.delay
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlin.concurrent.atomics.AtomicInt
import kotlin.concurrent.atomics.AtomicLong
import kotlin.concurrent.atomics.ExperimentalAtomicApi
/**
* Adaptive per-relay back-pressure with TWO independent controls, because relays
* push back for two different reasons that need two different responses:
*
* 1. **Subscription-count limit** a max on how many subscriptions may be OPEN
* at once ("too many subscriptions", "maximum concurrent subscription count",
* "number of subscriptions exceeds limit"). The fix is fewer *concurrent*
* subs, so we demote the relay's concurrency cap down [subLadder]
* (100 20 10).
* 2. **Rate limit** too many subscription *changes per second* ("rate-limited:
* too many messages", "burst exhausted", "slow down"). Fewer concurrent subs
* wouldn't help; the fix is to *space the REQs out in time*, so we impose a
* minimum interval between opens to that relay, growing it up [rateLadder]
* (250ms 500ms 1s 2s).
*
* Mixing the two mishandles the relay: capping concurrency does nothing for a
* rate limit, and slowing the rate does nothing for a subscription-count cap. So
* each complaint is routed to its own actuator by matching the notice text.
*
* A well-behaved relay starts at [startCap] concurrent subs with no rate delay,
* and only the ones that push back get throttled each only as far, and in the
* dimension, they keep pushing.
*
* Registered as a [RelayConnectionListener] on the shared client, so both signals
* are driven straight off the incoming NOTICE/CLOSED frames (which fire on the
* per-relay socket threads all state here is concurrent). Drains gate through
* [withPermit]: the gated-drain path holds a relay's permit for the lifetime of
* that relay's subscription, and passes the rate gate before it opens, so we
* respect both limits at once.
*/
@OptIn(ExperimentalAtomicApi::class)
class AdaptiveRelayLimiter(
private val startCap: Int = 100,
private val subLadder: List<Int> = listOf(20, 10),
private val rateLadder: List<Long> = listOf(250L, 500L, 1000L, 2000L),
) : RelayConnectionListener {
private val gates = ConcurrentMap<NormalizedRelayUrl, Gate>()
// Concurrency-cap demotions per relay (== index+1 into subLadder). Capped at
// subLadder.size: past the floor we stop demoting.
private val subDemotions = ConcurrentMap<NormalizedRelayUrl, Int>()
// Rate-limit state per relay: how far down rateLadder we've stepped, the
// current min interval between opens, and the next epoch-ms an open may fire.
private val rateSteps = ConcurrentMap<NormalizedRelayUrl, Int>()
private val rateDelayMs = ConcurrentMap<NormalizedRelayUrl, Long>()
private val nextAllowedAtMs = ConcurrentMap<NormalizedRelayUrl, AtomicLong>()
private fun gate(relay: NormalizedRelayUrl): Gate = gates.getOrPut(relay) { Gate(startCap) }
/** The concurrency cap currently enforced for [relay] ([startCap] unless demoted). */
fun concurrencyCapOf(relay: NormalizedRelayUrl): Int {
val step = subDemotions[relay] ?: 0
return if (step == 0) startCap else subLadder[(step - 1).coerceIn(0, subLadder.size - 1)]
}
/** The min interval (ms) between opens enforced for [relay]; 0 if not rate-limited. */
fun rateDelayOf(relay: NormalizedRelayUrl): Long = rateDelayMs[relay] ?: 0L
/** True if we lowered [relay]'s concurrency cap or imposed a rate delay (it pushed back). */
fun isThrottled(relay: NormalizedRelayUrl): Boolean = (subDemotions[relay] ?: 0) > 0 || (rateDelayMs[relay] ?: 0L) > 0L
/**
* Run [block] against [relay] respecting both limits: first wait out any rate
* delay (spacing opens in time), then hold one of the relay's concurrency
* permits for the duration.
*/
suspend fun <T> withPermit(
relay: NormalizedRelayUrl,
block: suspend () -> T,
): T {
rateGate(relay)
val g = gate(relay)
g.acquire()
try {
return block()
} finally {
g.release()
}
}
/** If [relay] is rate-limited, reserve and wait for its next allowed open slot. */
private suspend fun rateGate(relay: NormalizedRelayUrl) {
val delayMs = rateDelayMs[relay] ?: return
if (delayMs <= 0L) return
val now = TimeUtils.nowMillis()
// Atomically claim the next slot: my turn is max(prevSlot, now); the next
// caller can't fire until delayMs after me. Serializes opens to this relay
// at one per delayMs, in arrival order.
val slot = nextAllowedAtMs.getOrPut(relay) { AtomicLong(now) }
var myTurn: Long
while (true) {
val prev = slot.load()
myTurn = maxOf(prev, now)
if (slot.compareAndSet(prev, myTurn + delayMs)) break
}
val wait = myTurn - now
if (wait > 0) delay(wait)
}
override fun onIncomingMessage(
relay: IRelayClient,
msgStr: String,
msg: Message,
) {
val text =
when (msg) {
is ClosedMessage -> msg.message
is NoticeMessage -> msg.message
else -> return
}
val t = text.lowercase()
// Route each complaint to the matching actuator. Not mutually exclusive:
// if a relay somehow reports both, we act on both (they don't conflict).
if (RATE_LIMIT_MARKERS.any { it in t }) throttleRate(relay.url)
if (SUB_LIMIT_MARKERS.any { it in t }) demoteConcurrency(relay.url)
}
/** Step [relay] one rung down the concurrency-cap ladder, unless already at the floor. */
private fun demoteConcurrency(relay: NormalizedRelayUrl) {
if ((subDemotions[relay] ?: 0) >= subLadder.size) return
val step = subDemotions.merge(relay, 1) { a, b -> a + b }
val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)]
gate(relay).lower(cap)
if (step <= subLadder.size) {
Log.d("AdaptiveRelayLimiter") { "${relay.url} concurrency capped at $cap subs (sub-limit #$step)" }
}
}
/** Step [relay] one rung down the rate ladder, unless already at the slowest. */
private fun throttleRate(relay: NormalizedRelayUrl) {
if ((rateSteps[relay] ?: 0) >= rateLadder.size) return
val step = rateSteps.merge(relay, 1) { a, b -> a + b }
val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)]
rateDelayMs[relay] = d
if (step <= rateLadder.size) {
Log.d("AdaptiveRelayLimiter") { "${relay.url} rate-throttled to 1 REQ / ${d}ms (rate-limit #$step)" }
}
}
/** JSON-friendly view of which relays we throttled, in which dimension, how far. */
fun snapshot(): Map<String, Any?> {
val capCounts = HashMap<Int, Int>()
for ((_, step) in subDemotions.snapshot()) {
val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)]
capCounts[cap] = (capCounts[cap] ?: 0) + 1
}
val cappedAt = capCounts.toList().sortedBy { it.first }.toMap()
val rateCounts = HashMap<Long, Int>()
for ((_, step) in rateSteps.snapshot()) {
val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)]
rateCounts[d] = (rateCounts[d] ?: 0) + 1
}
val rateAt = rateCounts.toList().sortedBy { it.first }.toMap()
return mapOf(
"start_cap" to startCap,
"sub_ladder" to subLadder,
"rate_ladder_ms" to rateLadder,
"concurrency_capped_relays" to subDemotions.size(),
"concurrency_capped_at" to cappedAt,
"rate_limited_relays" to rateSteps.size(),
"rate_limited_at_ms" to rateAt,
)
}
fun hadThrottling(): Boolean = subDemotions.size() > 0 || rateSteps.size() > 0
/**
* A bounded-concurrency gate whose limit can only ever be *lowered* (relays
* never earn their cap back within a run). Fair FIFO hand-off: a released
* permit goes to the longest-waiting acquirer. Lowering the limit below the
* in-use count doesn't cancel live holders it just refuses to admit new
* ones until enough release that `inUse < limit` again, so the concurrency
* converges down to the new cap as the excess subscriptions finish.
*/
private class Gate(
initialLimit: Int,
) {
private val limit = AtomicInt(initialLimit)
private val mutex = Mutex()
private var inUse = 0
private val waiters = ArrayDeque<CompletableDeferred<Unit>>()
suspend fun acquire() {
val wait =
mutex.withLock {
if (inUse < limit.load()) {
inUse++
null
} else {
CompletableDeferred<Unit>().also { waiters.addLast(it) }
}
}
wait?.await()
}
suspend fun release() {
mutex.withLock {
inUse--
while (inUse < limit.load() && waiters.isNotEmpty()) {
waiters.removeFirst().complete(Unit)
inUse++
}
}
}
/** Monotonically shrink the cap. Safe to call from any thread. */
fun lower(newLimit: Int) {
while (true) {
val cur = limit.load()
if (newLimit >= cur) return
if (limit.compareAndSet(cur, newLimit)) return
}
}
}
companion object {
// A cap on how many subscriptions may be OPEN at once. Fix: fewer
// concurrent subs (demote the concurrency cap).
private val SUB_LIMIT_MARKERS =
listOf(
"too many concurrent",
"concurrent req",
"too many subscription",
"number of subscriptions",
"subscriptions exceeds",
"subscription limit",
"subscription count",
"maximum concurrent subscription",
"max subscription",
"too many req",
)
// Too many subscription CHANGES per second. Fix: space the REQs out in
// time (a per-relay min interval), not fewer concurrent subs.
private val RATE_LIMIT_MARKERS =
listOf(
"rate-limit",
"rate limit",
"ratelimit",
"too many messages",
"too many requests",
"burst exhausted",
"throttl",
"slow down",
)
}
}
@@ -0,0 +1,68 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
/**
* A drain per-relay failure worth acting on: the relay will not serve us THIS run,
* so drop it from further routing on the first occurrence. There is only one such
* verdict [DEAD] because re-probing hop-8's failed relays fresh, outside the
* crawl, showed the old "might clear, retry a few times" (TRANSIENT) bucket almost
* never clears: 503 Service Unavailable was 0/12 reachable, 502 Bad Gateway 3/15,
* connection-establishment failures 0/30, and the codes that WERE alive (403/402)
* are gated and will never hand us events. Spending extra dials on them was waste.
*
* The only two connect failures that genuinely recover are kept OUT of this verdict
* by [classifyDrainFailure] returning null (retry, never dead):
* - a **read** timeout the relay accepted the handshake but is slow to serve;
* 12/18 (67%) were reachable fresh, only overloaded by the crawl's fan-out. The
* crawler's per-authority timeout strikes, which CLEAR on success, shed the gone.
* - an HTTP **429 / too many requests** alive and rate-limiting; 4/4 reachable
* fresh. Retrying (spaced by the rate limiter) is how we eventually get its data.
*/
enum class DrainFailure { DEAD, }
/**
* Classify a drain per-relay terminal reason. Returns null when the relay should be
* retried rather than dropped a read/generic timeout, an alive 429 rate-limit, or
* a non-failure like eose/closed. Any other `cannot:<message>` (see
* `BasicRelayClient.onCannotConnect`) is [DrainFailure.DEAD]: it will not serve us
* this run, so drop it now instead of paying repeated connect attempts.
*/
fun classifyDrainFailure(reason: String): DrainFailure? {
if (!reason.startsWith("cannot")) return null
val m = reason.removePrefix("cannot:").lowercase()
// Alive, only asking us to slow down: an HTTP 429 / "too many requests" reliably
// clears — 4/4 such relays were reachable when re-probed fresh. Retry it (the
// rate limiter spaces our opens); never drop it.
if ("429" in m || "too many requests" in m) return null
// A READ timeout means the relay accepted the handshake but is slow to serve —
// 12/18 (67%) reachable fresh, alive but overloaded by the fan-out. Retry; the
// crawler's per-authority timeout strikes, which clear on success, shed the gone.
// A *connect* timeout is the opposite (the socket never opened, 0/30 reachable),
// so it is excluded here and falls through to DEAD with every other failure.
if (("timeout" in m || "timed out" in m) && "connect timed out" !in m) return null
// Everything else won't serve us this run: connect refused / unroutable / the
// proxy couldn't tunnel the CONNECT, a DNS or TLS failure, a dead-or-not-a-relay
// HTTP upgrade (502/503/500/504/410/404/200/…), or a mid-stream reset. Measured
// mostly dead (503 0%, 502 20% reachable) and, when alive, gated (402/403) or not
// a relay (200). Drop it now rather than burn more dials on it.
return DrainFailure.DEAD
}
@@ -0,0 +1,304 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip01Core.store.verifyAndInsert
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.joinAll
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import kotlin.concurrent.atomics.AtomicInt
import kotlin.concurrent.atomics.ExperimentalAtomicApi
import kotlin.coroutines.cancellation.CancellationException
/**
* Two-pass NIP-77 sync of ANY filter set between a relay and a local [IEventStore],
* with a paged fallback the reusable engine behind `amy sync` and the GrapeRank
* outbox updater, generalized so any caller can reconcile arbitrary
* `relay -> filters` work against their store.
*
* A **group** is one `(relay, filter)`: [syncGroup] runs the full two-pass sync for it.
*
* 1. **Content pass** [negentropyReconcile] diffs the relay's matched set for the
* filter against the store's ids, then:
* - [Config.down] downloads the residual **needs** (relay has, store lacks) by id
* and verifies+inserts them into [store];
* - [Config.up] uploads the residual **haves** (store has, relay lacks) as EVENTs.
* 2. **Deletion pass** [negentropySettleDeletions] over what the content pass could
* not converge ([Config.syncDeletions]). Its **applyDown** direction is the
* "download the deletion when our upload was rejected" case: an event pushed up
* that the relay keeps rejecting (it deleted it) is a residual *have*, so the
* relay's covering kind:5 is pulled down and applied and [store] drops the
* retracted event. **sendUp** publishes the store's covering deletions for records
* deleted locally that the relay still serves.
*
* If the content pass can't reconcile ([NegentropySyncException] no NIP-77, an
* over-cap minimal window, a mid-sync disconnect) and [Config.pageFallback] is on, the
* group pages the same filter ([fetchAllPages]) into the store instead; only the
* negentropy-only deletion settle is skipped there. Every group is best-effort a
* failure lands in [GroupResult.error], it never throws so one bad relay can't abort
* a multi-relay [sync].
*
* [sync] runs many groups: relays go up to [Config.concurrency] at once, and a single
* relay's filters run sequentially (so one relay never opens more than one group's worth
* of negentropy sessions at a time keeping under its subscription budget). Progress is
* emitted through [log].
*/
@OptIn(ExperimentalAtomicApi::class)
class NegentropyStoreSync(
private val client: INostrClient,
private val store: IEventStore,
private val config: Config = Config(),
private val log: (String) -> Unit = {},
) {
/**
* @param down download records the relay has that the store lacks.
* @param up upload records the store has that the relay lacks (also arms the
* deletion **applyDown** path a rejected upload pulls the relay's kind:5 down).
* @param syncDeletions run the deletion settle over the reconcile residual.
* @param pageFallback page the filter when negentropy can't reconcile the relay.
* @param idChunk ids per reconcile chunk and per by-id fetch.
* @param downloadWorkers concurrent by-id download fetches per group.
* @param reconcileConcurrency overlapped `created_at`-window reconciles after an over-cap split.
* @param maxDeletionRounds hard cap on deletion-settle rounds (converges in 12).
* @param concurrency relays synced at once by [sync] (a relay's own filters stay sequential).
* @param idleTimeoutMs idle watchdog for reconciles / fetches / pages.
* @param publishTimeoutSecs OK-confirmation wait per uploaded event.
*/
class Config(
val down: Boolean = true,
val up: Boolean = false,
val syncDeletions: Boolean = true,
val pageFallback: Boolean = true,
val idChunk: Int = 500,
val downloadWorkers: Int = 4,
val reconcileConcurrency: Int = 2,
val maxDeletionRounds: Int = 4,
val concurrency: Int = 4,
val idleTimeoutMs: Long = 30_000L,
val publishTimeoutSecs: Long = 15,
)
/** Outcome of one `(relay, filter)` group. `error` is null on success. */
class GroupResult(
val relay: NormalizedRelayUrl,
val filter: Filter,
val need: Int,
val have: Int,
val downloaded: Int,
val uploaded: Int,
val deletionsSentUp: Int,
val deletionsAppliedDown: Int,
/** True when negentropy couldn't reconcile and the filter was paged instead. */
val pagedFallback: Boolean,
val error: String?,
)
/**
* Sync every `(relay, filter)` in [filtersByRelay]: relays run up to
* [Config.concurrency] at once; each relay's filters run sequentially. Returns one
* [GroupResult] per relay+filter (relay order preserved, filters in list order).
*/
suspend fun sync(filtersByRelay: Map<NormalizedRelayUrl, List<Filter>>): List<GroupResult> {
if (filtersByRelay.isEmpty()) return emptyList()
val gate = Semaphore(config.concurrency.coerceAtLeast(1))
return coroutineScope {
filtersByRelay.entries
.map { (relay, filters) ->
async { gate.withPermit { filters.map { syncGroupSafely(relay, it) } } }
}.awaitAll()
.flatten()
}
}
/**
* [syncGroup] with a best-effort guard so an unexpected failure in one group
* (store I/O, a relay throwing outside the NIP-77 path, ) is recorded rather than
* cancelling every other relay in a [sync]. Cancellation is propagated, not caught.
*/
private suspend fun syncGroupSafely(
relay: NormalizedRelayUrl,
filter: Filter,
): GroupResult =
try {
syncGroup(relay, filter)
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
log("[store-sync] ${relay.url}: group failed: ${e::class.simpleName}: ${e.message}")
GroupResult(relay, filter, 0, 0, 0, 0, 0, 0, pagedFallback = false, error = "${e::class.simpleName}: ${e.message}")
}
/** Content pass + deletion settle (+ page fallback) for one relay + one filter. */
suspend fun syncGroup(
relay: NormalizedRelayUrl,
filter: Filter,
): GroupResult {
// Only the id+created_at snapshot is needed to reconcile — never the decoded
// events (~40 B/entry vs ~1 KB), which matters when a relay hosts a large
// matched set. The events the reconcile decides to UP-publish (the small
// residual haves) are fetched by id on demand in the uploader below.
val localEntries = store.snapshotIdsForNegentropy(listOf(filter))
val downloaded = AtomicInt(0)
val uploaded = AtomicInt(0)
val reconcileResult =
try {
coroutineScope {
// needIds = relay has, store lacks; haveIds = store has, relay lacks.
val needBatches = Channel<List<HexKey>>(config.downloadWorkers * 2)
val haveBatches = Channel<List<HexKey>>(Channel.UNLIMITED)
val downloaders =
List(config.downloadWorkers.coerceAtLeast(1)) {
launch {
for (batch in needBatches) {
for (event in client.fetchAll(relay, Filter(ids = batch), config.idleTimeoutMs)) {
if (store.verifyAndInsert(event)) downloaded.addAndFetch(1)
}
}
}
}
val uploader =
launch {
for (batch in haveBatches) {
// Fetch just the residual haves from the store (not the
// whole matched set) and publish them up.
for (ev in store.query<Event>(Filter(ids = batch))) {
if (client.publishAndConfirm(ev, setOf(relay), config.publishTimeoutSecs)) uploaded.addAndFetch(1)
}
}
}
val result =
try {
client.negentropyReconcile(
relay = relay,
filter = filter,
localEntries = localEntries,
batchSize = config.idChunk,
idleTimeoutMs = config.idleTimeoutMs,
reconcileConcurrency = config.reconcileConcurrency,
onHaveIds = if (config.up) { batch -> haveBatches.send(batch) } else null,
onNeedIds = { batch -> if (config.down) needBatches.send(batch) },
)
} finally {
needBatches.close()
haveBatches.close()
}
downloaders.joinAll()
uploader.join()
result
}
} catch (e: NegentropySyncException) {
// Negentropy couldn't reconcile — page the same filter so the records
// still refresh. Deletion settle is negentropy-only, so it is skipped.
var pageError: String? = e.message ?: "negentropy sync failed"
if (config.pageFallback && config.down) {
pageError =
try {
downloaded.addAndFetch(pageDownload(relay, filter))
null
} catch (pe: CancellationException) {
throw pe
} catch (pe: Exception) {
"negentropy: ${e.message}; page fallback: ${pe::class.simpleName}: ${pe.message}"
}
}
log("[store-sync] ${relay.url}: paged fallback, ${downloaded.load()} stored${pageError?.let { " (error: $it)" } ?: ""}")
return GroupResult(relay, filter, 0, 0, downloaded.load(), uploaded.load(), 0, 0, pagedFallback = true, error = pageError)
}
val deletions =
if (config.syncDeletions) {
client.negentropySettleDeletions(
relay = relay,
filter = filter,
store = store,
sendUp = config.down,
applyDown = config.up,
batchSize = config.idChunk,
idleTimeoutMs = config.idleTimeoutMs,
maxRounds = config.maxDeletionRounds,
reconcileConcurrency = config.reconcileConcurrency,
)
} else {
null
}
log(
"[store-sync] ${relay.url}: down ${downloaded.load()}, up ${uploaded.load()}, " +
"del↑ ${deletions?.sentUp ?: 0}, del↓ ${deletions?.appliedDown ?: 0}",
)
return GroupResult(
relay = relay,
filter = filter,
need = reconcileResult.needCount,
have = reconcileResult.haveCount,
downloaded = downloaded.load(),
uploaded = uploaded.load(),
deletionsSentUp = deletions?.sentUp ?: 0,
deletionsAppliedDown = deletions?.appliedDown ?: 0,
pagedFallback = false,
error = null,
)
}
/**
* Paged fallback: walk [relay] past its per-REQ cap for [filter], verifying and
* inserting each event into [store]. [fetchAllPages]'s `onEvent` can't suspend, so
* events funnel through a channel to a single inserter. Returns how many were newly stored.
*/
private suspend fun pageDownload(
relay: NormalizedRelayUrl,
filter: Filter,
): Int {
val stored = AtomicInt(0)
val events = Channel<Event>(Channel.UNLIMITED)
coroutineScope {
val inserter =
launch {
for (event in events) {
if (store.verifyAndInsert(event)) stored.addAndFetch(1)
}
}
try {
client.fetchAllPages(relay, listOf(filter), config.idleTimeoutMs) { event -> events.trySend(event) }
} finally {
events.close()
}
inserter.join()
}
return stored.load()
}
}
@@ -120,7 +120,17 @@ suspend fun INostrClient.fetchFirst(
remaining.clear()
}
doneChannel.onReceive { relay ->
remaining.remove(relay)
// A relay sends its matching events before its EOSE, so an event may
// already be buffered when this completion fires. select() picks a ready
// clause at random, so without this drain we could treat the relay as done
// and exit while its event still sits unread in the channel.
val buffered = eventChannel.tryReceive().getOrNull()
if (buffered != null) {
result = buffered
remaining.clear()
} else {
remaining.remove(relay)
}
}
}
}
@@ -0,0 +1,145 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.crypto.verify
import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip01Core.store.deletionsCovering
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
/**
* Outcome of a [negentropySettleDeletions] run.
*
* @property sentUp distinct local deletions published to the relay (up direction).
* @property appliedDown distinct relay deletions ingested into [store] (down direction).
* @property rounds reconcile rounds run before convergence (or the cap).
*/
class DeletionSettleResult(
val sentUp: Int,
val appliedDown: Int,
val rounds: Int,
)
/**
* Converge deletions between [store] and [relay] AFTER a content sync has settled the
* two sides the second half of a two-pass sync. NIP-77 reconciles by id, so a plain
* content sync converges everything except events a deletion physically stops from
* moving; those survive as the reconcile's residual, which this resolves:
*
* - **[sendUp]** a residual **need** (relay has it, [store] still lacks it after the
* content pass tried to download it) means we deleted it. Publish OUR covering
* deletion up ([IEventStore.deletionsCovering]) so the relay drops it.
* - **[applyDown]** a residual **have** ([store] has it, relay still lacks it after
* the content pass tried to upload it) means the relay deleted it. Pull the RELAY'S
* covering **kind-5** down and ingest it, so [store] drops it too. A NIP-62 vanish is
* deliberately NOT applied on pull its blast radius is the author's whole account.
*
* Because it works off the residual not every id the cost is one cheap reconcile
* per round plus the (small) residual, independent of database size. It loops until a
* round resolves nothing (converged, and thereby self-verified) or [maxRounds] is hit.
*
* **Direction requires the matching content pass.** A residual need is a clean signal
* only after the content sync attempted the download ([sendUp] pairs with a `--down`
* content pass); a residual have only after it attempted the upload ([applyDown] pairs
* with `--up`). Passing a direction whose content pass didn't run makes its residual the
* full unsettled set, not a deletion signal so drive this with the same directions the
* content pass used.
*
* Best-effort: a reconcile failure ([NegentropySyncException]) stops the loop and returns
* what already settled rather than throwing the content sync is the primary work.
*
* @param batchSize ids per reconcile chunk and per by-id fetch.
* @param idleTimeoutMs idle watchdog for the reconciles and fetches.
* @param maxRounds hard cap on rounds; the "resolved nothing" check usually stops first.
* @param reconcileConcurrency overlapped `created_at`-window reconciles after an over-cap split.
*/
suspend fun INostrClient.negentropySettleDeletions(
relay: NormalizedRelayUrl,
filter: Filter,
store: IEventStore,
sendUp: Boolean,
applyDown: Boolean,
batchSize: Int = 500,
idleTimeoutMs: Long = 120_000L,
maxRounds: Int = 4,
reconcileConcurrency: Int = 1,
): DeletionSettleResult {
if ((!sendUp && !applyDown) || maxRounds <= 0) return DeletionSettleResult(0, 0, 0)
val publishTimeoutSecs = (idleTimeoutMs / 1000).coerceAtLeast(1)
val sentUp = HashSet<HexKey>()
val appliedDown = HashSet<HexKey>()
var rounds = 0
while (rounds < maxRounds) {
rounds++
val diff =
try {
negentropyReconcileIds(
relay = relay,
filter = filter,
localEntries = store.snapshotIdsForNegentropy(listOf(filter)),
batchSize = batchSize,
idleTimeoutMs = idleTimeoutMs,
reconcileConcurrency = reconcileConcurrency,
)
} catch (e: NegentropySyncException) {
break
}
var resolved = 0
// residual needs → publish our covering deletions up.
if (sendUp) {
for (chunk in diff.needIds.chunked(batchSize)) {
val events = fetchAll(relay, Filter(ids = chunk), idleTimeoutMs)
for (del in store.deletionsCovering(events, relay)) {
if (sentUp.add(del.id)) {
if (publishAndConfirm(del, setOf(relay), publishTimeoutSecs)) resolved++
}
}
}
}
// residual haves → ingest the relay's covering kind-5 (never a vanish).
if (applyDown) {
for (chunk in diff.haveIds.chunked(batchSize)) {
val ours = store.query<Event>(Filter(ids = chunk))
val relayDeletions = deletionsCovering(ours, relay) { f -> fetchAll(relay, f, idleTimeoutMs) }
for (del in relayDeletions.filterIsInstance<DeletionEvent>()) {
if (del.verify() && appliedDown.add(del.id)) {
store.insert(del)
resolved++
}
}
}
}
if (resolved == 0) break
}
return DeletionSettleResult(sentUp.size, appliedDown.size, rounds)
}
@@ -0,0 +1,62 @@
# `INostrClient` accessories
One-shot / high-level relay operations, written as **extension functions** on
`INostrClient`. They live here (and in `../reqs/`) rather than on the client class,
so they don't show up under "usages of `NostrClient`" or in method completion — you
only find them by knowing this package exists.
**Before writing a new subscribe / REQ / publish loop, look here first.** Most of what
a caller needs (fetch a set, fetch one, page past the relay cap, publish-and-confirm,
count, negentropy sync/reconcile) already exists.
Import as `com.vitorpamplona.quartz.nip01Core.relay.client.accessories.<name>` (or
`...client.reqs.<name>` for the flow/subscribe helpers).
## One-shot reads (subscribe → collect → return)
| Function | File | Use when |
| --- | --- | --- |
| `fetchAll(relay, filter, timeoutMs)` | `NostrClientFetchAllExt` | Get every event matching a filter in one REQ, deduped by id, until EOSE or timeout. **No verify, no store** — just the events. |
| `fetchFirst(relay, filter, timeoutMs)` | `NostrClientFetchFirstExt` | Get the first matching event and stop (returns `null` on none/timeout). |
| `fetchAllPages(relay, filters, timeoutMs)` | `NostrClientFetchAllPagesExt` | Fully retrieve a result set larger than the relay's per-REQ cap (strfry `limit`, ~500) by walking a `created_at` cursor. Bound it with the filter's `limit`. |
| `fetchAllPagesFromPool(filters, ...)` | `NostrClientFetchAllPagesPoolExt` | Same paging, across several relays at once, deduped across them. |
## Streaming (`Flow`)
| Function | File | Use when |
| --- | --- | --- |
| `fetchAsFlow(relay, filter)` | `../reqs/NostrClientFetchAsFlowExt` | Emit the accumulating list on each arrival; completes on EOSE. One-shot query as a flow. |
| `subscribeAsFlow(relay, filter)` | `../reqs/NostrClientSubscribeAsFlowExt` | Live subscription as a flow (stays open past EOSE; re-sends the REQ on reconnect). |
| `subscribe(subId, filters, listener)` | `../reqs/StaticSubscription`, `DynamicSubscription` | Raw live subscription with a `SubscriptionListener`. The lowest-level primitive the above build on. |
## Publish
| Function | File | Use when |
| --- | --- | --- |
| `publishAndConfirm(event, relays, timeout)` | `NostrClientPublishExt` | Send an EVENT and wait for `OK`; returns whether any relay accepted it. |
| `publishAndConfirmDetailed(event, relays, timeout)` | `NostrClientPublishExt` | Same, but returns the per-relay accepted/rejected map. |
## Count (NIP-45)
| Function | File | Use when |
| --- | --- | --- |
| `count(relay, filter, timeoutMs)` | `NostrClientCountExt` | NIP-45 `COUNT` against one relay (`null` on timeout / no support). |
| `countMerged(relays, filter, ...)` | `NostrClientCountExt` | Merged count across relays. |
## Negentropy (NIP-77)
| Function | File | Use when |
| --- | --- | --- |
| `negentropySync(relay, filter, ...)` | `NostrClientNegentropySyncExt` | Download everything a relay holds for a filter, diffing against `localEntries` and by-id downloading only the diff. Throws `NegentropySyncException` if the relay can't reconcile (no fallback). |
| `negentropySyncOrFetch(relay, filter, ...)` | `NostrClientNegentropySyncExt` | Same, but transparently falls back to `fetchAllPages` when the relay can't reconcile. The "just get the events" combinator. |
| `negentropySyncEvents` / `negentropySyncOrFetchEvents` | `NostrClientNegentropySyncEventsExt` | The two above as an O(1)-memory `Flow<Event>`. |
| `negentropyReconcile(relay, filter, localEntries, onNeedIds, onHaveIds)` | `NostrClientNegentropySyncExt` | **Pure diff, no I/O** — streams the two directions (`need` = relay has & we lack; `have` = we have & relay lacks) to callbacks. Compose your own download/upload on top. |
| `negentropyReconcileIds(relay, filter, localEntries)` | `NostrClientNegentropySyncExt` | Same diff, materialized into `needIds` / `haveIds` lists (small sets only). |
| `negentropySettleDeletions(relay, filter, store, sendUp, applyDown)` | `NostrClientNegentropyDeletionSettleExt` | Second pass of a two-pass sync: after a content sync settles, re-reconcile and resolve only the residual — send our covering deletions up (`sendUp`) and/or apply the relay's kind-5 down (`applyDown`), looping until stable. Cost is O(residual), not O(db). Pairs with `IEventStore.deletionsCovering`. |
`fetchByIds`, `reconcileStreaming`, `syncPipeline` in `NostrClientNegentropySyncExt`
are `internal` implementation details — not part of the public surface.
---
_Keep this table in sync when you add a public `INostrClient` extension here._
@@ -0,0 +1,58 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.relay.client.auth
import androidx.compose.runtime.Immutable
/**
* Compose-stable per-relay AUTH snapshot exposed by [RelayAuthenticator].
*
* The internal [RelayAuthStatus] is a mutable holder around concurrent LRU
* caches necessary for the per-relay OkHttp dispatcher, but unsuitable as
* a [kotlinx.coroutines.flow.StateFlow] value (mutating it doesn't change
* identity, so distinct-until-changed swallows updates).
*
* [RelayAuthSnapshot] is the immutable view downstream consumers (UI banner,
* retry coordinator, indexer-fan-out gate) subscribe to.
*/
@Immutable
data class RelayAuthSnapshot(
val phase: Phase,
val lastAuthSuccessAt: Long?,
) {
enum class Phase {
/** Connected; no AUTH challenge has been received yet. */
IDLE,
/** Signed AUTH event in flight; awaiting OK from the relay. */
AUTHENTICATING,
/** Last AUTH succeeded; relay accepts authenticated REQs. */
AUTHENTICATED,
/** Last AUTH attempt failed; subsequent challenges may still arrive. */
AUTH_FAILED,
}
companion object {
val IDLE = RelayAuthSnapshot(Phase.IDLE, lastAuthSuccessAt = null)
}
}
@@ -23,6 +23,8 @@ package com.vitorpamplona.quartz.nip01Core.relay.client.auth
import androidx.collection.LruCache
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip42RelayAuth.RelayAuthEvent
import com.vitorpamplona.quartz.utils.TimeUtils
import kotlin.concurrent.Volatile
class RelayAuthStatus {
// Keeps track of auth responses to update the relay with all filters
@@ -32,6 +34,12 @@ class RelayAuthStatus {
// Avoids sending multiple replies for each auth.
private val uniqueAuthChallengesSent: LruCache<ChallengePair, ChallengePair> = LruCache(10)
// Latest epoch-second at which a tracked AUTH event received a successful OK.
// Read by RelayAuthSnapshot consumers for staleness checks (e.g. proactive
// re-AUTH on window focus).
@Volatile
private var lastAuthSuccessAt: Long? = null
enum class AuthEventReceiptStatus {
AUTHENTICATING,
AUTHENTICATED,
@@ -66,6 +74,7 @@ class RelayAuthStatus {
return if (wasAlreadyAuthenticated != null) {
if (success) {
authResponseWatcher.put(eventId, AuthEventReceiptStatus.AUTHENTICATED)
lastAuthSuccessAt = TimeUtils.now()
} else {
authResponseWatcher.put(eventId, AuthEventReceiptStatus.NOT_AUTHENTICATED)
}
@@ -77,4 +86,29 @@ class RelayAuthStatus {
}
fun hasFinishedAllAuths() = authResponseWatcher.snapshot().all { it.value != AuthEventReceiptStatus.AUTHENTICATING }
/**
* Build an immutable Compose-stable snapshot of the current per-relay AUTH
* state. The phase is derived from the response watcher:
*
* - any AUTHENTICATING entry [RelayAuthSnapshot.Phase.AUTHENTICATING]
* - else any AUTHENTICATED entry [RelayAuthSnapshot.Phase.AUTHENTICATED]
* - else any NOT_AUTHENTICATED entry [RelayAuthSnapshot.Phase.AUTH_FAILED]
* - else (no tracked challenges) [RelayAuthSnapshot.Phase.IDLE]
*
* The watcher LRU caps at 10 entries; a long-running connection that has
* already AUTHed will still report AUTHENTICATED even after older entries
* roll off, because the LRU keeps the most recent.
*/
fun snapshot(): RelayAuthSnapshot {
val entries = authResponseWatcher.snapshot()
val phase =
when {
entries.isEmpty() -> RelayAuthSnapshot.Phase.IDLE
entries.values.any { it == AuthEventReceiptStatus.AUTHENTICATING } -> RelayAuthSnapshot.Phase.AUTHENTICATING
entries.values.any { it == AuthEventReceiptStatus.AUTHENTICATED } -> RelayAuthSnapshot.Phase.AUTHENTICATED
else -> RelayAuthSnapshot.Phase.AUTH_FAILED
}
return RelayAuthSnapshot(phase, lastAuthSuccessAt)
}
}
@@ -33,10 +33,16 @@ import com.vitorpamplona.quartz.nip01Core.signers.SignerExceptions
import com.vitorpamplona.quartz.nip42RelayAuth.RelayAuthEvent
import com.vitorpamplona.quartz.utils.Log
import com.vitorpamplona.quartz.utils.cache.LargeCache
import kotlinx.collections.immutable.PersistentMap
import kotlinx.collections.immutable.persistentMapOf
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.IO
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
import kotlin.coroutines.cancellation.CancellationException
@@ -61,8 +67,32 @@ class RelayAuthenticator(
// Connection callbacks fire on the per-relay OkHttp dispatcher thread, so
// this state is mutated concurrently — LargeCache wraps a platform-tuned
// concurrent map (ConcurrentSkipListMap on jvmAndroid, CacheMap on Apple).
//
// This stays mutable because RelayAuthStatus carries an LruCache that has
// to be addressable from the dispatcher thread. The Compose-observable
// view of the same data is published on [authStateFlow] below, sourced
// from RelayAuthStatus.snapshot().
private val authStatus = LargeCache<NormalizedRelayUrl, RelayAuthStatus>()
private val _authStateFlow = MutableStateFlow<PersistentMap<NormalizedRelayUrl, RelayAuthSnapshot>>(persistentMapOf())
/**
* Per-relay AUTH state as an immutable Compose-stable snapshot map.
*
* Downstream consumers (UI banner, retry queue, indexer-fan-out gate)
* subscribe to this flow instead of polling [authStatus] directly.
* Identity changes on every mutation, so [kotlinx.coroutines.flow.distinctUntilChanged]
* downstream and Compose `@Immutable` skipping both work correctly.
*/
val authStateFlow: StateFlow<PersistentMap<NormalizedRelayUrl, RelayAuthSnapshot>> = _authStateFlow.asStateFlow()
private fun publishSnapshot(relayUrl: NormalizedRelayUrl) {
val status = authStatus.get(relayUrl)
_authStateFlow.update { current ->
if (status == null) current.remove(relayUrl) else current.put(relayUrl, status.snapshot())
}
}
private val clientListener =
object : RelayConnectionListener {
override fun onIncomingMessage(
@@ -78,10 +108,12 @@ class RelayAuthenticator(
override fun onConnecting(relay: IRelayClient) {
authStatus.put(relay.url, RelayAuthStatus())
publishSnapshot(relay.url)
}
override fun onDisconnected(relay: IRelayClient) {
authStatus.remove(relay.url)
publishSnapshot(relay.url)
}
}
@@ -102,6 +134,7 @@ class RelayAuthenticator(
// only send replies to new challenges to avoid infinite loop:
if (authStatus.get(relay.url)?.saveAuthSubmission(authEvent) == true) {
relay.sendIfConnected(AuthCmd(authEvent))
publishSnapshot(relay.url)
}
}
} catch (e: CancellationException) {
@@ -118,8 +151,12 @@ class RelayAuthenticator(
relay: IRelayClient,
msg: OkMessage,
) {
val transitioned = authStatus.get(relay.url)?.checkAuthResults(msg.eventId, msg.success) == true
// Publish even on failure transitions so the UI can clear "AUTHENTICATING"
// banners and reflect AUTH_FAILED state.
publishSnapshot(relay.url)
// if this is the OK of an auth event, renew all subscriptions and resend all outgoing events.
if (authStatus.get(relay.url)?.checkAuthResults(msg.eventId, msg.success) == true) {
if (transitioned) {
client.syncFilters(relay)
}
}
@@ -66,11 +66,15 @@ class PoolEventOutboxState(
success: Boolean,
message: String,
) {
val currentTries = failures[url]
if (success || message.shouldDiscard()) {
relaysRemaining = relaysRemaining - url
failures = failures - url
} else if (message.isAuthRequired()) {
// NIP-42 AUTH challenge in flight — don't count toward the try cap.
// RelayAuthenticator signs + relay re-issues OK; syncFilters() then
// re-pumps this outbox so the original publish is retried.
} else {
val currentTries = failures[url]
if (currentTries != null) {
currentTries.addResponse(message)
} else {
@@ -91,6 +95,8 @@ class PoolEventOutboxState(
this.startsWith("deleted:") ||
this.startsWith("invalid:")
fun String.isAuthRequired() = this.startsWith("auth-required:")
// Tries 3 times
class Tries(
var tries: List<Long> = listOf(),
@@ -136,7 +136,9 @@ open class BasicRelayClient(
socket?.connect()
} catch (e: Exception) {
if (e is CancellationException) throw e
listener.onCannotConnect(this, "Error when trying to connect: ${e.message ?: e::class.simpleName}")
val typeName = e::class.simpleName
val detail = e.message?.let { "$it ($typeName)" } ?: (typeName ?: "unknown error")
listener.onCannotConnect(this, "Error when trying to connect: $detail")
listener.onDisconnected(this)
dontTryAgainForALongTime()
markConnectionAsClosed()
@@ -187,9 +189,15 @@ open class BasicRelayClient(
} else {
socket?.disconnect()
// suppression rules below must match the raw message; displayMsg is for listener output only
// suppression rules below must match the raw message; displayMsg is for listener output only.
// Always include the exception's class name: message text is
// localized and inconsistent across platforms, but the type
// (SocketTimeoutException / UnknownHostException / SSLHandshakeException /
// ConnectException …) is stable and lets listeners classify a failure
// reliably — a busy relay (timeout) vs a dead one (bad domain / TLS).
val msg = t.message
val displayMsg = msg ?: t::class.simpleName
val typeName = t::class.simpleName
val displayMsg = if (msg != null) "$msg ($typeName)" else (typeName ?: "unknown error")
// checks if this is an actual failure. Closing the socket generates an onFailure as well.
// ignore tor errors.
@@ -156,7 +156,7 @@ class RelayUrlNormalizer {
if (trimmed.contains("://")) {
// some other scheme we cannot connect to.
Log.w("RelayUrlNormalizer") { "Rejected $url" }
Log.d("RelayUrlNormalizer") { "Rejected $url" }
return null
}
@@ -189,14 +189,14 @@ class RelayUrlNormalizer {
normalizedUrls.put(url, NormalizationResult.Success(normalized))
normalized
} else {
Log.w("NormalizedRelayUrl") { "Rejected $url" }
Log.d("NormalizedRelayUrl") { "Rejected $url" }
normalizedUrls.put(url, NormalizationResult.Error)
null
}
} catch (e: Exception) {
if (e is CancellationException) throw e
normalizedUrls.put(url, NormalizationResult.Error)
Log.w("NormalizedRelayUrl") { "Rejected $url" }
Log.d("NormalizedRelayUrl") { "Rejected $url" }
null
}
}
@@ -0,0 +1,104 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.store
import com.vitorpamplona.quartz.nip01Core.core.Address
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.core.isAddressable
import com.vitorpamplona.quartz.nip01Core.core.isReplaceable
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent
/** The addressable/replaceable coordinate of [event] as a NIP-01 `a`-tag value. */
private fun addressValue(event: Event): String {
val dTag = if (event.kind.isAddressable()) event.tags.firstOrNull { it.size > 1 && it[0] == "d" }?.get(1) ?: "" else ""
return Address.assemble(event.kind, event.pubKey, dTag)
}
/**
* The local deletion events that would make [relay] remove one of [serverEvents] the
* events the relay HAS that we LACK. Used by sync to push *only* the deletions that
* actually apply to what the relay holds, and nothing else (not other deletions by the
* same author). Covers every way a stored deletion can reach an event:
*
* - **NIP-09, id-based** a kind-5 with an `e` tag naming a server event's id.
* - **NIP-09, address-based** a kind-5 with an `a` tag naming a server event's
* addressable/replaceable coordinate, at or after that event's `created_at`
* (NIP-09 only deletes `created_at <= deletion.created_at`).
* - **NIP-62 vanish** a kind-62 by a server event's author, targeting [relay] (its
* `relay` tags name the URL or `ALL_RELAYS`), issued after that event (a vanish
* deletes `created_at < vanish.created_at`).
*
* Deduped by event id; a single deletion covering several events is returned once.
*
* [query] is where the deletions are looked up it is source-agnostic on purpose, so
* the same coverage rule runs in both sync directions:
* - **up** (send our deletions): `events` are the relay's, `query` is the local store
* which of OUR deletions would delete what the relay still holds.
* - **down** (apply the relay's deletions): `events` are ours, `query` fetches from the
* relay which of the RELAY'S deletions would delete what we still hold.
*/
suspend fun deletionsCovering(
events: List<Event>,
relay: NormalizedRelayUrl,
query: suspend (Filter) -> List<Event>,
): List<Event> {
if (events.isEmpty()) return emptyList()
val covering = LinkedHashMap<HexKey, Event>()
// 1. id-based NIP-09: a kind-5 `e`-tagging an event's id.
query(Filter(kinds = listOf(DeletionEvent.KIND), tags = mapOf("e" to events.map { it.id })))
.forEach { covering[it.id] = it }
// 2. address-based NIP-09: a kind-5 `a`-tagging an event's coordinate, cutoff-checked.
val byAddress = events.filter { it.kind.isAddressable() || it.kind.isReplaceable() }.groupBy(::addressValue)
if (byAddress.isNotEmpty()) {
query(Filter(kinds = listOf(DeletionEvent.KIND), tags = mapOf("a" to byAddress.keys.toList())))
.forEach { del ->
if (del !is DeletionEvent) return@forEach
for (addr in del.deleteAddresses()) {
val hit = byAddress[addr.toValue()] ?: continue
if (hit.any { it.createdAt <= del.createdAt }) {
covering[del.id] = del
break
}
}
}
}
// 3. NIP-62 vanish: a kind-62 by an event's author, targeting this relay, issued after it.
query(Filter(kinds = listOf(RequestToVanishEvent.KIND), authors = events.mapTo(HashSet()) { it.pubKey }.toList()))
.forEach { vanish ->
if (vanish !is RequestToVanishEvent || !vanish.shouldVanishFrom(relay)) return@forEach
if (events.any { it.pubKey == vanish.pubKey && it.createdAt < vanish.createdAt }) covering[vanish.id] = vanish
}
return covering.values.toList()
}
/** [deletionsCovering] with the local store as the deletion source (the "up" direction). */
suspend fun IEventStore.deletionsCovering(
serverEvents: List<Event>,
relay: NormalizedRelayUrl,
): List<Event> = deletionsCovering(serverEvents, relay) { query<Event>(it) }
@@ -22,8 +22,11 @@ package com.vitorpamplona.quartz.nip01Core.store
import com.vitorpamplona.negentropy.storage.IStorage
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
interface IEventStore : AutoCloseable {
companion object {
@@ -121,6 +124,41 @@ interface IEventStore : AutoCloseable {
suspend fun count(filters: List<Filter>): Int
/**
* Every distinct identity author with at least one stored event that has
* NO NIP-65 relay list (kind 10002 / "outbox") in this store.
*
* This is a whole-store anti-join the set of all authors minus the
* authors who already have an outbox which the positive-only nostr
* [Filter] grammar cannot express (there is no "NOT kind 10002"), so
* it is its own method rather than a [query]. "Missing" is relative to
* what THIS store holds (see [relay]); an author whose only 10002 was
* deleted (NIP-09) or expired (NIP-40) is reported as missing, because
* no row remains for it. Order is unspecified.
*
* GiftWraps (kind 1059) are NOT counted as authors: their `pubkey` is a
* random one-time key, so including them would return an unbounded set of
* ephemeral keys that can never own a 10002 useless to the outbox model
* this feeds.
*
* The default implementation walks the store: it collects the authors
* that DO have an outbox, then streams every event and keeps the
* authors not in that set. Correct for any store but O(events), and it
* decodes every event just to read its pubkey. SQLite overrides it with
* an index-only `EXCEPT` over `event_headers` that never materialises an
* event (see `QueryBuilder.authorsMissingKind`).
*/
suspend fun authorsMissingOutbox(): List<HexKey> {
val withOutbox = HashSet<HexKey>()
query<Event>(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND))) { withOutbox.add(it.pubKey) }
val missing = LinkedHashSet<HexKey>()
query<Event>(Filter()) { event ->
if (event.kind != GiftWrapEvent.KIND && event.pubKey !in withOutbox) missing.add(event.pubKey)
}
return missing.toList()
}
/**
* NIP-77 negentropy snapshot. Returns `(created_at, id)` pairs
* for every event matching [filters], with no content/tags/sig
@@ -0,0 +1,55 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.store
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.crypto.verify
import com.vitorpamplona.quartz.utils.Log
/**
* Verify [event]'s NIP-01 id + signature and, if valid, persist it to this store.
* Returns `true` when the event was accepted (verified) even if the insert was a
* no-op so callers can gate "surface this event" on the return.
*
* A UNIQUE-constraint rejection is normal, not a failure: the store already holds
* this id, or a newer version of a replaceable (kind 0/3/10000-19999). The outbox
* model routinely delivers the same event from several of a user's write relays, so
* a crawl produces these by the hundred-thousand so only genuine persistence
* failures (I/O, full disk, corruption) are logged. Persistence is best-effort: an
* insert error is swallowed, not propagated, so it can't break a live subscription.
*
* This is the single verify-then-store sink every event-arrival path should funnel
* through, so the store stays the authoritative cache of what has been seen.
*/
suspend fun IEventStore.verifyAndInsert(event: Event): Boolean {
if (!event.verify()) {
Log.w("EventStore") { "dropped event ${event.id.take(8)} kind=${event.kind} — bad signature" }
return false
}
try {
insert(event)
} catch (t: Throwable) {
if (t.message?.contains("UNIQUE constraint", ignoreCase = true) != true) {
Log.w("EventStore") { "store insert failed for ${event.id.take(8)}: ${t.message}" }
}
}
return true
}
@@ -77,6 +77,8 @@ class EventStore(
override suspend fun count(filters: List<Filter>) = store.count(filters)
override suspend fun authorsMissingOutbox() = store.authorsMissingOutbox()
override suspend fun snapshotIdsForNegentropy(
filters: List<Filter>,
maxEntries: Int?,
@@ -204,9 +204,7 @@ class FullTextSearchModule(
val kinds = searchableKindsPresent(db)
if (kinds.isEmpty()) return
val selectSql =
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
"FROM event_headers WHERE kind IN (${kinds.joinToString(",")})"
val selectSql = "$SELECT_EVENT_COLUMNS WHERE kind IN (${kinds.joinToString(",")})"
db.prepare(insertFTS).use { write ->
db.prepare(selectSql).use { read ->
@@ -259,10 +257,7 @@ class FullTextSearchModule(
val kinds = searchableKindsPresent(db)
if (kinds.isEmpty()) return FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true)
val selectSql =
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
"FROM event_headers WHERE row_id > ? AND kind IN (${kinds.joinToString(",")}) " +
"ORDER BY row_id LIMIT ?"
val selectSql = selectSearchablePageSql(kinds)
var last = afterRowId
var processed = 0
@@ -346,10 +341,7 @@ class FullTextSearchModule(
var last = watermark
var processed = 0
if (kinds.isNotEmpty()) {
val selectSql =
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
"FROM event_headers WHERE row_id > ? AND kind IN (${kinds.joinToString(",")}) " +
"ORDER BY row_id LIMIT ?"
val selectSql = selectSearchablePageSql(kinds)
db.prepare(insertFTS).use { write ->
db.prepare(selectSql).use { read ->
read.bindLong(1, watermark)
@@ -426,5 +418,12 @@ class FullTextSearchModule(
// inspects the resulting runtime type.
private const val PROBE_ID = "0"
private val EMPTY_TAGS = emptyArray<Array<String>>()
/** Column order matches the positional `read.getText(1)`…`getText(7)` event rebuilds. */
private const val SELECT_EVENT_COLUMNS =
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig FROM event_headers"
/** One `row_id`-cursored page of searchable events; binds: cursor, limit. */
private fun selectSearchablePageSql(kinds: List<Int>) = "$SELECT_EVENT_COLUMNS WHERE row_id > ? AND kind IN (${kinds.joinToString(",")}) ORDER BY row_id LIMIT ?"
}
}
@@ -31,6 +31,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.store.IdAndTime
import com.vitorpamplona.quartz.nip01Core.store.RawEvent
import com.vitorpamplona.quartz.nip01Core.store.sqlite.sql.where
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
import com.vitorpamplona.quartz.utils.EventFactory
class QueryBuilder(
@@ -594,6 +595,53 @@ class QueryBuilder(
return db.countIn(rowIdSubqueries.sql, rowIdSubqueries.args)
}
// -----------------------------------------------------------------
// Anti-join projections
//
// Set-difference over authors — "who is missing an event of kind K"
// — which the positive-only nostr Filter grammar can't express, so
// it lives here as a dedicated SELECT rather than going through the
// filter → SQL path.
// -----------------------------------------------------------------
/**
* Distinct identity authors with at least one stored event that have NO
* stored event of [kind], as an `EXCEPT` of two sets over `event_headers`:
* all authors, minus the authors that have a [kind]. Both sides are
* answered index-only off `query_by_kind_pubkey_created`
* (kind, pubkey, ) which is created unconditionally, so this does not
* depend on the optional pubkey-alone index and `EXCEPT` diffs them
* through one temp b-tree. That is ~3× faster than a
* `DISTINCT NOT EXISTS` correlated scan, which pays one index seek per
* distinct author; the gap widens with author cardinality. Order is
* unspecified (`EXCEPT` returns pubkey-sorted, which callers must not rely
* on).
*
* GiftWraps (kind 1059) are excluded from the "authors" set: their
* `pubkey` is a random one-time key (the real recipient lives only in
* `pubkey_owner_hash`), so counting them would return an unbounded set of
* ephemeral keys that can never own a [kind] event.
*/
fun authorsMissingKind(
kind: Int,
db: SQLiteConnection,
): List<HexKey> {
val sql =
"""
SELECT DISTINCT pubkey FROM event_headers WHERE kind <> ${GiftWrapEvent.KIND}
EXCEPT
SELECT pubkey FROM event_headers WHERE kind = ?
""".trimIndent()
return db.prepare(sql).use { stmt ->
stmt.bindLong(1, kind.toLong())
val out = ArrayList<HexKey>()
while (stmt.step()) {
out.add(stmt.getText(0))
}
out
}
}
private fun SQLiteConnection.countEverything() = runCount("SELECT count(*) as count FROM event_headers")
private fun SQLiteConnection.countIn(
@@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.nip01Core.store.RawEvent
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
import com.vitorpamplona.quartz.nip40Expiration.isExpired
import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
import com.vitorpamplona.quartz.nip77Negentropy.LiveNegentropyIndex
class SQLiteEventStore(
@@ -555,6 +556,8 @@ class SQLiteEventStore(
suspend fun count(filters: List<Filter>): Int = pool.useReader { queryBuilder.count(filters, it) }
suspend fun authorsMissingOutbox(): List<HexKey> = pool.useReader { queryBuilder.authorsMissingKind(AdvertisedRelayListEvent.KIND, it) }
suspend fun snapshotIdsForNegentropy(
filters: List<Filter>,
maxEntries: Int? = null,
@@ -23,6 +23,7 @@ package com.vitorpamplona.quartz.nip17Dm
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.hints.EventHintBundle
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
import com.vitorpamplona.quartz.nip01Core.tags.people.taggedUserIds
@@ -33,9 +34,12 @@ import com.vitorpamplona.quartz.nip17Dm.messages.ChatMessageEvent
import com.vitorpamplona.quartz.nip25Reactions.ReactionEvent
import com.vitorpamplona.quartz.nip30CustomEmoji.EmojiUrlTag
import com.vitorpamplona.quartz.nip40Expiration.expiration
import com.vitorpamplona.quartz.nip46RemoteSigner.signer.NostrSignerRemote
import com.vitorpamplona.quartz.nip59Giftwrap.seals.SealedRumorEvent
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
import com.vitorpamplona.quartz.utils.mapNotNullAsync
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
class NIP17Factory {
data class Result(
@@ -43,10 +47,34 @@ class NIP17Factory {
val wraps: List<GiftWrapEvent>,
)
/**
* Build one NIP-59 gift wrap per recipient.
*
* The rumor (kind 14) `created_at` is implicitly shared across all wraps
* because [event] is signed once by the caller before the per-recipient
* loop runs every seal encodes the same rumor `id`. This anchors
* cross-recipient dedupe + reaction/receipt targeting on group sends.
*
* Per NIP-17, the gift wrap's `p` tag MAY carry the recipient's primary
* DM inbox relay as a hint. Pass [recipientRelayHints] to surface those;
* the default `{ null }` lambda preserves the historical 2-element tag
* shape for every recipient.
*
* When [signer] is a [NostrSignerRemote] (NIP-46 bunker), seal building
* is rate-limited to [BUNKER_PARALLELISM] concurrent operations. Each
* seal needs `nip44_encrypt` + `sign` round-trips against the bunker; a
* 5-recipient group otherwise launches 10 concurrent in-flight RPCs and
* saturates the bunker socket. Local signers (NostrSignerInternal,
* NostrSignerSync) run fully parallel no semaphore overhead.
*
* The proper fix is the batched `nip44_get_conversation_keys` NIP-46
* RPC (separate plan); this is the interim throttle until that lands.
*/
private suspend fun createWraps(
event: Event,
to: Set<HexKey>,
signer: NostrSigner,
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
): List<GiftWrapEvent> {
val innerExpDelta =
event.expiration()?.let {
@@ -57,29 +85,47 @@ class NIP17Factory {
}
}
val bunkerLimiter = if (signer is NostrSignerRemote) Semaphore(BUNKER_PARALLELISM) else null
return mapNotNullAsync(
to.toList(),
) { next ->
GiftWrapEvent.create(
event =
SealedRumorEvent.create(
event = event,
encryptTo = next,
expirationDelta = innerExpDelta,
signer = signer,
),
recipientPubKey = next,
expirationDelta = innerExpDelta,
)
val build: suspend () -> GiftWrapEvent = {
GiftWrapEvent.create(
event =
SealedRumorEvent.create(
event = event,
encryptTo = next,
expirationDelta = innerExpDelta,
signer = signer,
),
recipientPubKey = next,
expirationDelta = innerExpDelta,
recipientRelayHint = recipientRelayHints(next),
)
}
bunkerLimiter?.withPermit { build() } ?: build()
}
}
companion object {
/**
* Max concurrent in-flight NIP-46 RPCs when building wraps via a
* remote signer. Empirically a sweet spot covers parallelism
* speedup for 24 recipient sends without saturating typical
* bunker apps (nsec.app, Amber, Keychat) that serialize requests
* internally past ~10 in-flight.
*/
const val BUNKER_PARALLELISM = 4
}
suspend fun createMessageNIP17(
template: EventTemplate<ChatMessageEvent>,
signer: NostrSigner,
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
): Result {
val senderMessage = signer.sign(template)
val wraps = createWraps(senderMessage, senderMessage.groupMembers(), signer)
val wraps = createWraps(senderMessage, senderMessage.groupMembers(), signer, recipientRelayHints)
return Result(
msg = senderMessage,
wraps = wraps,
@@ -108,9 +154,10 @@ class NIP17Factory {
suspend fun createEncryptedFileNIP17(
template: EventTemplate<ChatMessageEncryptedFileHeaderEvent>,
signer: NostrSigner,
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
): Result {
val senderMessage = signer.sign(template)
val wraps = createWraps(senderMessage, senderMessage.groupMembers(), signer)
val wraps = createWraps(senderMessage, senderMessage.groupMembers(), signer, recipientRelayHints)
return Result(
msg = senderMessage,
@@ -142,12 +189,13 @@ class NIP17Factory {
originalNote: EventHintBundle<Event>,
to: List<HexKey>,
signer: NostrSigner,
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
): Result {
val senderPublicKey = signer.pubKey
val template = ReactionEvent.build(content, originalNote)
val senderReaction = signer.sign(template)
val wraps = createWraps(senderReaction, to.plus(senderPublicKey).toSet(), signer)
val wraps = createWraps(senderReaction, to.plus(senderPublicKey).toSet(), signer, recipientRelayHints)
return Result(
msg = senderReaction,
wraps = wraps,
@@ -159,12 +207,13 @@ class NIP17Factory {
originalNote: EventHintBundle<Event>,
to: List<HexKey>,
signer: NostrSigner,
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
): Result {
val senderPublicKey = signer.pubKey
val template = ReactionEvent.build(emojiUrl, originalNote)
val senderReaction = signer.sign(template)
val wraps = createWraps(senderReaction, to.plus(senderPublicKey).toSet(), signer)
val wraps = createWraps(senderReaction, to.plus(senderPublicKey).toSet(), signer, recipientRelayHints)
return Result(
msg = senderReaction,
@@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.core.firstTagValue
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
import com.vitorpamplona.quartz.nip01Core.tags.people.PTag
@@ -96,11 +97,22 @@ open class GiftWrapEvent(
const val KIND = 1059
const val ALT = "Encrypted event"
/**
* Build a NIP-59 gift wrap addressed to `recipientPubKey`.
*
* Per NIP-17 §Publishing, the `p` tag on the wrap MAY carry the
* recipient's primary DM inbox relay as a hint, so other clients
* the recipient runs (or relays acting as inbox routers) can locate
* the wrap without a separate kind:10050 lookup. Pass it via
* [recipientRelayHint] `null` (the default) preserves the
* historical 2-element `["p", pubkey]` shape.
*/
fun create(
event: Event,
recipientPubKey: HexKey,
expirationDelta: Long? = null,
createdAt: Long = TimeUtils.randomWithTwoDays(),
recipientRelayHint: NormalizedRelayUrl? = null,
): GiftWrapEvent {
val signer = NostrSignerSync(KeyPair()) // GiftWrap is always a random key
@@ -109,11 +121,11 @@ open class GiftWrapEvent(
// minimum expiration is two days in the future due to the random created at
// this will make sure the even arrives and is not deleted because of the 2 days.
arrayOf(
PTag.assemble(recipientPubKey, null),
PTag.assemble(recipientPubKey, recipientRelayHint),
ExpirationTag.assemble(createdAt + it + TimeUtils.twoDays()),
)
} ?: arrayOf(
PTag.assemble(recipientPubKey, null),
PTag.assemble(recipientPubKey, recipientRelayHint),
)
return signer.sign(
@@ -0,0 +1,171 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip66RelayMonitor.reachability
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.RelayDiscoveryEvent
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.networkType
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.rtt
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkType
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RttType
import com.vitorpamplona.quartz.utils.TimeUtils
/**
* A durable, shareable relay-reachability cache backed by an [IEventStore] as
* NIP-66 **kind:30166 Relay Discovery** events so the crawler, the WoT updater,
* and future runs all read and write the *same* liveness knowledge instead of each
* rediscovering dead relays from an in-memory set that is wiped when the process ends.
*
* ## Why NIP-66 / the event store
* A 30166 event is addressable by its `d`-tag (the normalized relay URL), so the
* store keeps exactly **one replaceable record per (monitor, relay)** a natural
* per-relay status slot with a `created_at` timestamp that gives us a free TTL. The
* event store gives us persistence, cross-procedure sharing, and interop for free:
* 30166 events published by *other* monitors (nostr.watch et al.) can be ingested to
* seed reachability without probing, and our own records can be published back.
*
* ## How "dead" is represented
* NIP-66 has no explicit offline field; liveness is inferred from a fresh record that
* carries an `rtt-open` (a successful connection). This cache follows that convention:
* - **reachable** a 30166 **with** `rtt-open`, `created_at` = probe time.
* - **dead** a 30166 **without** `rtt-open` ("we checked, could not open"),
* `created_at` = probe time.
*
* So a fresh rtt-less record distinguishes *checked-and-dead* from *never-checked*
* (no record). When both a dead and a live record exist within the TTL for the same
* relay, **live wins** any recent successful open overrides an earlier failure,
* whether the two came from us across time or from two different monitors.
*
* ## Not a replacement for the hot path
* [snapshot] is meant to be loaded ONCE at the start of a run into whatever in-memory
* structure the caller already uses for per-request `isDead` checks; [record] flushes
* a run's findings back at the end. It is deliberately not queried per routing decision.
*
* A relay is only ever skipped for the TTL window, never permanently consistent with
* the outbox rule that every advertised write relay must be tried: a TTL'd record is
* "skip for now", not "ignore this author's home forever".
*
* ## The signer is a dedicated monitor service identity
* [signer] should be a **machine-level monitor key**, NOT a user/observer account: per
* NIP-66 a monitor is its own pubkey (which also publishes a kind:10166 announcement,
* a kind:0 profile and a kind:10002). Publishing these under the observer's key would
* conflate the WoT identity with a relay-monitoring service. [snapshot] still honours
* records from ANY author (so third-party monitors can be ingested); only [record]
* writes under this monitor key.
*/
class RelayReachabilityStore(
private val store: IEventStore,
private val signer: NostrSigner,
private val ttlSeconds: Long = DEFAULT_TTL_SECONDS,
) {
/**
* An in-memory view of the fresh (within-TTL) reachability records. [dead] holds
* relays proven unreachable and not since seen live; [live] holds relays with a
* recent successful open. A relay absent from both is simply unknown re-probe it.
*/
class Snapshot(
val dead: Set<NormalizedRelayUrl>,
val live: Set<NormalizedRelayUrl>,
) {
fun isKnownDead(relay: NormalizedRelayUrl) = relay in dead
val size: Int get() = dead.size + live.size
}
/**
* Load every 30166 record fresher than [ttlSeconds] and fold it into a [Snapshot].
* Records from any monitor are honoured (live-wins), so ingesting third-party
* monitors' 30166 into [store] transparently improves the result.
*/
suspend fun snapshot(now: Long = TimeUtils.now()): Snapshot {
val since = now - ttlSeconds
val events =
store.query<RelayDiscoveryEvent>(
Filter(kinds = listOf(RelayDiscoveryEvent.KIND), since = since),
)
val live = HashSet<NormalizedRelayUrl>()
val dead = HashSet<NormalizedRelayUrl>()
for (ev in events) {
val relay = ev.relay() ?: continue
if (ev.rttOpen() != null) live.add(relay) else dead.add(relay)
}
// A recent successful open (from us later, or from another monitor) overrides
// an earlier dead mark for the same relay.
dead.removeAll(live)
return Snapshot(dead, live)
}
/**
* Persist a run's reachability findings as 30166 events: each [reachable] relay as
* a record WITH `rtt-open`, each [dead] relay (that is not also reachable) as one
* WITHOUT. Signed by [signer] and inserted into [store]; being addressable, each
* replaces this monitor's prior record for that relay, so the store stays bounded
* at roughly the number of distinct relays.
*
* [rttOpenMs] is the measured open round-trip in ms. It defaults to 0 as a **liveness
* flag only** presence of the `rtt-open` tag, not its magnitude, is what [snapshot]
* reads as "reachable", and a caller that merely proved a relay served events (like
* the crawler) has no dedicated probe latency to report. A `0` therefore means
* "reachable, latency not probed by this writer", NOT a real 0 ms measurement. Do NOT
* publish these records to the wider network as authoritative latency data until a
* dedicated monitor probe supplies a real [rttOpenMs]; aggregators rank by it.
*/
suspend fun record(
reachable: Set<NormalizedRelayUrl>,
dead: Set<NormalizedRelayUrl>,
now: Long = TimeUtils.now(),
rttOpenMs: Long = 0,
) {
for (relay in reachable) writeOne(relay, up = true, now, rttOpenMs)
for (relay in dead) if (relay !in reachable) writeOne(relay, up = false, now, rttOpenMs)
}
private suspend fun writeOne(
relay: NormalizedRelayUrl,
up: Boolean,
now: Long,
rttOpenMs: Long,
) {
val template =
RelayDiscoveryEvent.build(relay, createdAt = now) {
networkType(networkTypeOf(relay))
if (up) rtt(RttType.OPEN, rttOpenMs)
}
store.insert(signer.sign(template))
}
companion object {
/** Default freshness window: a relay's status is trusted for a day, then re-probed. */
const val DEFAULT_TTL_SECONDS = 24L * 60 * 60
/** NIP-66 `n` network type inferred from the URL, so a `.onion`/i2p relay is tagged correctly. */
fun networkTypeOf(relay: NormalizedRelayUrl): NetworkType =
when {
RelayUrlNormalizer.isOnion(relay.url) -> NetworkType.TOR
relay.url.contains(".i2p") -> NetworkType.I2P
else -> NetworkType.CLEARNET
}
}
}
@@ -0,0 +1,68 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.concurrent
/**
* A thread-safe hash map whose compound operations [getOrPut] and [merge]
* apply their update **atomically**, not merely one-lock-per-primitive-op. This
* is the contract a concurrent producer/consumer pipeline needs: two coroutines
* racing `getOrPut` on the same key must agree on a single value, and racing
* `merge` must not lose an increment.
*
* commonMain has no `java.util.concurrent.ConcurrentHashMap`, so this is
* expect/actual, matching the split already used by [com.vitorpamplona.quartz.utils.cache.ConcurrentHashCache]:
* - JVM / Android `ConcurrentHashMap` (lock-free, true atomic `computeIfAbsent` / `merge`).
* - Native (Apple + Linux) copy-on-write over an atomic reference, with a
* CAS retry loop giving the same atomicity. Correct but O(n)-per-write; the
* native targets never run the heavy crawl this backs, they only compile it.
*
* Only the operations the crawl actually uses are exposed no full [MutableMap]
* surface so the native copy-on-write actual stays small and obviously correct.
*/
expect class ConcurrentMap<K : Any, V : Any>() {
operator fun get(key: K): V?
operator fun set(
key: K,
value: V,
)
/** Atomically return the value for [key], computing and inserting [defaultValue] once if absent. */
fun getOrPut(
key: K,
defaultValue: () -> V,
): V
/**
* Atomically insert [value] if [key] is absent, else replace the existing
* value with `remap(existing, value)`. Returns the value now stored.
*/
fun merge(
key: K,
value: V,
remap: (old: V, new: V) -> V,
): V
fun size(): Int
/** A point-in-time copy of the entries — safe to iterate without holding a lock. */
fun snapshot(): Map<K, V>
}
@@ -0,0 +1,43 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.concurrent
/**
* A thread-safe hash set for the crawl's cross-coroutine membership tracking
* (dead relays struck by drain workers while the router reads them, relay hints
* written by the ingest consumer while the producer reads them).
*
* commonMain has no `java.util.concurrent.ConcurrentHashMap.newKeySet()`, so this
* is expect/actual with the same JVM-vs-native split as [ConcurrentMap]:
* - JVM / Android `ConcurrentHashMap.newKeySet()`.
* - Native copy-on-write over an atomic reference (compile-only, never the hot path).
*/
expect class ConcurrentSet<E : Any>() {
/** Add [element]; returns true if it was not already present. */
fun add(element: E): Boolean
operator fun contains(element: E): Boolean
fun size(): Int
/** A point-in-time copy — safe to iterate or diff against without a lock. */
fun snapshot(): Set<E>
}
@@ -0,0 +1,66 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.experimental.graperank
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotEquals
/**
* [GrapeRankCrawler.authorityOf] is the key the crawl's timeout-eviction counts
* on. It must collapse the many per-user path URLs the outbox model mints for one
* server into a single host, WITHOUT folding a distinct sibling host (e.g. a
* `filter.` subdomain) into its parent.
*/
class GrapeRankAuthorityTest {
private fun auth(url: String) = GrapeRankCrawler.authorityOf(url)
@Test
fun bareHostIsItsOwnAuthority() {
assertEquals("relay.damus.io", auth("wss://relay.damus.io"))
assertEquals("relay.damus.io", auth("wss://relay.damus.io/"))
assertEquals("nos.lol", auth("ws://nos.lol"))
}
@Test
fun perUserPathUrlsOnOneHostCollapseToOneAuthority() {
val a = auth("wss://filter.nostr.wine/npub1aaaa?broadcast=true")
val b = auth("wss://filter.nostr.wine/npub1bbbb?broadcast=true&global=all")
val c = auth("wss://filter.nostr.wine/?global=all")
assertEquals("filter.nostr.wine", a)
assertEquals(a, b)
assertEquals(a, c)
}
@Test
fun filterSubdomainIsNotFoldedIntoBareHost() {
// nostr.wine reads are open; filter.nostr.wine is a different server that may
// stall — evicting one must never take out the other.
assertNotEquals(auth("wss://filter.nostr.wine/npub1x"), auth("wss://nostr.wine"))
}
@Test
fun portIsPartOfTheAuthority() {
assertEquals("relay.veganostr.com:443", auth("wss://relay.veganostr.com:443/npub1z"))
assertEquals("81.68.170.122:7114", auth("ws://81.68.170.122:7114/"))
assertNotEquals(auth("wss://example.com:443"), auth("wss://example.com:8080"))
}
}
@@ -0,0 +1,246 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.experimental.graperank
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import kotlin.math.abs
import kotlin.math.exp
import kotlin.math.ln
import kotlin.math.max
import kotlin.random.Random
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class GrapeRankTest {
private val obs = "observer"
private fun graphOf(edges: List<Triple<HexKey, HexKey, TrustRelation>>): TrustGraph {
val b = TrustGraphBuilder()
for ((source, target, relation) in edges) {
when (relation) {
TrustRelation.FOLLOW -> b.addFollows(source, listOf(target))
TrustRelation.MUTE -> b.addMutes(source, listOf(target))
TrustRelation.REPORT -> b.addReports(source, listOf(target))
}
}
return b.build()
}
private fun graphOf(vararg edges: Triple<HexKey, HexKey, TrustRelation>) = graphOf(edges.toList())
/** Score for a pubkey (0.0 if absent from the graph). */
private fun DoubleArray.of(
graph: TrustGraph,
pubkey: HexKey,
): Double {
val id = graph.idOf(pubkey)
return if (id < 0) 0.0 else this[id]
}
@Test
fun observerIsPinnedAtFullSelfTrust() {
val graph = graphOf(Triple(obs, "a", TrustRelation.FOLLOW))
val scores = GrapeRank().compute(graph, obs)
assertEquals(1.0, scores.of(graph, obs), 1e-12)
}
@Test
fun directFollowMatchesHandComputedValue() {
val graph = graphOf(Triple(obs, "a", TrustRelation.FOLLOW))
val scores = GrapeRank().compute(graph, obs)
// weight = 0.5 * 1.0 * 0.85 = 0.425 ; score = conf(0.425) = 0.2551612...
assertEquals(0.25516127, scores.of(graph, "a"), 1e-6)
}
@Test
fun trustDecaysSteeplyAcrossHops() {
val graph =
graphOf(
Triple(obs, "a", TrustRelation.FOLLOW),
Triple("a", "b", TrustRelation.FOLLOW),
)
val scores = GrapeRank().compute(graph, obs)
val a = scores.of(graph, "a")
val b = scores.of(graph, "b")
assertEquals(0.004499, b, 1e-5)
assertTrue(b < a / 10.0, "two-hop trust should be far below one-hop trust")
}
@Test
fun aMuteFromAnEndorsedUserLowersTheScore() {
val followOnlyGraph = graphOf(Triple(obs, "b", TrustRelation.FOLLOW))
val followOnly = GrapeRank().compute(followOnlyGraph, obs).of(followOnlyGraph, "b")
val muteGraph =
graphOf(
Triple(obs, "a", TrustRelation.FOLLOW),
Triple(obs, "b", TrustRelation.FOLLOW),
Triple("a", "b", TrustRelation.MUTE),
)
val withMute = GrapeRank().compute(muteGraph, obs).of(muteGraph, "b")
assertTrue(withMute < followOnly, "a mute from a trusted user should pull b below the follow-only baseline")
}
@Test
fun purelyReportedUserFloorsAtZero() {
val graph =
graphOf(
Triple(obs, "a", TrustRelation.FOLLOW),
Triple("a", "d", TrustRelation.REPORT),
)
val scores = GrapeRank().compute(graph, obs)
assertEquals(0.0, scores.of(graph, "d"), 1e-9)
}
@Test
fun unreachableUsersAreNotScored() {
val graph =
graphOf(
Triple(obs, "a", TrustRelation.FOLLOW),
Triple("x", "y", TrustRelation.FOLLOW),
)
val scores = GrapeRank().compute(graph, obs)
assertTrue(scores.of(graph, "a") > 0.0)
assertEquals(0.0, scores.of(graph, "y"), 1e-12, "a user with no path from the observer stays 0")
}
@Test
fun cyclesConverge() {
val graph =
graphOf(
Triple(obs, "a", TrustRelation.FOLLOW),
Triple("a", "b", TrustRelation.FOLLOW),
Triple("b", "a", TrustRelation.FOLLOW),
)
val scores = GrapeRank().compute(graph, obs)
assertTrue(scores.of(graph, "a") > 0.0)
assertTrue(scores.of(graph, "b") > 0.0)
}
@Test
fun deduplicatesRepeatedReportEdges() {
// Two report edges a->d collapse to one; the score matches a single report.
val once = graphOf(Triple(obs, "a", TrustRelation.FOLLOW), Triple("a", "d", TrustRelation.REPORT))
val twice =
graphOf(
Triple(obs, "a", TrustRelation.FOLLOW),
Triple("a", "d", TrustRelation.REPORT),
Triple("a", "d", TrustRelation.REPORT),
)
assertEquals(2, twice.edgeCount(), "duplicate report edge should be dropped")
assertEquals(
GrapeRank().compute(once, obs).of(once, "d"),
GrapeRank().compute(twice, obs).of(twice, "d"),
1e-12,
)
}
/**
* Adversarial cross-check: the worklist propagation must reach the same fixed
* point as a naive full-sweep (the reference `v1FullSweep`) on random graphs.
*/
@Test
fun worklistMatchesFullSweepOnRandomGraphs() {
val params = GrapeRankParams(convergence = 1e-10)
val engine = GrapeRank(params)
repeat(50) { seed ->
val rng = Random(seed)
val n = 3 + rng.nextInt(12)
val nodes = (0 until n).map { "u$it" }
val edges = ArrayList<Triple<HexKey, HexKey, TrustRelation>>()
for (src in nodes) {
for (dst in nodes) {
if (src == dst) continue
if (rng.nextDouble() < 0.25) {
val relation =
when (rng.nextInt(5)) {
0 -> TrustRelation.MUTE
1 -> TrustRelation.REPORT
else -> TrustRelation.FOLLOW
}
edges.add(Triple(src, dst, relation))
}
}
}
val observer = nodes.first()
val graph = graphOf(edges)
val scores = engine.compute(graph, observer)
val reference = fullSweep(edges, nodes, observer, params)
for (node in nodes) {
if (node == observer) continue // observer self-trust is not part of a ranking
val a = scores.of(graph, node)
val b = reference[node] ?: 0.0
assertEquals(b, a, 1e-5, "seed=$seed node=$node worklist=$a fullSweep=$b")
}
}
}
// Reference: blind full sweep over every user until nothing changes.
private fun fullSweep(
edges: List<Triple<HexKey, HexKey, TrustRelation>>,
nodes: List<HexKey>,
observer: HexKey,
params: GrapeRankParams,
): Map<HexKey, Double> {
// Dedup identical edges (mirrors the builder: report edges dedup; follow/mute
// sets are unique per source anyway).
val incoming = HashMap<HexKey, MutableSet<Pair<HexKey, TrustRelation>>>()
for ((s, t, r) in edges) {
if (s == t) continue
incoming.getOrPut(t) { LinkedHashSet() }.add(s to r)
}
fun confidence(
r: TrustRelation,
source: HexKey,
) = when (r) {
TrustRelation.FOLLOW -> if (source == observer) params.directFollowConfidence else params.indirectFollowConfidence
TrustRelation.MUTE -> params.muteConfidence
TrustRelation.REPORT -> params.reportConfidence
}
fun weightToConfidence(w: Double) = 1.0 - exp(-w * -ln(params.rigor))
val scores = HashMap<HexKey, Double>()
scores[observer] = 1.0
do {
var changed = false
for (target in nodes) {
if (target == observer) continue
var sumW = 0.0
var sumWR = 0.0
for ((source, r) in incoming[target] ?: emptySet()) {
val s = scores[source] ?: continue
val w = confidence(r, source) * s * params.attenuation
sumW += w
sumWR += w * r.rating
}
val newScore = if (abs(sumW) < 0.00001) 0.0 else max(weightToConfidence(sumW) * sumWR / sumW, 0.0)
val old = scores.put(target, newScore) ?: 0.0
changed = changed || abs(newScore - old) > params.convergence
}
} while (changed)
return scores
}
}
@@ -0,0 +1,107 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.experimental.graperank
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class TrustGraphBuilderTest {
private val alice = "alice"
private val bob = "bob"
private val carol = "carol"
private val dave = "dave"
/** Decode a node's incoming edges back to (source, relation) pairs from the CSR. */
private fun TrustGraph.incomingOf(pubkey: HexKey): Set<Pair<HexKey, TrustRelation>> {
val t = idOf(pubkey)
if (t < 0) return emptySet()
val out = HashSet<Pair<HexKey, TrustRelation>>()
var i = inOffsets[t]
val end = inOffsets[t + 1]
while (i < end) {
val packed = inPacked[i]
val source = pubkeyOf(packed and TrustGraph.SOURCE_MASK)
val relation = TrustRelation.entries.first { it.code == (packed ushr TrustGraph.SOURCE_BITS) }
out.add(source to relation)
i++
}
return out
}
@Test
fun buildsFollowMuteAndReportEdges() {
val b = TrustGraphBuilder()
b.addFollows(alice, listOf(bob, carol))
b.addMutes(bob, listOf(dave))
b.addReports(carol, listOf(dave))
val graph = b.build()
assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(bob))
assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(carol))
assertEquals(
setOf(bob to TrustRelation.MUTE, carol to TrustRelation.REPORT),
graph.incomingOf(dave),
)
}
@Test
fun dropsSelfEdges() {
val b = TrustGraphBuilder()
b.addFollows(alice, listOf(alice, bob))
val graph = b.build()
assertTrue(graph.incomingOf(alice).isEmpty(), "a self-follow must not become an edge")
assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(bob))
}
@Test
fun dedupesRepeatedReports() {
val b = TrustGraphBuilder()
b.addReports(alice, listOf(dave))
b.addReports(alice, listOf(dave))
val graph = b.build()
assertEquals(1, graph.edgeCount())
assertEquals(setOf(alice to TrustRelation.REPORT), graph.incomingOf(dave))
}
@Test
fun keepsFollowAndMuteFromSameSourceAsDistinctEdges() {
val b = TrustGraphBuilder()
b.addFollows(alice, listOf(bob))
b.addMutes(alice, listOf(bob))
val graph = b.build()
assertEquals(
setOf(alice to TrustRelation.FOLLOW, alice to TrustRelation.MUTE),
graph.incomingOf(bob),
)
}
@Test
fun internsEachPubkeyOnce() {
val b = TrustGraphBuilder()
b.addFollows(alice, listOf(bob, carol))
b.addFollows(bob, listOf(carol))
val graph = b.build()
assertEquals(3, graph.nodeCount, "alice, bob, carol interned once each")
assertEquals(3, graph.edgeCount())
}
}
@@ -0,0 +1,86 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
class DrainFailureTest {
// Non-failure and non-"cannot" terminals are never dead signals.
@Test
fun nonFailureTerminalsAreNull() {
assertNull(classifyDrainFailure("eose"))
assertNull(classifyDrainFailure("closed:duplicate: sub"))
assertNull(classifyDrainFailure("timeout"))
}
// A READ timeout (or generic post-handshake timeout) is alive-but-slow: never
// dead. Measured 67% of these relays were reachable when re-probed fresh.
@Test
fun readTimeoutsStayRetryable() {
assertNull(classifyDrainFailure("cannot:Read timed out (SocketTimeoutException)"))
assertNull(classifyDrainFailure("cannot:timeout (SocketTimeoutException)"))
}
// An HTTP 429 rate-limit is alive and will serve us after backoff: never dead.
// Measured 4/4 such relays reachable when re-probed fresh.
@Test
fun rateLimitStaysRetryable() {
assertNull(classifyDrainFailure("cannot:Server Misconfigured. Response: 429 Too Many Requests (ProtocolException)"))
}
// Failing to ESTABLISH the connection is dead (0/30 reachable fresh). "connect
// timed out" must be caught as DEAD and NOT slip into the read-timeout branch.
@Test
fun connectEstablishmentFailuresAreDead() {
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Connect timed out (SocketTimeoutException)"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Unexpected response code for CONNECT: (IOException)"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Connection refused (ConnectException)"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Failed to connect to /1.2.3.4:443"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:No route to host (NoRouteToHostException)"))
}
// DNS and TLS misconfig can never work: DEAD.
@Test
fun dnsAndTlsAreDead() {
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Unable to resolve host (UnknownHostException)"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Received fatal alert: unrecognized_name (SSLHandshakeException)"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:PKIX path building failed: certificate (CertificateException)"))
}
// Every other bad HTTP upgrade won't serve us this run (measured 503 0%, 502 20%
// reachable; 402/403 gated; 200 not a relay) — DEAD, dropped on the first strike.
@Test
fun deadOrGatedHttpUpgradesAreDead() {
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Server Misconfigured. not a websocket"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Server Misconfigured. Response: 503 Service Unavailable (ProtocolException)"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Server Misconfigured. Response: 502 Bad Gateway (ProtocolException)"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Server Misconfigured. Response: 402 Payment Required (ProtocolException)"))
}
// A mid-stream reset won't hand us events this run either: DEAD.
@Test
fun midStreamResetIsDead() {
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Connection reset (SocketException)"))
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Broken pipe (SocketException)"))
}
}
@@ -0,0 +1,97 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.relay.client.pool
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import kotlin.test.Test
import kotlin.test.assertContains
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class PoolEventOutboxStateTest {
private val relay = NormalizedRelayUrl("wss://relay.example/")
private fun fakeEvent() =
Event(
id = "0".repeat(64),
pubKey = "0".repeat(64),
createdAt = 0L,
kind = 1,
tags = emptyArray(),
content = "",
sig = "0".repeat(128),
)
@Test
fun authRequiredResponseDoesNotConsumeTryBudget() {
val state = PoolEventOutboxState(fakeEvent(), setOf(relay))
// Simulate 5 `auth-required:` responses — relay keeps challenging while
// RelayAuthenticator signs + sends AUTH events asynchronously. None of
// these should be counted against the 3-response try cap.
repeat(5) {
state.newResponse(relay, success = false, message = "auth-required: please authenticate")
}
// Even after a follow-up newTry, the relay must remain in the outbox so
// syncFilters() can re-publish once AUTH succeeds.
state.newTry(relay)
assertContains(state.relaysLeft(), relay)
assertFalse(state.isDone())
}
@Test
fun regularRejectionStillBoundedByTryCap() {
val state = PoolEventOutboxState(fakeEvent(), setOf(relay))
// 3 non-AUTH rejections accumulate normally.
repeat(3) {
state.newResponse(relay, success = false, message = "error: rate limited")
}
state.newTry(relay)
// After the 4th newTry (with 3 prior responses already in flight), the
// Tries cap kicks in and the relay is dropped from the outbox.
assertFalse(state.relaysLeft().contains(relay))
}
@Test
fun terminalRejectionImmediatelyDropsRelay() {
val state = PoolEventOutboxState(fakeEvent(), setOf(relay))
state.newResponse(relay, success = false, message = "invalid: malformed event")
assertFalse(state.relaysLeft().contains(relay))
assertTrue(state.isDone())
}
@Test
fun successDropsRelayFromOutbox() {
val state = PoolEventOutboxState(fakeEvent(), setOf(relay))
state.newResponse(relay, success = true, message = "")
assertEquals(emptySet(), state.relaysLeft())
assertTrue(state.isDone())
}
}
@@ -72,13 +72,15 @@ class BasicRelayClientTest {
}
@Test
fun onFailureWithMessageKeepsExistingFormat() {
fun onFailureWithMessageAppendsExceptionClassName() {
val (socket, listener) = connectAndCapture()
socket.onFailure(Exception("Connection reset"), null, null)
// The exception type is appended so listeners can classify the failure by
// its stable class name rather than by localized message text.
assertEquals(
listOf("WebSocket Failure: Connection reset"),
listOf("WebSocket Failure: Connection reset (Exception)"),
listener.cannotConnectMessages,
)
}
@@ -0,0 +1,127 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.store.sqlite
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
import com.vitorpamplona.quartz.utils.EventFactory
import kotlin.test.Test
import kotlin.test.assertEquals
class AuthorsMissingOutboxTest : BaseDBTest() {
@Test
fun emptyStoreReturnsNoAuthors() =
forEachDB { db ->
assertEquals(emptySet(), db.authorsMissingOutbox().toSet())
}
@Test
fun authorWithEventButNoOutboxIsMissing() =
forEachDB { db ->
val signer = NostrSignerSync()
db.insert(signer.sign(TextNoteEvent.build("hello")))
assertEquals(setOf(signer.pubKey), db.authorsMissingOutbox().toSet())
}
@Test
fun authorWithOutboxIsNotMissing() =
forEachDB { db ->
val hasOutbox = NostrSignerSync()
val noOutbox = NostrSignerSync()
// Both authors have content; only one advertises a 10002.
db.insert(hasOutbox.sign(TextNoteEvent.build("with relays")))
db.insert(AdvertisedRelayListEvent.create(emptyList(), hasOutbox))
db.insert(noOutbox.sign(TextNoteEvent.build("no relays")))
assertEquals(setOf(noOutbox.pubKey), db.authorsMissingOutbox().toSet())
}
@Test
fun authorKnownOnlyByTheirOutboxIsNotMissing() =
forEachDB { db ->
// The only stored event for this author IS the 10002. They must
// not appear (the outer scan sees them, the NOT EXISTS excludes
// them) — the anti-join is symmetric on the same table.
val signer = NostrSignerSync()
db.insert(AdvertisedRelayListEvent.create(emptyList(), signer))
assertEquals(emptySet(), db.authorsMissingOutbox().toSet())
}
@Test
fun outboxDeletedMakesAuthorMissingAgain() =
forEachDB { db ->
val signer = NostrSignerSync()
db.insert(signer.sign(TextNoteEvent.build("content")))
val relayList = AdvertisedRelayListEvent.create(emptyList(), signer)
db.insert(relayList)
assertEquals(emptySet(), db.authorsMissingOutbox().toSet())
// NIP-09: the author deletes their own relay list. No 10002 row
// remains, so the anti-join reports them as missing again.
db.insert(signer.sign(DeletionEvent.build(listOf(relayList))))
assertEquals(setOf(signer.pubKey), db.authorsMissingOutbox().toSet())
}
@Test
fun giftWrapSenderIsNotCountedAsAuthor() =
forEachDB { db ->
val noteAuthor = NostrSignerSync()
db.insert(noteAuthor.sign(TextNoteEvent.build("hi")))
// A kind-1059 giftwrap stores an ephemeral one-time key as its
// pubkey (the real recipient is only a hash). It has no outbox and
// never will — but it must NOT be reported as "missing" one, or the
// result set would grow by one junk key per received DM.
val ephemeralSender = "aa".repeat(32)
db.insert(
EventFactory.create("bb".repeat(32), ephemeralSender, 1L, GiftWrapEvent.KIND, emptyArray(), "", "00".repeat(64)),
)
assertEquals(setOf(noteAuthor.pubKey), db.authorsMissingOutbox().toSet())
}
@Test
fun mixOfAuthorsReportsOnlyThoseWithoutOutbox() =
forEachDB { db ->
val a = NostrSignerSync()
val b = NostrSignerSync()
val c = NostrSignerSync()
db.insert(a.sign(TextNoteEvent.build("a1")))
db.insert(a.sign(TextNoteEvent.build("a2")))
db.insert(AdvertisedRelayListEvent.create(emptyList(), a))
db.insert(b.sign(TextNoteEvent.build("b1")))
db.insert(c.sign(TextNoteEvent.build("c1")))
db.insert(AdvertisedRelayListEvent.create(emptyList(), c))
assertEquals(setOf(b.pubKey), db.authorsMissingOutbox().toSet())
}
}
@@ -0,0 +1,104 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip59Giftwrap.wraps
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.toHexKey
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
import kotlinx.coroutines.test.runTest
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
/**
* NIP-17 relay-hint placement contract.
*
* Per NIP-17 §Publishing, the gift wrap's `p` tag MAY carry the recipient's
* primary DM inbox relay as a third element so other devices of the recipient
* can discover the wrap without a separate kind:10050 lookup. The hint
* deliberately lives on the public wrap, NOT on the encrypted seal putting
* it on the seal would hide the routing information inside the encryption
* envelope, defeating the purpose.
*/
class GiftWrapRelayHintTest {
private val recipient = KeyPair()
private fun innerEvent(): Event {
val signer = NostrSignerSync(KeyPair())
return signer.sign(
createdAt = 0L,
kind = 1,
tags = emptyArray(),
content = "hello",
)
}
@Test
fun defaultsToNoRelayHintForBackwardsCompat() =
runTest {
// Existing callers that don't pass a hint must continue to emit the
// historical ["p", recipientPubKey] two-element tag shape.
val wrap =
GiftWrapEvent.create(
event = innerEvent(),
recipientPubKey = recipient.pubKey.toHexKey(),
)
val pTag = wrap.tags.first { it.firstOrNull() == "p" }
assertEquals(2, pTag.size, "p tag must be 2 elements when no hint passed")
assertEquals(recipient.pubKey.toHexKey(), pTag[1])
}
@Test
fun relayHintLandsOnWrapPTagAsThirdElement() =
runTest {
// When a hint is passed, it must appear as the THIRD element of the
// wrap's p tag — NIP-17 spec. Not inside the encrypted seal.
val hint = NormalizedRelayUrl("wss://dm.relay.example/")
val wrap =
GiftWrapEvent.create(
event = innerEvent(),
recipientPubKey = recipient.pubKey.toHexKey(),
recipientRelayHint = hint,
)
val pTag = wrap.tags.first { it.firstOrNull() == "p" }
assertEquals(3, pTag.size, "p tag carries [tag, pubkey, relay-hint]")
assertEquals(recipient.pubKey.toHexKey(), pTag[1])
assertEquals(hint.url, pTag[2])
}
@Test
fun absentHintDoesNotAddTrailingEmptyElement() =
runTest {
// Defensive: a null hint must not produce `["p", pubkey, ""]` — that
// would be a leak (broadcasts the user has no canonical inbox) and
// a wire-format change from the historical shape.
val wrap =
GiftWrapEvent.create(
event = innerEvent(),
recipientPubKey = recipient.pubKey.toHexKey(),
recipientRelayHint = null,
)
val pTag = wrap.tags.first { it.firstOrNull() == "p" }
assertNull(pTag.getOrNull(2), "third element must be absent, not empty string")
}
}
@@ -0,0 +1,108 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.concurrent
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNull
import kotlin.test.assertTrue
class ConcurrentCollectionsTest {
@Test
fun mapGetSet() {
val m = ConcurrentMap<String, Int>()
assertNull(m["a"])
m["a"] = 1
assertEquals(1, m["a"])
m["a"] = 2
assertEquals(2, m["a"])
assertEquals(1, m.size())
}
@Test
fun mapGetOrPutComputesOnce() {
val m = ConcurrentMap<String, Int>()
var calls = 0
assertEquals(
7,
m.getOrPut("k") {
calls++
7
},
)
// Present now: the default must NOT be recomputed.
assertEquals(
7,
m.getOrPut("k") {
calls++
99
},
)
assertEquals(1, calls)
assertEquals(7, m["k"])
}
@Test
fun mapMergeInsertsThenCombines() {
val m = ConcurrentMap<String, Int>()
// Absent -> inserts the value verbatim, remap not applied.
assertEquals(1, m.merge("k", 1) { a, b -> a + b })
// Present -> remap(existing, value).
assertEquals(4, m.merge("k", 3) { a, b -> a + b })
assertEquals(4, m["k"])
}
@Test
fun mapSnapshotIsDetached() {
val m = ConcurrentMap<String, Int>()
m["a"] = 1
m["b"] = 2
val snap = m.snapshot()
assertEquals(mapOf("a" to 1, "b" to 2), snap)
// Mutating the map after the snapshot must not change the snapshot.
m["c"] = 3
assertEquals(2, snap.size)
assertEquals(3, m.size())
}
@Test
fun setAddContainsSize() {
val s = ConcurrentSet<String>()
assertFalse("x" in s)
assertTrue(s.add("x"))
// Re-adding is a no-op and reports it.
assertFalse(s.add("x"))
assertTrue("x" in s)
assertTrue(s.add("y"))
assertEquals(2, s.size())
}
@Test
fun setSnapshotIsDetached() {
val s = ConcurrentSet<String>()
s.add("a")
val snap = s.snapshot()
s.add("b")
assertEquals(setOf("a"), snap)
assertEquals(2, s.size())
}
}
@@ -127,7 +127,7 @@ actual object EventHasherSerializer {
content: String,
): Boolean {
val br: BufferRecycler = JacksonMapper.mapper.factory._getBufferRecycler()
val digest = threadLocalDigest.get()
val digest = threadLocalDigest.get()!!
val bb = HashingByteArrayBuilder(br, digest)
try {
val generator = JacksonMapper.mapper.createGenerator(bb, JsonEncoding.UTF8)
@@ -83,9 +83,7 @@ class CommandSerializer : StdSerializer<Command>(Command::class.java) {
gen.writeString(cmd.subId)
}
else -> {
null
}
else -> {}
}
gen.writeEndArray()
@@ -162,7 +162,7 @@ class GitHttpClient(
visited.add(start)
}
while (frontier.isNotEmpty() && result.size < depth) {
val commit = frontier.poll()
val commit = frontier.poll()!!
result.add(commit)
for (parent in commit.parents) {
if (parent !in visited) {
@@ -0,0 +1,55 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.concurrent
import java.util.concurrent.ConcurrentHashMap
actual class ConcurrentMap<K : Any, V : Any> {
private val map = ConcurrentHashMap<K, V>()
actual operator fun get(key: K): V? = map[key]
actual operator fun set(
key: K,
value: V,
) {
map[key] = value
}
actual fun getOrPut(
key: K,
defaultValue: () -> V,
): V =
// Fast-path the present-key hit (the common case in the crawl's hot
// relay-hint accumulation) so it never allocates the mapping-function
// closure; only an absent key pays for the atomic computeIfAbsent.
map[key] ?: map.computeIfAbsent(key) { defaultValue() }
actual fun merge(
key: K,
value: V,
remap: (old: V, new: V) -> V,
): V = map.merge(key, value) { old, new -> remap(old, new) }!!
actual fun size(): Int = map.size
actual fun snapshot(): Map<K, V> = HashMap(map)
}
@@ -0,0 +1,35 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.concurrent
import java.util.concurrent.ConcurrentHashMap
actual class ConcurrentSet<E : Any> {
private val set: MutableSet<E> = ConcurrentHashMap.newKeySet()
actual fun add(element: E): Boolean = set.add(element)
actual operator fun contains(element: E): Boolean = set.contains(element)
actual fun size(): Int = set.size
actual fun snapshot(): Set<E> = HashSet(set)
}
@@ -30,19 +30,19 @@ import java.security.MessageDigest
* (lock acquire + release) for ~2µs of actual hashing. ThreadLocal eliminates all locking
* since each thread gets its own MessageDigest instance. digest() implicitly resets state.
*/
val threadLocalDigest =
val threadLocalDigest: ThreadLocal<MessageDigest> =
ThreadLocal.withInitial {
MessageDigest.getInstance("SHA-256")
}
actual fun sha256(data: ByteArray): ByteArray = threadLocalDigest.get().digest(data)
actual fun sha256(data: ByteArray): ByteArray = threadLocalDigest.get()!!.digest(data)
actual fun sha256Into(
out: ByteArray,
data: ByteArray,
len: Int,
): ByteArray {
val md = threadLocalDigest.get()
val md = threadLocalDigest.get()!!
md.update(data, 0, len)
md.digest(out, 0, 32)
return out
@@ -62,7 +62,7 @@ fun sha256StreamWithCount(
bufferSize: Int = 8192,
): Pair<ByteArray, Long> {
val countingStream = CountingInputStream(inputStream)
val digest = threadLocalDigest.get()
val digest = threadLocalDigest.get()!!
try {
val buffer = ByteArray(bufferSize)
var bytesRead: Int
@@ -123,6 +123,46 @@ class MlsGroupManagerTest {
}
}
/**
* Regression: [MlsGroupManager.encrypt] must persist the advanced ratchet
* position, not just commits. A group state persists only at commits was
* the second half of the generation-reuse bug: sends between two commits
* advanced the SecretTree in memory but never hit the store, so a restart
* reloaded the pre-send ratchet and re-emitted an already-used generation.
*
* Alice and Bob share a group. Alice sends one message (Bob consumes
* generation 0), Alice "restarts" from the store WITHOUT any intervening
* commit, and her next send must be a fresh generation Bob accepts.
*/
@Test
fun testEncryptPersistsRatchetPositionBetweenCommits() {
runBlocking {
val aliceStore = InMemoryGroupStateStore()
val alice = MlsGroupManager(aliceStore)
val aliceGroup = alice.createGroup(groupId, "alice".encodeToByteArray())
// Bob joins as a low-level MlsGroup — a strict peer that tracks
// consumed generations. (The manager's processWelcome requires a
// NostrGroupData extension we don't set up here; the low-level
// group is enough to observe the ratchet behavior.)
val bobBundle = aliceGroup.createKeyPackage("bob".encodeToByteArray(), ByteArray(0))
val addResult = alice.addMember(groupId, bobBundle.keyPackage.toTlsBytes())
val bob = MlsGroup.processWelcome(addResult.welcomeBytes!!, bobBundle)
// Alice sends generation 0 (no commit); Bob consumes it.
val ct0 = alice.encrypt(groupId, "msg0".encodeToByteArray())
assertContentEquals("msg0".encodeToByteArray(), bob.decrypt(ct0).content)
// Alice restarts from the store — only encrypt() has run since the
// last commit, so this proves encrypt persisted the ratchet.
val aliceRestarted = MlsGroupManager(aliceStore)
aliceRestarted.restoreAll()
val ct1 = aliceRestarted.encrypt(groupId, "msg1".encodeToByteArray())
assertContentEquals("msg1".encodeToByteArray(), bob.decrypt(ct1).content)
}
}
@Test
fun testAddMemberPersistsState() {
runBlocking {
@@ -173,12 +173,12 @@ class MlsGroupStateTest {
val state = group.saveState()
val bytes = state.encodeTls()
// First two bytes should be the version (uint16 = 1)
// First two bytes should be the version (uint16 = 2)
val reader =
com.vitorpamplona.quartz.marmot.mls.codec
.TlsReader(bytes)
val version = reader.readUint16()
assertEquals(1, version)
assertEquals(2, version)
}
@Test
@@ -219,4 +219,114 @@ class MlsGroupStateTest {
val decrypted = restoredGroup.decrypt(encrypted)
assertContentEquals(plaintext, decrypted.content)
}
/**
* Regression: a restore must NOT rewind the SecretTree ratchet to
* generation 0. A peer that already consumed generation 0 in this epoch
* (like openmls / MDK / Whitenoise, which forbid generation reuse) would
* otherwise reject the restored sender's next message as a replay.
*/
@Test
fun testRestorePreservesSenderGeneration_peerAcceptsNextMessage() {
val alice = MlsGroup.create("alice".encodeToByteArray())
val bobBundle =
MlsGroup
.create("bob".encodeToByteArray())
.createKeyPackage("bob".encodeToByteArray(), ByteArray(0))
val bob = MlsGroup.processWelcome(alice.addMember(bobBundle.keyPackage.toTlsBytes()).welcomeBytes!!, bobBundle)
// Alice sends generation 0; Bob consumes it.
val ct0 = alice.encrypt("msg0".encodeToByteArray())
assertContentEquals("msg0".encodeToByteArray(), bob.decrypt(ct0).content)
// Alice "restarts": persist then restore.
val aliceRestored = MlsGroup.restore(MlsGroupState.decodeTls(alice.saveState().encodeTls()))
// Alice's next send must be generation 1, which Bob accepts. Before
// the fix this re-emitted generation 0 and Bob threw "Generation 0
// already consumed".
val ct1 = aliceRestored.encrypt("msg1".encodeToByteArray())
assertContentEquals("msg1".encodeToByteArray(), bob.decrypt(ct1).content)
}
/**
* The ratchet position must survive several sends across a restore, not
* just one. Covers the case where the app persists (at a commit) after N
* application messages have already advanced the ratchet.
*/
@Test
fun testRestorePreservesSenderGenerationAfterMultipleSends() {
val alice = MlsGroup.create("alice".encodeToByteArray())
val bobBundle =
MlsGroup
.create("bob".encodeToByteArray())
.createKeyPackage("bob".encodeToByteArray(), ByteArray(0))
val bob = MlsGroup.processWelcome(alice.addMember(bobBundle.keyPackage.toTlsBytes()).welcomeBytes!!, bobBundle)
for (i in 0 until 5) {
val ct = alice.encrypt("m$i".encodeToByteArray())
assertContentEquals("m$i".encodeToByteArray(), bob.decrypt(ct).content)
}
val aliceRestored = MlsGroup.restore(MlsGroupState.decodeTls(alice.saveState().encodeTls()))
// Continues at generation 5 — Bob (who consumed 0..4) accepts it.
val ct = aliceRestored.encrypt("m5".encodeToByteArray())
assertContentEquals("m5".encodeToByteArray(), bob.decrypt(ct).content)
}
/**
* Backward compatibility: a STATE_VERSION 1 blob (no persisted ratchet
* positions) must still decode, yielding an empty ratchet map and the
* legacy generation-0 restore behavior.
*/
@Test
fun testDecodeLegacyV1StateBlob() {
val group = MlsGroup.create("alice".encodeToByteArray())
group.encrypt("advance the ratchet".encodeToByteArray())
val state = group.saveState()
val v1Bytes = encodeAsV1(state)
val decoded = MlsGroupState.decodeTls(v1Bytes)
assertTrue(decoded.senderRatchetStates.isEmpty(), "v1 blob has no ratchet positions")
// Restores and can still encrypt/decrypt (legacy behavior).
val restored = MlsGroup.restore(decoded)
val ct = restored.encrypt("post-restore".encodeToByteArray())
assertContentEquals("post-restore".encodeToByteArray(), restored.decrypt(ct).content)
}
/**
* Re-encode a state in the original STATE_VERSION 1 layout: identical to
* v2 but with the version tag set to 1 and no trailing ratchet section.
*/
private fun encodeAsV1(state: MlsGroupState): ByteArray {
val writer =
com.vitorpamplona.quartz.marmot.mls.codec
.TlsWriter()
writer.putUint16(1)
state.groupContext.encodeTls(writer)
writer.putOpaqueVarInt(state.treeBytes)
writer.putUint32(state.myLeafIndex.toLong())
val es = state.epochSecrets
writer.putOpaqueVarInt(es.joinerSecret)
writer.putOpaqueVarInt(es.welcomeSecret)
writer.putOpaqueVarInt(es.epochSecret)
writer.putOpaqueVarInt(es.senderDataSecret)
writer.putOpaqueVarInt(es.encryptionSecret)
writer.putOpaqueVarInt(es.exporterSecret)
writer.putOpaqueVarInt(es.epochAuthenticator)
writer.putOpaqueVarInt(es.externalSecret)
writer.putOpaqueVarInt(es.confirmationKey)
writer.putOpaqueVarInt(es.membershipKey)
writer.putOpaqueVarInt(es.resumptionPsk)
writer.putOpaqueVarInt(es.initSecret)
writer.putOpaqueVarInt(state.initSecret)
writer.putOpaqueVarInt(state.signingPrivateKey)
writer.putOpaqueVarInt(state.encryptionPrivateKey)
writer.putOpaqueVarInt(state.interimTranscriptHash)
writer.putOpaqueVarInt(state.encryptionSecret)
return writer.toByteArray()
}
}
@@ -0,0 +1,251 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.relay.prodbench
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient
import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPages
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl
import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.BasicOkHttpWebSocket
import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
import com.vitorpamplona.quartz.utils.EventFactory
import kotlinx.coroutines.runBlocking
import okhttp3.OkHttpClient
import java.nio.file.Files
import java.util.concurrent.TimeUnit
import kotlin.test.Test
import kotlin.test.assertEquals
/**
* Head-to-head for `IEventStore.authorsMissingOutbox()` "give me every
* author with events but no NIP-65 relay list (kind 10002)" — at 1,000,000
* events, comparing the two implementations that ship:
*
* - **generic** the `IEventStore` interface default: query the 10002
* owners into a set, then stream EVERY event (`query(Filter())`) and keep
* the authors not in that set. Correct for any store, but it decodes all
* 1M events off SQLite into `Event` objects.
* - **sqlite** `EventStore.authorsMissingOutbox()`, an index-only `EXCEPT`
* over `event_headers` (all authors minus the 10002 owners) that never
* decodes an event, riding the `(kind, pubkey, created_at)` covering index.
*
* Corpus: the benchmark first **syncs a real sample from a popular relay**
* (kind 1 notes + kind 10002 relay lists from [RELAY]) so the pubkey
* cardinality, per-author event fan-out, tag/content sizes, and the fraction
* of authors that actually advertise relays are all real. It then replicates
* that sample cloning each real event with a fresh id and timestamp but the
* SAME pubkey/kind/tags/content up to [TARGET] rows. Replication preserves
* the real distinct-author set and the real 10002-owner set exactly (so the
* answer is unchanged), it only grows each author's history the way a
* long-lived relay would. A live 1M download is bandwidth-bound and isn't
* what we're measuring; the query is.
*
* The store keeps the `indexEventsByPubkeyAlone` index a relay actually keeps
* (this query is a relay / outbox-model concern) that is the index the
* `DISTINCT pubkey ... NOT EXISTS` scan rides. NIP-50 full-text indexing is
* turned off: it is pure insert-path cost that neither query touches, so
* dropping it just makes seeding 1M rows fast without changing either timing.
*
* Network + heavy, so gated like the other prod benches:
* ./gradlew :quartz:jvmTest --tests "*.AuthorsMissingOutboxBenchmark" -PprodRelayBench=1
*/
class AuthorsMissingOutboxBenchmark {
companion object {
const val RELAY = "wss://relay.damus.io"
const val TARGET = 1_000_000
const val SAMPLE_NOTES = 25_000
const val SAMPLE_RELAY_LISTS = 15_000
const val FETCH_TIMEOUT_MS = 90_000L
const val INSERT_CHUNK = 2_000
val SIG = "0".repeat(128)
}
private fun idFor(counter: Long): String = "%064x".format(counter)
/** The generic path — a verbatim copy of the `IEventStore` interface default. */
private suspend fun genericAuthorsMissingOutbox(store: EventStore): List<HexKey> {
val withOutbox = HashSet<HexKey>()
store.query<Event>(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND))) { withOutbox.add(it.pubKey) }
val missing = LinkedHashSet<HexKey>()
store.query<Event>(Filter()) { event ->
if (event.kind != GiftWrapEvent.KIND && event.pubKey !in withOutbox) missing.add(event.pubKey)
}
return missing.toList()
}
@Test
fun authorsMissingOutboxScaling() {
if (System.getenv("PROD_RELAY_BENCH") == null && System.getProperty("prodRelayBench") == null) {
println("AuthorsMissingOutboxBenchmark skipped. Run with -PprodRelayBench=1 to enable.")
return
}
val httpClient =
OkHttpClient
.Builder()
.connectTimeout(15, TimeUnit.SECONDS)
.readTimeout(120, TimeUnit.SECONDS)
.pingInterval(30, TimeUnit.SECONDS)
.build()
println("=== authorsMissingOutbox 1M benchmark === cores=${Runtime.getRuntime().availableProcessors()}")
// ── 1. SYNC a real sample from a popular relay ──────────────────────
val notes = ArrayList<Event>(SAMPLE_NOTES)
val relayLists = ArrayList<Event>(SAMPLE_RELAY_LISTS)
val relay = RELAY.normalizeRelayUrl()
runBlocking {
val client = NostrClient(BasicOkHttpWebSocket.Builder { httpClient })
try {
val t0 = System.nanoTime()
client.fetchAllPages(relay, listOf(Filter(kinds = listOf(1), limit = SAMPLE_NOTES)), FETCH_TIMEOUT_MS) { notes.add(it) }
client.fetchAllPages(relay, listOf(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), limit = SAMPLE_RELAY_LISTS)), FETCH_TIMEOUT_MS) { relayLists.add(it) }
println(" synced from $RELAY in %.1fs: %,d notes + %,d relay-lists".format((System.nanoTime() - t0) / 1e9, notes.size, relayLists.size))
} finally {
client.close()
}
}
httpClient.dispatcher.executorService.shutdown()
val pool = (notes + relayLists).distinctBy { it.id }
require(pool.isNotEmpty()) { "relay returned no events — cannot build corpus" }
val outboxOwners = relayLists.mapTo(HashSet()) { it.pubKey }
val allAuthors = pool.mapTo(HashSet()) { it.pubKey }
val expectedMissing = allAuthors - outboxOwners
println(
" real sample: %,d events, %,d distinct authors, %,d with a 10002 (%.1f%%) → %,d missing".format(
pool.size,
allAuthors.size,
outboxOwners.size,
100.0 * outboxOwners.size / allAuthors.size,
expectedMissing.size,
),
)
// ── 2. SCALE to TARGET by replicating the real sample ───────────────
// kind 10002 is replaceable — one row survives per owner no matter how
// many times it is re-cloned — so the store is filled with note (kind 1)
// clones and each owner's relay list is inserted exactly once. That
// lands a genuine TARGET rows while keeping the real author set and
// outbox-owner set intact.
val notePool = notes.distinctBy { it.id }
require(notePool.isNotEmpty()) { "relay returned no kind-1 notes — cannot fill the corpus" }
val relayListPerOwner = relayLists.associateBy { it.pubKey }.values.toList()
val noteCloneTarget = (TARGET - relayListPerOwner.size).coerceAtLeast(0)
// FS/FTS off, pubkey+created_at indexes on: representative of the query,
// fast to seed. See the class KDoc.
val strategy =
DefaultIndexingStrategy(
indexEventsByCreatedAtAlone = true,
indexEventsByPubkeyAlone = true,
useAndIndexIdOnOrderBy = true,
indexFullTextSearch = false,
)
val dbFile = Files.createTempFile("authors-missing-outbox-", ".db")
Files.deleteIfExists(dbFile)
val store = EventStore(dbName = dbFile.toAbsolutePath().toString(), relay = null, indexStrategy = strategy)
try {
val baseTime = 1_600_000_000L
var counter = 0L
val seedT0 = System.nanoTime()
val batch = ArrayList<Event>(INSERT_CHUNK)
suspend fun flush() {
if (batch.isNotEmpty()) {
store.batchInsert(batch)
batch.clear()
}
}
runBlocking {
// One relay list per owner (fresh id; content/tags preserved).
for (src in relayListPerOwner) {
batch.add(EventFactory.create(idFor(++counter), src.pubKey, baseTime + counter, src.kind, src.tags, src.content, SIG))
if (batch.size == INSERT_CHUNK) flush()
}
// Fill the rest with note clones cycling the real notes.
var made = 0L
while (made < noteCloneTarget) {
val src = notePool[(made % notePool.size).toInt()]
batch.add(EventFactory.create(idFor(++counter), src.pubKey, baseTime + counter, src.kind, src.tags, src.content, SIG))
made++
if (batch.size == INSERT_CHUNK) flush()
}
flush()
}
val total = runBlocking { store.count(Filter()) }
println(" seeded %,d rows (stored %,d) in %.1fs".format(counter, total, (System.nanoTime() - seedT0) / 1e9))
// ── 3. MEASURE both implementations on the same store ──────────
// Warm the page cache with one throwaway pass of each so neither
// eats the cold-cache penalty for the other.
runBlocking {
store.authorsMissingOutbox()
genericAuthorsMissingOutbox(store)
}
val runs = 3
var sqliteResult: List<HexKey> = emptyList()
var genericResult: List<HexKey> = emptyList()
val sqliteMs = DoubleArray(runs)
val genericMs = DoubleArray(runs)
runBlocking {
repeat(runs) { i ->
var t = System.nanoTime()
sqliteResult = store.authorsMissingOutbox()
sqliteMs[i] = (System.nanoTime() - t) / 1e6
t = System.nanoTime()
genericResult = genericAuthorsMissingOutbox(store)
genericMs[i] = (System.nanoTime() - t) / 1e6
}
}
// Correctness: both must return the same author set, and it must
// match the ground truth computed from the real sample.
assertEquals(sqliteResult.toSet(), genericResult.toSet(), "sqlite and generic disagree")
assertEquals(expectedMissing, sqliteResult.toSet(), "result does not match the seeded distribution")
val sqliteBest = sqliteMs.min()
val genericBest = genericMs.min()
println("\n result: %,d authors missing an outbox (of %,d distinct authors)".format(sqliteResult.size, allAuthors.size))
println(" ── timings over $runs runs (best-of) ──")
println(" generic (decode all %,d events) best=%,9.1f ms runs=%s".format(total, genericBest, genericMs.joinToString { "%.0f".format(it) }))
println(" sqlite (index-only EXCEPT) best=%,9.1f ms runs=%s".format(sqliteBest, sqliteMs.joinToString { "%.1f".format(it) }))
println(" → sqlite is %.1f× faster at %,d events".format(genericBest / sqliteBest, total))
} finally {
store.close()
listOf("", "-wal", "-shm").forEach {
Files.deleteIfExists(
java.nio.file.Path
.of(dbFile.toAbsolutePath().toString() + it),
)
}
}
}
}
@@ -0,0 +1,103 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.store.fs
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
import com.vitorpamplona.quartz.utils.EventFactory
import com.vitorpamplona.quartz.utils.Secp256k1Instance
import kotlinx.coroutines.runBlocking
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.exists
import kotlin.test.AfterTest
import kotlin.test.BeforeTest
import kotlin.test.Test
import kotlin.test.assertEquals
/**
* `authorsMissingOutbox()` against [FsEventStore], which does NOT override the
* method so this is the ONLY coverage of the `IEventStore` interface DEFAULT
* implementation (the SQLite tests always hit the override). It pins the
* default's behaviour, and asserts it agrees with the same scenarios the SQLite
* suite checks: 10002 exclusion, NIP-09 deletion re-exposing an author, and the
* giftwrap-sender carve-out.
*/
class FsAuthorsMissingOutboxTest {
private lateinit var root: Path
private lateinit var store: FsEventStore
@BeforeTest
fun setup() {
Secp256k1Instance
root = Files.createTempDirectory("fs-missing-outbox-")
store = FsEventStore(root)
}
@AfterTest
fun tearDown() {
store.close()
if (root.exists()) {
Files.walk(root).use { s -> s.sorted(Comparator.reverseOrder()).forEach { Files.deleteIfExists(it) } }
}
}
@Test
fun defaultImplReportsOnlyAuthorsWithoutOutbox() =
runBlocking {
val withOutbox = NostrSignerSync()
val noOutbox = NostrSignerSync()
store.insert(withOutbox.sign(TextNoteEvent.build("a")))
store.insert(AdvertisedRelayListEvent.create(emptyList(), withOutbox))
store.insert(noOutbox.sign(TextNoteEvent.build("b")))
assertEquals(setOf(noOutbox.pubKey), store.authorsMissingOutbox().toSet())
}
@Test
fun defaultImplExcludesGiftWrapSenders() =
runBlocking {
val noteAuthor = NostrSignerSync()
store.insert(noteAuthor.sign(TextNoteEvent.build("hi")))
store.insert(
EventFactory.create("bb".repeat(32), "aa".repeat(32), 1L, GiftWrapEvent.KIND, emptyArray(), "", "00".repeat(64)),
)
assertEquals(setOf(noteAuthor.pubKey), store.authorsMissingOutbox().toSet())
}
@Test
fun defaultImplReExposesAuthorAfterOutboxDeleted() =
runBlocking {
val signer = NostrSignerSync()
store.insert(signer.sign(TextNoteEvent.build("content")))
val relayList = AdvertisedRelayListEvent.create(emptyList(), signer)
store.insert(relayList)
assertEquals(emptySet(), store.authorsMissingOutbox().toSet())
store.insert(signer.sign(DeletionEvent.build(listOf(relayList))))
assertEquals(setOf(signer.pubKey), store.authorsMissingOutbox().toSet())
}
}
@@ -0,0 +1,117 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip66RelayMonitor.reachability
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal
import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkType
import com.vitorpamplona.quartz.utils.Secp256k1Instance
import kotlinx.coroutines.runBlocking
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class RelayReachabilityStoreTest {
private fun store() =
EventStore(
dbName = null,
indexStrategy = DefaultIndexingStrategy(),
)
private fun cache(store: EventStore) =
RelayReachabilityStore(
store = store,
signer = NostrSignerInternal(KeyPair()),
ttlSeconds = 3600,
)
private val live1 = RelayUrlNormalizer.normalize("wss://alive.example.com")
private val live2 = RelayUrlNormalizer.normalize("wss://also-alive.example.com")
private val dead1 = RelayUrlNormalizer.normalize("wss://dead.example.com")
private val dead2 = RelayUrlNormalizer.normalize("wss://gone.example.com")
private val onion = RelayUrlNormalizer.normalize("wss://abc.onion")
private val onionPath = RelayUrlNormalizer.normalize("wss://abc.onion/npub1x")
// Contains the literal ".onion" as a substring but is NOT a Tor host — a loose
// `contains(".onion")` would misclassify it; the normalizer's isOnion must not.
private val fakeOnion = RelayUrlNormalizer.normalize("wss://relay.onionfake.com")
@Test
fun recordsAndReloadsReachability() =
runBlocking {
Secp256k1Instance
val store = store()
val cache = cache(store)
val now = 1_000_000L
cache.record(reachable = setOf(live1, live2), dead = setOf(dead1, dead2), now = now)
val snap = cache.snapshot(now = now)
assertEquals(setOf(live1, live2), snap.live)
assertEquals(setOf(dead1, dead2), snap.dead)
assertTrue(snap.isKnownDead(dead1))
assertFalse(snap.isKnownDead(live1))
}
@Test
fun aFreshSuccessfulOpenOverridesAnEarlierDeadMark() =
runBlocking {
Secp256k1Instance
val store = store()
val cache = cache(store)
// Marked dead first, then seen alive a second later (addressable replace).
cache.record(reachable = emptySet(), dead = setOf(dead1), now = 1_000L)
cache.record(reachable = setOf(dead1), dead = emptySet(), now = 1_001L)
val snap = cache.snapshot(now = 1_001L)
assertTrue(dead1 in snap.live)
assertFalse(snap.isKnownDead(dead1))
}
@Test
fun recordsOlderThanTheTtlAreIgnored() =
runBlocking {
Secp256k1Instance
val store = store()
val cache = cache(store) // ttl = 3600s
cache.record(reachable = emptySet(), dead = setOf(dead1), now = 1_000L)
// "now" is well past the 1h TTL from when dead1 was recorded.
val snap = cache.snapshot(now = 1_000L + 3601L)
assertFalse(snap.isKnownDead(dead1))
assertEquals(0, snap.size)
}
@Test
fun onionRelayIsTaggedTorNetwork() {
assertEquals(NetworkType.TOR, RelayReachabilityStore.networkTypeOf(onion))
assertEquals(NetworkType.TOR, RelayReachabilityStore.networkTypeOf(onionPath))
assertEquals(NetworkType.CLEARNET, RelayReachabilityStore.networkTypeOf(live1))
// A host that merely contains ".onion" as a substring is clearnet, not Tor.
assertEquals(NetworkType.CLEARNET, RelayReachabilityStore.networkTypeOf(fakeOnion))
}
}
@@ -0,0 +1,80 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.concurrent
import kotlin.concurrent.atomics.AtomicReference
import kotlin.concurrent.atomics.ExperimentalAtomicApi
// Copy-on-write, mirroring ConcurrentHashCache.linux: correct and simple. The
// native targets never run the crawl this backs (it is JVM/Android-only work);
// they only compile it, so the O(n)-per-write cost is irrelevant. A CAS retry
// loop gives getOrPut/merge the same atomicity the JVM actual gets for free.
@OptIn(ExperimentalAtomicApi::class)
actual class ConcurrentMap<K : Any, V : Any> {
private val ref = AtomicReference(HashMap<K, V>())
actual operator fun get(key: K): V? = ref.load()[key]
actual operator fun set(
key: K,
value: V,
) {
while (true) {
val cur = ref.load()
val copy = HashMap(cur)
copy[key] = value
if (ref.compareAndSet(cur, copy)) return
}
}
actual fun getOrPut(
key: K,
defaultValue: () -> V,
): V {
while (true) {
val cur = ref.load()
cur[key]?.let { return it }
val value = defaultValue()
val copy = HashMap(cur)
copy[key] = value
if (ref.compareAndSet(cur, copy)) return value
}
}
actual fun merge(
key: K,
value: V,
remap: (old: V, new: V) -> V,
): V {
while (true) {
val cur = ref.load()
val old = cur[key]
val merged = if (old == null) value else remap(old, value)
val copy = HashMap(cur)
copy[key] = merged
if (ref.compareAndSet(cur, copy)) return merged
}
}
actual fun size(): Int = ref.load().size
actual fun snapshot(): Map<K, V> = HashMap(ref.load())
}
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.concurrent
import kotlin.concurrent.atomics.AtomicReference
import kotlin.concurrent.atomics.ExperimentalAtomicApi
// Copy-on-write native actual — see ConcurrentMap.native for the rationale.
@OptIn(ExperimentalAtomicApi::class)
actual class ConcurrentSet<E : Any> {
private val ref = AtomicReference(HashSet<E>())
actual fun add(element: E): Boolean {
while (true) {
val cur = ref.load()
if (element in cur) return false
val copy = HashSet(cur)
copy.add(element)
if (ref.compareAndSet(cur, copy)) return true
}
}
actual operator fun contains(element: E): Boolean = element in ref.load()
actual fun size(): Int = ref.load().size
actual fun snapshot(): Set<E> = HashSet(ref.load())
}