mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-11 16:57:39 +00:00
Merge remote-tracking branch 'origin/main' into claude/armada-nip29-integration-lwqard
# Conflicts: # cli/tests/.gitignore
This commit is contained in:
+2
-3
@@ -24,8 +24,7 @@ package com.vitorpamplona.quartz.utils.secp256k1
|
||||
internal actual class ScratchLocal<T> actual constructor(
|
||||
initializer: () -> T,
|
||||
) {
|
||||
private val tl = ThreadLocal.withInitial(initializer)
|
||||
private val tl: ThreadLocal<T> = ThreadLocal.withInitial(initializer)
|
||||
|
||||
@Suppress("NULLABILITY_MISMATCH_BASED_ON_JAVA_ANNOTATIONS")
|
||||
actual fun get(): T = tl.get()
|
||||
actual fun get(): T = tl.get()!!
|
||||
}
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.experimental.graperank
|
||||
|
||||
import androidx.compose.runtime.Immutable
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.exp
|
||||
import kotlin.math.ln
|
||||
|
||||
/**
|
||||
* Tunable GrapeRank parameters. Defaults mirror the reference implementation at
|
||||
* <https://github.com/vitorpamplona/graperank> and NosFabrica's Brainstorm
|
||||
* `DEFAULT` preset.
|
||||
*/
|
||||
@Immutable
|
||||
data class GrapeRankParams(
|
||||
val attenuation: Double = 0.85,
|
||||
val rigor: Double = 0.5,
|
||||
val directFollowConfidence: Double = 0.5,
|
||||
val indirectFollowConfidence: Double = 0.03,
|
||||
val muteConfidence: Double = 0.5,
|
||||
val reportConfidence: Double = 0.5,
|
||||
val convergence: Double = 0.0001,
|
||||
)
|
||||
|
||||
/**
|
||||
* GrapeRank — a subjective, observer-centric web-of-trust score in `[0, 1]` for
|
||||
* every user reachable from an observer in a [TrustGraph]. See the algorithm
|
||||
* notes in `TrustGraph`/`GrapeRankTest`; this is the single-observer
|
||||
* Gauss-Seidel form, operating on the compact int-CSR graph so it scales to the
|
||||
* whole network.
|
||||
*
|
||||
* [compute] returns a `DoubleArray` indexed by node id (`graph.idOf(pubkey)`),
|
||||
* not a map — at millions of nodes a boxed map would dwarf the graph itself. The
|
||||
* observer's own entry stays pinned at `1.0`; callers rank the others.
|
||||
*/
|
||||
class GrapeRank(
|
||||
val params: GrapeRankParams = GrapeRankParams(),
|
||||
) {
|
||||
private val rigidity = -ln(params.rigor)
|
||||
|
||||
/** Exponential saturation turning accumulated weight into a confidence in `[0, 1)`. */
|
||||
private fun weightToConfidence(weight: Double): Double = 1.0 - exp(-weight * rigidity)
|
||||
|
||||
private fun confidence(
|
||||
relationCode: Int,
|
||||
sourceIsObserver: Boolean,
|
||||
): Double =
|
||||
when (relationCode) {
|
||||
TrustRelation.FOLLOW.code -> if (sourceIsObserver) params.directFollowConfidence else params.indirectFollowConfidence
|
||||
TrustRelation.MUTE.code -> params.muteConfidence
|
||||
else -> params.reportConfidence
|
||||
}
|
||||
|
||||
private fun rating(relationCode: Int): Double =
|
||||
when (relationCode) {
|
||||
TrustRelation.FOLLOW.code -> TrustRelation.FOLLOW.rating
|
||||
TrustRelation.MUTE.code -> TrustRelation.MUTE.rating
|
||||
else -> TrustRelation.REPORT.rating
|
||||
}
|
||||
|
||||
/**
|
||||
* Score every node reachable from [observer]. Returns scores by node id, or an
|
||||
* all-zero array if the observer isn't in the graph. [onProgress] fires once
|
||||
* per sweep with `(totalNodeUpdates, nodesStillMoving)` — the second value is
|
||||
* how many nodes moved more than [GrapeRankParams.convergence] this sweep, so
|
||||
* it trends to 0 as the graph settles.
|
||||
*
|
||||
* Iterates synchronous **Gauss-Seidel** sweeps over every node, updating scores
|
||||
* in place so a value computed earlier in a sweep is already visible to nodes
|
||||
* later in the same sweep (this converges faster than a double-buffered Jacobi
|
||||
* pass). A sweep that moves no node by more than the convergence delta ends the
|
||||
* loop — the same per-node threshold and fixed point as NosFabrica's Brainstorm
|
||||
* reference. On a dense graph this is far less total work than a
|
||||
* change-propagating worklist: a worklist re-visits a node once per rater whose
|
||||
* score nudges, so its cost scales with the in-degree of the churning core,
|
||||
* whereas a sweep touches each node exactly once per iteration. Attenuation < 1
|
||||
* makes the update a contraction, so the fixed point is unique regardless of
|
||||
* sweep order; ids run in roughly BFS order from the observer, which lets
|
||||
* trust flow outward within a single sweep and keeps the iteration count low.
|
||||
*
|
||||
* Nodes unreachable from the observer settle to 0 for free: all of their raters
|
||||
* stay at 0, so the inner loop's `sourceScore != 0.0` guard skips every edge.
|
||||
*/
|
||||
fun compute(
|
||||
graph: TrustGraph,
|
||||
observer: HexKey,
|
||||
onProgress: ((visited: Long, queued: Int) -> Unit)? = null,
|
||||
): DoubleArray {
|
||||
val n = graph.nodeCount
|
||||
val scores = DoubleArray(n)
|
||||
val observerId = graph.idOf(observer)
|
||||
if (observerId < 0) return scores
|
||||
|
||||
scores[observerId] = 1.0
|
||||
|
||||
val attenuation = params.attenuation
|
||||
val convergence = params.convergence
|
||||
val inOffsets = graph.inOffsets
|
||||
val inPacked = graph.inPacked
|
||||
|
||||
var visited = 0L
|
||||
while (true) {
|
||||
var stillMoving = 0
|
||||
var target = 0
|
||||
while (target < n) {
|
||||
if (target != observerId) {
|
||||
var sumOfWeights = 0.0
|
||||
var sumOfWeightedRatings = 0.0
|
||||
var i = inOffsets[target]
|
||||
val end = inOffsets[target + 1]
|
||||
while (i < end) {
|
||||
val packed = inPacked[i]
|
||||
val source = packed and TrustGraph.SOURCE_MASK
|
||||
val sourceScore = scores[source]
|
||||
if (sourceScore != 0.0) {
|
||||
val relationCode = packed ushr TrustGraph.SOURCE_BITS
|
||||
val weight = confidence(relationCode, source == observerId) * sourceScore * attenuation
|
||||
sumOfWeights += weight
|
||||
sumOfWeightedRatings += weight * rating(relationCode)
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
val newScore =
|
||||
if (abs(sumOfWeights) < 0.00001) {
|
||||
0.0
|
||||
} else {
|
||||
val s = weightToConfidence(sumOfWeights) * sumOfWeightedRatings / sumOfWeights
|
||||
if (s > 0.0) s else 0.0
|
||||
}
|
||||
|
||||
val oldScore = scores[target]
|
||||
if (newScore != oldScore) {
|
||||
scores[target] = newScore
|
||||
if (abs(newScore - oldScore) > convergence) stillMoving++
|
||||
}
|
||||
visited++
|
||||
}
|
||||
target++
|
||||
}
|
||||
|
||||
onProgress?.invoke(visited, stillMoving)
|
||||
if (stillMoving == 0) break
|
||||
}
|
||||
|
||||
return scores
|
||||
}
|
||||
}
|
||||
+1990
File diff suppressed because it is too large
Load Diff
+207
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.experimental.graperank
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
|
||||
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
|
||||
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.users.ContactCardEvent
|
||||
import com.vitorpamplona.quartz.nip85TrustedAssertions.users.tags.RankTag
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.awaitAll
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
|
||||
/**
|
||||
* Publishes a set of GrapeRank scores as NIP-85 kind:30382 [ContactCardEvent]
|
||||
* trusted assertions (one `rank` card per scored user), reconciled against what
|
||||
* this provider key has already published so a repeat run only writes what moved.
|
||||
*
|
||||
* Reconciliation, given the desired `(target, rank)` set the scorer produced:
|
||||
* - **skip** a target whose stored card already carries the same rank string —
|
||||
* re-signing an unchanged card would churn a new event id for no client benefit;
|
||||
* - **upsert** a target whose rank changed (or that has no card yet), up to a
|
||||
* publish limit;
|
||||
* - **retract** every stored card whose target is no longer in the desired set
|
||||
* (it fell below the caller's cutoff, or dropped out of the graph) with a NIP-09
|
||||
* kind:5 deletion, batched so the frame stays under the ~64KB event cap.
|
||||
*
|
||||
* Transport-agnostic like [GrapeRankCrawler]: it reads prior cards from an
|
||||
* [IEventStore] and emits through an injected [publish] function (event + relays →
|
||||
* per-relay ack), so the store/relay wiring stays in the application while the
|
||||
* reconcile + card-construction logic is reusable (e.g. by the Android app).
|
||||
*/
|
||||
class GrapeRankPublisher(
|
||||
private val store: IEventStore,
|
||||
private val publish: suspend (Event, Set<NormalizedRelayUrl>) -> Map<NormalizedRelayUrl, Boolean>,
|
||||
) {
|
||||
/** Outcome counts for one reconcile: what was written, retracted, and skipped. */
|
||||
class Result(
|
||||
val published: Int,
|
||||
val publishRejected: Int,
|
||||
val deleted: Int,
|
||||
val deleteRejected: Int,
|
||||
val skippedUnchanged: Int,
|
||||
/** Changed cards beyond [publishLimit] that were not upserted this run. */
|
||||
val truncated: Int,
|
||||
)
|
||||
|
||||
/**
|
||||
* Reconcile the desired [scored] `(target, rank)` set (the caller has already
|
||||
* applied any rank cutoff) against the cards [providerPubkey] previously
|
||||
* published, then upsert the changes and retract the stale cards, all signed by
|
||||
* [providerSigner]. At most [publishLimit] changed cards are upserted per run.
|
||||
*/
|
||||
suspend fun reconcileAndPublish(
|
||||
providerSigner: NostrSigner,
|
||||
providerPubkey: HexKey,
|
||||
scored: List<Pair<HexKey, Int>>,
|
||||
relays: Set<NormalizedRelayUrl>,
|
||||
publishLimit: Int,
|
||||
publishConcurrency: Int = PUBLISH_CONCURRENCY,
|
||||
): Result {
|
||||
// Newest card per target this provider already published (read back from
|
||||
// the store, which every published card was persisted to).
|
||||
val existing = existingCards(providerPubkey)
|
||||
val publishableTargets = scored.mapTo(HashSet()) { it.first }
|
||||
|
||||
// Upsert publishable targets whose rank tag STRING would change (or that
|
||||
// have no card yet). RankTag.assemble writes rank.toString(), so we diff
|
||||
// that exact string — an unchanged score is skipped so clients only sync
|
||||
// ranks that moved.
|
||||
val changed = scored.filter { (target, rank) -> existing[target]?.let(::rankTagValue) != rank.toString() }
|
||||
val toUpsert = changed.take(publishLimit)
|
||||
|
||||
// Retract existing cards whose target is no longer publishable — it dropped
|
||||
// out of the graph, or fell below the caller's cutoff. We won't leave a
|
||||
// stale assertion standing.
|
||||
val toDelete = existing.filterKeys { it !in publishableTargets }.values.toList()
|
||||
|
||||
val (ok, rejected) = publishCards(providerSigner, toUpsert, relays, publishConcurrency)
|
||||
val (deleted, deleteRejected) = publishDeletions(providerSigner, toDelete, relays)
|
||||
|
||||
return Result(
|
||||
published = ok,
|
||||
publishRejected = rejected,
|
||||
deleted = deleted,
|
||||
deleteRejected = deleteRejected,
|
||||
skippedUnchanged = scored.size - changed.size,
|
||||
truncated = (changed.size - toUpsert.size).coerceAtLeast(0),
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The newest kind:30382 card [providerPubkey] published per target, read from
|
||||
* the store (every card [publish] sends is persisted first, so on repeat runs
|
||||
* this reflects what is already out there).
|
||||
*/
|
||||
private suspend fun existingCards(providerPubkey: HexKey): Map<HexKey, ContactCardEvent> =
|
||||
store
|
||||
.query<Event>(Filter(kinds = listOf(ContactCardEvent.KIND), authors = listOf(providerPubkey)))
|
||||
.filterIsInstance<ContactCardEvent>()
|
||||
.groupBy { it.aboutUser() }
|
||||
.mapNotNull { (target, cards) ->
|
||||
val t = target.ifBlank { return@mapNotNull null }
|
||||
t to (cards.maxByOrNull { it.createdAt } ?: return@mapNotNull null)
|
||||
}.toMap()
|
||||
|
||||
/**
|
||||
* The raw `rank` tag value string on a card — exactly what a client diffs, so an
|
||||
* unchanged score never produces a new signature. Our cards carry only a `rank`
|
||||
* tag (plus the d-tag target), so this one value decides whether a re-publish
|
||||
* would differ.
|
||||
*/
|
||||
private fun rankTagValue(card: ContactCardEvent): String? =
|
||||
card.tags.firstNotNullOfOrNull { tag ->
|
||||
if (tag.size > 1 && tag[0] == RankTag.TAG_NAME) tag[1] else null
|
||||
}
|
||||
|
||||
/** Build + publish one kind:30382 card per (target, rank), bounded-concurrently. */
|
||||
private suspend fun publishCards(
|
||||
signer: NostrSigner,
|
||||
cards: List<Pair<HexKey, Int>>,
|
||||
relays: Set<NormalizedRelayUrl>,
|
||||
concurrency: Int,
|
||||
): Pair<Int, Int> {
|
||||
var published = 0
|
||||
var rejected = 0
|
||||
for (batch in cards.chunked(concurrency)) {
|
||||
val acks =
|
||||
coroutineScope {
|
||||
batch
|
||||
.map { (pubkey, rank) ->
|
||||
async {
|
||||
val card =
|
||||
ContactCardEvent.create(
|
||||
targetUser = pubkey,
|
||||
signer = signer,
|
||||
publicInitializer = { add(RankTag.assemble(rank)) },
|
||||
)
|
||||
publish(card, relays)
|
||||
}
|
||||
}.awaitAll()
|
||||
}
|
||||
for (ack in acks) {
|
||||
if (ack.values.any { it }) published++ else rejected++
|
||||
}
|
||||
}
|
||||
return published to rejected
|
||||
}
|
||||
|
||||
/**
|
||||
* Retract stale cards with NIP-09 kind:5 deletions signed by [signer] (the same
|
||||
* key that signed the cards). Batches [DELETE_PER_EVENT] addressable coordinates
|
||||
* per deletion so the kind:5 frame stays under the ~64KB event cap; each carries
|
||||
* the card's `a` tag (30382:provider:target), so re-publishing a newer version
|
||||
* later isn't blocked. Returns (deleted, rejected) card counts.
|
||||
*/
|
||||
private suspend fun publishDeletions(
|
||||
signer: NostrSigner,
|
||||
cards: List<ContactCardEvent>,
|
||||
relays: Set<NormalizedRelayUrl>,
|
||||
): Pair<Int, Int> {
|
||||
if (cards.isEmpty()) return 0 to 0
|
||||
var deleted = 0
|
||||
var rejected = 0
|
||||
for (chunk in cards.chunked(DELETE_PER_EVENT)) {
|
||||
val event = signer.sign(DeletionEvent.build(chunk))
|
||||
val ack = publish(event, relays)
|
||||
if (ack.values.any { it }) deleted += chunk.size else rejected += chunk.size
|
||||
}
|
||||
return deleted to rejected
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** Concurrent card publishes when upserting. */
|
||||
const val PUBLISH_CONCURRENCY = 16
|
||||
|
||||
/**
|
||||
* Addressable coordinates cited per kind:5 retraction. Each `a` tag is
|
||||
* ~130 bytes (30382:<64hex>:<64hex>), so 400 keeps the whole event ~52KB —
|
||||
* under the 64KB event-size cap many relays enforce (stricter than the
|
||||
* 256KB message cap).
|
||||
*/
|
||||
const val DELETE_PER_EVENT = 400
|
||||
}
|
||||
}
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.experimental.graperank
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.NegentropyStoreSync
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
|
||||
import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent
|
||||
import com.vitorpamplona.quartz.nip56Reports.ReportEvent
|
||||
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
|
||||
|
||||
/**
|
||||
* Store-driven, outbox-model refresh of the record kinds a [GrapeRank] score is a
|
||||
* function of — the profiles (kind:0), follows (kind:3), outbox relay lists
|
||||
* (kind:10002), and reports (kind:1984) of every author already known to the
|
||||
* local [store].
|
||||
*
|
||||
* Where [GrapeRankCrawler] discovers the graph by walking follows outward from
|
||||
* an observer, this refreshes what is *already* known: it reads every kind:10002 in
|
||||
* the store, inverts them into a `write-relay -> authors` map (the outbox model — an
|
||||
* author's events live on the relays they write to), fans those into one filter per
|
||||
* `(write relay, author chunk)`, and hands them to [NegentropyStoreSync] — the generic
|
||||
* two-pass sync engine. So each relay is asked only for the authors it hosts, and each
|
||||
* author is reconciled only against their own relays. Run it periodically to keep a
|
||||
* scored network current without paying a full from-scratch crawl.
|
||||
*
|
||||
* The engine does the work per `(relay, filter)` group: a bidirectional NIP-77
|
||||
* reconcile against [store] ([Config.down] / [Config.up]), a deletion settle over the
|
||||
* residual ([Config.syncDeletions] — its applyDown direction downloads the relay's
|
||||
* kind:5 when an uploaded record was rejected because the author retracted it), and a
|
||||
* paged-download fallback when a relay can't reconcile ([Config.pageFallback]). This
|
||||
* class only builds the outbox filter set and folds the engine's per-group results back
|
||||
* up per relay.
|
||||
*
|
||||
* Transport-agnostic within quartz: it takes an [INostrClient] and an [IEventStore].
|
||||
* Progress is emitted through [log]; a headless caller routes it to stderr, a UI ignores it.
|
||||
*/
|
||||
class GrapeRankUpdater(
|
||||
private val client: INostrClient,
|
||||
private val store: IEventStore,
|
||||
private val config: Config = Config(),
|
||||
private val log: (String) -> Unit = {},
|
||||
) {
|
||||
/**
|
||||
* @param kinds the record kinds refreshed per author (default: the WoT set
|
||||
* 0 / 3 / 10002 / 1984).
|
||||
* @param down download records the relay has that the store lacks.
|
||||
* @param up upload records the store has that the relay lacks (also arms the
|
||||
* deletion **applyDown** path — a rejected upload pulls the relay's kind:5 down).
|
||||
* @param syncDeletions run the deletion settle over the reconcile residual.
|
||||
* @param pageFallback page the filter when negentropy can't reconcile a relay.
|
||||
* @param idChunk ids per reconcile chunk and per by-id fetch.
|
||||
* @param downloadWorkers concurrent by-id download fetches per group.
|
||||
* @param reconcileConcurrency overlapped `created_at`-window reconciles after an over-cap split.
|
||||
* @param maxDeletionRounds hard cap on deletion-settle rounds (converges in 1–2).
|
||||
* @param relayConcurrency write relays synced at once.
|
||||
* @param authorChunk authors per reconcile filter (a relay with more is split into several).
|
||||
* @param minAuthors skip relays hosting fewer than this many of the store's authors.
|
||||
* @param idleTimeoutMs idle watchdog for reconciles / fetches / pages.
|
||||
* @param publishTimeoutSecs OK-confirmation wait per uploaded event.
|
||||
*/
|
||||
class Config(
|
||||
val kinds: List<Int> = DEFAULT_KINDS,
|
||||
val down: Boolean = true,
|
||||
val up: Boolean = true,
|
||||
val syncDeletions: Boolean = true,
|
||||
val pageFallback: Boolean = true,
|
||||
val idChunk: Int = 500,
|
||||
val downloadWorkers: Int = 4,
|
||||
val reconcileConcurrency: Int = 2,
|
||||
val maxDeletionRounds: Int = 4,
|
||||
val relayConcurrency: Int = 4,
|
||||
val authorChunk: Int = 500,
|
||||
val minAuthors: Int = 1,
|
||||
val idleTimeoutMs: Long = 30_000L,
|
||||
val publishTimeoutSecs: Long = 15,
|
||||
/**
|
||||
* Relays proven unreachable within the reachability cache's TTL (kind:30166).
|
||||
* Skipped from the reconcile plan so we don't burn a connect timeout per dead
|
||||
* relay — the crawl already found them dead, and a dead relay cannot serve its
|
||||
* authors anyway. TTL'd, so a recovered relay is retried once the record ages
|
||||
* out; this never drops a *live* author-advertised relay. See RelayReachabilityStore.
|
||||
*/
|
||||
val knownDead: Set<NormalizedRelayUrl> = emptySet(),
|
||||
) {
|
||||
/** Project the shared engine knobs onto a [NegentropyStoreSync.Config]. */
|
||||
internal fun toEngineConfig() =
|
||||
NegentropyStoreSync.Config(
|
||||
down = down,
|
||||
up = up,
|
||||
syncDeletions = syncDeletions,
|
||||
pageFallback = pageFallback,
|
||||
idChunk = idChunk,
|
||||
downloadWorkers = downloadWorkers,
|
||||
reconcileConcurrency = reconcileConcurrency,
|
||||
maxDeletionRounds = maxDeletionRounds,
|
||||
concurrency = relayConcurrency,
|
||||
idleTimeoutMs = idleTimeoutMs,
|
||||
publishTimeoutSecs = publishTimeoutSecs,
|
||||
)
|
||||
}
|
||||
|
||||
/** Per-write-relay outcome of an [update] (folded from the engine's group results). */
|
||||
class RelayResult(
|
||||
val relay: NormalizedRelayUrl,
|
||||
val authors: Int,
|
||||
val need: Int,
|
||||
val have: Int,
|
||||
val downloaded: Int,
|
||||
val uploaded: Int,
|
||||
val deletionsSentUp: Int,
|
||||
val deletionsAppliedDown: Int,
|
||||
val pagedFallback: Boolean,
|
||||
/** null when every chunk of this relay succeeded; the first failure otherwise. */
|
||||
val error: String?,
|
||||
)
|
||||
|
||||
/** Aggregate outcome of an [update], plus the per-relay breakdown. */
|
||||
class Result(
|
||||
val relayListsInStore: Int,
|
||||
val authorsWithOutbox: Int,
|
||||
val relays: Int,
|
||||
val relaysOk: Int,
|
||||
val relaysFailed: Int,
|
||||
val relaysPagedFallback: Int,
|
||||
val downloaded: Int,
|
||||
val uploaded: Int,
|
||||
val deletionsSentUp: Int,
|
||||
val deletionsAppliedDown: Int,
|
||||
val perRelay: List<RelayResult>,
|
||||
)
|
||||
|
||||
/**
|
||||
* Group the store's authors by their kind:10002 write relays (the outbox model).
|
||||
* The latest kind:10002 per author wins; an author with no write-marked relays
|
||||
* contributes nothing (there is nowhere to reconcile them). Public so callers can
|
||||
* inspect the plan (relay count, largest groups) before running [update].
|
||||
*/
|
||||
suspend fun writeRelayGroups(): Map<NormalizedRelayUrl, Set<HexKey>> = groupByWriteRelay(loadLatestRelayLists())
|
||||
|
||||
/**
|
||||
* Run the full outbox-model refresh: [writeRelayGroups] then hand every group
|
||||
* hosting at least [Config.minAuthors] authors (largest first, chunked to
|
||||
* [Config.authorChunk]) to [NegentropyStoreSync], folding its per-group results back
|
||||
* up per relay. Best-effort — a relay that fails is recorded in [RelayResult.error]
|
||||
* and never aborts the run.
|
||||
*/
|
||||
suspend fun update(): Result {
|
||||
val latest = loadLatestRelayLists()
|
||||
val groups = groupByWriteRelay(latest)
|
||||
val authorsWithOutbox = groups.values.flatMapTo(HashSet()) { it }.size
|
||||
|
||||
// Plan: relays with enough authors, largest first so the heaviest groups start
|
||||
// while engine permits are free.
|
||||
val plan =
|
||||
groups.entries
|
||||
.filter { it.value.size >= config.minAuthors }
|
||||
.filterNot { it.key in config.knownDead }
|
||||
.sortedByDescending { it.value.size }
|
||||
.map { it.key to it.value }
|
||||
|
||||
// Fan each relay's authors into one filter per authorChunk-sized slice.
|
||||
val authorChunk = config.authorChunk.coerceAtLeast(1)
|
||||
val filtersByRelay =
|
||||
plan.associate { (relay, authors) ->
|
||||
relay to authors.toList().chunked(authorChunk).map { Filter(kinds = config.kinds, authors = it) }
|
||||
}
|
||||
|
||||
val groupResults = NegentropyStoreSync(client, store, config.toEngineConfig(), log).sync(filtersByRelay)
|
||||
val byRelay = groupResults.groupBy { it.relay }
|
||||
|
||||
// Fold each relay's chunk results back into one RelayResult (plan order preserved).
|
||||
val perRelay =
|
||||
plan.map { (relay, authors) ->
|
||||
val chunks = byRelay[relay].orEmpty()
|
||||
RelayResult(
|
||||
relay = relay,
|
||||
authors = authors.size,
|
||||
need = chunks.sumOf { it.need },
|
||||
have = chunks.sumOf { it.have },
|
||||
downloaded = chunks.sumOf { it.downloaded },
|
||||
uploaded = chunks.sumOf { it.uploaded },
|
||||
deletionsSentUp = chunks.sumOf { it.deletionsSentUp },
|
||||
deletionsAppliedDown = chunks.sumOf { it.deletionsAppliedDown },
|
||||
pagedFallback = chunks.any { it.pagedFallback },
|
||||
error = chunks.firstNotNullOfOrNull { it.error },
|
||||
)
|
||||
}
|
||||
|
||||
return Result(
|
||||
relayListsInStore = latest.size,
|
||||
authorsWithOutbox = authorsWithOutbox,
|
||||
relays = perRelay.size,
|
||||
relaysOk = perRelay.count { it.error == null },
|
||||
relaysFailed = perRelay.count { it.error != null },
|
||||
relaysPagedFallback = perRelay.count { it.pagedFallback },
|
||||
downloaded = perRelay.sumOf { it.downloaded },
|
||||
uploaded = perRelay.sumOf { it.uploaded },
|
||||
deletionsSentUp = perRelay.sumOf { it.deletionsSentUp },
|
||||
deletionsAppliedDown = perRelay.sumOf { it.deletionsAppliedDown },
|
||||
perRelay = perRelay,
|
||||
)
|
||||
}
|
||||
|
||||
/** Latest kind:10002 per author from the store (replaceable — newest createdAt wins). */
|
||||
private suspend fun loadLatestRelayLists(): Map<HexKey, AdvertisedRelayListEvent> {
|
||||
val latest = HashMap<HexKey, AdvertisedRelayListEvent>()
|
||||
for (event in store.query<Event>(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND)))) {
|
||||
if (event !is AdvertisedRelayListEvent) continue
|
||||
val prev = latest[event.pubKey]
|
||||
if (prev == null || event.createdAt > prev.createdAt) latest[event.pubKey] = event
|
||||
}
|
||||
return latest
|
||||
}
|
||||
|
||||
/** Invert the per-author relay lists into `write-relay -> authors`. */
|
||||
private fun groupByWriteRelay(latest: Map<HexKey, AdvertisedRelayListEvent>): Map<NormalizedRelayUrl, Set<HexKey>> {
|
||||
val relayToAuthors = HashMap<NormalizedRelayUrl, MutableSet<HexKey>>()
|
||||
for ((author, list) in latest) {
|
||||
val writes = list.writeRelaysNorm() ?: continue
|
||||
for (relay in writes) relayToAuthors.getOrPut(relay) { HashSet() }.add(author)
|
||||
}
|
||||
return relayToAuthors
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** The record kinds a GrapeRank score is a function of. */
|
||||
val DEFAULT_KINDS =
|
||||
listOf(
|
||||
MetadataEvent.KIND, // 0 — profiles
|
||||
ContactListEvent.KIND, // 3 — follows
|
||||
AdvertisedRelayListEvent.KIND, // 10002 — outbox relay lists
|
||||
ReportEvent.KIND, // 1984 — reports
|
||||
)
|
||||
}
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.experimental.graperank
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
|
||||
/**
|
||||
* A trust relationship kind and the GrapeRank rating it carries. [code] is the
|
||||
* 2-bit tag packed alongside a source node id in the edge arrays (see
|
||||
* [TrustGraph]); keep it in `0..3`.
|
||||
*/
|
||||
enum class TrustRelation(
|
||||
val rating: Double,
|
||||
val code: Int,
|
||||
) {
|
||||
FOLLOW(1.0, 0),
|
||||
MUTE(-0.1, 1),
|
||||
REPORT(-0.1, 2),
|
||||
}
|
||||
|
||||
/**
|
||||
* A web-of-trust graph over Nostr pubkeys, stored compactly so it scales to the
|
||||
* whole network (millions of edges) without a `String`-keyed edge object per
|
||||
* relationship.
|
||||
*
|
||||
* Pubkeys are interned to dense `Int` node ids. Edges live in two
|
||||
* compressed-sparse-row (CSR) layouts backed by flat `IntArray`s — one indexed
|
||||
* by target (what [GrapeRank] reads to score a node) and one by source (what the
|
||||
* propagation worklist follows). Each incoming entry packs the source id in the
|
||||
* low 29 bits and the [TrustRelation.code] in the top bits, so an edge is a
|
||||
* single `int`. A 100M-edge graph is then ~0.8 GB of primitive arrays instead of
|
||||
* tens of GB of objects.
|
||||
*
|
||||
* Build one with [TrustGraphBuilder], feeding contact lists / mutes / reports in
|
||||
* as they stream off the relays.
|
||||
*/
|
||||
class TrustGraph internal constructor(
|
||||
val nodeCount: Int,
|
||||
private val pubkeys: Array<HexKey>,
|
||||
private val ids: HashMap<HexKey, Int>,
|
||||
// CSR by target: incoming edges of node t are inPacked[inOffsets[t] until inOffsets[t+1]],
|
||||
// each packing source id (low 29 bits) + relation code (top bits).
|
||||
internal val inOffsets: IntArray,
|
||||
internal val inPacked: IntArray,
|
||||
// CSR by source: out-neighbour targets of node s are outTargets[outOffsets[s] until outOffsets[s+1]].
|
||||
internal val outOffsets: IntArray,
|
||||
internal val outTargets: IntArray,
|
||||
) {
|
||||
/** Node id for [pubkey], or `-1` if it never appeared in the graph. */
|
||||
fun idOf(pubkey: HexKey): Int = ids[pubkey] ?: -1
|
||||
|
||||
/** Pubkey for a node [id]. */
|
||||
fun pubkeyOf(id: Int): HexKey = pubkeys[id]
|
||||
|
||||
fun edgeCount(): Int = inPacked.size
|
||||
|
||||
companion object {
|
||||
const val SOURCE_BITS = 29
|
||||
const val SOURCE_MASK = (1 shl SOURCE_BITS) - 1
|
||||
const val MAX_NODES = SOURCE_MASK // ids must fit in the low 29 bits
|
||||
}
|
||||
}
|
||||
|
||||
/** A minimal growable `int[]` — avoids boxing `Int`s in an `ArrayList` at graph scale. */
|
||||
internal class IntArrayList(
|
||||
initialCapacity: Int = 16,
|
||||
) {
|
||||
var data: IntArray = IntArray(initialCapacity.coerceAtLeast(1))
|
||||
private set
|
||||
var size: Int = 0
|
||||
private set
|
||||
|
||||
fun add(value: Int) {
|
||||
if (size == data.size) data = data.copyOf(data.size * 2)
|
||||
data[size++] = value
|
||||
}
|
||||
|
||||
fun get(index: Int): Int = data[index]
|
||||
|
||||
fun removeLast(): Int = data[--size]
|
||||
|
||||
fun isNotEmpty(): Boolean = size > 0
|
||||
}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.experimental.graperank
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
|
||||
/**
|
||||
* Builds a [TrustGraph] incrementally so callers never have to hold every contact
|
||||
* list in memory at once — feed each user's follows / mutes / reports as they
|
||||
* stream off the relays (or out of the store), then call [build].
|
||||
*
|
||||
* Interns pubkeys to dense ids on the fly and accumulates edges in flat growable
|
||||
* int arrays. Follows and mutes are replaceable (one list per author, deduped by
|
||||
* the caller via latest-per-author + set-valued tags); reports are regular events,
|
||||
* so `(reporter → reported)` report edges are deduped here. Self-edges are dropped.
|
||||
*/
|
||||
class TrustGraphBuilder {
|
||||
private val ids = HashMap<HexKey, Int>()
|
||||
private val pubkeys = ArrayList<HexKey>()
|
||||
|
||||
// Parallel edge arrays: edge i is source edgeSource[i] --relation--> edgeTarget[i],
|
||||
// with the relation packed into the top bits of edgeSource[i].
|
||||
private val edgeTargets = IntArrayList()
|
||||
private val edgeSourcesPacked = IntArrayList()
|
||||
|
||||
// Dedup for report edges only (reporters can file many kind:1984 for one target).
|
||||
private val reportSeen = HashSet<Long>()
|
||||
|
||||
private fun intern(pubkey: HexKey): Int =
|
||||
ids.getOrPut(pubkey) {
|
||||
val id = pubkeys.size
|
||||
pubkeys.add(pubkey)
|
||||
id
|
||||
}
|
||||
|
||||
private fun addEdge(
|
||||
source: HexKey,
|
||||
target: HexKey,
|
||||
relation: TrustRelation,
|
||||
) {
|
||||
if (source == target) return
|
||||
val s = intern(source)
|
||||
val t = intern(target)
|
||||
if (relation == TrustRelation.REPORT) {
|
||||
val key = (s.toLong() shl 32) or (t.toLong() and 0xFFFFFFFFL)
|
||||
if (!reportSeen.add(key)) return
|
||||
}
|
||||
edgeTargets.add(t)
|
||||
edgeSourcesPacked.add(s or (relation.code shl TrustGraph.SOURCE_BITS))
|
||||
}
|
||||
|
||||
fun addFollows(
|
||||
source: HexKey,
|
||||
follows: Iterable<HexKey>,
|
||||
) {
|
||||
for (target in follows) addEdge(source, target, TrustRelation.FOLLOW)
|
||||
}
|
||||
|
||||
fun addMutes(
|
||||
source: HexKey,
|
||||
muted: Iterable<HexKey>,
|
||||
) {
|
||||
for (target in muted) addEdge(source, target, TrustRelation.MUTE)
|
||||
}
|
||||
|
||||
fun addReports(
|
||||
source: HexKey,
|
||||
reported: Iterable<HexKey>,
|
||||
) {
|
||||
for (target in reported) addEdge(source, target, TrustRelation.REPORT)
|
||||
}
|
||||
|
||||
fun nodeCount(): Int = pubkeys.size
|
||||
|
||||
fun edgeCount(): Int = edgeTargets.size
|
||||
|
||||
/** Freeze the accumulated edges into the two CSR layouts. */
|
||||
fun build(): TrustGraph {
|
||||
val n = pubkeys.size
|
||||
val m = edgeTargets.size
|
||||
|
||||
// Incoming CSR (by target).
|
||||
val inOffsets = IntArray(n + 1)
|
||||
for (i in 0 until m) inOffsets[edgeTargets.get(i) + 1]++
|
||||
for (i in 1..n) inOffsets[i] += inOffsets[i - 1]
|
||||
val inPacked = IntArray(m)
|
||||
val inCursor = inOffsets.copyOf()
|
||||
for (i in 0 until m) {
|
||||
val t = edgeTargets.get(i)
|
||||
inPacked[inCursor[t]++] = edgeSourcesPacked.get(i)
|
||||
}
|
||||
|
||||
// Outgoing CSR (by source).
|
||||
val outOffsets = IntArray(n + 1)
|
||||
for (i in 0 until m) {
|
||||
val s = edgeSourcesPacked.get(i) and TrustGraph.SOURCE_MASK
|
||||
outOffsets[s + 1]++
|
||||
}
|
||||
for (i in 1..n) outOffsets[i] += outOffsets[i - 1]
|
||||
val outTargets = IntArray(m)
|
||||
val outCursor = outOffsets.copyOf()
|
||||
for (i in 0 until m) {
|
||||
val s = edgeSourcesPacked.get(i) and TrustGraph.SOURCE_MASK
|
||||
outTargets[outCursor[s]++] = edgeTargets.get(i)
|
||||
}
|
||||
|
||||
return TrustGraph(n, pubkeys.toTypedArray(), ids, inOffsets, inPacked, outOffsets, outTargets)
|
||||
}
|
||||
}
|
||||
@@ -327,6 +327,7 @@ data class KindName(
|
||||
* platform concern layered on top, never a fork of this data.
|
||||
*/
|
||||
object KindNames {
|
||||
@Suppress("DEPRECATION") // registry intentionally names deprecated kinds (GitReply, TorrentComment) for display
|
||||
val names: Map<Int, KindName> =
|
||||
mapOf(
|
||||
AcceptedBadgeSetEvent.KIND to KindName("Accepted Badge Set", "58"),
|
||||
|
||||
@@ -215,6 +215,10 @@ class MlsGroup private constructor(
|
||||
encryptionPrivateKey = encryptionPrivateKey,
|
||||
interimTranscriptHash = interimTranscriptHash,
|
||||
encryptionSecret = epochSecrets.encryptionSecret,
|
||||
// Preserve the SecretTree ratchet positions so a restore doesn't
|
||||
// rewind our own generation counter to 0 and reuse an AEAD
|
||||
// key+nonce within this epoch (RFC 9420 §9).
|
||||
senderRatchetStates = secretTree.exportSenderStates(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3501,15 +3505,23 @@ class MlsGroup private constructor(
|
||||
/**
|
||||
* Restore a group from a previously saved [MlsGroupState].
|
||||
*
|
||||
* The SecretTree is reconstructed from the stored encryption_secret.
|
||||
* Note: SecretTree ratchet state (per-sender generation counters) is
|
||||
* NOT preserved — messages sent/received before the save point cannot
|
||||
* be re-decrypted, which is acceptable because they would already
|
||||
* have been processed.
|
||||
* The SecretTree is reconstructed from the stored encryption_secret,
|
||||
* then seeded with the persisted per-sender ratchet positions
|
||||
* ([MlsGroupState.senderRatchetStates]). Seeding is what keeps the
|
||||
* local member's generation counter monotonic across a restart — a
|
||||
* fresh SecretTree would restart every sender at generation 0, so our
|
||||
* next send would reuse generation 0's AEAD key+nonce within the same
|
||||
* epoch and be rejected by strict receivers (openmls / MDK /
|
||||
* Whitenoise) that forbid generation reuse.
|
||||
*
|
||||
* Receive-only ratchets that weren't persisted (STATE_VERSION 1 blobs,
|
||||
* or senders we never decrypted) simply re-derive from generation 0 on
|
||||
* first use — safe, because those messages were already processed.
|
||||
*/
|
||||
fun restore(state: MlsGroupState): MlsGroup {
|
||||
val tree = RatchetTree.decodeTls(TlsReader(state.treeBytes))
|
||||
val secretTree = SecretTree(state.encryptionSecret, tree.leafCount)
|
||||
secretTree.importSenderStates(state.senderRatchetStates)
|
||||
|
||||
return MlsGroup(
|
||||
groupContext = state.groupContext,
|
||||
|
||||
+11
-1
@@ -348,13 +348,23 @@ class MlsGroupManager(
|
||||
/**
|
||||
* Encrypt an application message.
|
||||
* Synchronized to prevent nonce reuse from concurrent encryption.
|
||||
*
|
||||
* The group state is persisted after every send. Encrypting advances the
|
||||
* SecretTree ratchet (RFC 9420 §9) but does not change the epoch, so
|
||||
* without this save a restart between two messages would reload the
|
||||
* pre-send ratchet position and re-emit an already-used generation —
|
||||
* reusing the AEAD key+nonce and getting rejected by strict receivers.
|
||||
* State was previously persisted only at commits, which left every
|
||||
* inter-commit send unprotected.
|
||||
*/
|
||||
suspend fun encrypt(
|
||||
nostrGroupId: HexKey,
|
||||
plaintext: ByteArray,
|
||||
): ByteArray =
|
||||
mutex.withLock {
|
||||
requireGroup(nostrGroupId).encrypt(plaintext)
|
||||
val ciphertext = requireGroup(nostrGroupId).encrypt(plaintext)
|
||||
persistGroup(nostrGroupId)
|
||||
ciphertext
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+56
-2
@@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.marmot.mls.codec.TlsReader
|
||||
import com.vitorpamplona.quartz.marmot.mls.codec.TlsWriter
|
||||
import com.vitorpamplona.quartz.marmot.mls.messages.GroupContext
|
||||
import com.vitorpamplona.quartz.marmot.mls.schedule.EpochSecrets
|
||||
import com.vitorpamplona.quartz.marmot.mls.schedule.SenderRatchetState
|
||||
|
||||
/**
|
||||
* Serializable snapshot of an MLS group's complete state.
|
||||
@@ -40,6 +41,13 @@ import com.vitorpamplona.quartz.marmot.mls.schedule.EpochSecrets
|
||||
*
|
||||
* Security: This blob contains secret key material (signing key, encryption key,
|
||||
* epoch secrets). It MUST be stored in encrypted local storage.
|
||||
*
|
||||
* [senderRatchetStates] carries each sender's live SecretTree ratchet position
|
||||
* (RFC 9420 §9). Preserving it is what stops the restored local member from
|
||||
* re-emitting an already-used generation within the same epoch — see
|
||||
* [com.vitorpamplona.quartz.marmot.mls.schedule.SecretTree.exportSenderStates].
|
||||
* It is optional (empty for STATE_VERSION 1 blobs) so older persisted state
|
||||
* still decodes.
|
||||
*/
|
||||
data class MlsGroupState(
|
||||
val groupContext: GroupContext,
|
||||
@@ -51,6 +59,7 @@ data class MlsGroupState(
|
||||
val encryptionPrivateKey: ByteArray,
|
||||
val interimTranscriptHash: ByteArray,
|
||||
val encryptionSecret: ByteArray,
|
||||
val senderRatchetStates: Map<Int, SenderRatchetState> = emptyMap(),
|
||||
) {
|
||||
fun encodeTls(): ByteArray {
|
||||
val writer = TlsWriter()
|
||||
@@ -94,6 +103,18 @@ data class MlsGroupState(
|
||||
// Encryption secret for SecretTree reconstruction
|
||||
writer.putOpaqueVarInt(encryptionSecret)
|
||||
|
||||
// Per-sender SecretTree ratchet positions (STATE_VERSION 2+).
|
||||
// Preserving the local sender's generation counter is what prevents
|
||||
// AEAD key+nonce reuse (and strict-receiver rejection) after a restore.
|
||||
writer.putUint32(senderRatchetStates.size.toLong())
|
||||
for ((leafIndex, ratchet) in senderRatchetStates) {
|
||||
writer.putUint32(leafIndex.toLong())
|
||||
writer.putOpaqueVarInt(ratchet.handshakeSecret)
|
||||
writer.putUint32(ratchet.handshakeGeneration.toLong())
|
||||
writer.putOpaqueVarInt(ratchet.applicationSecret)
|
||||
writer.putUint32(ratchet.applicationGeneration.toLong())
|
||||
}
|
||||
|
||||
return writer.toByteArray()
|
||||
}
|
||||
|
||||
@@ -110,13 +131,18 @@ data class MlsGroupState(
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val STATE_VERSION = 1
|
||||
/**
|
||||
* v1: original layout (no SecretTree ratchet positions).
|
||||
* v2: appends [senderRatchetStates] so restores don't reset the
|
||||
* ratchet to generation 0. v1 blobs still decode (empty map).
|
||||
*/
|
||||
private const val STATE_VERSION = 2
|
||||
|
||||
fun decodeTls(data: ByteArray): MlsGroupState {
|
||||
val reader = TlsReader(data)
|
||||
|
||||
val version = reader.readUint16()
|
||||
require(version == STATE_VERSION) { "Unsupported state version: $version" }
|
||||
require(version in 1..STATE_VERSION) { "Unsupported state version: $version" }
|
||||
|
||||
val groupContext = GroupContext.decodeTls(reader)
|
||||
val treeBytes = reader.readOpaqueVarInt()
|
||||
@@ -144,6 +170,33 @@ data class MlsGroupState(
|
||||
val interimTranscriptHash = reader.readOpaqueVarInt()
|
||||
val encryptionSecret = reader.readOpaqueVarInt()
|
||||
|
||||
// v2+: per-sender SecretTree ratchet positions. Absent (or an
|
||||
// empty count) for v1 blobs, which restore at generation 0.
|
||||
val senderRatchetStates =
|
||||
if (version >= 2 && reader.hasRemaining) {
|
||||
val count = reader.readUint32().toInt()
|
||||
buildMap {
|
||||
repeat(count) {
|
||||
val leafIndex = reader.readUint32().toInt()
|
||||
val handshakeSecret = reader.readOpaqueVarInt()
|
||||
val handshakeGeneration = reader.readUint32().toInt()
|
||||
val applicationSecret = reader.readOpaqueVarInt()
|
||||
val applicationGeneration = reader.readUint32().toInt()
|
||||
put(
|
||||
leafIndex,
|
||||
SenderRatchetState(
|
||||
handshakeSecret = handshakeSecret,
|
||||
handshakeGeneration = handshakeGeneration,
|
||||
applicationSecret = applicationSecret,
|
||||
applicationGeneration = applicationGeneration,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
emptyMap()
|
||||
}
|
||||
|
||||
return MlsGroupState(
|
||||
groupContext = groupContext,
|
||||
treeBytes = treeBytes,
|
||||
@@ -154,6 +207,7 @@ data class MlsGroupState(
|
||||
encryptionPrivateKey = encryptionPrivateKey,
|
||||
interimTranscriptHash = interimTranscriptHash,
|
||||
encryptionSecret = encryptionSecret,
|
||||
senderRatchetStates = senderRatchetStates,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+26
@@ -389,6 +389,32 @@ class SecretTree(
|
||||
|
||||
return currentSecret
|
||||
}
|
||||
|
||||
/**
|
||||
* Snapshot every sender's current ratchet position so the enclosing
|
||||
* group state can be persisted (RFC 9420 §9).
|
||||
*
|
||||
* Without this, a restore rebuilds the tree at generation 0 for every
|
||||
* sender, and the LOCAL member then re-emits generation 0 within the
|
||||
* same epoch on its next send — reusing the AEAD key+nonce (a
|
||||
* confidentiality break) and getting rejected by strict receivers
|
||||
* (openmls / MDK / Whitenoise) that forbid generation reuse.
|
||||
*
|
||||
* Only the live ratchet position (secret + generation) per sender is
|
||||
* captured. The replay-detection and skipped-key caches are runtime-only
|
||||
* and deliberately excluded — they are safe to drop across a restart.
|
||||
*/
|
||||
fun exportSenderStates(): Map<Int, SenderRatchetState> = senderState.toMap()
|
||||
|
||||
/**
|
||||
* Seed per-sender ratchet positions from an [exportSenderStates]
|
||||
* snapshot. Called by `MlsGroup.restore`. Any sender absent from
|
||||
* [states] simply re-derives from generation 0 on first use, which is
|
||||
* correct for receive-only ratchets.
|
||||
*/
|
||||
fun importSenderStates(states: Map<Int, SenderRatchetState>) {
|
||||
senderState.putAll(states)
|
||||
}
|
||||
}
|
||||
|
||||
data class SenderRatchetState(
|
||||
|
||||
+287
@@ -0,0 +1,287 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.ClosedMessage
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NoticeMessage
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.utils.Log
|
||||
import com.vitorpamplona.quartz.utils.TimeUtils
|
||||
import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlin.concurrent.atomics.AtomicInt
|
||||
import kotlin.concurrent.atomics.AtomicLong
|
||||
import kotlin.concurrent.atomics.ExperimentalAtomicApi
|
||||
|
||||
/**
|
||||
* Adaptive per-relay back-pressure with TWO independent controls, because relays
|
||||
* push back for two different reasons that need two different responses:
|
||||
*
|
||||
* 1. **Subscription-count limit** — a max on how many subscriptions may be OPEN
|
||||
* at once ("too many subscriptions", "maximum concurrent subscription count",
|
||||
* "number of subscriptions exceeds limit"). The fix is fewer *concurrent*
|
||||
* subs, so we demote the relay's concurrency cap down [subLadder]
|
||||
* (100 → 20 → 10).
|
||||
* 2. **Rate limit** — too many subscription *changes per second* ("rate-limited:
|
||||
* too many messages", "burst exhausted", "slow down"). Fewer concurrent subs
|
||||
* wouldn't help; the fix is to *space the REQs out in time*, so we impose a
|
||||
* minimum interval between opens to that relay, growing it up [rateLadder]
|
||||
* (250ms → 500ms → 1s → 2s).
|
||||
*
|
||||
* Mixing the two mishandles the relay: capping concurrency does nothing for a
|
||||
* rate limit, and slowing the rate does nothing for a subscription-count cap. So
|
||||
* each complaint is routed to its own actuator by matching the notice text.
|
||||
*
|
||||
* A well-behaved relay starts at [startCap] concurrent subs with no rate delay,
|
||||
* and only the ones that push back get throttled — each only as far, and in the
|
||||
* dimension, they keep pushing.
|
||||
*
|
||||
* Registered as a [RelayConnectionListener] on the shared client, so both signals
|
||||
* are driven straight off the incoming NOTICE/CLOSED frames (which fire on the
|
||||
* per-relay socket threads — all state here is concurrent). Drains gate through
|
||||
* [withPermit]: the gated-drain path holds a relay's permit for the lifetime of
|
||||
* that relay's subscription, and passes the rate gate before it opens, so we
|
||||
* respect both limits at once.
|
||||
*/
|
||||
@OptIn(ExperimentalAtomicApi::class)
|
||||
class AdaptiveRelayLimiter(
|
||||
private val startCap: Int = 100,
|
||||
private val subLadder: List<Int> = listOf(20, 10),
|
||||
private val rateLadder: List<Long> = listOf(250L, 500L, 1000L, 2000L),
|
||||
) : RelayConnectionListener {
|
||||
private val gates = ConcurrentMap<NormalizedRelayUrl, Gate>()
|
||||
|
||||
// Concurrency-cap demotions per relay (== index+1 into subLadder). Capped at
|
||||
// subLadder.size: past the floor we stop demoting.
|
||||
private val subDemotions = ConcurrentMap<NormalizedRelayUrl, Int>()
|
||||
|
||||
// Rate-limit state per relay: how far down rateLadder we've stepped, the
|
||||
// current min interval between opens, and the next epoch-ms an open may fire.
|
||||
private val rateSteps = ConcurrentMap<NormalizedRelayUrl, Int>()
|
||||
private val rateDelayMs = ConcurrentMap<NormalizedRelayUrl, Long>()
|
||||
private val nextAllowedAtMs = ConcurrentMap<NormalizedRelayUrl, AtomicLong>()
|
||||
|
||||
private fun gate(relay: NormalizedRelayUrl): Gate = gates.getOrPut(relay) { Gate(startCap) }
|
||||
|
||||
/** The concurrency cap currently enforced for [relay] ([startCap] unless demoted). */
|
||||
fun concurrencyCapOf(relay: NormalizedRelayUrl): Int {
|
||||
val step = subDemotions[relay] ?: 0
|
||||
return if (step == 0) startCap else subLadder[(step - 1).coerceIn(0, subLadder.size - 1)]
|
||||
}
|
||||
|
||||
/** The min interval (ms) between opens enforced for [relay]; 0 if not rate-limited. */
|
||||
fun rateDelayOf(relay: NormalizedRelayUrl): Long = rateDelayMs[relay] ?: 0L
|
||||
|
||||
/** True if we lowered [relay]'s concurrency cap or imposed a rate delay (it pushed back). */
|
||||
fun isThrottled(relay: NormalizedRelayUrl): Boolean = (subDemotions[relay] ?: 0) > 0 || (rateDelayMs[relay] ?: 0L) > 0L
|
||||
|
||||
/**
|
||||
* Run [block] against [relay] respecting both limits: first wait out any rate
|
||||
* delay (spacing opens in time), then hold one of the relay's concurrency
|
||||
* permits for the duration.
|
||||
*/
|
||||
suspend fun <T> withPermit(
|
||||
relay: NormalizedRelayUrl,
|
||||
block: suspend () -> T,
|
||||
): T {
|
||||
rateGate(relay)
|
||||
val g = gate(relay)
|
||||
g.acquire()
|
||||
try {
|
||||
return block()
|
||||
} finally {
|
||||
g.release()
|
||||
}
|
||||
}
|
||||
|
||||
/** If [relay] is rate-limited, reserve and wait for its next allowed open slot. */
|
||||
private suspend fun rateGate(relay: NormalizedRelayUrl) {
|
||||
val delayMs = rateDelayMs[relay] ?: return
|
||||
if (delayMs <= 0L) return
|
||||
val now = TimeUtils.nowMillis()
|
||||
// Atomically claim the next slot: my turn is max(prevSlot, now); the next
|
||||
// caller can't fire until delayMs after me. Serializes opens to this relay
|
||||
// at one per delayMs, in arrival order.
|
||||
val slot = nextAllowedAtMs.getOrPut(relay) { AtomicLong(now) }
|
||||
var myTurn: Long
|
||||
while (true) {
|
||||
val prev = slot.load()
|
||||
myTurn = maxOf(prev, now)
|
||||
if (slot.compareAndSet(prev, myTurn + delayMs)) break
|
||||
}
|
||||
val wait = myTurn - now
|
||||
if (wait > 0) delay(wait)
|
||||
}
|
||||
|
||||
override fun onIncomingMessage(
|
||||
relay: IRelayClient,
|
||||
msgStr: String,
|
||||
msg: Message,
|
||||
) {
|
||||
val text =
|
||||
when (msg) {
|
||||
is ClosedMessage -> msg.message
|
||||
is NoticeMessage -> msg.message
|
||||
else -> return
|
||||
}
|
||||
val t = text.lowercase()
|
||||
// Route each complaint to the matching actuator. Not mutually exclusive:
|
||||
// if a relay somehow reports both, we act on both (they don't conflict).
|
||||
if (RATE_LIMIT_MARKERS.any { it in t }) throttleRate(relay.url)
|
||||
if (SUB_LIMIT_MARKERS.any { it in t }) demoteConcurrency(relay.url)
|
||||
}
|
||||
|
||||
/** Step [relay] one rung down the concurrency-cap ladder, unless already at the floor. */
|
||||
private fun demoteConcurrency(relay: NormalizedRelayUrl) {
|
||||
if ((subDemotions[relay] ?: 0) >= subLadder.size) return
|
||||
val step = subDemotions.merge(relay, 1) { a, b -> a + b }
|
||||
val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)]
|
||||
gate(relay).lower(cap)
|
||||
if (step <= subLadder.size) {
|
||||
Log.d("AdaptiveRelayLimiter") { "${relay.url} concurrency capped at $cap subs (sub-limit #$step)" }
|
||||
}
|
||||
}
|
||||
|
||||
/** Step [relay] one rung down the rate ladder, unless already at the slowest. */
|
||||
private fun throttleRate(relay: NormalizedRelayUrl) {
|
||||
if ((rateSteps[relay] ?: 0) >= rateLadder.size) return
|
||||
val step = rateSteps.merge(relay, 1) { a, b -> a + b }
|
||||
val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)]
|
||||
rateDelayMs[relay] = d
|
||||
if (step <= rateLadder.size) {
|
||||
Log.d("AdaptiveRelayLimiter") { "${relay.url} rate-throttled to 1 REQ / ${d}ms (rate-limit #$step)" }
|
||||
}
|
||||
}
|
||||
|
||||
/** JSON-friendly view of which relays we throttled, in which dimension, how far. */
|
||||
fun snapshot(): Map<String, Any?> {
|
||||
val capCounts = HashMap<Int, Int>()
|
||||
for ((_, step) in subDemotions.snapshot()) {
|
||||
val cap = subLadder[(step - 1).coerceIn(0, subLadder.size - 1)]
|
||||
capCounts[cap] = (capCounts[cap] ?: 0) + 1
|
||||
}
|
||||
val cappedAt = capCounts.toList().sortedBy { it.first }.toMap()
|
||||
val rateCounts = HashMap<Long, Int>()
|
||||
for ((_, step) in rateSteps.snapshot()) {
|
||||
val d = rateLadder[(step - 1).coerceIn(0, rateLadder.size - 1)]
|
||||
rateCounts[d] = (rateCounts[d] ?: 0) + 1
|
||||
}
|
||||
val rateAt = rateCounts.toList().sortedBy { it.first }.toMap()
|
||||
return mapOf(
|
||||
"start_cap" to startCap,
|
||||
"sub_ladder" to subLadder,
|
||||
"rate_ladder_ms" to rateLadder,
|
||||
"concurrency_capped_relays" to subDemotions.size(),
|
||||
"concurrency_capped_at" to cappedAt,
|
||||
"rate_limited_relays" to rateSteps.size(),
|
||||
"rate_limited_at_ms" to rateAt,
|
||||
)
|
||||
}
|
||||
|
||||
fun hadThrottling(): Boolean = subDemotions.size() > 0 || rateSteps.size() > 0
|
||||
|
||||
/**
|
||||
* A bounded-concurrency gate whose limit can only ever be *lowered* (relays
|
||||
* never earn their cap back within a run). Fair FIFO hand-off: a released
|
||||
* permit goes to the longest-waiting acquirer. Lowering the limit below the
|
||||
* in-use count doesn't cancel live holders — it just refuses to admit new
|
||||
* ones until enough release that `inUse < limit` again, so the concurrency
|
||||
* converges down to the new cap as the excess subscriptions finish.
|
||||
*/
|
||||
private class Gate(
|
||||
initialLimit: Int,
|
||||
) {
|
||||
private val limit = AtomicInt(initialLimit)
|
||||
private val mutex = Mutex()
|
||||
private var inUse = 0
|
||||
private val waiters = ArrayDeque<CompletableDeferred<Unit>>()
|
||||
|
||||
suspend fun acquire() {
|
||||
val wait =
|
||||
mutex.withLock {
|
||||
if (inUse < limit.load()) {
|
||||
inUse++
|
||||
null
|
||||
} else {
|
||||
CompletableDeferred<Unit>().also { waiters.addLast(it) }
|
||||
}
|
||||
}
|
||||
wait?.await()
|
||||
}
|
||||
|
||||
suspend fun release() {
|
||||
mutex.withLock {
|
||||
inUse--
|
||||
while (inUse < limit.load() && waiters.isNotEmpty()) {
|
||||
waiters.removeFirst().complete(Unit)
|
||||
inUse++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Monotonically shrink the cap. Safe to call from any thread. */
|
||||
fun lower(newLimit: Int) {
|
||||
while (true) {
|
||||
val cur = limit.load()
|
||||
if (newLimit >= cur) return
|
||||
if (limit.compareAndSet(cur, newLimit)) return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
// A cap on how many subscriptions may be OPEN at once. Fix: fewer
|
||||
// concurrent subs (demote the concurrency cap).
|
||||
private val SUB_LIMIT_MARKERS =
|
||||
listOf(
|
||||
"too many concurrent",
|
||||
"concurrent req",
|
||||
"too many subscription",
|
||||
"number of subscriptions",
|
||||
"subscriptions exceeds",
|
||||
"subscription limit",
|
||||
"subscription count",
|
||||
"maximum concurrent subscription",
|
||||
"max subscription",
|
||||
"too many req",
|
||||
)
|
||||
|
||||
// Too many subscription CHANGES per second. Fix: space the REQs out in
|
||||
// time (a per-relay min interval), not fewer concurrent subs.
|
||||
private val RATE_LIMIT_MARKERS =
|
||||
listOf(
|
||||
"rate-limit",
|
||||
"rate limit",
|
||||
"ratelimit",
|
||||
"too many messages",
|
||||
"too many requests",
|
||||
"burst exhausted",
|
||||
"throttl",
|
||||
"slow down",
|
||||
)
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
|
||||
|
||||
/**
|
||||
* A drain per-relay failure worth acting on: the relay will not serve us THIS run,
|
||||
* so drop it from further routing on the first occurrence. There is only one such
|
||||
* verdict — [DEAD] — because re-probing hop-8's failed relays fresh, outside the
|
||||
* crawl, showed the old "might clear, retry a few times" (TRANSIENT) bucket almost
|
||||
* never clears: 503 Service Unavailable was 0/12 reachable, 502 Bad Gateway 3/15,
|
||||
* connection-establishment failures 0/30, and the codes that WERE alive (403/402)
|
||||
* are gated and will never hand us events. Spending extra dials on them was waste.
|
||||
*
|
||||
* The only two connect failures that genuinely recover are kept OUT of this verdict
|
||||
* by [classifyDrainFailure] returning null (retry, never dead):
|
||||
* - a **read** timeout — the relay accepted the handshake but is slow to serve;
|
||||
* 12/18 (67%) were reachable fresh, only overloaded by the crawl's fan-out. The
|
||||
* crawler's per-authority timeout strikes, which CLEAR on success, shed the gone.
|
||||
* - an HTTP **429 / too many requests** — alive and rate-limiting; 4/4 reachable
|
||||
* fresh. Retrying (spaced by the rate limiter) is how we eventually get its data.
|
||||
*/
|
||||
enum class DrainFailure { DEAD, }
|
||||
|
||||
/**
|
||||
* Classify a drain per-relay terminal reason. Returns null when the relay should be
|
||||
* retried rather than dropped — a read/generic timeout, an alive 429 rate-limit, or
|
||||
* a non-failure like eose/closed. Any other `cannot:<message>` (see
|
||||
* `BasicRelayClient.onCannotConnect`) is [DrainFailure.DEAD]: it will not serve us
|
||||
* this run, so drop it now instead of paying repeated connect attempts.
|
||||
*/
|
||||
fun classifyDrainFailure(reason: String): DrainFailure? {
|
||||
if (!reason.startsWith("cannot")) return null
|
||||
val m = reason.removePrefix("cannot:").lowercase()
|
||||
// Alive, only asking us to slow down: an HTTP 429 / "too many requests" reliably
|
||||
// clears — 4/4 such relays were reachable when re-probed fresh. Retry it (the
|
||||
// rate limiter spaces our opens); never drop it.
|
||||
if ("429" in m || "too many requests" in m) return null
|
||||
// A READ timeout means the relay accepted the handshake but is slow to serve —
|
||||
// 12/18 (67%) reachable fresh, alive but overloaded by the fan-out. Retry; the
|
||||
// crawler's per-authority timeout strikes, which clear on success, shed the gone.
|
||||
// A *connect* timeout is the opposite (the socket never opened, 0/30 reachable),
|
||||
// so it is excluded here and falls through to DEAD with every other failure.
|
||||
if (("timeout" in m || "timed out" in m) && "connect timed out" !in m) return null
|
||||
// Everything else won't serve us this run: connect refused / unroutable / the
|
||||
// proxy couldn't tunnel the CONNECT, a DNS or TLS failure, a dead-or-not-a-relay
|
||||
// HTTP upgrade (502/503/500/504/410/404/200/…), or a mid-stream reset. Measured
|
||||
// mostly dead (503 0%, 502 20% reachable) and, when alive, gated (402/403) or not
|
||||
// a relay (200). Drop it now rather than burn more dials on it.
|
||||
return DrainFailure.DEAD
|
||||
}
|
||||
+304
@@ -0,0 +1,304 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
|
||||
import com.vitorpamplona.quartz.nip01Core.store.verifyAndInsert
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.awaitAll
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.joinAll
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.sync.Semaphore
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
import kotlin.concurrent.atomics.AtomicInt
|
||||
import kotlin.concurrent.atomics.ExperimentalAtomicApi
|
||||
import kotlin.coroutines.cancellation.CancellationException
|
||||
|
||||
/**
|
||||
* Two-pass NIP-77 sync of ANY filter set between a relay and a local [IEventStore],
|
||||
* with a paged fallback — the reusable engine behind `amy sync` and the GrapeRank
|
||||
* outbox updater, generalized so any caller can reconcile arbitrary
|
||||
* `relay -> filters` work against their store.
|
||||
*
|
||||
* A **group** is one `(relay, filter)`: [syncGroup] runs the full two-pass sync for it.
|
||||
*
|
||||
* 1. **Content pass** — [negentropyReconcile] diffs the relay's matched set for the
|
||||
* filter against the store's ids, then:
|
||||
* - [Config.down] downloads the residual **needs** (relay has, store lacks) by id
|
||||
* and verifies+inserts them into [store];
|
||||
* - [Config.up] uploads the residual **haves** (store has, relay lacks) as EVENTs.
|
||||
* 2. **Deletion pass** — [negentropySettleDeletions] over what the content pass could
|
||||
* not converge ([Config.syncDeletions]). Its **applyDown** direction is the
|
||||
* "download the deletion when our upload was rejected" case: an event pushed up
|
||||
* that the relay keeps rejecting (it deleted it) is a residual *have*, so the
|
||||
* relay's covering kind:5 is pulled down and applied and [store] drops the
|
||||
* retracted event. **sendUp** publishes the store's covering deletions for records
|
||||
* deleted locally that the relay still serves.
|
||||
*
|
||||
* If the content pass can't reconcile ([NegentropySyncException] — no NIP-77, an
|
||||
* over-cap minimal window, a mid-sync disconnect) and [Config.pageFallback] is on, the
|
||||
* group pages the same filter ([fetchAllPages]) into the store instead; only the
|
||||
* negentropy-only deletion settle is skipped there. Every group is best-effort — a
|
||||
* failure lands in [GroupResult.error], it never throws — so one bad relay can't abort
|
||||
* a multi-relay [sync].
|
||||
*
|
||||
* [sync] runs many groups: relays go up to [Config.concurrency] at once, and a single
|
||||
* relay's filters run sequentially (so one relay never opens more than one group's worth
|
||||
* of negentropy sessions at a time — keeping under its subscription budget). Progress is
|
||||
* emitted through [log].
|
||||
*/
|
||||
@OptIn(ExperimentalAtomicApi::class)
|
||||
class NegentropyStoreSync(
|
||||
private val client: INostrClient,
|
||||
private val store: IEventStore,
|
||||
private val config: Config = Config(),
|
||||
private val log: (String) -> Unit = {},
|
||||
) {
|
||||
/**
|
||||
* @param down download records the relay has that the store lacks.
|
||||
* @param up upload records the store has that the relay lacks (also arms the
|
||||
* deletion **applyDown** path — a rejected upload pulls the relay's kind:5 down).
|
||||
* @param syncDeletions run the deletion settle over the reconcile residual.
|
||||
* @param pageFallback page the filter when negentropy can't reconcile the relay.
|
||||
* @param idChunk ids per reconcile chunk and per by-id fetch.
|
||||
* @param downloadWorkers concurrent by-id download fetches per group.
|
||||
* @param reconcileConcurrency overlapped `created_at`-window reconciles after an over-cap split.
|
||||
* @param maxDeletionRounds hard cap on deletion-settle rounds (converges in 1–2).
|
||||
* @param concurrency relays synced at once by [sync] (a relay's own filters stay sequential).
|
||||
* @param idleTimeoutMs idle watchdog for reconciles / fetches / pages.
|
||||
* @param publishTimeoutSecs OK-confirmation wait per uploaded event.
|
||||
*/
|
||||
class Config(
|
||||
val down: Boolean = true,
|
||||
val up: Boolean = false,
|
||||
val syncDeletions: Boolean = true,
|
||||
val pageFallback: Boolean = true,
|
||||
val idChunk: Int = 500,
|
||||
val downloadWorkers: Int = 4,
|
||||
val reconcileConcurrency: Int = 2,
|
||||
val maxDeletionRounds: Int = 4,
|
||||
val concurrency: Int = 4,
|
||||
val idleTimeoutMs: Long = 30_000L,
|
||||
val publishTimeoutSecs: Long = 15,
|
||||
)
|
||||
|
||||
/** Outcome of one `(relay, filter)` group. `error` is null on success. */
|
||||
class GroupResult(
|
||||
val relay: NormalizedRelayUrl,
|
||||
val filter: Filter,
|
||||
val need: Int,
|
||||
val have: Int,
|
||||
val downloaded: Int,
|
||||
val uploaded: Int,
|
||||
val deletionsSentUp: Int,
|
||||
val deletionsAppliedDown: Int,
|
||||
/** True when negentropy couldn't reconcile and the filter was paged instead. */
|
||||
val pagedFallback: Boolean,
|
||||
val error: String?,
|
||||
)
|
||||
|
||||
/**
|
||||
* Sync every `(relay, filter)` in [filtersByRelay]: relays run up to
|
||||
* [Config.concurrency] at once; each relay's filters run sequentially. Returns one
|
||||
* [GroupResult] per relay+filter (relay order preserved, filters in list order).
|
||||
*/
|
||||
suspend fun sync(filtersByRelay: Map<NormalizedRelayUrl, List<Filter>>): List<GroupResult> {
|
||||
if (filtersByRelay.isEmpty()) return emptyList()
|
||||
val gate = Semaphore(config.concurrency.coerceAtLeast(1))
|
||||
return coroutineScope {
|
||||
filtersByRelay.entries
|
||||
.map { (relay, filters) ->
|
||||
async { gate.withPermit { filters.map { syncGroupSafely(relay, it) } } }
|
||||
}.awaitAll()
|
||||
.flatten()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* [syncGroup] with a best-effort guard so an unexpected failure in one group
|
||||
* (store I/O, a relay throwing outside the NIP-77 path, …) is recorded rather than
|
||||
* cancelling every other relay in a [sync]. Cancellation is propagated, not caught.
|
||||
*/
|
||||
private suspend fun syncGroupSafely(
|
||||
relay: NormalizedRelayUrl,
|
||||
filter: Filter,
|
||||
): GroupResult =
|
||||
try {
|
||||
syncGroup(relay, filter)
|
||||
} catch (e: CancellationException) {
|
||||
throw e
|
||||
} catch (e: Exception) {
|
||||
log("[store-sync] ${relay.url}: group failed: ${e::class.simpleName}: ${e.message}")
|
||||
GroupResult(relay, filter, 0, 0, 0, 0, 0, 0, pagedFallback = false, error = "${e::class.simpleName}: ${e.message}")
|
||||
}
|
||||
|
||||
/** Content pass + deletion settle (+ page fallback) for one relay + one filter. */
|
||||
suspend fun syncGroup(
|
||||
relay: NormalizedRelayUrl,
|
||||
filter: Filter,
|
||||
): GroupResult {
|
||||
// Only the id+created_at snapshot is needed to reconcile — never the decoded
|
||||
// events (~40 B/entry vs ~1 KB), which matters when a relay hosts a large
|
||||
// matched set. The events the reconcile decides to UP-publish (the small
|
||||
// residual haves) are fetched by id on demand in the uploader below.
|
||||
val localEntries = store.snapshotIdsForNegentropy(listOf(filter))
|
||||
|
||||
val downloaded = AtomicInt(0)
|
||||
val uploaded = AtomicInt(0)
|
||||
|
||||
val reconcileResult =
|
||||
try {
|
||||
coroutineScope {
|
||||
// needIds = relay has, store lacks; haveIds = store has, relay lacks.
|
||||
val needBatches = Channel<List<HexKey>>(config.downloadWorkers * 2)
|
||||
val haveBatches = Channel<List<HexKey>>(Channel.UNLIMITED)
|
||||
|
||||
val downloaders =
|
||||
List(config.downloadWorkers.coerceAtLeast(1)) {
|
||||
launch {
|
||||
for (batch in needBatches) {
|
||||
for (event in client.fetchAll(relay, Filter(ids = batch), config.idleTimeoutMs)) {
|
||||
if (store.verifyAndInsert(event)) downloaded.addAndFetch(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val uploader =
|
||||
launch {
|
||||
for (batch in haveBatches) {
|
||||
// Fetch just the residual haves from the store (not the
|
||||
// whole matched set) and publish them up.
|
||||
for (ev in store.query<Event>(Filter(ids = batch))) {
|
||||
if (client.publishAndConfirm(ev, setOf(relay), config.publishTimeoutSecs)) uploaded.addAndFetch(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val result =
|
||||
try {
|
||||
client.negentropyReconcile(
|
||||
relay = relay,
|
||||
filter = filter,
|
||||
localEntries = localEntries,
|
||||
batchSize = config.idChunk,
|
||||
idleTimeoutMs = config.idleTimeoutMs,
|
||||
reconcileConcurrency = config.reconcileConcurrency,
|
||||
onHaveIds = if (config.up) { batch -> haveBatches.send(batch) } else null,
|
||||
onNeedIds = { batch -> if (config.down) needBatches.send(batch) },
|
||||
)
|
||||
} finally {
|
||||
needBatches.close()
|
||||
haveBatches.close()
|
||||
}
|
||||
|
||||
downloaders.joinAll()
|
||||
uploader.join()
|
||||
result
|
||||
}
|
||||
} catch (e: NegentropySyncException) {
|
||||
// Negentropy couldn't reconcile — page the same filter so the records
|
||||
// still refresh. Deletion settle is negentropy-only, so it is skipped.
|
||||
var pageError: String? = e.message ?: "negentropy sync failed"
|
||||
if (config.pageFallback && config.down) {
|
||||
pageError =
|
||||
try {
|
||||
downloaded.addAndFetch(pageDownload(relay, filter))
|
||||
null
|
||||
} catch (pe: CancellationException) {
|
||||
throw pe
|
||||
} catch (pe: Exception) {
|
||||
"negentropy: ${e.message}; page fallback: ${pe::class.simpleName}: ${pe.message}"
|
||||
}
|
||||
}
|
||||
log("[store-sync] ${relay.url}: paged fallback, ${downloaded.load()} stored${pageError?.let { " (error: $it)" } ?: ""}")
|
||||
return GroupResult(relay, filter, 0, 0, downloaded.load(), uploaded.load(), 0, 0, pagedFallback = true, error = pageError)
|
||||
}
|
||||
|
||||
val deletions =
|
||||
if (config.syncDeletions) {
|
||||
client.negentropySettleDeletions(
|
||||
relay = relay,
|
||||
filter = filter,
|
||||
store = store,
|
||||
sendUp = config.down,
|
||||
applyDown = config.up,
|
||||
batchSize = config.idChunk,
|
||||
idleTimeoutMs = config.idleTimeoutMs,
|
||||
maxRounds = config.maxDeletionRounds,
|
||||
reconcileConcurrency = config.reconcileConcurrency,
|
||||
)
|
||||
} else {
|
||||
null
|
||||
}
|
||||
|
||||
log(
|
||||
"[store-sync] ${relay.url}: down ${downloaded.load()}, up ${uploaded.load()}, " +
|
||||
"del↑ ${deletions?.sentUp ?: 0}, del↓ ${deletions?.appliedDown ?: 0}",
|
||||
)
|
||||
return GroupResult(
|
||||
relay = relay,
|
||||
filter = filter,
|
||||
need = reconcileResult.needCount,
|
||||
have = reconcileResult.haveCount,
|
||||
downloaded = downloaded.load(),
|
||||
uploaded = uploaded.load(),
|
||||
deletionsSentUp = deletions?.sentUp ?: 0,
|
||||
deletionsAppliedDown = deletions?.appliedDown ?: 0,
|
||||
pagedFallback = false,
|
||||
error = null,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Paged fallback: walk [relay] past its per-REQ cap for [filter], verifying and
|
||||
* inserting each event into [store]. [fetchAllPages]'s `onEvent` can't suspend, so
|
||||
* events funnel through a channel to a single inserter. Returns how many were newly stored.
|
||||
*/
|
||||
private suspend fun pageDownload(
|
||||
relay: NormalizedRelayUrl,
|
||||
filter: Filter,
|
||||
): Int {
|
||||
val stored = AtomicInt(0)
|
||||
val events = Channel<Event>(Channel.UNLIMITED)
|
||||
coroutineScope {
|
||||
val inserter =
|
||||
launch {
|
||||
for (event in events) {
|
||||
if (store.verifyAndInsert(event)) stored.addAndFetch(1)
|
||||
}
|
||||
}
|
||||
try {
|
||||
client.fetchAllPages(relay, listOf(filter), config.idleTimeoutMs) { event -> events.trySend(event) }
|
||||
} finally {
|
||||
events.close()
|
||||
}
|
||||
inserter.join()
|
||||
}
|
||||
return stored.load()
|
||||
}
|
||||
}
|
||||
+11
-1
@@ -120,7 +120,17 @@ suspend fun INostrClient.fetchFirst(
|
||||
remaining.clear()
|
||||
}
|
||||
doneChannel.onReceive { relay ->
|
||||
remaining.remove(relay)
|
||||
// A relay sends its matching events before its EOSE, so an event may
|
||||
// already be buffered when this completion fires. select() picks a ready
|
||||
// clause at random, so without this drain we could treat the relay as done
|
||||
// and exit while its event still sits unread in the channel.
|
||||
val buffered = eventChannel.tryReceive().getOrNull()
|
||||
if (buffered != null) {
|
||||
result = buffered
|
||||
remaining.clear()
|
||||
} else {
|
||||
remaining.remove(relay)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.verify
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
|
||||
import com.vitorpamplona.quartz.nip01Core.store.deletionsCovering
|
||||
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
|
||||
|
||||
/**
|
||||
* Outcome of a [negentropySettleDeletions] run.
|
||||
*
|
||||
* @property sentUp distinct local deletions published to the relay (up direction).
|
||||
* @property appliedDown distinct relay deletions ingested into [store] (down direction).
|
||||
* @property rounds reconcile rounds run before convergence (or the cap).
|
||||
*/
|
||||
class DeletionSettleResult(
|
||||
val sentUp: Int,
|
||||
val appliedDown: Int,
|
||||
val rounds: Int,
|
||||
)
|
||||
|
||||
/**
|
||||
* Converge deletions between [store] and [relay] AFTER a content sync has settled the
|
||||
* two sides — the second half of a two-pass sync. NIP-77 reconciles by id, so a plain
|
||||
* content sync converges everything except events a deletion physically stops from
|
||||
* moving; those survive as the reconcile's residual, which this resolves:
|
||||
*
|
||||
* - **[sendUp]** — a residual **need** (relay has it, [store] still lacks it after the
|
||||
* content pass tried to download it) means we deleted it. Publish OUR covering
|
||||
* deletion up ([IEventStore.deletionsCovering]) so the relay drops it.
|
||||
* - **[applyDown]** — a residual **have** ([store] has it, relay still lacks it after
|
||||
* the content pass tried to upload it) means the relay deleted it. Pull the RELAY'S
|
||||
* covering **kind-5** down and ingest it, so [store] drops it too. A NIP-62 vanish is
|
||||
* deliberately NOT applied on pull — its blast radius is the author's whole account.
|
||||
*
|
||||
* Because it works off the residual — not every id — the cost is one cheap reconcile
|
||||
* per round plus the (small) residual, independent of database size. It loops until a
|
||||
* round resolves nothing (converged, and thereby self-verified) or [maxRounds] is hit.
|
||||
*
|
||||
* **Direction requires the matching content pass.** A residual need is a clean signal
|
||||
* only after the content sync attempted the download ([sendUp] pairs with a `--down`
|
||||
* content pass); a residual have only after it attempted the upload ([applyDown] pairs
|
||||
* with `--up`). Passing a direction whose content pass didn't run makes its residual the
|
||||
* full unsettled set, not a deletion signal — so drive this with the same directions the
|
||||
* content pass used.
|
||||
*
|
||||
* Best-effort: a reconcile failure ([NegentropySyncException]) stops the loop and returns
|
||||
* what already settled rather than throwing — the content sync is the primary work.
|
||||
*
|
||||
* @param batchSize ids per reconcile chunk and per by-id fetch.
|
||||
* @param idleTimeoutMs idle watchdog for the reconciles and fetches.
|
||||
* @param maxRounds hard cap on rounds; the "resolved nothing" check usually stops first.
|
||||
* @param reconcileConcurrency overlapped `created_at`-window reconciles after an over-cap split.
|
||||
*/
|
||||
suspend fun INostrClient.negentropySettleDeletions(
|
||||
relay: NormalizedRelayUrl,
|
||||
filter: Filter,
|
||||
store: IEventStore,
|
||||
sendUp: Boolean,
|
||||
applyDown: Boolean,
|
||||
batchSize: Int = 500,
|
||||
idleTimeoutMs: Long = 120_000L,
|
||||
maxRounds: Int = 4,
|
||||
reconcileConcurrency: Int = 1,
|
||||
): DeletionSettleResult {
|
||||
if ((!sendUp && !applyDown) || maxRounds <= 0) return DeletionSettleResult(0, 0, 0)
|
||||
|
||||
val publishTimeoutSecs = (idleTimeoutMs / 1000).coerceAtLeast(1)
|
||||
val sentUp = HashSet<HexKey>()
|
||||
val appliedDown = HashSet<HexKey>()
|
||||
var rounds = 0
|
||||
|
||||
while (rounds < maxRounds) {
|
||||
rounds++
|
||||
val diff =
|
||||
try {
|
||||
negentropyReconcileIds(
|
||||
relay = relay,
|
||||
filter = filter,
|
||||
localEntries = store.snapshotIdsForNegentropy(listOf(filter)),
|
||||
batchSize = batchSize,
|
||||
idleTimeoutMs = idleTimeoutMs,
|
||||
reconcileConcurrency = reconcileConcurrency,
|
||||
)
|
||||
} catch (e: NegentropySyncException) {
|
||||
break
|
||||
}
|
||||
|
||||
var resolved = 0
|
||||
|
||||
// residual needs → publish our covering deletions up.
|
||||
if (sendUp) {
|
||||
for (chunk in diff.needIds.chunked(batchSize)) {
|
||||
val events = fetchAll(relay, Filter(ids = chunk), idleTimeoutMs)
|
||||
for (del in store.deletionsCovering(events, relay)) {
|
||||
if (sentUp.add(del.id)) {
|
||||
if (publishAndConfirm(del, setOf(relay), publishTimeoutSecs)) resolved++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// residual haves → ingest the relay's covering kind-5 (never a vanish).
|
||||
if (applyDown) {
|
||||
for (chunk in diff.haveIds.chunked(batchSize)) {
|
||||
val ours = store.query<Event>(Filter(ids = chunk))
|
||||
val relayDeletions = deletionsCovering(ours, relay) { f -> fetchAll(relay, f, idleTimeoutMs) }
|
||||
for (del in relayDeletions.filterIsInstance<DeletionEvent>()) {
|
||||
if (del.verify() && appliedDown.add(del.id)) {
|
||||
store.insert(del)
|
||||
resolved++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (resolved == 0) break
|
||||
}
|
||||
|
||||
return DeletionSettleResult(sentUp.size, appliedDown.size, rounds)
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
# `INostrClient` accessories
|
||||
|
||||
One-shot / high-level relay operations, written as **extension functions** on
|
||||
`INostrClient`. They live here (and in `../reqs/`) rather than on the client class,
|
||||
so they don't show up under "usages of `NostrClient`" or in method completion — you
|
||||
only find them by knowing this package exists.
|
||||
|
||||
**Before writing a new subscribe / REQ / publish loop, look here first.** Most of what
|
||||
a caller needs (fetch a set, fetch one, page past the relay cap, publish-and-confirm,
|
||||
count, negentropy sync/reconcile) already exists.
|
||||
|
||||
Import as `com.vitorpamplona.quartz.nip01Core.relay.client.accessories.<name>` (or
|
||||
`...client.reqs.<name>` for the flow/subscribe helpers).
|
||||
|
||||
## One-shot reads (subscribe → collect → return)
|
||||
|
||||
| Function | File | Use when |
|
||||
| --- | --- | --- |
|
||||
| `fetchAll(relay, filter, timeoutMs)` | `NostrClientFetchAllExt` | Get every event matching a filter in one REQ, deduped by id, until EOSE or timeout. **No verify, no store** — just the events. |
|
||||
| `fetchFirst(relay, filter, timeoutMs)` | `NostrClientFetchFirstExt` | Get the first matching event and stop (returns `null` on none/timeout). |
|
||||
| `fetchAllPages(relay, filters, timeoutMs)` | `NostrClientFetchAllPagesExt` | Fully retrieve a result set larger than the relay's per-REQ cap (strfry `limit`, ~500) by walking a `created_at` cursor. Bound it with the filter's `limit`. |
|
||||
| `fetchAllPagesFromPool(filters, ...)` | `NostrClientFetchAllPagesPoolExt` | Same paging, across several relays at once, deduped across them. |
|
||||
|
||||
## Streaming (`Flow`)
|
||||
|
||||
| Function | File | Use when |
|
||||
| --- | --- | --- |
|
||||
| `fetchAsFlow(relay, filter)` | `../reqs/NostrClientFetchAsFlowExt` | Emit the accumulating list on each arrival; completes on EOSE. One-shot query as a flow. |
|
||||
| `subscribeAsFlow(relay, filter)` | `../reqs/NostrClientSubscribeAsFlowExt` | Live subscription as a flow (stays open past EOSE; re-sends the REQ on reconnect). |
|
||||
| `subscribe(subId, filters, listener)` | `../reqs/StaticSubscription`, `DynamicSubscription` | Raw live subscription with a `SubscriptionListener`. The lowest-level primitive the above build on. |
|
||||
|
||||
## Publish
|
||||
|
||||
| Function | File | Use when |
|
||||
| --- | --- | --- |
|
||||
| `publishAndConfirm(event, relays, timeout)` | `NostrClientPublishExt` | Send an EVENT and wait for `OK`; returns whether any relay accepted it. |
|
||||
| `publishAndConfirmDetailed(event, relays, timeout)` | `NostrClientPublishExt` | Same, but returns the per-relay accepted/rejected map. |
|
||||
|
||||
## Count (NIP-45)
|
||||
|
||||
| Function | File | Use when |
|
||||
| --- | --- | --- |
|
||||
| `count(relay, filter, timeoutMs)` | `NostrClientCountExt` | NIP-45 `COUNT` against one relay (`null` on timeout / no support). |
|
||||
| `countMerged(relays, filter, ...)` | `NostrClientCountExt` | Merged count across relays. |
|
||||
|
||||
## Negentropy (NIP-77)
|
||||
|
||||
| Function | File | Use when |
|
||||
| --- | --- | --- |
|
||||
| `negentropySync(relay, filter, ...)` | `NostrClientNegentropySyncExt` | Download everything a relay holds for a filter, diffing against `localEntries` and by-id downloading only the diff. Throws `NegentropySyncException` if the relay can't reconcile (no fallback). |
|
||||
| `negentropySyncOrFetch(relay, filter, ...)` | `NostrClientNegentropySyncExt` | Same, but transparently falls back to `fetchAllPages` when the relay can't reconcile. The "just get the events" combinator. |
|
||||
| `negentropySyncEvents` / `negentropySyncOrFetchEvents` | `NostrClientNegentropySyncEventsExt` | The two above as an O(1)-memory `Flow<Event>`. |
|
||||
| `negentropyReconcile(relay, filter, localEntries, onNeedIds, onHaveIds)` | `NostrClientNegentropySyncExt` | **Pure diff, no I/O** — streams the two directions (`need` = relay has & we lack; `have` = we have & relay lacks) to callbacks. Compose your own download/upload on top. |
|
||||
| `negentropyReconcileIds(relay, filter, localEntries)` | `NostrClientNegentropySyncExt` | Same diff, materialized into `needIds` / `haveIds` lists (small sets only). |
|
||||
| `negentropySettleDeletions(relay, filter, store, sendUp, applyDown)` | `NostrClientNegentropyDeletionSettleExt` | Second pass of a two-pass sync: after a content sync settles, re-reconcile and resolve only the residual — send our covering deletions up (`sendUp`) and/or apply the relay's kind-5 down (`applyDown`), looping until stable. Cost is O(residual), not O(db). Pairs with `IEventStore.deletionsCovering`. |
|
||||
|
||||
`fetchByIds`, `reconcileStreaming`, `syncPipeline` in `NostrClientNegentropySyncExt`
|
||||
are `internal` implementation details — not part of the public surface.
|
||||
|
||||
---
|
||||
|
||||
_Keep this table in sync when you add a public `INostrClient` extension here._
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.relay.client.auth
|
||||
|
||||
import androidx.compose.runtime.Immutable
|
||||
|
||||
/**
|
||||
* Compose-stable per-relay AUTH snapshot exposed by [RelayAuthenticator].
|
||||
*
|
||||
* The internal [RelayAuthStatus] is a mutable holder around concurrent LRU
|
||||
* caches — necessary for the per-relay OkHttp dispatcher, but unsuitable as
|
||||
* a [kotlinx.coroutines.flow.StateFlow] value (mutating it doesn't change
|
||||
* identity, so distinct-until-changed swallows updates).
|
||||
*
|
||||
* [RelayAuthSnapshot] is the immutable view downstream consumers (UI banner,
|
||||
* retry coordinator, indexer-fan-out gate) subscribe to.
|
||||
*/
|
||||
@Immutable
|
||||
data class RelayAuthSnapshot(
|
||||
val phase: Phase,
|
||||
val lastAuthSuccessAt: Long?,
|
||||
) {
|
||||
enum class Phase {
|
||||
/** Connected; no AUTH challenge has been received yet. */
|
||||
IDLE,
|
||||
|
||||
/** Signed AUTH event in flight; awaiting OK from the relay. */
|
||||
AUTHENTICATING,
|
||||
|
||||
/** Last AUTH succeeded; relay accepts authenticated REQs. */
|
||||
AUTHENTICATED,
|
||||
|
||||
/** Last AUTH attempt failed; subsequent challenges may still arrive. */
|
||||
AUTH_FAILED,
|
||||
}
|
||||
|
||||
companion object {
|
||||
val IDLE = RelayAuthSnapshot(Phase.IDLE, lastAuthSuccessAt = null)
|
||||
}
|
||||
}
|
||||
+34
@@ -23,6 +23,8 @@ package com.vitorpamplona.quartz.nip01Core.relay.client.auth
|
||||
import androidx.collection.LruCache
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip42RelayAuth.RelayAuthEvent
|
||||
import com.vitorpamplona.quartz.utils.TimeUtils
|
||||
import kotlin.concurrent.Volatile
|
||||
|
||||
class RelayAuthStatus {
|
||||
// Keeps track of auth responses to update the relay with all filters
|
||||
@@ -32,6 +34,12 @@ class RelayAuthStatus {
|
||||
// Avoids sending multiple replies for each auth.
|
||||
private val uniqueAuthChallengesSent: LruCache<ChallengePair, ChallengePair> = LruCache(10)
|
||||
|
||||
// Latest epoch-second at which a tracked AUTH event received a successful OK.
|
||||
// Read by RelayAuthSnapshot consumers for staleness checks (e.g. proactive
|
||||
// re-AUTH on window focus).
|
||||
@Volatile
|
||||
private var lastAuthSuccessAt: Long? = null
|
||||
|
||||
enum class AuthEventReceiptStatus {
|
||||
AUTHENTICATING,
|
||||
AUTHENTICATED,
|
||||
@@ -66,6 +74,7 @@ class RelayAuthStatus {
|
||||
return if (wasAlreadyAuthenticated != null) {
|
||||
if (success) {
|
||||
authResponseWatcher.put(eventId, AuthEventReceiptStatus.AUTHENTICATED)
|
||||
lastAuthSuccessAt = TimeUtils.now()
|
||||
} else {
|
||||
authResponseWatcher.put(eventId, AuthEventReceiptStatus.NOT_AUTHENTICATED)
|
||||
}
|
||||
@@ -77,4 +86,29 @@ class RelayAuthStatus {
|
||||
}
|
||||
|
||||
fun hasFinishedAllAuths() = authResponseWatcher.snapshot().all { it.value != AuthEventReceiptStatus.AUTHENTICATING }
|
||||
|
||||
/**
|
||||
* Build an immutable Compose-stable snapshot of the current per-relay AUTH
|
||||
* state. The phase is derived from the response watcher:
|
||||
*
|
||||
* - any AUTHENTICATING entry → [RelayAuthSnapshot.Phase.AUTHENTICATING]
|
||||
* - else any AUTHENTICATED entry → [RelayAuthSnapshot.Phase.AUTHENTICATED]
|
||||
* - else any NOT_AUTHENTICATED entry → [RelayAuthSnapshot.Phase.AUTH_FAILED]
|
||||
* - else (no tracked challenges) → [RelayAuthSnapshot.Phase.IDLE]
|
||||
*
|
||||
* The watcher LRU caps at 10 entries; a long-running connection that has
|
||||
* already AUTHed will still report AUTHENTICATED even after older entries
|
||||
* roll off, because the LRU keeps the most recent.
|
||||
*/
|
||||
fun snapshot(): RelayAuthSnapshot {
|
||||
val entries = authResponseWatcher.snapshot()
|
||||
val phase =
|
||||
when {
|
||||
entries.isEmpty() -> RelayAuthSnapshot.Phase.IDLE
|
||||
entries.values.any { it == AuthEventReceiptStatus.AUTHENTICATING } -> RelayAuthSnapshot.Phase.AUTHENTICATING
|
||||
entries.values.any { it == AuthEventReceiptStatus.AUTHENTICATED } -> RelayAuthSnapshot.Phase.AUTHENTICATED
|
||||
else -> RelayAuthSnapshot.Phase.AUTH_FAILED
|
||||
}
|
||||
return RelayAuthSnapshot(phase, lastAuthSuccessAt)
|
||||
}
|
||||
}
|
||||
|
||||
+38
-1
@@ -33,10 +33,16 @@ import com.vitorpamplona.quartz.nip01Core.signers.SignerExceptions
|
||||
import com.vitorpamplona.quartz.nip42RelayAuth.RelayAuthEvent
|
||||
import com.vitorpamplona.quartz.utils.Log
|
||||
import com.vitorpamplona.quartz.utils.cache.LargeCache
|
||||
import kotlinx.collections.immutable.PersistentMap
|
||||
import kotlinx.collections.immutable.persistentMapOf
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.IO
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.flow.update
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlin.coroutines.cancellation.CancellationException
|
||||
|
||||
@@ -61,8 +67,32 @@ class RelayAuthenticator(
|
||||
// Connection callbacks fire on the per-relay OkHttp dispatcher thread, so
|
||||
// this state is mutated concurrently — LargeCache wraps a platform-tuned
|
||||
// concurrent map (ConcurrentSkipListMap on jvmAndroid, CacheMap on Apple).
|
||||
//
|
||||
// This stays mutable because RelayAuthStatus carries an LruCache that has
|
||||
// to be addressable from the dispatcher thread. The Compose-observable
|
||||
// view of the same data is published on [authStateFlow] below, sourced
|
||||
// from RelayAuthStatus.snapshot().
|
||||
private val authStatus = LargeCache<NormalizedRelayUrl, RelayAuthStatus>()
|
||||
|
||||
private val _authStateFlow = MutableStateFlow<PersistentMap<NormalizedRelayUrl, RelayAuthSnapshot>>(persistentMapOf())
|
||||
|
||||
/**
|
||||
* Per-relay AUTH state as an immutable Compose-stable snapshot map.
|
||||
*
|
||||
* Downstream consumers (UI banner, retry queue, indexer-fan-out gate)
|
||||
* subscribe to this flow instead of polling [authStatus] directly.
|
||||
* Identity changes on every mutation, so [kotlinx.coroutines.flow.distinctUntilChanged]
|
||||
* downstream and Compose `@Immutable` skipping both work correctly.
|
||||
*/
|
||||
val authStateFlow: StateFlow<PersistentMap<NormalizedRelayUrl, RelayAuthSnapshot>> = _authStateFlow.asStateFlow()
|
||||
|
||||
private fun publishSnapshot(relayUrl: NormalizedRelayUrl) {
|
||||
val status = authStatus.get(relayUrl)
|
||||
_authStateFlow.update { current ->
|
||||
if (status == null) current.remove(relayUrl) else current.put(relayUrl, status.snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
private val clientListener =
|
||||
object : RelayConnectionListener {
|
||||
override fun onIncomingMessage(
|
||||
@@ -78,10 +108,12 @@ class RelayAuthenticator(
|
||||
|
||||
override fun onConnecting(relay: IRelayClient) {
|
||||
authStatus.put(relay.url, RelayAuthStatus())
|
||||
publishSnapshot(relay.url)
|
||||
}
|
||||
|
||||
override fun onDisconnected(relay: IRelayClient) {
|
||||
authStatus.remove(relay.url)
|
||||
publishSnapshot(relay.url)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +134,7 @@ class RelayAuthenticator(
|
||||
// only send replies to new challenges to avoid infinite loop:
|
||||
if (authStatus.get(relay.url)?.saveAuthSubmission(authEvent) == true) {
|
||||
relay.sendIfConnected(AuthCmd(authEvent))
|
||||
publishSnapshot(relay.url)
|
||||
}
|
||||
}
|
||||
} catch (e: CancellationException) {
|
||||
@@ -118,8 +151,12 @@ class RelayAuthenticator(
|
||||
relay: IRelayClient,
|
||||
msg: OkMessage,
|
||||
) {
|
||||
val transitioned = authStatus.get(relay.url)?.checkAuthResults(msg.eventId, msg.success) == true
|
||||
// Publish even on failure transitions so the UI can clear "AUTHENTICATING"
|
||||
// banners and reflect AUTH_FAILED state.
|
||||
publishSnapshot(relay.url)
|
||||
// if this is the OK of an auth event, renew all subscriptions and resend all outgoing events.
|
||||
if (authStatus.get(relay.url)?.checkAuthResults(msg.eventId, msg.success) == true) {
|
||||
if (transitioned) {
|
||||
client.syncFilters(relay)
|
||||
}
|
||||
}
|
||||
|
||||
+7
-1
@@ -66,11 +66,15 @@ class PoolEventOutboxState(
|
||||
success: Boolean,
|
||||
message: String,
|
||||
) {
|
||||
val currentTries = failures[url]
|
||||
if (success || message.shouldDiscard()) {
|
||||
relaysRemaining = relaysRemaining - url
|
||||
failures = failures - url
|
||||
} else if (message.isAuthRequired()) {
|
||||
// NIP-42 AUTH challenge in flight — don't count toward the try cap.
|
||||
// RelayAuthenticator signs + relay re-issues OK; syncFilters() then
|
||||
// re-pumps this outbox so the original publish is retried.
|
||||
} else {
|
||||
val currentTries = failures[url]
|
||||
if (currentTries != null) {
|
||||
currentTries.addResponse(message)
|
||||
} else {
|
||||
@@ -91,6 +95,8 @@ class PoolEventOutboxState(
|
||||
this.startsWith("deleted:") ||
|
||||
this.startsWith("invalid:")
|
||||
|
||||
fun String.isAuthRequired() = this.startsWith("auth-required:")
|
||||
|
||||
// Tries 3 times
|
||||
class Tries(
|
||||
var tries: List<Long> = listOf(),
|
||||
|
||||
+11
-3
@@ -136,7 +136,9 @@ open class BasicRelayClient(
|
||||
socket?.connect()
|
||||
} catch (e: Exception) {
|
||||
if (e is CancellationException) throw e
|
||||
listener.onCannotConnect(this, "Error when trying to connect: ${e.message ?: e::class.simpleName}")
|
||||
val typeName = e::class.simpleName
|
||||
val detail = e.message?.let { "$it ($typeName)" } ?: (typeName ?: "unknown error")
|
||||
listener.onCannotConnect(this, "Error when trying to connect: $detail")
|
||||
listener.onDisconnected(this)
|
||||
dontTryAgainForALongTime()
|
||||
markConnectionAsClosed()
|
||||
@@ -187,9 +189,15 @@ open class BasicRelayClient(
|
||||
} else {
|
||||
socket?.disconnect()
|
||||
|
||||
// suppression rules below must match the raw message; displayMsg is for listener output only
|
||||
// suppression rules below must match the raw message; displayMsg is for listener output only.
|
||||
// Always include the exception's class name: message text is
|
||||
// localized and inconsistent across platforms, but the type
|
||||
// (SocketTimeoutException / UnknownHostException / SSLHandshakeException /
|
||||
// ConnectException …) is stable and lets listeners classify a failure
|
||||
// reliably — a busy relay (timeout) vs a dead one (bad domain / TLS).
|
||||
val msg = t.message
|
||||
val displayMsg = msg ?: t::class.simpleName
|
||||
val typeName = t::class.simpleName
|
||||
val displayMsg = if (msg != null) "$msg ($typeName)" else (typeName ?: "unknown error")
|
||||
|
||||
// checks if this is an actual failure. Closing the socket generates an onFailure as well.
|
||||
// ignore tor errors.
|
||||
|
||||
+3
-3
@@ -156,7 +156,7 @@ class RelayUrlNormalizer {
|
||||
|
||||
if (trimmed.contains("://")) {
|
||||
// some other scheme we cannot connect to.
|
||||
Log.w("RelayUrlNormalizer") { "Rejected $url" }
|
||||
Log.d("RelayUrlNormalizer") { "Rejected $url" }
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -189,14 +189,14 @@ class RelayUrlNormalizer {
|
||||
normalizedUrls.put(url, NormalizationResult.Success(normalized))
|
||||
normalized
|
||||
} else {
|
||||
Log.w("NormalizedRelayUrl") { "Rejected $url" }
|
||||
Log.d("NormalizedRelayUrl") { "Rejected $url" }
|
||||
normalizedUrls.put(url, NormalizationResult.Error)
|
||||
null
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
if (e is CancellationException) throw e
|
||||
normalizedUrls.put(url, NormalizationResult.Error)
|
||||
Log.w("NormalizedRelayUrl") { "Rejected $url" }
|
||||
Log.d("NormalizedRelayUrl") { "Rejected $url" }
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.store
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Address
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.core.isAddressable
|
||||
import com.vitorpamplona.quartz.nip01Core.core.isReplaceable
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
|
||||
import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent
|
||||
|
||||
/** The addressable/replaceable coordinate of [event] as a NIP-01 `a`-tag value. */
|
||||
private fun addressValue(event: Event): String {
|
||||
val dTag = if (event.kind.isAddressable()) event.tags.firstOrNull { it.size > 1 && it[0] == "d" }?.get(1) ?: "" else ""
|
||||
return Address.assemble(event.kind, event.pubKey, dTag)
|
||||
}
|
||||
|
||||
/**
|
||||
* The local deletion events that would make [relay] remove one of [serverEvents] — the
|
||||
* events the relay HAS that we LACK. Used by sync to push *only* the deletions that
|
||||
* actually apply to what the relay holds, and nothing else (not other deletions by the
|
||||
* same author). Covers every way a stored deletion can reach an event:
|
||||
*
|
||||
* - **NIP-09, id-based** — a kind-5 with an `e` tag naming a server event's id.
|
||||
* - **NIP-09, address-based** — a kind-5 with an `a` tag naming a server event's
|
||||
* addressable/replaceable coordinate, at or after that event's `created_at`
|
||||
* (NIP-09 only deletes `created_at <= deletion.created_at`).
|
||||
* - **NIP-62 vanish** — a kind-62 by a server event's author, targeting [relay] (its
|
||||
* `relay` tags name the URL or `ALL_RELAYS`), issued after that event (a vanish
|
||||
* deletes `created_at < vanish.created_at`).
|
||||
*
|
||||
* Deduped by event id; a single deletion covering several events is returned once.
|
||||
*
|
||||
* [query] is where the deletions are looked up — it is source-agnostic on purpose, so
|
||||
* the same coverage rule runs in both sync directions:
|
||||
* - **up** (send our deletions): `events` are the relay's, `query` is the local store —
|
||||
* which of OUR deletions would delete what the relay still holds.
|
||||
* - **down** (apply the relay's deletions): `events` are ours, `query` fetches from the
|
||||
* relay — which of the RELAY'S deletions would delete what we still hold.
|
||||
*/
|
||||
suspend fun deletionsCovering(
|
||||
events: List<Event>,
|
||||
relay: NormalizedRelayUrl,
|
||||
query: suspend (Filter) -> List<Event>,
|
||||
): List<Event> {
|
||||
if (events.isEmpty()) return emptyList()
|
||||
val covering = LinkedHashMap<HexKey, Event>()
|
||||
|
||||
// 1. id-based NIP-09: a kind-5 `e`-tagging an event's id.
|
||||
query(Filter(kinds = listOf(DeletionEvent.KIND), tags = mapOf("e" to events.map { it.id })))
|
||||
.forEach { covering[it.id] = it }
|
||||
|
||||
// 2. address-based NIP-09: a kind-5 `a`-tagging an event's coordinate, cutoff-checked.
|
||||
val byAddress = events.filter { it.kind.isAddressable() || it.kind.isReplaceable() }.groupBy(::addressValue)
|
||||
if (byAddress.isNotEmpty()) {
|
||||
query(Filter(kinds = listOf(DeletionEvent.KIND), tags = mapOf("a" to byAddress.keys.toList())))
|
||||
.forEach { del ->
|
||||
if (del !is DeletionEvent) return@forEach
|
||||
for (addr in del.deleteAddresses()) {
|
||||
val hit = byAddress[addr.toValue()] ?: continue
|
||||
if (hit.any { it.createdAt <= del.createdAt }) {
|
||||
covering[del.id] = del
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. NIP-62 vanish: a kind-62 by an event's author, targeting this relay, issued after it.
|
||||
query(Filter(kinds = listOf(RequestToVanishEvent.KIND), authors = events.mapTo(HashSet()) { it.pubKey }.toList()))
|
||||
.forEach { vanish ->
|
||||
if (vanish !is RequestToVanishEvent || !vanish.shouldVanishFrom(relay)) return@forEach
|
||||
if (events.any { it.pubKey == vanish.pubKey && it.createdAt < vanish.createdAt }) covering[vanish.id] = vanish
|
||||
}
|
||||
|
||||
return covering.values.toList()
|
||||
}
|
||||
|
||||
/** [deletionsCovering] with the local store as the deletion source (the "up" direction). */
|
||||
suspend fun IEventStore.deletionsCovering(
|
||||
serverEvents: List<Event>,
|
||||
relay: NormalizedRelayUrl,
|
||||
): List<Event> = deletionsCovering(serverEvents, relay) { query<Event>(it) }
|
||||
@@ -22,8 +22,11 @@ package com.vitorpamplona.quartz.nip01Core.store
|
||||
|
||||
import com.vitorpamplona.negentropy.storage.IStorage
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
|
||||
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
|
||||
|
||||
interface IEventStore : AutoCloseable {
|
||||
companion object {
|
||||
@@ -121,6 +124,41 @@ interface IEventStore : AutoCloseable {
|
||||
|
||||
suspend fun count(filters: List<Filter>): Int
|
||||
|
||||
/**
|
||||
* Every distinct identity author with at least one stored event that has
|
||||
* NO NIP-65 relay list (kind 10002 / "outbox") in this store.
|
||||
*
|
||||
* This is a whole-store anti-join — the set of all authors minus the
|
||||
* authors who already have an outbox — which the positive-only nostr
|
||||
* [Filter] grammar cannot express (there is no "NOT kind 10002"), so
|
||||
* it is its own method rather than a [query]. "Missing" is relative to
|
||||
* what THIS store holds (see [relay]); an author whose only 10002 was
|
||||
* deleted (NIP-09) or expired (NIP-40) is reported as missing, because
|
||||
* no row remains for it. Order is unspecified.
|
||||
*
|
||||
* GiftWraps (kind 1059) are NOT counted as authors: their `pubkey` is a
|
||||
* random one-time key, so including them would return an unbounded set of
|
||||
* ephemeral keys that can never own a 10002 — useless to the outbox model
|
||||
* this feeds.
|
||||
*
|
||||
* The default implementation walks the store: it collects the authors
|
||||
* that DO have an outbox, then streams every event and keeps the
|
||||
* authors not in that set. Correct for any store but O(events), and it
|
||||
* decodes every event just to read its pubkey. SQLite overrides it with
|
||||
* an index-only `EXCEPT` over `event_headers` that never materialises an
|
||||
* event (see `QueryBuilder.authorsMissingKind`).
|
||||
*/
|
||||
suspend fun authorsMissingOutbox(): List<HexKey> {
|
||||
val withOutbox = HashSet<HexKey>()
|
||||
query<Event>(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND))) { withOutbox.add(it.pubKey) }
|
||||
|
||||
val missing = LinkedHashSet<HexKey>()
|
||||
query<Event>(Filter()) { event ->
|
||||
if (event.kind != GiftWrapEvent.KIND && event.pubKey !in withOutbox) missing.add(event.pubKey)
|
||||
}
|
||||
return missing.toList()
|
||||
}
|
||||
|
||||
/**
|
||||
* NIP-77 negentropy snapshot. Returns `(created_at, id)` pairs
|
||||
* for every event matching [filters], with no content/tags/sig
|
||||
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.store
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.verify
|
||||
import com.vitorpamplona.quartz.utils.Log
|
||||
|
||||
/**
|
||||
* Verify [event]'s NIP-01 id + signature and, if valid, persist it to this store.
|
||||
* Returns `true` when the event was accepted (verified) — even if the insert was a
|
||||
* no-op — so callers can gate "surface this event" on the return.
|
||||
*
|
||||
* A UNIQUE-constraint rejection is normal, not a failure: the store already holds
|
||||
* this id, or a newer version of a replaceable (kind 0/3/10000-19999). The outbox
|
||||
* model routinely delivers the same event from several of a user's write relays, so
|
||||
* a crawl produces these by the hundred-thousand — so only genuine persistence
|
||||
* failures (I/O, full disk, corruption) are logged. Persistence is best-effort: an
|
||||
* insert error is swallowed, not propagated, so it can't break a live subscription.
|
||||
*
|
||||
* This is the single verify-then-store sink every event-arrival path should funnel
|
||||
* through, so the store stays the authoritative cache of what has been seen.
|
||||
*/
|
||||
suspend fun IEventStore.verifyAndInsert(event: Event): Boolean {
|
||||
if (!event.verify()) {
|
||||
Log.w("EventStore") { "dropped event ${event.id.take(8)} kind=${event.kind} — bad signature" }
|
||||
return false
|
||||
}
|
||||
try {
|
||||
insert(event)
|
||||
} catch (t: Throwable) {
|
||||
if (t.message?.contains("UNIQUE constraint", ignoreCase = true) != true) {
|
||||
Log.w("EventStore") { "store insert failed for ${event.id.take(8)}: ${t.message}" }
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
+2
@@ -77,6 +77,8 @@ class EventStore(
|
||||
|
||||
override suspend fun count(filters: List<Filter>) = store.count(filters)
|
||||
|
||||
override suspend fun authorsMissingOutbox() = store.authorsMissingOutbox()
|
||||
|
||||
override suspend fun snapshotIdsForNegentropy(
|
||||
filters: List<Filter>,
|
||||
maxEntries: Int?,
|
||||
|
||||
+10
-11
@@ -204,9 +204,7 @@ class FullTextSearchModule(
|
||||
val kinds = searchableKindsPresent(db)
|
||||
if (kinds.isEmpty()) return
|
||||
|
||||
val selectSql =
|
||||
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
|
||||
"FROM event_headers WHERE kind IN (${kinds.joinToString(",")})"
|
||||
val selectSql = "$SELECT_EVENT_COLUMNS WHERE kind IN (${kinds.joinToString(",")})"
|
||||
|
||||
db.prepare(insertFTS).use { write ->
|
||||
db.prepare(selectSql).use { read ->
|
||||
@@ -259,10 +257,7 @@ class FullTextSearchModule(
|
||||
val kinds = searchableKindsPresent(db)
|
||||
if (kinds.isEmpty()) return FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true)
|
||||
|
||||
val selectSql =
|
||||
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
|
||||
"FROM event_headers WHERE row_id > ? AND kind IN (${kinds.joinToString(",")}) " +
|
||||
"ORDER BY row_id LIMIT ?"
|
||||
val selectSql = selectSearchablePageSql(kinds)
|
||||
|
||||
var last = afterRowId
|
||||
var processed = 0
|
||||
@@ -346,10 +341,7 @@ class FullTextSearchModule(
|
||||
var last = watermark
|
||||
var processed = 0
|
||||
if (kinds.isNotEmpty()) {
|
||||
val selectSql =
|
||||
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
|
||||
"FROM event_headers WHERE row_id > ? AND kind IN (${kinds.joinToString(",")}) " +
|
||||
"ORDER BY row_id LIMIT ?"
|
||||
val selectSql = selectSearchablePageSql(kinds)
|
||||
db.prepare(insertFTS).use { write ->
|
||||
db.prepare(selectSql).use { read ->
|
||||
read.bindLong(1, watermark)
|
||||
@@ -426,5 +418,12 @@ class FullTextSearchModule(
|
||||
// inspects the resulting runtime type.
|
||||
private const val PROBE_ID = "0"
|
||||
private val EMPTY_TAGS = emptyArray<Array<String>>()
|
||||
|
||||
/** Column order matches the positional `read.getText(1)`…`getText(7)` event rebuilds. */
|
||||
private const val SELECT_EVENT_COLUMNS =
|
||||
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig FROM event_headers"
|
||||
|
||||
/** One `row_id`-cursored page of searchable events; binds: cursor, limit. */
|
||||
private fun selectSearchablePageSql(kinds: List<Int>) = "$SELECT_EVENT_COLUMNS WHERE row_id > ? AND kind IN (${kinds.joinToString(",")}) ORDER BY row_id LIMIT ?"
|
||||
}
|
||||
}
|
||||
|
||||
+48
@@ -31,6 +31,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.store.IdAndTime
|
||||
import com.vitorpamplona.quartz.nip01Core.store.RawEvent
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.sql.where
|
||||
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
|
||||
import com.vitorpamplona.quartz.utils.EventFactory
|
||||
|
||||
class QueryBuilder(
|
||||
@@ -594,6 +595,53 @@ class QueryBuilder(
|
||||
return db.countIn(rowIdSubqueries.sql, rowIdSubqueries.args)
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Anti-join projections
|
||||
//
|
||||
// Set-difference over authors — "who is missing an event of kind K"
|
||||
// — which the positive-only nostr Filter grammar can't express, so
|
||||
// it lives here as a dedicated SELECT rather than going through the
|
||||
// filter → SQL path.
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Distinct identity authors with at least one stored event that have NO
|
||||
* stored event of [kind], as an `EXCEPT` of two sets over `event_headers`:
|
||||
* all authors, minus the authors that have a [kind]. Both sides are
|
||||
* answered index-only off `query_by_kind_pubkey_created`
|
||||
* (kind, pubkey, …) — which is created unconditionally, so this does not
|
||||
* depend on the optional pubkey-alone index — and `EXCEPT` diffs them
|
||||
* through one temp b-tree. That is ~3× faster than a
|
||||
* `DISTINCT … NOT EXISTS` correlated scan, which pays one index seek per
|
||||
* distinct author; the gap widens with author cardinality. Order is
|
||||
* unspecified (`EXCEPT` returns pubkey-sorted, which callers must not rely
|
||||
* on).
|
||||
*
|
||||
* GiftWraps (kind 1059) are excluded from the "authors" set: their
|
||||
* `pubkey` is a random one-time key (the real recipient lives only in
|
||||
* `pubkey_owner_hash`), so counting them would return an unbounded set of
|
||||
* ephemeral keys that can never own a [kind] event.
|
||||
*/
|
||||
fun authorsMissingKind(
|
||||
kind: Int,
|
||||
db: SQLiteConnection,
|
||||
): List<HexKey> {
|
||||
val sql =
|
||||
"""
|
||||
SELECT DISTINCT pubkey FROM event_headers WHERE kind <> ${GiftWrapEvent.KIND}
|
||||
EXCEPT
|
||||
SELECT pubkey FROM event_headers WHERE kind = ?
|
||||
""".trimIndent()
|
||||
return db.prepare(sql).use { stmt ->
|
||||
stmt.bindLong(1, kind.toLong())
|
||||
val out = ArrayList<HexKey>()
|
||||
while (stmt.step()) {
|
||||
out.add(stmt.getText(0))
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
private fun SQLiteConnection.countEverything() = runCount("SELECT count(*) as count FROM event_headers")
|
||||
|
||||
private fun SQLiteConnection.countIn(
|
||||
|
||||
+3
@@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.nip01Core.store.RawEvent
|
||||
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
|
||||
import com.vitorpamplona.quartz.nip40Expiration.isExpired
|
||||
import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent
|
||||
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
|
||||
import com.vitorpamplona.quartz.nip77Negentropy.LiveNegentropyIndex
|
||||
|
||||
class SQLiteEventStore(
|
||||
@@ -555,6 +556,8 @@ class SQLiteEventStore(
|
||||
|
||||
suspend fun count(filters: List<Filter>): Int = pool.useReader { queryBuilder.count(filters, it) }
|
||||
|
||||
suspend fun authorsMissingOutbox(): List<HexKey> = pool.useReader { queryBuilder.authorsMissingKind(AdvertisedRelayListEvent.KIND, it) }
|
||||
|
||||
suspend fun snapshotIdsForNegentropy(
|
||||
filters: List<Filter>,
|
||||
maxEntries: Int? = null,
|
||||
|
||||
@@ -23,6 +23,7 @@ package com.vitorpamplona.quartz.nip17Dm
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.hints.EventHintBundle
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
|
||||
import com.vitorpamplona.quartz.nip01Core.tags.people.taggedUserIds
|
||||
@@ -33,9 +34,12 @@ import com.vitorpamplona.quartz.nip17Dm.messages.ChatMessageEvent
|
||||
import com.vitorpamplona.quartz.nip25Reactions.ReactionEvent
|
||||
import com.vitorpamplona.quartz.nip30CustomEmoji.EmojiUrlTag
|
||||
import com.vitorpamplona.quartz.nip40Expiration.expiration
|
||||
import com.vitorpamplona.quartz.nip46RemoteSigner.signer.NostrSignerRemote
|
||||
import com.vitorpamplona.quartz.nip59Giftwrap.seals.SealedRumorEvent
|
||||
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
|
||||
import com.vitorpamplona.quartz.utils.mapNotNullAsync
|
||||
import kotlinx.coroutines.sync.Semaphore
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
|
||||
class NIP17Factory {
|
||||
data class Result(
|
||||
@@ -43,10 +47,34 @@ class NIP17Factory {
|
||||
val wraps: List<GiftWrapEvent>,
|
||||
)
|
||||
|
||||
/**
|
||||
* Build one NIP-59 gift wrap per recipient.
|
||||
*
|
||||
* The rumor (kind 14) `created_at` is implicitly shared across all wraps
|
||||
* because [event] is signed once by the caller before the per-recipient
|
||||
* loop runs — every seal encodes the same rumor `id`. This anchors
|
||||
* cross-recipient dedupe + reaction/receipt targeting on group sends.
|
||||
*
|
||||
* Per NIP-17, the gift wrap's `p` tag MAY carry the recipient's primary
|
||||
* DM inbox relay as a hint. Pass [recipientRelayHints] to surface those;
|
||||
* the default `{ null }` lambda preserves the historical 2-element tag
|
||||
* shape for every recipient.
|
||||
*
|
||||
* When [signer] is a [NostrSignerRemote] (NIP-46 bunker), seal building
|
||||
* is rate-limited to [BUNKER_PARALLELISM] concurrent operations. Each
|
||||
* seal needs `nip44_encrypt` + `sign` round-trips against the bunker; a
|
||||
* 5-recipient group otherwise launches 10 concurrent in-flight RPCs and
|
||||
* saturates the bunker socket. Local signers (NostrSignerInternal,
|
||||
* NostrSignerSync) run fully parallel — no semaphore overhead.
|
||||
*
|
||||
* The proper fix is the batched `nip44_get_conversation_keys` NIP-46
|
||||
* RPC (separate plan); this is the interim throttle until that lands.
|
||||
*/
|
||||
private suspend fun createWraps(
|
||||
event: Event,
|
||||
to: Set<HexKey>,
|
||||
signer: NostrSigner,
|
||||
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
|
||||
): List<GiftWrapEvent> {
|
||||
val innerExpDelta =
|
||||
event.expiration()?.let {
|
||||
@@ -57,29 +85,47 @@ class NIP17Factory {
|
||||
}
|
||||
}
|
||||
|
||||
val bunkerLimiter = if (signer is NostrSignerRemote) Semaphore(BUNKER_PARALLELISM) else null
|
||||
|
||||
return mapNotNullAsync(
|
||||
to.toList(),
|
||||
) { next ->
|
||||
GiftWrapEvent.create(
|
||||
event =
|
||||
SealedRumorEvent.create(
|
||||
event = event,
|
||||
encryptTo = next,
|
||||
expirationDelta = innerExpDelta,
|
||||
signer = signer,
|
||||
),
|
||||
recipientPubKey = next,
|
||||
expirationDelta = innerExpDelta,
|
||||
)
|
||||
val build: suspend () -> GiftWrapEvent = {
|
||||
GiftWrapEvent.create(
|
||||
event =
|
||||
SealedRumorEvent.create(
|
||||
event = event,
|
||||
encryptTo = next,
|
||||
expirationDelta = innerExpDelta,
|
||||
signer = signer,
|
||||
),
|
||||
recipientPubKey = next,
|
||||
expirationDelta = innerExpDelta,
|
||||
recipientRelayHint = recipientRelayHints(next),
|
||||
)
|
||||
}
|
||||
bunkerLimiter?.withPermit { build() } ?: build()
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
/**
|
||||
* Max concurrent in-flight NIP-46 RPCs when building wraps via a
|
||||
* remote signer. Empirically a sweet spot — covers parallelism
|
||||
* speedup for 2–4 recipient sends without saturating typical
|
||||
* bunker apps (nsec.app, Amber, Keychat) that serialize requests
|
||||
* internally past ~10 in-flight.
|
||||
*/
|
||||
const val BUNKER_PARALLELISM = 4
|
||||
}
|
||||
|
||||
suspend fun createMessageNIP17(
|
||||
template: EventTemplate<ChatMessageEvent>,
|
||||
signer: NostrSigner,
|
||||
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
|
||||
): Result {
|
||||
val senderMessage = signer.sign(template)
|
||||
val wraps = createWraps(senderMessage, senderMessage.groupMembers(), signer)
|
||||
val wraps = createWraps(senderMessage, senderMessage.groupMembers(), signer, recipientRelayHints)
|
||||
return Result(
|
||||
msg = senderMessage,
|
||||
wraps = wraps,
|
||||
@@ -108,9 +154,10 @@ class NIP17Factory {
|
||||
suspend fun createEncryptedFileNIP17(
|
||||
template: EventTemplate<ChatMessageEncryptedFileHeaderEvent>,
|
||||
signer: NostrSigner,
|
||||
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
|
||||
): Result {
|
||||
val senderMessage = signer.sign(template)
|
||||
val wraps = createWraps(senderMessage, senderMessage.groupMembers(), signer)
|
||||
val wraps = createWraps(senderMessage, senderMessage.groupMembers(), signer, recipientRelayHints)
|
||||
|
||||
return Result(
|
||||
msg = senderMessage,
|
||||
@@ -142,12 +189,13 @@ class NIP17Factory {
|
||||
originalNote: EventHintBundle<Event>,
|
||||
to: List<HexKey>,
|
||||
signer: NostrSigner,
|
||||
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
|
||||
): Result {
|
||||
val senderPublicKey = signer.pubKey
|
||||
val template = ReactionEvent.build(content, originalNote)
|
||||
|
||||
val senderReaction = signer.sign(template)
|
||||
val wraps = createWraps(senderReaction, to.plus(senderPublicKey).toSet(), signer)
|
||||
val wraps = createWraps(senderReaction, to.plus(senderPublicKey).toSet(), signer, recipientRelayHints)
|
||||
return Result(
|
||||
msg = senderReaction,
|
||||
wraps = wraps,
|
||||
@@ -159,12 +207,13 @@ class NIP17Factory {
|
||||
originalNote: EventHintBundle<Event>,
|
||||
to: List<HexKey>,
|
||||
signer: NostrSigner,
|
||||
recipientRelayHints: (HexKey) -> NormalizedRelayUrl? = { null },
|
||||
): Result {
|
||||
val senderPublicKey = signer.pubKey
|
||||
val template = ReactionEvent.build(emojiUrl, originalNote)
|
||||
|
||||
val senderReaction = signer.sign(template)
|
||||
val wraps = createWraps(senderReaction, to.plus(senderPublicKey).toSet(), signer)
|
||||
val wraps = createWraps(senderReaction, to.plus(senderPublicKey).toSet(), signer, recipientRelayHints)
|
||||
|
||||
return Result(
|
||||
msg = senderReaction,
|
||||
|
||||
+14
-2
@@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.core.firstTagValue
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
|
||||
import com.vitorpamplona.quartz.nip01Core.tags.people.PTag
|
||||
@@ -96,11 +97,22 @@ open class GiftWrapEvent(
|
||||
const val KIND = 1059
|
||||
const val ALT = "Encrypted event"
|
||||
|
||||
/**
|
||||
* Build a NIP-59 gift wrap addressed to `recipientPubKey`.
|
||||
*
|
||||
* Per NIP-17 §Publishing, the `p` tag on the wrap MAY carry the
|
||||
* recipient's primary DM inbox relay as a hint, so other clients
|
||||
* the recipient runs (or relays acting as inbox routers) can locate
|
||||
* the wrap without a separate kind:10050 lookup. Pass it via
|
||||
* [recipientRelayHint] — `null` (the default) preserves the
|
||||
* historical 2-element `["p", pubkey]` shape.
|
||||
*/
|
||||
fun create(
|
||||
event: Event,
|
||||
recipientPubKey: HexKey,
|
||||
expirationDelta: Long? = null,
|
||||
createdAt: Long = TimeUtils.randomWithTwoDays(),
|
||||
recipientRelayHint: NormalizedRelayUrl? = null,
|
||||
): GiftWrapEvent {
|
||||
val signer = NostrSignerSync(KeyPair()) // GiftWrap is always a random key
|
||||
|
||||
@@ -109,11 +121,11 @@ open class GiftWrapEvent(
|
||||
// minimum expiration is two days in the future due to the random created at
|
||||
// this will make sure the even arrives and is not deleted because of the 2 days.
|
||||
arrayOf(
|
||||
PTag.assemble(recipientPubKey, null),
|
||||
PTag.assemble(recipientPubKey, recipientRelayHint),
|
||||
ExpirationTag.assemble(createdAt + it + TimeUtils.twoDays()),
|
||||
)
|
||||
} ?: arrayOf(
|
||||
PTag.assemble(recipientPubKey, null),
|
||||
PTag.assemble(recipientPubKey, recipientRelayHint),
|
||||
)
|
||||
|
||||
return signer.sign(
|
||||
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip66RelayMonitor.reachability
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner
|
||||
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
|
||||
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.RelayDiscoveryEvent
|
||||
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.networkType
|
||||
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.rtt
|
||||
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkType
|
||||
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RttType
|
||||
import com.vitorpamplona.quartz.utils.TimeUtils
|
||||
|
||||
/**
|
||||
* A durable, shareable relay-reachability cache backed by an [IEventStore] as
|
||||
* NIP-66 **kind:30166 Relay Discovery** events — so the crawler, the WoT updater,
|
||||
* and future runs all read and write the *same* liveness knowledge instead of each
|
||||
* rediscovering dead relays from an in-memory set that is wiped when the process ends.
|
||||
*
|
||||
* ## Why NIP-66 / the event store
|
||||
* A 30166 event is addressable by its `d`-tag (the normalized relay URL), so the
|
||||
* store keeps exactly **one replaceable record per (monitor, relay)** — a natural
|
||||
* per-relay status slot with a `created_at` timestamp that gives us a free TTL. The
|
||||
* event store gives us persistence, cross-procedure sharing, and interop for free:
|
||||
* 30166 events published by *other* monitors (nostr.watch et al.) can be ingested to
|
||||
* seed reachability without probing, and our own records can be published back.
|
||||
*
|
||||
* ## How "dead" is represented
|
||||
* NIP-66 has no explicit offline field; liveness is inferred from a fresh record that
|
||||
* carries an `rtt-open` (a successful connection). This cache follows that convention:
|
||||
* - **reachable** → a 30166 **with** `rtt-open`, `created_at` = probe time.
|
||||
* - **dead** → a 30166 **without** `rtt-open` ("we checked, could not open"),
|
||||
* `created_at` = probe time.
|
||||
*
|
||||
* So a fresh rtt-less record distinguishes *checked-and-dead* from *never-checked*
|
||||
* (no record). When both a dead and a live record exist within the TTL for the same
|
||||
* relay, **live wins** — any recent successful open overrides an earlier failure,
|
||||
* whether the two came from us across time or from two different monitors.
|
||||
*
|
||||
* ## Not a replacement for the hot path
|
||||
* [snapshot] is meant to be loaded ONCE at the start of a run into whatever in-memory
|
||||
* structure the caller already uses for per-request `isDead` checks; [record] flushes
|
||||
* a run's findings back at the end. It is deliberately not queried per routing decision.
|
||||
*
|
||||
* A relay is only ever skipped for the TTL window, never permanently — consistent with
|
||||
* the outbox rule that every advertised write relay must be tried: a TTL'd record is
|
||||
* "skip for now", not "ignore this author's home forever".
|
||||
*
|
||||
* ## The signer is a dedicated monitor service identity
|
||||
* [signer] should be a **machine-level monitor key**, NOT a user/observer account: per
|
||||
* NIP-66 a monitor is its own pubkey (which also publishes a kind:10166 announcement,
|
||||
* a kind:0 profile and a kind:10002). Publishing these under the observer's key would
|
||||
* conflate the WoT identity with a relay-monitoring service. [snapshot] still honours
|
||||
* records from ANY author (so third-party monitors can be ingested); only [record]
|
||||
* writes under this monitor key.
|
||||
*/
|
||||
class RelayReachabilityStore(
|
||||
private val store: IEventStore,
|
||||
private val signer: NostrSigner,
|
||||
private val ttlSeconds: Long = DEFAULT_TTL_SECONDS,
|
||||
) {
|
||||
/**
|
||||
* An in-memory view of the fresh (within-TTL) reachability records. [dead] holds
|
||||
* relays proven unreachable and not since seen live; [live] holds relays with a
|
||||
* recent successful open. A relay absent from both is simply unknown — re-probe it.
|
||||
*/
|
||||
class Snapshot(
|
||||
val dead: Set<NormalizedRelayUrl>,
|
||||
val live: Set<NormalizedRelayUrl>,
|
||||
) {
|
||||
fun isKnownDead(relay: NormalizedRelayUrl) = relay in dead
|
||||
|
||||
val size: Int get() = dead.size + live.size
|
||||
}
|
||||
|
||||
/**
|
||||
* Load every 30166 record fresher than [ttlSeconds] and fold it into a [Snapshot].
|
||||
* Records from any monitor are honoured (live-wins), so ingesting third-party
|
||||
* monitors' 30166 into [store] transparently improves the result.
|
||||
*/
|
||||
suspend fun snapshot(now: Long = TimeUtils.now()): Snapshot {
|
||||
val since = now - ttlSeconds
|
||||
val events =
|
||||
store.query<RelayDiscoveryEvent>(
|
||||
Filter(kinds = listOf(RelayDiscoveryEvent.KIND), since = since),
|
||||
)
|
||||
val live = HashSet<NormalizedRelayUrl>()
|
||||
val dead = HashSet<NormalizedRelayUrl>()
|
||||
for (ev in events) {
|
||||
val relay = ev.relay() ?: continue
|
||||
if (ev.rttOpen() != null) live.add(relay) else dead.add(relay)
|
||||
}
|
||||
// A recent successful open (from us later, or from another monitor) overrides
|
||||
// an earlier dead mark for the same relay.
|
||||
dead.removeAll(live)
|
||||
return Snapshot(dead, live)
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a run's reachability findings as 30166 events: each [reachable] relay as
|
||||
* a record WITH `rtt-open`, each [dead] relay (that is not also reachable) as one
|
||||
* WITHOUT. Signed by [signer] and inserted into [store]; being addressable, each
|
||||
* replaces this monitor's prior record for that relay, so the store stays bounded
|
||||
* at roughly the number of distinct relays.
|
||||
*
|
||||
* [rttOpenMs] is the measured open round-trip in ms. It defaults to 0 as a **liveness
|
||||
* flag only** — presence of the `rtt-open` tag, not its magnitude, is what [snapshot]
|
||||
* reads as "reachable", and a caller that merely proved a relay served events (like
|
||||
* the crawler) has no dedicated probe latency to report. A `0` therefore means
|
||||
* "reachable, latency not probed by this writer", NOT a real 0 ms measurement. Do NOT
|
||||
* publish these records to the wider network as authoritative latency data until a
|
||||
* dedicated monitor probe supplies a real [rttOpenMs]; aggregators rank by it.
|
||||
*/
|
||||
suspend fun record(
|
||||
reachable: Set<NormalizedRelayUrl>,
|
||||
dead: Set<NormalizedRelayUrl>,
|
||||
now: Long = TimeUtils.now(),
|
||||
rttOpenMs: Long = 0,
|
||||
) {
|
||||
for (relay in reachable) writeOne(relay, up = true, now, rttOpenMs)
|
||||
for (relay in dead) if (relay !in reachable) writeOne(relay, up = false, now, rttOpenMs)
|
||||
}
|
||||
|
||||
private suspend fun writeOne(
|
||||
relay: NormalizedRelayUrl,
|
||||
up: Boolean,
|
||||
now: Long,
|
||||
rttOpenMs: Long,
|
||||
) {
|
||||
val template =
|
||||
RelayDiscoveryEvent.build(relay, createdAt = now) {
|
||||
networkType(networkTypeOf(relay))
|
||||
if (up) rtt(RttType.OPEN, rttOpenMs)
|
||||
}
|
||||
store.insert(signer.sign(template))
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** Default freshness window: a relay's status is trusted for a day, then re-probed. */
|
||||
const val DEFAULT_TTL_SECONDS = 24L * 60 * 60
|
||||
|
||||
/** NIP-66 `n` network type inferred from the URL, so a `.onion`/i2p relay is tagged correctly. */
|
||||
fun networkTypeOf(relay: NormalizedRelayUrl): NetworkType =
|
||||
when {
|
||||
RelayUrlNormalizer.isOnion(relay.url) -> NetworkType.TOR
|
||||
relay.url.contains(".i2p") -> NetworkType.I2P
|
||||
else -> NetworkType.CLEARNET
|
||||
}
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.utils.concurrent
|
||||
|
||||
/**
|
||||
* A thread-safe hash map whose compound operations — [getOrPut] and [merge] —
|
||||
* apply their update **atomically**, not merely one-lock-per-primitive-op. This
|
||||
* is the contract a concurrent producer/consumer pipeline needs: two coroutines
|
||||
* racing `getOrPut` on the same key must agree on a single value, and racing
|
||||
* `merge` must not lose an increment.
|
||||
*
|
||||
* commonMain has no `java.util.concurrent.ConcurrentHashMap`, so this is
|
||||
* expect/actual, matching the split already used by [com.vitorpamplona.quartz.utils.cache.ConcurrentHashCache]:
|
||||
* - JVM / Android → `ConcurrentHashMap` (lock-free, true atomic `computeIfAbsent` / `merge`).
|
||||
* - Native (Apple + Linux) → copy-on-write over an atomic reference, with a
|
||||
* CAS retry loop giving the same atomicity. Correct but O(n)-per-write; the
|
||||
* native targets never run the heavy crawl this backs, they only compile it.
|
||||
*
|
||||
* Only the operations the crawl actually uses are exposed — no full [MutableMap]
|
||||
* surface — so the native copy-on-write actual stays small and obviously correct.
|
||||
*/
|
||||
expect class ConcurrentMap<K : Any, V : Any>() {
|
||||
operator fun get(key: K): V?
|
||||
|
||||
operator fun set(
|
||||
key: K,
|
||||
value: V,
|
||||
)
|
||||
|
||||
/** Atomically return the value for [key], computing and inserting [defaultValue] once if absent. */
|
||||
fun getOrPut(
|
||||
key: K,
|
||||
defaultValue: () -> V,
|
||||
): V
|
||||
|
||||
/**
|
||||
* Atomically insert [value] if [key] is absent, else replace the existing
|
||||
* value with `remap(existing, value)`. Returns the value now stored.
|
||||
*/
|
||||
fun merge(
|
||||
key: K,
|
||||
value: V,
|
||||
remap: (old: V, new: V) -> V,
|
||||
): V
|
||||
|
||||
fun size(): Int
|
||||
|
||||
/** A point-in-time copy of the entries — safe to iterate without holding a lock. */
|
||||
fun snapshot(): Map<K, V>
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.utils.concurrent
|
||||
|
||||
/**
|
||||
* A thread-safe hash set for the crawl's cross-coroutine membership tracking
|
||||
* (dead relays struck by drain workers while the router reads them, relay hints
|
||||
* written by the ingest consumer while the producer reads them).
|
||||
*
|
||||
* commonMain has no `java.util.concurrent.ConcurrentHashMap.newKeySet()`, so this
|
||||
* is expect/actual with the same JVM-vs-native split as [ConcurrentMap]:
|
||||
* - JVM / Android → `ConcurrentHashMap.newKeySet()`.
|
||||
* - Native → copy-on-write over an atomic reference (compile-only, never the hot path).
|
||||
*/
|
||||
expect class ConcurrentSet<E : Any>() {
|
||||
/** Add [element]; returns true if it was not already present. */
|
||||
fun add(element: E): Boolean
|
||||
|
||||
operator fun contains(element: E): Boolean
|
||||
|
||||
fun size(): Int
|
||||
|
||||
/** A point-in-time copy — safe to iterate or diff against without a lock. */
|
||||
fun snapshot(): Set<E>
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.experimental.graperank
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertNotEquals
|
||||
|
||||
/**
|
||||
* [GrapeRankCrawler.authorityOf] is the key the crawl's timeout-eviction counts
|
||||
* on. It must collapse the many per-user path URLs the outbox model mints for one
|
||||
* server into a single host, WITHOUT folding a distinct sibling host (e.g. a
|
||||
* `filter.` subdomain) into its parent.
|
||||
*/
|
||||
class GrapeRankAuthorityTest {
|
||||
private fun auth(url: String) = GrapeRankCrawler.authorityOf(url)
|
||||
|
||||
@Test
|
||||
fun bareHostIsItsOwnAuthority() {
|
||||
assertEquals("relay.damus.io", auth("wss://relay.damus.io"))
|
||||
assertEquals("relay.damus.io", auth("wss://relay.damus.io/"))
|
||||
assertEquals("nos.lol", auth("ws://nos.lol"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun perUserPathUrlsOnOneHostCollapseToOneAuthority() {
|
||||
val a = auth("wss://filter.nostr.wine/npub1aaaa?broadcast=true")
|
||||
val b = auth("wss://filter.nostr.wine/npub1bbbb?broadcast=true&global=all")
|
||||
val c = auth("wss://filter.nostr.wine/?global=all")
|
||||
assertEquals("filter.nostr.wine", a)
|
||||
assertEquals(a, b)
|
||||
assertEquals(a, c)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun filterSubdomainIsNotFoldedIntoBareHost() {
|
||||
// nostr.wine reads are open; filter.nostr.wine is a different server that may
|
||||
// stall — evicting one must never take out the other.
|
||||
assertNotEquals(auth("wss://filter.nostr.wine/npub1x"), auth("wss://nostr.wine"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun portIsPartOfTheAuthority() {
|
||||
assertEquals("relay.veganostr.com:443", auth("wss://relay.veganostr.com:443/npub1z"))
|
||||
assertEquals("81.68.170.122:7114", auth("ws://81.68.170.122:7114/"))
|
||||
assertNotEquals(auth("wss://example.com:443"), auth("wss://example.com:8080"))
|
||||
}
|
||||
}
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.experimental.graperank
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.exp
|
||||
import kotlin.math.ln
|
||||
import kotlin.math.max
|
||||
import kotlin.random.Random
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class GrapeRankTest {
|
||||
private val obs = "observer"
|
||||
|
||||
private fun graphOf(edges: List<Triple<HexKey, HexKey, TrustRelation>>): TrustGraph {
|
||||
val b = TrustGraphBuilder()
|
||||
for ((source, target, relation) in edges) {
|
||||
when (relation) {
|
||||
TrustRelation.FOLLOW -> b.addFollows(source, listOf(target))
|
||||
TrustRelation.MUTE -> b.addMutes(source, listOf(target))
|
||||
TrustRelation.REPORT -> b.addReports(source, listOf(target))
|
||||
}
|
||||
}
|
||||
return b.build()
|
||||
}
|
||||
|
||||
private fun graphOf(vararg edges: Triple<HexKey, HexKey, TrustRelation>) = graphOf(edges.toList())
|
||||
|
||||
/** Score for a pubkey (0.0 if absent from the graph). */
|
||||
private fun DoubleArray.of(
|
||||
graph: TrustGraph,
|
||||
pubkey: HexKey,
|
||||
): Double {
|
||||
val id = graph.idOf(pubkey)
|
||||
return if (id < 0) 0.0 else this[id]
|
||||
}
|
||||
|
||||
@Test
|
||||
fun observerIsPinnedAtFullSelfTrust() {
|
||||
val graph = graphOf(Triple(obs, "a", TrustRelation.FOLLOW))
|
||||
val scores = GrapeRank().compute(graph, obs)
|
||||
assertEquals(1.0, scores.of(graph, obs), 1e-12)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun directFollowMatchesHandComputedValue() {
|
||||
val graph = graphOf(Triple(obs, "a", TrustRelation.FOLLOW))
|
||||
val scores = GrapeRank().compute(graph, obs)
|
||||
// weight = 0.5 * 1.0 * 0.85 = 0.425 ; score = conf(0.425) = 0.2551612...
|
||||
assertEquals(0.25516127, scores.of(graph, "a"), 1e-6)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun trustDecaysSteeplyAcrossHops() {
|
||||
val graph =
|
||||
graphOf(
|
||||
Triple(obs, "a", TrustRelation.FOLLOW),
|
||||
Triple("a", "b", TrustRelation.FOLLOW),
|
||||
)
|
||||
val scores = GrapeRank().compute(graph, obs)
|
||||
val a = scores.of(graph, "a")
|
||||
val b = scores.of(graph, "b")
|
||||
assertEquals(0.004499, b, 1e-5)
|
||||
assertTrue(b < a / 10.0, "two-hop trust should be far below one-hop trust")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aMuteFromAnEndorsedUserLowersTheScore() {
|
||||
val followOnlyGraph = graphOf(Triple(obs, "b", TrustRelation.FOLLOW))
|
||||
val followOnly = GrapeRank().compute(followOnlyGraph, obs).of(followOnlyGraph, "b")
|
||||
|
||||
val muteGraph =
|
||||
graphOf(
|
||||
Triple(obs, "a", TrustRelation.FOLLOW),
|
||||
Triple(obs, "b", TrustRelation.FOLLOW),
|
||||
Triple("a", "b", TrustRelation.MUTE),
|
||||
)
|
||||
val withMute = GrapeRank().compute(muteGraph, obs).of(muteGraph, "b")
|
||||
|
||||
assertTrue(withMute < followOnly, "a mute from a trusted user should pull b below the follow-only baseline")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun purelyReportedUserFloorsAtZero() {
|
||||
val graph =
|
||||
graphOf(
|
||||
Triple(obs, "a", TrustRelation.FOLLOW),
|
||||
Triple("a", "d", TrustRelation.REPORT),
|
||||
)
|
||||
val scores = GrapeRank().compute(graph, obs)
|
||||
assertEquals(0.0, scores.of(graph, "d"), 1e-9)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun unreachableUsersAreNotScored() {
|
||||
val graph =
|
||||
graphOf(
|
||||
Triple(obs, "a", TrustRelation.FOLLOW),
|
||||
Triple("x", "y", TrustRelation.FOLLOW),
|
||||
)
|
||||
val scores = GrapeRank().compute(graph, obs)
|
||||
assertTrue(scores.of(graph, "a") > 0.0)
|
||||
assertEquals(0.0, scores.of(graph, "y"), 1e-12, "a user with no path from the observer stays 0")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun cyclesConverge() {
|
||||
val graph =
|
||||
graphOf(
|
||||
Triple(obs, "a", TrustRelation.FOLLOW),
|
||||
Triple("a", "b", TrustRelation.FOLLOW),
|
||||
Triple("b", "a", TrustRelation.FOLLOW),
|
||||
)
|
||||
val scores = GrapeRank().compute(graph, obs)
|
||||
assertTrue(scores.of(graph, "a") > 0.0)
|
||||
assertTrue(scores.of(graph, "b") > 0.0)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun deduplicatesRepeatedReportEdges() {
|
||||
// Two report edges a->d collapse to one; the score matches a single report.
|
||||
val once = graphOf(Triple(obs, "a", TrustRelation.FOLLOW), Triple("a", "d", TrustRelation.REPORT))
|
||||
val twice =
|
||||
graphOf(
|
||||
Triple(obs, "a", TrustRelation.FOLLOW),
|
||||
Triple("a", "d", TrustRelation.REPORT),
|
||||
Triple("a", "d", TrustRelation.REPORT),
|
||||
)
|
||||
assertEquals(2, twice.edgeCount(), "duplicate report edge should be dropped")
|
||||
assertEquals(
|
||||
GrapeRank().compute(once, obs).of(once, "d"),
|
||||
GrapeRank().compute(twice, obs).of(twice, "d"),
|
||||
1e-12,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Adversarial cross-check: the worklist propagation must reach the same fixed
|
||||
* point as a naive full-sweep (the reference `v1FullSweep`) on random graphs.
|
||||
*/
|
||||
@Test
|
||||
fun worklistMatchesFullSweepOnRandomGraphs() {
|
||||
val params = GrapeRankParams(convergence = 1e-10)
|
||||
val engine = GrapeRank(params)
|
||||
repeat(50) { seed ->
|
||||
val rng = Random(seed)
|
||||
val n = 3 + rng.nextInt(12)
|
||||
val nodes = (0 until n).map { "u$it" }
|
||||
val edges = ArrayList<Triple<HexKey, HexKey, TrustRelation>>()
|
||||
for (src in nodes) {
|
||||
for (dst in nodes) {
|
||||
if (src == dst) continue
|
||||
if (rng.nextDouble() < 0.25) {
|
||||
val relation =
|
||||
when (rng.nextInt(5)) {
|
||||
0 -> TrustRelation.MUTE
|
||||
1 -> TrustRelation.REPORT
|
||||
else -> TrustRelation.FOLLOW
|
||||
}
|
||||
edges.add(Triple(src, dst, relation))
|
||||
}
|
||||
}
|
||||
}
|
||||
val observer = nodes.first()
|
||||
val graph = graphOf(edges)
|
||||
val scores = engine.compute(graph, observer)
|
||||
val reference = fullSweep(edges, nodes, observer, params)
|
||||
|
||||
for (node in nodes) {
|
||||
if (node == observer) continue // observer self-trust is not part of a ranking
|
||||
val a = scores.of(graph, node)
|
||||
val b = reference[node] ?: 0.0
|
||||
assertEquals(b, a, 1e-5, "seed=$seed node=$node worklist=$a fullSweep=$b")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reference: blind full sweep over every user until nothing changes.
|
||||
private fun fullSweep(
|
||||
edges: List<Triple<HexKey, HexKey, TrustRelation>>,
|
||||
nodes: List<HexKey>,
|
||||
observer: HexKey,
|
||||
params: GrapeRankParams,
|
||||
): Map<HexKey, Double> {
|
||||
// Dedup identical edges (mirrors the builder: report edges dedup; follow/mute
|
||||
// sets are unique per source anyway).
|
||||
val incoming = HashMap<HexKey, MutableSet<Pair<HexKey, TrustRelation>>>()
|
||||
for ((s, t, r) in edges) {
|
||||
if (s == t) continue
|
||||
incoming.getOrPut(t) { LinkedHashSet() }.add(s to r)
|
||||
}
|
||||
|
||||
fun confidence(
|
||||
r: TrustRelation,
|
||||
source: HexKey,
|
||||
) = when (r) {
|
||||
TrustRelation.FOLLOW -> if (source == observer) params.directFollowConfidence else params.indirectFollowConfidence
|
||||
TrustRelation.MUTE -> params.muteConfidence
|
||||
TrustRelation.REPORT -> params.reportConfidence
|
||||
}
|
||||
|
||||
fun weightToConfidence(w: Double) = 1.0 - exp(-w * -ln(params.rigor))
|
||||
|
||||
val scores = HashMap<HexKey, Double>()
|
||||
scores[observer] = 1.0
|
||||
do {
|
||||
var changed = false
|
||||
for (target in nodes) {
|
||||
if (target == observer) continue
|
||||
var sumW = 0.0
|
||||
var sumWR = 0.0
|
||||
for ((source, r) in incoming[target] ?: emptySet()) {
|
||||
val s = scores[source] ?: continue
|
||||
val w = confidence(r, source) * s * params.attenuation
|
||||
sumW += w
|
||||
sumWR += w * r.rating
|
||||
}
|
||||
val newScore = if (abs(sumW) < 0.00001) 0.0 else max(weightToConfidence(sumW) * sumWR / sumW, 0.0)
|
||||
val old = scores.put(target, newScore) ?: 0.0
|
||||
changed = changed || abs(newScore - old) > params.convergence
|
||||
}
|
||||
} while (changed)
|
||||
return scores
|
||||
}
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.experimental.graperank
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class TrustGraphBuilderTest {
|
||||
private val alice = "alice"
|
||||
private val bob = "bob"
|
||||
private val carol = "carol"
|
||||
private val dave = "dave"
|
||||
|
||||
/** Decode a node's incoming edges back to (source, relation) pairs from the CSR. */
|
||||
private fun TrustGraph.incomingOf(pubkey: HexKey): Set<Pair<HexKey, TrustRelation>> {
|
||||
val t = idOf(pubkey)
|
||||
if (t < 0) return emptySet()
|
||||
val out = HashSet<Pair<HexKey, TrustRelation>>()
|
||||
var i = inOffsets[t]
|
||||
val end = inOffsets[t + 1]
|
||||
while (i < end) {
|
||||
val packed = inPacked[i]
|
||||
val source = pubkeyOf(packed and TrustGraph.SOURCE_MASK)
|
||||
val relation = TrustRelation.entries.first { it.code == (packed ushr TrustGraph.SOURCE_BITS) }
|
||||
out.add(source to relation)
|
||||
i++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@Test
|
||||
fun buildsFollowMuteAndReportEdges() {
|
||||
val b = TrustGraphBuilder()
|
||||
b.addFollows(alice, listOf(bob, carol))
|
||||
b.addMutes(bob, listOf(dave))
|
||||
b.addReports(carol, listOf(dave))
|
||||
val graph = b.build()
|
||||
|
||||
assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(bob))
|
||||
assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(carol))
|
||||
assertEquals(
|
||||
setOf(bob to TrustRelation.MUTE, carol to TrustRelation.REPORT),
|
||||
graph.incomingOf(dave),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dropsSelfEdges() {
|
||||
val b = TrustGraphBuilder()
|
||||
b.addFollows(alice, listOf(alice, bob))
|
||||
val graph = b.build()
|
||||
assertTrue(graph.incomingOf(alice).isEmpty(), "a self-follow must not become an edge")
|
||||
assertEquals(setOf(alice to TrustRelation.FOLLOW), graph.incomingOf(bob))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dedupesRepeatedReports() {
|
||||
val b = TrustGraphBuilder()
|
||||
b.addReports(alice, listOf(dave))
|
||||
b.addReports(alice, listOf(dave))
|
||||
val graph = b.build()
|
||||
assertEquals(1, graph.edgeCount())
|
||||
assertEquals(setOf(alice to TrustRelation.REPORT), graph.incomingOf(dave))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun keepsFollowAndMuteFromSameSourceAsDistinctEdges() {
|
||||
val b = TrustGraphBuilder()
|
||||
b.addFollows(alice, listOf(bob))
|
||||
b.addMutes(alice, listOf(bob))
|
||||
val graph = b.build()
|
||||
assertEquals(
|
||||
setOf(alice to TrustRelation.FOLLOW, alice to TrustRelation.MUTE),
|
||||
graph.incomingOf(bob),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun internsEachPubkeyOnce() {
|
||||
val b = TrustGraphBuilder()
|
||||
b.addFollows(alice, listOf(bob, carol))
|
||||
b.addFollows(bob, listOf(carol))
|
||||
val graph = b.build()
|
||||
assertEquals(3, graph.nodeCount, "alice, bob, carol interned once each")
|
||||
assertEquals(3, graph.edgeCount())
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.relay.client.accessories
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertNull
|
||||
|
||||
class DrainFailureTest {
|
||||
// Non-failure and non-"cannot" terminals are never dead signals.
|
||||
@Test
|
||||
fun nonFailureTerminalsAreNull() {
|
||||
assertNull(classifyDrainFailure("eose"))
|
||||
assertNull(classifyDrainFailure("closed:duplicate: sub"))
|
||||
assertNull(classifyDrainFailure("timeout"))
|
||||
}
|
||||
|
||||
// A READ timeout (or generic post-handshake timeout) is alive-but-slow: never
|
||||
// dead. Measured 67% of these relays were reachable when re-probed fresh.
|
||||
@Test
|
||||
fun readTimeoutsStayRetryable() {
|
||||
assertNull(classifyDrainFailure("cannot:Read timed out (SocketTimeoutException)"))
|
||||
assertNull(classifyDrainFailure("cannot:timeout (SocketTimeoutException)"))
|
||||
}
|
||||
|
||||
// An HTTP 429 rate-limit is alive and will serve us after backoff: never dead.
|
||||
// Measured 4/4 such relays reachable when re-probed fresh.
|
||||
@Test
|
||||
fun rateLimitStaysRetryable() {
|
||||
assertNull(classifyDrainFailure("cannot:Server Misconfigured. Response: 429 Too Many Requests (ProtocolException)"))
|
||||
}
|
||||
|
||||
// Failing to ESTABLISH the connection is dead (0/30 reachable fresh). "connect
|
||||
// timed out" must be caught as DEAD and NOT slip into the read-timeout branch.
|
||||
@Test
|
||||
fun connectEstablishmentFailuresAreDead() {
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Connect timed out (SocketTimeoutException)"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Unexpected response code for CONNECT: (IOException)"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Connection refused (ConnectException)"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Failed to connect to /1.2.3.4:443"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:No route to host (NoRouteToHostException)"))
|
||||
}
|
||||
|
||||
// DNS and TLS misconfig can never work: DEAD.
|
||||
@Test
|
||||
fun dnsAndTlsAreDead() {
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Unable to resolve host (UnknownHostException)"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Received fatal alert: unrecognized_name (SSLHandshakeException)"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:PKIX path building failed: certificate (CertificateException)"))
|
||||
}
|
||||
|
||||
// Every other bad HTTP upgrade won't serve us this run (measured 503 0%, 502 20%
|
||||
// reachable; 402/403 gated; 200 not a relay) — DEAD, dropped on the first strike.
|
||||
@Test
|
||||
fun deadOrGatedHttpUpgradesAreDead() {
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Server Misconfigured. not a websocket"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Server Misconfigured. Response: 503 Service Unavailable (ProtocolException)"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Server Misconfigured. Response: 502 Bad Gateway (ProtocolException)"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Server Misconfigured. Response: 402 Payment Required (ProtocolException)"))
|
||||
}
|
||||
|
||||
// A mid-stream reset won't hand us events this run either: DEAD.
|
||||
@Test
|
||||
fun midStreamResetIsDead() {
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Connection reset (SocketException)"))
|
||||
assertEquals(DrainFailure.DEAD, classifyDrainFailure("cannot:Broken pipe (SocketException)"))
|
||||
}
|
||||
}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.relay.client.pool
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertContains
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class PoolEventOutboxStateTest {
|
||||
private val relay = NormalizedRelayUrl("wss://relay.example/")
|
||||
|
||||
private fun fakeEvent() =
|
||||
Event(
|
||||
id = "0".repeat(64),
|
||||
pubKey = "0".repeat(64),
|
||||
createdAt = 0L,
|
||||
kind = 1,
|
||||
tags = emptyArray(),
|
||||
content = "",
|
||||
sig = "0".repeat(128),
|
||||
)
|
||||
|
||||
@Test
|
||||
fun authRequiredResponseDoesNotConsumeTryBudget() {
|
||||
val state = PoolEventOutboxState(fakeEvent(), setOf(relay))
|
||||
|
||||
// Simulate 5 `auth-required:` responses — relay keeps challenging while
|
||||
// RelayAuthenticator signs + sends AUTH events asynchronously. None of
|
||||
// these should be counted against the 3-response try cap.
|
||||
repeat(5) {
|
||||
state.newResponse(relay, success = false, message = "auth-required: please authenticate")
|
||||
}
|
||||
|
||||
// Even after a follow-up newTry, the relay must remain in the outbox so
|
||||
// syncFilters() can re-publish once AUTH succeeds.
|
||||
state.newTry(relay)
|
||||
assertContains(state.relaysLeft(), relay)
|
||||
assertFalse(state.isDone())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun regularRejectionStillBoundedByTryCap() {
|
||||
val state = PoolEventOutboxState(fakeEvent(), setOf(relay))
|
||||
|
||||
// 3 non-AUTH rejections accumulate normally.
|
||||
repeat(3) {
|
||||
state.newResponse(relay, success = false, message = "error: rate limited")
|
||||
}
|
||||
state.newTry(relay)
|
||||
|
||||
// After the 4th newTry (with 3 prior responses already in flight), the
|
||||
// Tries cap kicks in and the relay is dropped from the outbox.
|
||||
assertFalse(state.relaysLeft().contains(relay))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun terminalRejectionImmediatelyDropsRelay() {
|
||||
val state = PoolEventOutboxState(fakeEvent(), setOf(relay))
|
||||
|
||||
state.newResponse(relay, success = false, message = "invalid: malformed event")
|
||||
|
||||
assertFalse(state.relaysLeft().contains(relay))
|
||||
assertTrue(state.isDone())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun successDropsRelayFromOutbox() {
|
||||
val state = PoolEventOutboxState(fakeEvent(), setOf(relay))
|
||||
|
||||
state.newResponse(relay, success = true, message = "")
|
||||
|
||||
assertEquals(emptySet(), state.relaysLeft())
|
||||
assertTrue(state.isDone())
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -72,13 +72,15 @@ class BasicRelayClientTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun onFailureWithMessageKeepsExistingFormat() {
|
||||
fun onFailureWithMessageAppendsExceptionClassName() {
|
||||
val (socket, listener) = connectAndCapture()
|
||||
|
||||
socket.onFailure(Exception("Connection reset"), null, null)
|
||||
|
||||
// The exception type is appended so listeners can classify the failure by
|
||||
// its stable class name rather than by localized message text.
|
||||
assertEquals(
|
||||
listOf("WebSocket Failure: Connection reset"),
|
||||
listOf("WebSocket Failure: Connection reset (Exception)"),
|
||||
listener.cannotConnectMessages,
|
||||
)
|
||||
}
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.store.sqlite
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
|
||||
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
|
||||
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
|
||||
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
|
||||
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
|
||||
import com.vitorpamplona.quartz.utils.EventFactory
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
class AuthorsMissingOutboxTest : BaseDBTest() {
|
||||
@Test
|
||||
fun emptyStoreReturnsNoAuthors() =
|
||||
forEachDB { db ->
|
||||
assertEquals(emptySet(), db.authorsMissingOutbox().toSet())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun authorWithEventButNoOutboxIsMissing() =
|
||||
forEachDB { db ->
|
||||
val signer = NostrSignerSync()
|
||||
db.insert(signer.sign(TextNoteEvent.build("hello")))
|
||||
|
||||
assertEquals(setOf(signer.pubKey), db.authorsMissingOutbox().toSet())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun authorWithOutboxIsNotMissing() =
|
||||
forEachDB { db ->
|
||||
val hasOutbox = NostrSignerSync()
|
||||
val noOutbox = NostrSignerSync()
|
||||
|
||||
// Both authors have content; only one advertises a 10002.
|
||||
db.insert(hasOutbox.sign(TextNoteEvent.build("with relays")))
|
||||
db.insert(AdvertisedRelayListEvent.create(emptyList(), hasOutbox))
|
||||
db.insert(noOutbox.sign(TextNoteEvent.build("no relays")))
|
||||
|
||||
assertEquals(setOf(noOutbox.pubKey), db.authorsMissingOutbox().toSet())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun authorKnownOnlyByTheirOutboxIsNotMissing() =
|
||||
forEachDB { db ->
|
||||
// The only stored event for this author IS the 10002. They must
|
||||
// not appear (the outer scan sees them, the NOT EXISTS excludes
|
||||
// them) — the anti-join is symmetric on the same table.
|
||||
val signer = NostrSignerSync()
|
||||
db.insert(AdvertisedRelayListEvent.create(emptyList(), signer))
|
||||
|
||||
assertEquals(emptySet(), db.authorsMissingOutbox().toSet())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun outboxDeletedMakesAuthorMissingAgain() =
|
||||
forEachDB { db ->
|
||||
val signer = NostrSignerSync()
|
||||
db.insert(signer.sign(TextNoteEvent.build("content")))
|
||||
val relayList = AdvertisedRelayListEvent.create(emptyList(), signer)
|
||||
db.insert(relayList)
|
||||
|
||||
assertEquals(emptySet(), db.authorsMissingOutbox().toSet())
|
||||
|
||||
// NIP-09: the author deletes their own relay list. No 10002 row
|
||||
// remains, so the anti-join reports them as missing again.
|
||||
db.insert(signer.sign(DeletionEvent.build(listOf(relayList))))
|
||||
|
||||
assertEquals(setOf(signer.pubKey), db.authorsMissingOutbox().toSet())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun giftWrapSenderIsNotCountedAsAuthor() =
|
||||
forEachDB { db ->
|
||||
val noteAuthor = NostrSignerSync()
|
||||
db.insert(noteAuthor.sign(TextNoteEvent.build("hi")))
|
||||
|
||||
// A kind-1059 giftwrap stores an ephemeral one-time key as its
|
||||
// pubkey (the real recipient is only a hash). It has no outbox and
|
||||
// never will — but it must NOT be reported as "missing" one, or the
|
||||
// result set would grow by one junk key per received DM.
|
||||
val ephemeralSender = "aa".repeat(32)
|
||||
db.insert(
|
||||
EventFactory.create("bb".repeat(32), ephemeralSender, 1L, GiftWrapEvent.KIND, emptyArray(), "", "00".repeat(64)),
|
||||
)
|
||||
|
||||
assertEquals(setOf(noteAuthor.pubKey), db.authorsMissingOutbox().toSet())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun mixOfAuthorsReportsOnlyThoseWithoutOutbox() =
|
||||
forEachDB { db ->
|
||||
val a = NostrSignerSync()
|
||||
val b = NostrSignerSync()
|
||||
val c = NostrSignerSync()
|
||||
|
||||
db.insert(a.sign(TextNoteEvent.build("a1")))
|
||||
db.insert(a.sign(TextNoteEvent.build("a2")))
|
||||
db.insert(AdvertisedRelayListEvent.create(emptyList(), a))
|
||||
|
||||
db.insert(b.sign(TextNoteEvent.build("b1")))
|
||||
|
||||
db.insert(c.sign(TextNoteEvent.build("c1")))
|
||||
db.insert(AdvertisedRelayListEvent.create(emptyList(), c))
|
||||
|
||||
assertEquals(setOf(b.pubKey), db.authorsMissingOutbox().toSet())
|
||||
}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip59Giftwrap.wraps
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.toHexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertNull
|
||||
|
||||
/**
|
||||
* NIP-17 relay-hint placement contract.
|
||||
*
|
||||
* Per NIP-17 §Publishing, the gift wrap's `p` tag MAY carry the recipient's
|
||||
* primary DM inbox relay as a third element so other devices of the recipient
|
||||
* can discover the wrap without a separate kind:10050 lookup. The hint
|
||||
* deliberately lives on the public wrap, NOT on the encrypted seal — putting
|
||||
* it on the seal would hide the routing information inside the encryption
|
||||
* envelope, defeating the purpose.
|
||||
*/
|
||||
class GiftWrapRelayHintTest {
|
||||
private val recipient = KeyPair()
|
||||
|
||||
private fun innerEvent(): Event {
|
||||
val signer = NostrSignerSync(KeyPair())
|
||||
return signer.sign(
|
||||
createdAt = 0L,
|
||||
kind = 1,
|
||||
tags = emptyArray(),
|
||||
content = "hello",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun defaultsToNoRelayHintForBackwardsCompat() =
|
||||
runTest {
|
||||
// Existing callers that don't pass a hint must continue to emit the
|
||||
// historical ["p", recipientPubKey] two-element tag shape.
|
||||
val wrap =
|
||||
GiftWrapEvent.create(
|
||||
event = innerEvent(),
|
||||
recipientPubKey = recipient.pubKey.toHexKey(),
|
||||
)
|
||||
val pTag = wrap.tags.first { it.firstOrNull() == "p" }
|
||||
assertEquals(2, pTag.size, "p tag must be 2 elements when no hint passed")
|
||||
assertEquals(recipient.pubKey.toHexKey(), pTag[1])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun relayHintLandsOnWrapPTagAsThirdElement() =
|
||||
runTest {
|
||||
// When a hint is passed, it must appear as the THIRD element of the
|
||||
// wrap's p tag — NIP-17 spec. Not inside the encrypted seal.
|
||||
val hint = NormalizedRelayUrl("wss://dm.relay.example/")
|
||||
val wrap =
|
||||
GiftWrapEvent.create(
|
||||
event = innerEvent(),
|
||||
recipientPubKey = recipient.pubKey.toHexKey(),
|
||||
recipientRelayHint = hint,
|
||||
)
|
||||
val pTag = wrap.tags.first { it.firstOrNull() == "p" }
|
||||
assertEquals(3, pTag.size, "p tag carries [tag, pubkey, relay-hint]")
|
||||
assertEquals(recipient.pubKey.toHexKey(), pTag[1])
|
||||
assertEquals(hint.url, pTag[2])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun absentHintDoesNotAddTrailingEmptyElement() =
|
||||
runTest {
|
||||
// Defensive: a null hint must not produce `["p", pubkey, ""]` — that
|
||||
// would be a leak (broadcasts the user has no canonical inbox) and
|
||||
// a wire-format change from the historical shape.
|
||||
val wrap =
|
||||
GiftWrapEvent.create(
|
||||
event = innerEvent(),
|
||||
recipientPubKey = recipient.pubKey.toHexKey(),
|
||||
recipientRelayHint = null,
|
||||
)
|
||||
val pTag = wrap.tags.first { it.firstOrNull() == "p" }
|
||||
assertNull(pTag.getOrNull(2), "third element must be absent, not empty string")
|
||||
}
|
||||
}
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.utils.concurrent
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertNull
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class ConcurrentCollectionsTest {
|
||||
@Test
|
||||
fun mapGetSet() {
|
||||
val m = ConcurrentMap<String, Int>()
|
||||
assertNull(m["a"])
|
||||
m["a"] = 1
|
||||
assertEquals(1, m["a"])
|
||||
m["a"] = 2
|
||||
assertEquals(2, m["a"])
|
||||
assertEquals(1, m.size())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun mapGetOrPutComputesOnce() {
|
||||
val m = ConcurrentMap<String, Int>()
|
||||
var calls = 0
|
||||
assertEquals(
|
||||
7,
|
||||
m.getOrPut("k") {
|
||||
calls++
|
||||
7
|
||||
},
|
||||
)
|
||||
// Present now: the default must NOT be recomputed.
|
||||
assertEquals(
|
||||
7,
|
||||
m.getOrPut("k") {
|
||||
calls++
|
||||
99
|
||||
},
|
||||
)
|
||||
assertEquals(1, calls)
|
||||
assertEquals(7, m["k"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun mapMergeInsertsThenCombines() {
|
||||
val m = ConcurrentMap<String, Int>()
|
||||
// Absent -> inserts the value verbatim, remap not applied.
|
||||
assertEquals(1, m.merge("k", 1) { a, b -> a + b })
|
||||
// Present -> remap(existing, value).
|
||||
assertEquals(4, m.merge("k", 3) { a, b -> a + b })
|
||||
assertEquals(4, m["k"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun mapSnapshotIsDetached() {
|
||||
val m = ConcurrentMap<String, Int>()
|
||||
m["a"] = 1
|
||||
m["b"] = 2
|
||||
val snap = m.snapshot()
|
||||
assertEquals(mapOf("a" to 1, "b" to 2), snap)
|
||||
// Mutating the map after the snapshot must not change the snapshot.
|
||||
m["c"] = 3
|
||||
assertEquals(2, snap.size)
|
||||
assertEquals(3, m.size())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun setAddContainsSize() {
|
||||
val s = ConcurrentSet<String>()
|
||||
assertFalse("x" in s)
|
||||
assertTrue(s.add("x"))
|
||||
// Re-adding is a no-op and reports it.
|
||||
assertFalse(s.add("x"))
|
||||
assertTrue("x" in s)
|
||||
assertTrue(s.add("y"))
|
||||
assertEquals(2, s.size())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun setSnapshotIsDetached() {
|
||||
val s = ConcurrentSet<String>()
|
||||
s.add("a")
|
||||
val snap = s.snapshot()
|
||||
s.add("b")
|
||||
assertEquals(setOf("a"), snap)
|
||||
assertEquals(2, s.size())
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -127,7 +127,7 @@ actual object EventHasherSerializer {
|
||||
content: String,
|
||||
): Boolean {
|
||||
val br: BufferRecycler = JacksonMapper.mapper.factory._getBufferRecycler()
|
||||
val digest = threadLocalDigest.get()
|
||||
val digest = threadLocalDigest.get()!!
|
||||
val bb = HashingByteArrayBuilder(br, digest)
|
||||
try {
|
||||
val generator = JacksonMapper.mapper.createGenerator(bb, JsonEncoding.UTF8)
|
||||
|
||||
+1
-3
@@ -83,9 +83,7 @@ class CommandSerializer : StdSerializer<Command>(Command::class.java) {
|
||||
gen.writeString(cmd.subId)
|
||||
}
|
||||
|
||||
else -> {
|
||||
null
|
||||
}
|
||||
else -> {}
|
||||
}
|
||||
|
||||
gen.writeEndArray()
|
||||
|
||||
@@ -162,7 +162,7 @@ class GitHttpClient(
|
||||
visited.add(start)
|
||||
}
|
||||
while (frontier.isNotEmpty() && result.size < depth) {
|
||||
val commit = frontier.poll()
|
||||
val commit = frontier.poll()!!
|
||||
result.add(commit)
|
||||
for (parent in commit.parents) {
|
||||
if (parent !in visited) {
|
||||
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.utils.concurrent
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
actual class ConcurrentMap<K : Any, V : Any> {
|
||||
private val map = ConcurrentHashMap<K, V>()
|
||||
|
||||
actual operator fun get(key: K): V? = map[key]
|
||||
|
||||
actual operator fun set(
|
||||
key: K,
|
||||
value: V,
|
||||
) {
|
||||
map[key] = value
|
||||
}
|
||||
|
||||
actual fun getOrPut(
|
||||
key: K,
|
||||
defaultValue: () -> V,
|
||||
): V =
|
||||
// Fast-path the present-key hit (the common case in the crawl's hot
|
||||
// relay-hint accumulation) so it never allocates the mapping-function
|
||||
// closure; only an absent key pays for the atomic computeIfAbsent.
|
||||
map[key] ?: map.computeIfAbsent(key) { defaultValue() }
|
||||
|
||||
actual fun merge(
|
||||
key: K,
|
||||
value: V,
|
||||
remap: (old: V, new: V) -> V,
|
||||
): V = map.merge(key, value) { old, new -> remap(old, new) }!!
|
||||
|
||||
actual fun size(): Int = map.size
|
||||
|
||||
actual fun snapshot(): Map<K, V> = HashMap(map)
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.utils.concurrent
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
actual class ConcurrentSet<E : Any> {
|
||||
private val set: MutableSet<E> = ConcurrentHashMap.newKeySet()
|
||||
|
||||
actual fun add(element: E): Boolean = set.add(element)
|
||||
|
||||
actual operator fun contains(element: E): Boolean = set.contains(element)
|
||||
|
||||
actual fun size(): Int = set.size
|
||||
|
||||
actual fun snapshot(): Set<E> = HashSet(set)
|
||||
}
|
||||
+4
-4
@@ -30,19 +30,19 @@ import java.security.MessageDigest
|
||||
* (lock acquire + release) for ~2µs of actual hashing. ThreadLocal eliminates all locking
|
||||
* since each thread gets its own MessageDigest instance. digest() implicitly resets state.
|
||||
*/
|
||||
val threadLocalDigest =
|
||||
val threadLocalDigest: ThreadLocal<MessageDigest> =
|
||||
ThreadLocal.withInitial {
|
||||
MessageDigest.getInstance("SHA-256")
|
||||
}
|
||||
|
||||
actual fun sha256(data: ByteArray): ByteArray = threadLocalDigest.get().digest(data)
|
||||
actual fun sha256(data: ByteArray): ByteArray = threadLocalDigest.get()!!.digest(data)
|
||||
|
||||
actual fun sha256Into(
|
||||
out: ByteArray,
|
||||
data: ByteArray,
|
||||
len: Int,
|
||||
): ByteArray {
|
||||
val md = threadLocalDigest.get()
|
||||
val md = threadLocalDigest.get()!!
|
||||
md.update(data, 0, len)
|
||||
md.digest(out, 0, 32)
|
||||
return out
|
||||
@@ -62,7 +62,7 @@ fun sha256StreamWithCount(
|
||||
bufferSize: Int = 8192,
|
||||
): Pair<ByteArray, Long> {
|
||||
val countingStream = CountingInputStream(inputStream)
|
||||
val digest = threadLocalDigest.get()
|
||||
val digest = threadLocalDigest.get()!!
|
||||
try {
|
||||
val buffer = ByteArray(bufferSize)
|
||||
var bytesRead: Int
|
||||
|
||||
+40
@@ -123,6 +123,46 @@ class MlsGroupManagerTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Regression: [MlsGroupManager.encrypt] must persist the advanced ratchet
|
||||
* position, not just commits. A group state persists only at commits was
|
||||
* the second half of the generation-reuse bug: sends between two commits
|
||||
* advanced the SecretTree in memory but never hit the store, so a restart
|
||||
* reloaded the pre-send ratchet and re-emitted an already-used generation.
|
||||
*
|
||||
* Alice and Bob share a group. Alice sends one message (Bob consumes
|
||||
* generation 0), Alice "restarts" from the store WITHOUT any intervening
|
||||
* commit, and her next send must be a fresh generation Bob accepts.
|
||||
*/
|
||||
@Test
|
||||
fun testEncryptPersistsRatchetPositionBetweenCommits() {
|
||||
runBlocking {
|
||||
val aliceStore = InMemoryGroupStateStore()
|
||||
val alice = MlsGroupManager(aliceStore)
|
||||
val aliceGroup = alice.createGroup(groupId, "alice".encodeToByteArray())
|
||||
|
||||
// Bob joins as a low-level MlsGroup — a strict peer that tracks
|
||||
// consumed generations. (The manager's processWelcome requires a
|
||||
// NostrGroupData extension we don't set up here; the low-level
|
||||
// group is enough to observe the ratchet behavior.)
|
||||
val bobBundle = aliceGroup.createKeyPackage("bob".encodeToByteArray(), ByteArray(0))
|
||||
val addResult = alice.addMember(groupId, bobBundle.keyPackage.toTlsBytes())
|
||||
val bob = MlsGroup.processWelcome(addResult.welcomeBytes!!, bobBundle)
|
||||
|
||||
// Alice sends generation 0 (no commit); Bob consumes it.
|
||||
val ct0 = alice.encrypt(groupId, "msg0".encodeToByteArray())
|
||||
assertContentEquals("msg0".encodeToByteArray(), bob.decrypt(ct0).content)
|
||||
|
||||
// Alice restarts from the store — only encrypt() has run since the
|
||||
// last commit, so this proves encrypt persisted the ratchet.
|
||||
val aliceRestarted = MlsGroupManager(aliceStore)
|
||||
aliceRestarted.restoreAll()
|
||||
|
||||
val ct1 = aliceRestarted.encrypt(groupId, "msg1".encodeToByteArray())
|
||||
assertContentEquals("msg1".encodeToByteArray(), bob.decrypt(ct1).content)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testAddMemberPersistsState() {
|
||||
runBlocking {
|
||||
|
||||
+112
-2
@@ -173,12 +173,12 @@ class MlsGroupStateTest {
|
||||
val state = group.saveState()
|
||||
val bytes = state.encodeTls()
|
||||
|
||||
// First two bytes should be the version (uint16 = 1)
|
||||
// First two bytes should be the version (uint16 = 2)
|
||||
val reader =
|
||||
com.vitorpamplona.quartz.marmot.mls.codec
|
||||
.TlsReader(bytes)
|
||||
val version = reader.readUint16()
|
||||
assertEquals(1, version)
|
||||
assertEquals(2, version)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -219,4 +219,114 @@ class MlsGroupStateTest {
|
||||
val decrypted = restoredGroup.decrypt(encrypted)
|
||||
assertContentEquals(plaintext, decrypted.content)
|
||||
}
|
||||
|
||||
/**
|
||||
* Regression: a restore must NOT rewind the SecretTree ratchet to
|
||||
* generation 0. A peer that already consumed generation 0 in this epoch
|
||||
* (like openmls / MDK / Whitenoise, which forbid generation reuse) would
|
||||
* otherwise reject the restored sender's next message as a replay.
|
||||
*/
|
||||
@Test
|
||||
fun testRestorePreservesSenderGeneration_peerAcceptsNextMessage() {
|
||||
val alice = MlsGroup.create("alice".encodeToByteArray())
|
||||
val bobBundle =
|
||||
MlsGroup
|
||||
.create("bob".encodeToByteArray())
|
||||
.createKeyPackage("bob".encodeToByteArray(), ByteArray(0))
|
||||
val bob = MlsGroup.processWelcome(alice.addMember(bobBundle.keyPackage.toTlsBytes()).welcomeBytes!!, bobBundle)
|
||||
|
||||
// Alice sends generation 0; Bob consumes it.
|
||||
val ct0 = alice.encrypt("msg0".encodeToByteArray())
|
||||
assertContentEquals("msg0".encodeToByteArray(), bob.decrypt(ct0).content)
|
||||
|
||||
// Alice "restarts": persist then restore.
|
||||
val aliceRestored = MlsGroup.restore(MlsGroupState.decodeTls(alice.saveState().encodeTls()))
|
||||
|
||||
// Alice's next send must be generation 1, which Bob accepts. Before
|
||||
// the fix this re-emitted generation 0 and Bob threw "Generation 0
|
||||
// already consumed".
|
||||
val ct1 = aliceRestored.encrypt("msg1".encodeToByteArray())
|
||||
assertContentEquals("msg1".encodeToByteArray(), bob.decrypt(ct1).content)
|
||||
}
|
||||
|
||||
/**
|
||||
* The ratchet position must survive several sends across a restore, not
|
||||
* just one. Covers the case where the app persists (at a commit) after N
|
||||
* application messages have already advanced the ratchet.
|
||||
*/
|
||||
@Test
|
||||
fun testRestorePreservesSenderGenerationAfterMultipleSends() {
|
||||
val alice = MlsGroup.create("alice".encodeToByteArray())
|
||||
val bobBundle =
|
||||
MlsGroup
|
||||
.create("bob".encodeToByteArray())
|
||||
.createKeyPackage("bob".encodeToByteArray(), ByteArray(0))
|
||||
val bob = MlsGroup.processWelcome(alice.addMember(bobBundle.keyPackage.toTlsBytes()).welcomeBytes!!, bobBundle)
|
||||
|
||||
for (i in 0 until 5) {
|
||||
val ct = alice.encrypt("m$i".encodeToByteArray())
|
||||
assertContentEquals("m$i".encodeToByteArray(), bob.decrypt(ct).content)
|
||||
}
|
||||
|
||||
val aliceRestored = MlsGroup.restore(MlsGroupState.decodeTls(alice.saveState().encodeTls()))
|
||||
|
||||
// Continues at generation 5 — Bob (who consumed 0..4) accepts it.
|
||||
val ct = aliceRestored.encrypt("m5".encodeToByteArray())
|
||||
assertContentEquals("m5".encodeToByteArray(), bob.decrypt(ct).content)
|
||||
}
|
||||
|
||||
/**
|
||||
* Backward compatibility: a STATE_VERSION 1 blob (no persisted ratchet
|
||||
* positions) must still decode, yielding an empty ratchet map and the
|
||||
* legacy generation-0 restore behavior.
|
||||
*/
|
||||
@Test
|
||||
fun testDecodeLegacyV1StateBlob() {
|
||||
val group = MlsGroup.create("alice".encodeToByteArray())
|
||||
group.encrypt("advance the ratchet".encodeToByteArray())
|
||||
val state = group.saveState()
|
||||
|
||||
val v1Bytes = encodeAsV1(state)
|
||||
val decoded = MlsGroupState.decodeTls(v1Bytes)
|
||||
|
||||
assertTrue(decoded.senderRatchetStates.isEmpty(), "v1 blob has no ratchet positions")
|
||||
|
||||
// Restores and can still encrypt/decrypt (legacy behavior).
|
||||
val restored = MlsGroup.restore(decoded)
|
||||
val ct = restored.encrypt("post-restore".encodeToByteArray())
|
||||
assertContentEquals("post-restore".encodeToByteArray(), restored.decrypt(ct).content)
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-encode a state in the original STATE_VERSION 1 layout: identical to
|
||||
* v2 but with the version tag set to 1 and no trailing ratchet section.
|
||||
*/
|
||||
private fun encodeAsV1(state: MlsGroupState): ByteArray {
|
||||
val writer =
|
||||
com.vitorpamplona.quartz.marmot.mls.codec
|
||||
.TlsWriter()
|
||||
writer.putUint16(1)
|
||||
state.groupContext.encodeTls(writer)
|
||||
writer.putOpaqueVarInt(state.treeBytes)
|
||||
writer.putUint32(state.myLeafIndex.toLong())
|
||||
val es = state.epochSecrets
|
||||
writer.putOpaqueVarInt(es.joinerSecret)
|
||||
writer.putOpaqueVarInt(es.welcomeSecret)
|
||||
writer.putOpaqueVarInt(es.epochSecret)
|
||||
writer.putOpaqueVarInt(es.senderDataSecret)
|
||||
writer.putOpaqueVarInt(es.encryptionSecret)
|
||||
writer.putOpaqueVarInt(es.exporterSecret)
|
||||
writer.putOpaqueVarInt(es.epochAuthenticator)
|
||||
writer.putOpaqueVarInt(es.externalSecret)
|
||||
writer.putOpaqueVarInt(es.confirmationKey)
|
||||
writer.putOpaqueVarInt(es.membershipKey)
|
||||
writer.putOpaqueVarInt(es.resumptionPsk)
|
||||
writer.putOpaqueVarInt(es.initSecret)
|
||||
writer.putOpaqueVarInt(state.initSecret)
|
||||
writer.putOpaqueVarInt(state.signingPrivateKey)
|
||||
writer.putOpaqueVarInt(state.encryptionPrivateKey)
|
||||
writer.putOpaqueVarInt(state.interimTranscriptHash)
|
||||
writer.putOpaqueVarInt(state.encryptionSecret)
|
||||
return writer.toByteArray()
|
||||
}
|
||||
}
|
||||
|
||||
+251
@@ -0,0 +1,251 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.relay.prodbench
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.HexKey
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPages
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.BasicOkHttpWebSocket
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
|
||||
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
|
||||
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
|
||||
import com.vitorpamplona.quartz.utils.EventFactory
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import okhttp3.OkHttpClient
|
||||
import java.nio.file.Files
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
/**
|
||||
* Head-to-head for `IEventStore.authorsMissingOutbox()` — "give me every
|
||||
* author with events but no NIP-65 relay list (kind 10002)" — at 1,000,000
|
||||
* events, comparing the two implementations that ship:
|
||||
*
|
||||
* - **generic** — the `IEventStore` interface default: query the 10002
|
||||
* owners into a set, then stream EVERY event (`query(Filter())`) and keep
|
||||
* the authors not in that set. Correct for any store, but it decodes all
|
||||
* 1M events off SQLite into `Event` objects.
|
||||
* - **sqlite** — `EventStore.authorsMissingOutbox()`, an index-only `EXCEPT`
|
||||
* over `event_headers` (all authors minus the 10002 owners) that never
|
||||
* decodes an event, riding the `(kind, pubkey, created_at)` covering index.
|
||||
*
|
||||
* Corpus: the benchmark first **syncs a real sample from a popular relay**
|
||||
* (kind 1 notes + kind 10002 relay lists from [RELAY]) so the pubkey
|
||||
* cardinality, per-author event fan-out, tag/content sizes, and the fraction
|
||||
* of authors that actually advertise relays are all real. It then replicates
|
||||
* that sample — cloning each real event with a fresh id and timestamp but the
|
||||
* SAME pubkey/kind/tags/content — up to [TARGET] rows. Replication preserves
|
||||
* the real distinct-author set and the real 10002-owner set exactly (so the
|
||||
* answer is unchanged), it only grows each author's history the way a
|
||||
* long-lived relay would. A live 1M download is bandwidth-bound and isn't
|
||||
* what we're measuring; the query is.
|
||||
*
|
||||
* The store keeps the `indexEventsByPubkeyAlone` index a relay actually keeps
|
||||
* (this query is a relay / outbox-model concern) — that is the index the
|
||||
* `DISTINCT pubkey ... NOT EXISTS` scan rides. NIP-50 full-text indexing is
|
||||
* turned off: it is pure insert-path cost that neither query touches, so
|
||||
* dropping it just makes seeding 1M rows fast without changing either timing.
|
||||
*
|
||||
* Network + heavy, so gated like the other prod benches:
|
||||
* ./gradlew :quartz:jvmTest --tests "*.AuthorsMissingOutboxBenchmark" -PprodRelayBench=1
|
||||
*/
|
||||
class AuthorsMissingOutboxBenchmark {
|
||||
companion object {
|
||||
const val RELAY = "wss://relay.damus.io"
|
||||
const val TARGET = 1_000_000
|
||||
const val SAMPLE_NOTES = 25_000
|
||||
const val SAMPLE_RELAY_LISTS = 15_000
|
||||
const val FETCH_TIMEOUT_MS = 90_000L
|
||||
const val INSERT_CHUNK = 2_000
|
||||
val SIG = "0".repeat(128)
|
||||
}
|
||||
|
||||
private fun idFor(counter: Long): String = "%064x".format(counter)
|
||||
|
||||
/** The generic path — a verbatim copy of the `IEventStore` interface default. */
|
||||
private suspend fun genericAuthorsMissingOutbox(store: EventStore): List<HexKey> {
|
||||
val withOutbox = HashSet<HexKey>()
|
||||
store.query<Event>(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND))) { withOutbox.add(it.pubKey) }
|
||||
|
||||
val missing = LinkedHashSet<HexKey>()
|
||||
store.query<Event>(Filter()) { event ->
|
||||
if (event.kind != GiftWrapEvent.KIND && event.pubKey !in withOutbox) missing.add(event.pubKey)
|
||||
}
|
||||
return missing.toList()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun authorsMissingOutboxScaling() {
|
||||
if (System.getenv("PROD_RELAY_BENCH") == null && System.getProperty("prodRelayBench") == null) {
|
||||
println("AuthorsMissingOutboxBenchmark skipped. Run with -PprodRelayBench=1 to enable.")
|
||||
return
|
||||
}
|
||||
|
||||
val httpClient =
|
||||
OkHttpClient
|
||||
.Builder()
|
||||
.connectTimeout(15, TimeUnit.SECONDS)
|
||||
.readTimeout(120, TimeUnit.SECONDS)
|
||||
.pingInterval(30, TimeUnit.SECONDS)
|
||||
.build()
|
||||
|
||||
println("=== authorsMissingOutbox 1M benchmark === cores=${Runtime.getRuntime().availableProcessors()}")
|
||||
|
||||
// ── 1. SYNC a real sample from a popular relay ──────────────────────
|
||||
val notes = ArrayList<Event>(SAMPLE_NOTES)
|
||||
val relayLists = ArrayList<Event>(SAMPLE_RELAY_LISTS)
|
||||
val relay = RELAY.normalizeRelayUrl()
|
||||
runBlocking {
|
||||
val client = NostrClient(BasicOkHttpWebSocket.Builder { httpClient })
|
||||
try {
|
||||
val t0 = System.nanoTime()
|
||||
client.fetchAllPages(relay, listOf(Filter(kinds = listOf(1), limit = SAMPLE_NOTES)), FETCH_TIMEOUT_MS) { notes.add(it) }
|
||||
client.fetchAllPages(relay, listOf(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND), limit = SAMPLE_RELAY_LISTS)), FETCH_TIMEOUT_MS) { relayLists.add(it) }
|
||||
println(" synced from $RELAY in %.1fs: %,d notes + %,d relay-lists".format((System.nanoTime() - t0) / 1e9, notes.size, relayLists.size))
|
||||
} finally {
|
||||
client.close()
|
||||
}
|
||||
}
|
||||
httpClient.dispatcher.executorService.shutdown()
|
||||
|
||||
val pool = (notes + relayLists).distinctBy { it.id }
|
||||
require(pool.isNotEmpty()) { "relay returned no events — cannot build corpus" }
|
||||
|
||||
val outboxOwners = relayLists.mapTo(HashSet()) { it.pubKey }
|
||||
val allAuthors = pool.mapTo(HashSet()) { it.pubKey }
|
||||
val expectedMissing = allAuthors - outboxOwners
|
||||
println(
|
||||
" real sample: %,d events, %,d distinct authors, %,d with a 10002 (%.1f%%) → %,d missing".format(
|
||||
pool.size,
|
||||
allAuthors.size,
|
||||
outboxOwners.size,
|
||||
100.0 * outboxOwners.size / allAuthors.size,
|
||||
expectedMissing.size,
|
||||
),
|
||||
)
|
||||
|
||||
// ── 2. SCALE to TARGET by replicating the real sample ───────────────
|
||||
// kind 10002 is replaceable — one row survives per owner no matter how
|
||||
// many times it is re-cloned — so the store is filled with note (kind 1)
|
||||
// clones and each owner's relay list is inserted exactly once. That
|
||||
// lands a genuine TARGET rows while keeping the real author set and
|
||||
// outbox-owner set intact.
|
||||
val notePool = notes.distinctBy { it.id }
|
||||
require(notePool.isNotEmpty()) { "relay returned no kind-1 notes — cannot fill the corpus" }
|
||||
val relayListPerOwner = relayLists.associateBy { it.pubKey }.values.toList()
|
||||
val noteCloneTarget = (TARGET - relayListPerOwner.size).coerceAtLeast(0)
|
||||
|
||||
// FS/FTS off, pubkey+created_at indexes on: representative of the query,
|
||||
// fast to seed. See the class KDoc.
|
||||
val strategy =
|
||||
DefaultIndexingStrategy(
|
||||
indexEventsByCreatedAtAlone = true,
|
||||
indexEventsByPubkeyAlone = true,
|
||||
useAndIndexIdOnOrderBy = true,
|
||||
indexFullTextSearch = false,
|
||||
)
|
||||
val dbFile = Files.createTempFile("authors-missing-outbox-", ".db")
|
||||
Files.deleteIfExists(dbFile)
|
||||
val store = EventStore(dbName = dbFile.toAbsolutePath().toString(), relay = null, indexStrategy = strategy)
|
||||
try {
|
||||
val baseTime = 1_600_000_000L
|
||||
var counter = 0L
|
||||
val seedT0 = System.nanoTime()
|
||||
val batch = ArrayList<Event>(INSERT_CHUNK)
|
||||
|
||||
suspend fun flush() {
|
||||
if (batch.isNotEmpty()) {
|
||||
store.batchInsert(batch)
|
||||
batch.clear()
|
||||
}
|
||||
}
|
||||
runBlocking {
|
||||
// One relay list per owner (fresh id; content/tags preserved).
|
||||
for (src in relayListPerOwner) {
|
||||
batch.add(EventFactory.create(idFor(++counter), src.pubKey, baseTime + counter, src.kind, src.tags, src.content, SIG))
|
||||
if (batch.size == INSERT_CHUNK) flush()
|
||||
}
|
||||
// Fill the rest with note clones cycling the real notes.
|
||||
var made = 0L
|
||||
while (made < noteCloneTarget) {
|
||||
val src = notePool[(made % notePool.size).toInt()]
|
||||
batch.add(EventFactory.create(idFor(++counter), src.pubKey, baseTime + counter, src.kind, src.tags, src.content, SIG))
|
||||
made++
|
||||
if (batch.size == INSERT_CHUNK) flush()
|
||||
}
|
||||
flush()
|
||||
}
|
||||
val total = runBlocking { store.count(Filter()) }
|
||||
println(" seeded %,d rows (stored %,d) in %.1fs".format(counter, total, (System.nanoTime() - seedT0) / 1e9))
|
||||
|
||||
// ── 3. MEASURE both implementations on the same store ──────────
|
||||
// Warm the page cache with one throwaway pass of each so neither
|
||||
// eats the cold-cache penalty for the other.
|
||||
runBlocking {
|
||||
store.authorsMissingOutbox()
|
||||
genericAuthorsMissingOutbox(store)
|
||||
}
|
||||
|
||||
val runs = 3
|
||||
var sqliteResult: List<HexKey> = emptyList()
|
||||
var genericResult: List<HexKey> = emptyList()
|
||||
val sqliteMs = DoubleArray(runs)
|
||||
val genericMs = DoubleArray(runs)
|
||||
runBlocking {
|
||||
repeat(runs) { i ->
|
||||
var t = System.nanoTime()
|
||||
sqliteResult = store.authorsMissingOutbox()
|
||||
sqliteMs[i] = (System.nanoTime() - t) / 1e6
|
||||
|
||||
t = System.nanoTime()
|
||||
genericResult = genericAuthorsMissingOutbox(store)
|
||||
genericMs[i] = (System.nanoTime() - t) / 1e6
|
||||
}
|
||||
}
|
||||
|
||||
// Correctness: both must return the same author set, and it must
|
||||
// match the ground truth computed from the real sample.
|
||||
assertEquals(sqliteResult.toSet(), genericResult.toSet(), "sqlite and generic disagree")
|
||||
assertEquals(expectedMissing, sqliteResult.toSet(), "result does not match the seeded distribution")
|
||||
|
||||
val sqliteBest = sqliteMs.min()
|
||||
val genericBest = genericMs.min()
|
||||
println("\n result: %,d authors missing an outbox (of %,d distinct authors)".format(sqliteResult.size, allAuthors.size))
|
||||
println(" ── timings over $runs runs (best-of) ──")
|
||||
println(" generic (decode all %,d events) best=%,9.1f ms runs=%s".format(total, genericBest, genericMs.joinToString { "%.0f".format(it) }))
|
||||
println(" sqlite (index-only EXCEPT) best=%,9.1f ms runs=%s".format(sqliteBest, sqliteMs.joinToString { "%.1f".format(it) }))
|
||||
println(" → sqlite is %.1f× faster at %,d events".format(genericBest / sqliteBest, total))
|
||||
} finally {
|
||||
store.close()
|
||||
listOf("", "-wal", "-shm").forEach {
|
||||
Files.deleteIfExists(
|
||||
java.nio.file.Path
|
||||
.of(dbFile.toAbsolutePath().toString() + it),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.store.fs
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
|
||||
import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
|
||||
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
|
||||
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
|
||||
import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent
|
||||
import com.vitorpamplona.quartz.utils.EventFactory
|
||||
import com.vitorpamplona.quartz.utils.Secp256k1Instance
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.exists
|
||||
import kotlin.test.AfterTest
|
||||
import kotlin.test.BeforeTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
/**
|
||||
* `authorsMissingOutbox()` against [FsEventStore], which does NOT override the
|
||||
* method — so this is the ONLY coverage of the `IEventStore` interface DEFAULT
|
||||
* implementation (the SQLite tests always hit the override). It pins the
|
||||
* default's behaviour, and asserts it agrees with the same scenarios the SQLite
|
||||
* suite checks: 10002 exclusion, NIP-09 deletion re-exposing an author, and the
|
||||
* giftwrap-sender carve-out.
|
||||
*/
|
||||
class FsAuthorsMissingOutboxTest {
|
||||
private lateinit var root: Path
|
||||
private lateinit var store: FsEventStore
|
||||
|
||||
@BeforeTest
|
||||
fun setup() {
|
||||
Secp256k1Instance
|
||||
root = Files.createTempDirectory("fs-missing-outbox-")
|
||||
store = FsEventStore(root)
|
||||
}
|
||||
|
||||
@AfterTest
|
||||
fun tearDown() {
|
||||
store.close()
|
||||
if (root.exists()) {
|
||||
Files.walk(root).use { s -> s.sorted(Comparator.reverseOrder()).forEach { Files.deleteIfExists(it) } }
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun defaultImplReportsOnlyAuthorsWithoutOutbox() =
|
||||
runBlocking {
|
||||
val withOutbox = NostrSignerSync()
|
||||
val noOutbox = NostrSignerSync()
|
||||
|
||||
store.insert(withOutbox.sign(TextNoteEvent.build("a")))
|
||||
store.insert(AdvertisedRelayListEvent.create(emptyList(), withOutbox))
|
||||
store.insert(noOutbox.sign(TextNoteEvent.build("b")))
|
||||
|
||||
assertEquals(setOf(noOutbox.pubKey), store.authorsMissingOutbox().toSet())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun defaultImplExcludesGiftWrapSenders() =
|
||||
runBlocking {
|
||||
val noteAuthor = NostrSignerSync()
|
||||
store.insert(noteAuthor.sign(TextNoteEvent.build("hi")))
|
||||
store.insert(
|
||||
EventFactory.create("bb".repeat(32), "aa".repeat(32), 1L, GiftWrapEvent.KIND, emptyArray(), "", "00".repeat(64)),
|
||||
)
|
||||
|
||||
assertEquals(setOf(noteAuthor.pubKey), store.authorsMissingOutbox().toSet())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun defaultImplReExposesAuthorAfterOutboxDeleted() =
|
||||
runBlocking {
|
||||
val signer = NostrSignerSync()
|
||||
store.insert(signer.sign(TextNoteEvent.build("content")))
|
||||
val relayList = AdvertisedRelayListEvent.create(emptyList(), signer)
|
||||
store.insert(relayList)
|
||||
assertEquals(emptySet(), store.authorsMissingOutbox().toSet())
|
||||
|
||||
store.insert(signer.sign(DeletionEvent.build(listOf(relayList))))
|
||||
assertEquals(setOf(signer.pubKey), store.authorsMissingOutbox().toSet())
|
||||
}
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip66RelayMonitor.reachability
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
|
||||
import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkType
|
||||
import com.vitorpamplona.quartz.utils.Secp256k1Instance
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class RelayReachabilityStoreTest {
|
||||
private fun store() =
|
||||
EventStore(
|
||||
dbName = null,
|
||||
indexStrategy = DefaultIndexingStrategy(),
|
||||
)
|
||||
|
||||
private fun cache(store: EventStore) =
|
||||
RelayReachabilityStore(
|
||||
store = store,
|
||||
signer = NostrSignerInternal(KeyPair()),
|
||||
ttlSeconds = 3600,
|
||||
)
|
||||
|
||||
private val live1 = RelayUrlNormalizer.normalize("wss://alive.example.com")
|
||||
private val live2 = RelayUrlNormalizer.normalize("wss://also-alive.example.com")
|
||||
private val dead1 = RelayUrlNormalizer.normalize("wss://dead.example.com")
|
||||
private val dead2 = RelayUrlNormalizer.normalize("wss://gone.example.com")
|
||||
private val onion = RelayUrlNormalizer.normalize("wss://abc.onion")
|
||||
private val onionPath = RelayUrlNormalizer.normalize("wss://abc.onion/npub1x")
|
||||
|
||||
// Contains the literal ".onion" as a substring but is NOT a Tor host — a loose
|
||||
// `contains(".onion")` would misclassify it; the normalizer's isOnion must not.
|
||||
private val fakeOnion = RelayUrlNormalizer.normalize("wss://relay.onionfake.com")
|
||||
|
||||
@Test
|
||||
fun recordsAndReloadsReachability() =
|
||||
runBlocking {
|
||||
Secp256k1Instance
|
||||
val store = store()
|
||||
val cache = cache(store)
|
||||
val now = 1_000_000L
|
||||
|
||||
cache.record(reachable = setOf(live1, live2), dead = setOf(dead1, dead2), now = now)
|
||||
|
||||
val snap = cache.snapshot(now = now)
|
||||
assertEquals(setOf(live1, live2), snap.live)
|
||||
assertEquals(setOf(dead1, dead2), snap.dead)
|
||||
assertTrue(snap.isKnownDead(dead1))
|
||||
assertFalse(snap.isKnownDead(live1))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aFreshSuccessfulOpenOverridesAnEarlierDeadMark() =
|
||||
runBlocking {
|
||||
Secp256k1Instance
|
||||
val store = store()
|
||||
val cache = cache(store)
|
||||
|
||||
// Marked dead first, then seen alive a second later (addressable replace).
|
||||
cache.record(reachable = emptySet(), dead = setOf(dead1), now = 1_000L)
|
||||
cache.record(reachable = setOf(dead1), dead = emptySet(), now = 1_001L)
|
||||
|
||||
val snap = cache.snapshot(now = 1_001L)
|
||||
assertTrue(dead1 in snap.live)
|
||||
assertFalse(snap.isKnownDead(dead1))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun recordsOlderThanTheTtlAreIgnored() =
|
||||
runBlocking {
|
||||
Secp256k1Instance
|
||||
val store = store()
|
||||
val cache = cache(store) // ttl = 3600s
|
||||
|
||||
cache.record(reachable = emptySet(), dead = setOf(dead1), now = 1_000L)
|
||||
|
||||
// "now" is well past the 1h TTL from when dead1 was recorded.
|
||||
val snap = cache.snapshot(now = 1_000L + 3601L)
|
||||
assertFalse(snap.isKnownDead(dead1))
|
||||
assertEquals(0, snap.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun onionRelayIsTaggedTorNetwork() {
|
||||
assertEquals(NetworkType.TOR, RelayReachabilityStore.networkTypeOf(onion))
|
||||
assertEquals(NetworkType.TOR, RelayReachabilityStore.networkTypeOf(onionPath))
|
||||
assertEquals(NetworkType.CLEARNET, RelayReachabilityStore.networkTypeOf(live1))
|
||||
// A host that merely contains ".onion" as a substring is clearnet, not Tor.
|
||||
assertEquals(NetworkType.CLEARNET, RelayReachabilityStore.networkTypeOf(fakeOnion))
|
||||
}
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.utils.concurrent
|
||||
|
||||
import kotlin.concurrent.atomics.AtomicReference
|
||||
import kotlin.concurrent.atomics.ExperimentalAtomicApi
|
||||
|
||||
// Copy-on-write, mirroring ConcurrentHashCache.linux: correct and simple. The
|
||||
// native targets never run the crawl this backs (it is JVM/Android-only work);
|
||||
// they only compile it, so the O(n)-per-write cost is irrelevant. A CAS retry
|
||||
// loop gives getOrPut/merge the same atomicity the JVM actual gets for free.
|
||||
@OptIn(ExperimentalAtomicApi::class)
|
||||
actual class ConcurrentMap<K : Any, V : Any> {
|
||||
private val ref = AtomicReference(HashMap<K, V>())
|
||||
|
||||
actual operator fun get(key: K): V? = ref.load()[key]
|
||||
|
||||
actual operator fun set(
|
||||
key: K,
|
||||
value: V,
|
||||
) {
|
||||
while (true) {
|
||||
val cur = ref.load()
|
||||
val copy = HashMap(cur)
|
||||
copy[key] = value
|
||||
if (ref.compareAndSet(cur, copy)) return
|
||||
}
|
||||
}
|
||||
|
||||
actual fun getOrPut(
|
||||
key: K,
|
||||
defaultValue: () -> V,
|
||||
): V {
|
||||
while (true) {
|
||||
val cur = ref.load()
|
||||
cur[key]?.let { return it }
|
||||
val value = defaultValue()
|
||||
val copy = HashMap(cur)
|
||||
copy[key] = value
|
||||
if (ref.compareAndSet(cur, copy)) return value
|
||||
}
|
||||
}
|
||||
|
||||
actual fun merge(
|
||||
key: K,
|
||||
value: V,
|
||||
remap: (old: V, new: V) -> V,
|
||||
): V {
|
||||
while (true) {
|
||||
val cur = ref.load()
|
||||
val old = cur[key]
|
||||
val merged = if (old == null) value else remap(old, value)
|
||||
val copy = HashMap(cur)
|
||||
copy[key] = merged
|
||||
if (ref.compareAndSet(cur, copy)) return merged
|
||||
}
|
||||
}
|
||||
|
||||
actual fun size(): Int = ref.load().size
|
||||
|
||||
actual fun snapshot(): Map<K, V> = HashMap(ref.load())
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.utils.concurrent
|
||||
|
||||
import kotlin.concurrent.atomics.AtomicReference
|
||||
import kotlin.concurrent.atomics.ExperimentalAtomicApi
|
||||
|
||||
// Copy-on-write native actual — see ConcurrentMap.native for the rationale.
|
||||
@OptIn(ExperimentalAtomicApi::class)
|
||||
actual class ConcurrentSet<E : Any> {
|
||||
private val ref = AtomicReference(HashSet<E>())
|
||||
|
||||
actual fun add(element: E): Boolean {
|
||||
while (true) {
|
||||
val cur = ref.load()
|
||||
if (element in cur) return false
|
||||
val copy = HashSet(cur)
|
||||
copy.add(element)
|
||||
if (ref.compareAndSet(cur, copy)) return true
|
||||
}
|
||||
}
|
||||
|
||||
actual operator fun contains(element: E): Boolean = element in ref.load()
|
||||
|
||||
actual fun size(): Int = ref.load().size
|
||||
|
||||
actual fun snapshot(): Set<E> = HashSet(ref.load())
|
||||
}
|
||||
Reference in New Issue
Block a user