From 5988db010d18bd9f94374346dce9eb7a00d29ea6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 14:34:57 +0000 Subject: [PATCH 01/34] =?UTF-8?q?test(store):=20add=20tag=E2=88=A9author?= =?UTF-8?q?=20and=20FS=20driver-selection=20benchmarks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-07 client filter-assembler survey mapped 551 Filter constructions to ~12 query archetypes. Two hot shapes had no benchmark coverage in prodbench or relayBench, and the FS store had none at all: - TagAuthorIndexBenchmark: the DM-room shape (kinds + authors + #p, 65 assembler call sites) with indexTagsWithKindAndPubkey off vs on, including the insert-cost delta of the extra index; plus the reactions watcher (kinds=[7], #e IN 300, limit) cold and since-bounded, which has no tag-side k-way merge today. - FsDriverSelectionBenchmark: FsQueryPlanner's fixed driver order (tags → kinds → authors) on authors+kinds+limit — the most common CLI shape — comparing the current kind-tree driver against an author-tree driver with kind post-filter. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01RzkoN3SJHCZWRAiadXXG4w --- .../prodbench/TagAuthorIndexBenchmark.kt | 184 ++++++++++++++++++ .../store/fs/FsDriverSelectionBenchmark.kt | 158 +++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/TagAuthorIndexBenchmark.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDriverSelectionBenchmark.kt diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/TagAuthorIndexBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/TagAuthorIndexBenchmark.kt new file mode 100644 index 0000000000..7ef0fc7ac8 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/TagAuthorIndexBenchmark.kt @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.prodbench + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.utils.EventFactory +import kotlinx.coroutines.runBlocking +import kotlin.test.Test + +/** + * Measures the two tag-path query shapes the client filter-assembler survey + * (2026-07) found hot but that no existing benchmark covers: + * + * 1. **tag ∩ author (DM-room shape)** — `kinds=[4] AND authors=[peer] AND + * #p=[me] LIMIT n`. 65 assembler call sites build this shape (every + * NIP-04 chat room, reports-by-follows, follows-scoped community feeds). + * [com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy.indexTagsWithKindAndPubkey] + * gates a covering `(tag_hash, kind, pubkey_hash, created_at)` index for + * it, but the flag is off everywhere (including geode). Without it the + * plan seeks `(tag_hash, kind)` and reads EVERY DM the user has ever + * received before filtering to the one peer. This compares query latency + * with the flag off vs on, and the batch-insert cost the extra index adds. + * + * 2. **large-IN tag watcher (reactions shape)** — `kinds=[7] AND + * #e=[hundreds of note ids] LIMIT n`. The per-value streams come sorted + * off `(tag_hash, kind, created_at)`, but their union does not, so SQLite + * collects every matching row and TEMP-B-TREE sorts to the limit — the + * tag-index analogue of the follow-feed regression + * [MergeQueryExecutor] fixed for author streams. Reported with and + * without a `since` bound to show what EOSE-warm steady state hides. + * + * Size the seed with `-DtagBenchScale=N` (default 1 ≈ ~200k events). + */ +class TagAuthorIndexBenchmark { + companion object { + val SCALE = System.getProperty("tagBenchScale")?.toInt() ?: 1 + } + + private val hex = "0123456789abcdef" + + private fun mix(seed: Long): Long { + var z = seed + -0x61c8864680b583ebL + z = (z xor (z ushr 30)) * -0x40a7b892e31b1a47L + z = (z xor (z ushr 27)) * -0x6b2fb644ecceee15L + return z xor (z ushr 31) + } + + private fun hex64( + salt: Long, + index: Int, + ): String { + val out = CharArray(64) + for (w in 0 until 4) { + val v = mix(salt * 1_000_003 + index.toLong() * 4 + w) + for (b in 0 until 8) { + val byte = ((v ushr (b * 8)) and 0xFF).toInt() + out[(w * 8 + b) * 2] = hex[byte ushr 4] + out[(w * 8 + b) * 2 + 1] = hex[byte and 0xF] + } + } + return String(out) + } + + private val sig = "0".repeat(128) + private var idSeq = 0 + + private fun ev( + pubkey: String, + createdAt: Long, + kind: Int, + tags: Array>, + ): Event = EventFactory.create(hex64(7, idSeq++), pubkey, createdAt, kind, tags, "", sig) + + private fun seedEvents(): List { + idSeq = 0 + val base = 1_700_000_000L + val span = 3_000_000L // ~35 days + val me = hex64(9, 0) + val events = ArrayList(220_000 * SCALE) + + // DM inbox: 200 peers, 300 DMs each → 60k kind-4 rows sharing the + // same (p:me) tag hash. The room query wants one peer's 300. + val peers = (0 until 200).map { hex64(2, it) } + for ((i, peer) in peers.withIndex()) { + repeat(300 * SCALE) { + val ts = base + (mix(i * 131L + it) and 0x7fffffff) % span + events.add(ev(peer, ts, 4, arrayOf(arrayOf("p", me)))) + } + } + + // Notification noise: 2000 authors mention me in kind-1 notes, so + // (p:me) spans multiple kinds like a real inbox does. + repeat(40_000 * SCALE) { + val author = hex64(3, it % 2_000) + val ts = base + (mix(it * 17L) and 0x7fffffff) % span + events.add(ev(author, ts, 1, arrayOf(arrayOf("p", me)))) + } + + // Reactions: 100k kind-7 events spread over 5000 target notes, for + // the large-IN watcher shape. + val noteIds = (0 until 5_000).map { hex64(5, it) } + repeat(100_000 * SCALE) { + val author = hex64(4, it % 3_000) + val ts = base + (mix(it * 29L) and 0x7fffffff) % span + events.add(ev(author, ts, 7, arrayOf(arrayOf("e", noteIds[it % noteIds.size])))) + } + return events + } + + @Test + fun compareTagAuthorIndex() = + runBlocking { + val events = seedEvents() + val me = hex64(9, 0) + val peers = (0 until 200).map { hex64(2, it) } + val noteIds = (0 until 5_000).map { hex64(5, it) } + + println("─ TagAuthorIndexBenchmark: ${events.size} events (scale=$SCALE) ─") + + val strategies = + listOf( + "flag-off" to DefaultIndexingStrategy(indexFullTextSearch = false), + "flag-on " to DefaultIndexingStrategy(indexFullTextSearch = false, indexTagsWithKindAndPubkey = true), + ) + + for ((label, strategy) in strategies) { + val store = EventStore(dbName = null, indexStrategy = strategy) + + val t0 = System.nanoTime() + events.chunked(10_000).forEach { store.batchInsert(it) } + val insertMs = (System.nanoTime() - t0) / 1e6 + println(" ═ $label ═ insert: %.0f ms (%.1f µs/event)".format(insertMs, insertMs * 1000 / events.size)) + + // 1. DM room: one peer's DMs out of the whole (p:me) inbox. + val room = Filter(kinds = listOf(4), authors = listOf(peers[42]), tags = mapOf("p" to listOf(me)), limit = 100) + time(store, "dm-room (#p ∩ author ∩ kind, limit 100)", room) + + // 2. Reactions watcher: 300 note ids, cold (no since). + val watcher = Filter(kinds = listOf(7), tags = mapOf("e" to noteIds.take(300)), limit = 500) + time(store, "reactions (#e IN 300, limit 500, cold)", watcher) + + // 3. Same watcher, EOSE-warm (since bounds the window). + val warm = watcher.copy(since = 1_700_000_000L + 2_900_000L) + time(store, "reactions (#e IN 300, limit 500, since)", warm) + + store.close() + } + } + + private suspend fun time( + store: EventStore, + label: String, + filter: Filter, + ) { + repeat(3) { store.query(filter) } + val runs = 10 + var rows = 0 + val start = System.nanoTime() + repeat(runs) { rows = store.query(filter).size } + val ms = (System.nanoTime() - start) / 1e6 / runs + println(" %-42s %8.2f ms (%d rows)".format(label, ms, rows)) + } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDriverSelectionBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDriverSelectionBenchmark.kt new file mode 100644 index 0000000000..54058191ba --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDriverSelectionBenchmark.kt @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.fs + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.utils.EventFactory +import kotlinx.coroutines.runBlocking +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.test.Test + +/** + * Quantifies [FsQueryPlanner]'s "first available driver wins" ordering + * (tags → kinds → authors) on the `authors + kinds + limit` shape — the + * most common CLI query (27 assembler call sites; every `amy feed`-style + * author timeline over non-replaceable kinds). + * + * `Filter(authors=[pk], kinds=[1], limit=n)` drives from `idx/kind/1/` + * (the biggest tree in any real store) and post-filters the author, even + * though `idx/author//` holds exactly that author's events. The + * benchmark times: + * + * - **kind-driver (current)**: the filter as the planner runs it today. + * - **author-driver (proposed)**: same result set, but driven from the + * author tree with the kind check as a post-filter — what a cost-based + * picker (compare candidate directory sizes) would choose. + * + * Also reports the author-only shape (`authors + limit`) as the floor: the + * planner already picks the author tree there, so its time is the target. + * + * Size the seed with `-DfsBenchScale=N` (default 1 ≈ ~30k events; each + * event is a file + ~3 hardlinks, so seeding dominates wall time). + */ +class FsDriverSelectionBenchmark { + companion object { + val SCALE = System.getProperty("fsBenchScale")?.toInt() ?: 1 + } + + private val hex = "0123456789abcdef" + + private fun mix(seed: Long): Long { + var z = seed + -0x61c8864680b583ebL + z = (z xor (z ushr 30)) * -0x40a7b892e31b1a47L + z = (z xor (z ushr 27)) * -0x6b2fb644ecceee15L + return z xor (z ushr 31) + } + + private fun hex64( + salt: Long, + index: Int, + ): String { + val out = CharArray(64) + for (w in 0 until 4) { + val v = mix(salt * 1_000_003 + index.toLong() * 4 + w) + for (b in 0 until 8) { + val byte = ((v ushr (b * 8)) and 0xFF).toInt() + out[(w * 8 + b) * 2] = hex[byte ushr 4] + out[(w * 8 + b) * 2 + 1] = hex[byte and 0xF] + } + } + return String(out) + } + + private val sig = "0".repeat(128) + private var idSeq = 0 + + private fun ev( + pubkey: String, + createdAt: Long, + kind: Int, + ): Event = EventFactory.create(hex64(7, idSeq++), pubkey, createdAt, kind, emptyArray(), "", sig) + + @Test + fun compareDrivers() = + runBlocking { + val root: Path = Files.createTempDirectory("fs-driver-bench-") + val store = FsEventStore(root) + try { + val base = 1_700_000_000L + val span = 3_000_000L + val target = hex64(9, 0) + + // Background: 300 authors × 100 kind-1 notes. + val bg = ArrayList(30_000 * SCALE + 300) + repeat(30_000 * SCALE) { + val author = hex64(1, it % 300) + bg.add(ev(author, base + (mix(it * 31L) and 0x7fffffff) % span, 1)) + } + // Target author: 200 kind-1 notes + 50 kind-7 reactions. + repeat(200) { bg.add(ev(target, base + (mix(it * 131L) and 0x7fffffff) % span, 1)) } + repeat(50) { bg.add(ev(target, base + (mix(it * 61L) and 0x7fffffff) % span, 7)) } + + val t0 = System.nanoTime() + store.transaction { bg.forEach { insert(it) } } + val insertMs = (System.nanoTime() - t0) / 1e6 + println("─ FsDriverSelectionBenchmark: ${bg.size} events (scale=$SCALE), seed %.0f ms ─".format(insertMs)) + + // Current planner: kinds present → kind tree drives, author + // is a post-filter over the whole kind-1 listing. + val kindDriven = Filter(authors = listOf(target), kinds = listOf(1), limit = 50) + time(store, "kind-driver (current planner)") { store.query(kindDriven).size } + + // Proposed: drive from the author tree, post-filter kind — + // same semantics, what a cost-based picker would run. + time(store, "author-driver (proposed)") { + store + .query(Filter(authors = listOf(target), limit = 250)) + .asSequence() + .filter { it.kind == 1 } + .take(50) + .count() + } + + // Floor: author-only shape, planner already optimal here. + val authorOnly = Filter(authors = listOf(target), limit = 50) + time(store, "author-only (planner floor)") { store.query(authorOnly).size } + } finally { + store.close() + if (root.exists()) { + Files.walk(root).use { it.sorted(Comparator.reverseOrder()).forEach { p -> Files.deleteIfExists(p) } } + } + } + } + + private inline fun time( + store: FsEventStore, + label: String, + run: () -> Int, + ) { + repeat(3) { run() } + val runs = 10 + var rows = 0 + val start = System.nanoTime() + repeat(runs) { rows = run() } + val ms = (System.nanoTime() - start) / 1e6 / runs + println(" %-32s %8.2f ms (%d rows)".format(label, ms, rows)) + } +} From 41240eeab9e9d32470379c0d15d89ba1f816ba21 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 14:39:21 +0000 Subject: [PATCH 02/34] docs(store): record measured index/planner gaps as TODOs in both stores Real numbers from the two new benchmarks, so the trade-offs are on the decision points instead of in a chat log: - IndexingStrategy.indexTagsWithKindAndPubkey: the KDoc called the kinds+authors+tags shape "rarely used", but the client assembler survey found 65 call sites. TagAuthorIndexBenchmark @ 200k events: DM-room query 9.4 ms -> 0.6 ms (~15x) with the flag on, insert cost +14% (41.5 -> 47.3 us/event). TODO: re-evaluate defaults (geode). - MergeQueryExecutor: tag-path analogue of the follow-feed collect-all sort (kinds + #e IN [hundreds] + limit never merges). Measured 12.8 ms cold / 6.0 ms since-bounded at 200k events; revisit if relayBench shows it at relay scale. - FsQueryPlanner: fixed driver order sends authors+kinds+limit (the most common CLI shape) through the kind tree. FsDriverSelection- Benchmark @ 30k events: 149 ms -> 3.4 ms (~44x) driving from the author tree; TODO: cost-based pick by directory entry counts. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01RzkoN3SJHCZWRAiadXXG4w --- .../nip01Core/store/sqlite/IndexingStrategy.kt | 15 ++++++++++++--- .../nip01Core/store/sqlite/MergeQueryExecutor.kt | 10 ++++++++++ .../quartz/nip01Core/store/fs/FsQueryPlanner.kt | 11 +++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt index 816f50f0f5..4f2e848c12 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt @@ -74,9 +74,18 @@ interface IndexingStrategy { * Activate this if you see too many Tag-centric Filters without * kind AND pubkey at the same time. * - * This is a rarely used index (reports by your follows or - * NIP-04 DMs for instance) that becomes quite large without - * major gains. + * This shape (reports by your follows, NIP-04 DM rooms, follows-scoped + * community feeds) is not rare on the client side: the 2026-07 filter + * assembler survey counted 65 call sites building + * `kinds + authors + tags`. Without this index the plan seeks + * `(tag_hash, kind)` and reads every row for that tag/kind before + * filtering the author. + * + * TODO: re-evaluate the off-by-default choice (especially for geode) + * with `TagAuthorIndexBenchmark` (jvmTest prodbench). At 200k events: + * DM-room query 9.4 ms → 0.6 ms (~15×) with the flag on, for a batch + * insert cost of 41.5 → 47.3 µs/event (+14%) — measure at target + * corpus size before flipping, since the index competes for page cache. * * Keep in mind that activating too many indexes increases the size of the * DB so much that the indexes themselves won't fit in memory, requiring diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt index 62660454db..24b62c56d5 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt @@ -52,6 +52,16 @@ import androidx.sqlite.SQLiteStatement * exactly at a same-second boundary. */ internal object MergeQueryExecutor { + // TODO: the same collect-all + TEMP-B-TREE-sort pattern exists one index + // over, on the tag path: `kinds + tags(#e IN [hundreds]) + limit` (the + // reactions/replies watcher archetype) unions per-value streams that are + // each sorted off `(tag_hash, kind, created_at)` and sorts the union. + // `streamCount` currently rejects any filter with tags, so those queries + // never merge. Measured by `TagAuthorIndexBenchmark` at 200k events: + // `#e IN 300, limit 500` costs 12.8 ms cold / 6.0 ms since-bounded — + // tolerable client-side, but it scales with matching history like the + // follow-feed shape did; extend the merge to per-tag-value streams if + // relay-scale runs (relayBench) show it in the profile. const val COLS = "id, pubkey, created_at, kind, tags, content, sig" /** diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt index 1c64f50360..f6958d8d80 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt @@ -46,6 +46,17 @@ import kotlin.io.path.exists * later without changing callers. All FilterMatcher semantics (tag * AND/OR, since/until, id, author, kind cross-checks) are enforced in * the orchestrator, so picking a loose driver is correctness-safe. + * + * TODO: add the cost-based pick. The fixed tags → kinds → authors order + * makes `authors + kinds + limit` — the most common CLI shape (27 + * assembler call sites, every `amy feed`-style author timeline) — drive + * from the kind tree and post-filter the author. Measured by + * `FsDriverSelectionBenchmark` at 30k events: 149 ms via `idx/kind/1/` + * vs 3.4 ms via `idx/author//` with kind post-filtered (~44×), and + * the gap grows with the kind tree, not the result. Comparing candidate + * directory entry counts before walking (kind dirs vs author dirs vs tag + * dirs) is enough; the slot shortcut already rescues the + * replaceable/addressable subset. */ internal class FsQueryPlanner( private val layout: FsLayout, From a3e239d33fc9e4a3cfa9c6ed1c1ca11745267f6a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 14:54:48 +0000 Subject: [PATCH 03/34] feat(store): cost-based FS driver pick, runtime index materialization, new relayBench shapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acts on the measured gaps from TagAuthorIndexBenchmark and FsDriverSelectionBenchmark: - FsQueryPlanner: replace the fixed tags -> kinds -> authors driver order with a cost-based pick. Every legal driver (each tagsAll value, each tags key's value union, the kind set, the author set) opens a lazy directory iterator; all are drained in lockstep and the first to exhaust (the smallest listing) drives, so a giant idx/kind tree is never read past ~the smallest candidate's size. Fixes the 149 ms vs 3.4 ms (~44x at 30k events) authors+kinds+limit regression. - EventIndexesModule.ensureOptionalIndexes + SQLiteEventStore: flag- gated indexes are runtime config, not schema. An idempotent CREATE INDEX IF NOT EXISTS pass now runs on every open, so flipping an IndexingStrategy flag on an existing DB builds the index without a user_version bump. - Desktop LocalRelayStore: enable indexEventsByPubkeyAlone. Shared ViewModels (Nip65RelayList, PrivateOutboxRelayList, VanishRequests) replay authors-only filters that full-scanned without (pubkey, created_at); existing DBs pick the index up on next open. - relayBench Scenarios: add "conversation" (tag ∩ author ∩ kind, the DM-room shape, 65 client assembler call sites) and "reactions-watch" (kind 7 + #e IN 150 hottest notes) so the uncovered archetypes get head-to-head numbers vs strfry. - quartz build: forward tagBenchScale/fsBenchScale to the test JVM. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01RzkoN3SJHCZWRAiadXXG4w --- .../amethyst/desktop/relay/LocalRelayStore.kt | 17 +- quartz/build.gradle.kts | 2 + .../store/sqlite/EventIndexesModule.kt | 39 ++++- .../store/sqlite/SQLiteEventStore.kt | 5 + .../nip01Core/store/fs/FsQueryPlanner.kt | 152 +++++++++++++----- .../com/vitorpamplona/relaybench/Scenarios.kt | 50 +++++- 6 files changed, 216 insertions(+), 49 deletions(-) diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/relay/LocalRelayStore.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/relay/LocalRelayStore.kt index 566df9e006..1300067464 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/relay/LocalRelayStore.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/relay/LocalRelayStore.kt @@ -26,6 +26,7 @@ import com.vitorpamplona.amethyst.desktop.cache.DesktopLocalCache import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CoroutineScope @@ -42,6 +43,18 @@ class LocalRelayStore( ) : AutoCloseable { companion object { val LOCAL_RELAY_URL: NormalizedRelayUrl = NormalizedRelayUrl("ws://localhost/amethyst-local/") + + /** + * Client defaults plus the authors-without-kinds index: shared + * ViewModels (`Nip65RelayListViewModel`, `PrivateOutboxRelayListViewModel`, + * `VanishRequestsState`) replay `authors`-only filters against this + * store, which full-scan without `(pubkey, created_at)` — the + * `(kind, pubkey, …)` index can't serve them, pubkey is its second + * column. A personal store is small, so the extra insert cost is + * negligible; existing DBs get the index built on next open via + * `ensureOptionalIndexes`. + */ + val INDEX_STRATEGY = DefaultIndexingStrategy(indexEventsByPubkeyAlone = true) } private fun dbDir(pubKeyHex: String): File = File(homeDir, ".amethyst/accounts/${pubKeyHex.take(8)}") @@ -87,14 +100,14 @@ class LocalRelayStore( dir.mkdirs() val path = File(dir, "events.db").absolutePath try { - store = EventStore(dbName = path, relay = LOCAL_RELAY_URL) + store = EventStore(dbName = path, relay = LOCAL_RELAY_URL, indexStrategy = INDEX_STRATEGY) _lastError.value = null refreshStats() } catch (e: Exception) { Log.w("LocalRelayStore") { "DB open failed, recreating: ${e.message}" } try { deleteDbFiles(path) - store = EventStore(dbName = path, relay = LOCAL_RELAY_URL) + store = EventStore(dbName = path, relay = LOCAL_RELAY_URL, indexStrategy = INDEX_STRATEGY) _lastError.value = "Database was recreated: ${e.message}" } catch (e2: Exception) { _lastError.value = "Cannot open local store: ${e2.message}" diff --git a/quartz/build.gradle.kts b/quartz/build.gradle.kts index 10c0817ee8..513bdd3cd5 100644 --- a/quartz/build.gradle.kts +++ b/quartz/build.gradle.kts @@ -94,6 +94,8 @@ kotlin { // Forward the negentropy-benchmark corpus size to the test JVM. System.getProperty("negBenchN")?.let { systemProperty("negBenchN", it) } System.getProperty("followBenchScale")?.let { systemProperty("followBenchScale", it) } + System.getProperty("tagBenchScale")?.let { systemProperty("tagBenchScale", it) } + System.getProperty("fsBenchScale")?.let { systemProperty("fsBenchScale", it) } // Opt-in JFR profiling of a benchmark run (-PnegProfile=/tmp/neg.jfr). (project.findProperty("negProfile") as? String)?.let { jvmArgs("-XX:+FlightRecorder", "-XX:StartFlightRecording=filename=$it,settings=profile,dumponexit=true") diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt index f0228217b0..dba6e2f9f3 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt @@ -147,13 +147,38 @@ class EventIndexesModule( */ fun migrateV2AddPubkeyIndex(db: SQLiteConnection) { if (!indexStrategy.indexEventsByPubkeyAlone) return - val orderBy = - if (indexStrategy.useAndIndexIdOnOrderBy) { - "created_at DESC, id ASC" - } else { - "created_at DESC" - } - db.execSQL("CREATE INDEX IF NOT EXISTS query_by_pubkey_created ON event_headers (pubkey, $orderBy)") + db.execSQL("CREATE INDEX IF NOT EXISTS query_by_pubkey_created ON event_headers (pubkey, ${orderByColumns()})") + } + + private fun orderByColumns() = + if (indexStrategy.useAndIndexIdOnOrderBy) { + "created_at DESC, id ASC" + } else { + "created_at DESC" + } + + /** + * Materializes any flag-gated index the current [indexStrategy] wants + * but the on-disk schema predates. Flags are runtime configuration, not + * schema — a deployment can flip one without a `user_version` bump — so + * this runs idempotently on every open. The first open after enabling a + * flag pays a one-time index build over the existing rows; subsequent + * opens are no-ops. A disabled flag never drops an existing index (that + * stays an operator decision). + */ + fun ensureOptionalIndexes(db: SQLiteConnection) { + if (indexStrategy.indexEventsByCreatedAtAlone) { + db.execSQL("CREATE INDEX IF NOT EXISTS query_by_created_at_id ON event_headers (${orderByColumns()})") + } + if (indexStrategy.indexEventsByPubkeyAlone) { + db.execSQL("CREATE INDEX IF NOT EXISTS query_by_pubkey_created ON event_headers (pubkey, ${orderByColumns()})") + } + if (indexStrategy.indexTagsByCreatedAtAlone) { + db.execSQL("CREATE INDEX IF NOT EXISTS query_by_tags_hash ON event_tags (tag_hash, created_at DESC)") + } + if (indexStrategy.indexTagsWithKindAndPubkey) { + db.execSQL("CREATE INDEX IF NOT EXISTS query_by_tags_hash_kind_pubkey ON event_tags (tag_hash, kind, pubkey_hash, created_at DESC)") + } } val sqlInsertHeader = diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index eea4950f40..3150b7625b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -160,6 +160,11 @@ class SQLiteEventStore( setUserVersion(this, DATABASE_VERSION) } } + // Flag-gated indexes are runtime config, not schema: a + // deployment that flips an IndexingStrategy flag on an + // existing DB gets the index built here (idempotent, + // one-time cost), with no user_version bump involved. + eventIndexModule.ensureOptionalIndexes(db) }, ) } diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt index f6958d8d80..431ccafe85 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.core.isAddressable import com.vitorpamplona.quartz.nip01Core.core.isReplaceable import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.store.sqlite.TagNameValueHasher +import java.nio.file.DirectoryStream import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -36,27 +37,22 @@ import kotlin.io.path.exists * * Step-2 coverage: * - `ids` → direct canonical opens - * - `tagsAll`/`tags` → tag index union (first key) - * - `kinds` → kind index union - * - `authors` → author index union + * - `tagsAll`/`tags`/`kinds`/`authors` → cheapest index tree drives + * (capped entry-count comparison), the rest post-filter * - otherwise → full scan via every `idx/kind//` subtree * - * The planner is intentionally dumb about selectivity — "first available - * driver wins". A cost-based picker (smallest listing) can slot in - * later without changing callers. All FilterMatcher semantics (tag - * AND/OR, since/until, id, author, kind cross-checks) are enforced in - * the orchestrator, so picking a loose driver is correctness-safe. - * - * TODO: add the cost-based pick. The fixed tags → kinds → authors order - * makes `authors + kinds + limit` — the most common CLI shape (27 - * assembler call sites, every `amy feed`-style author timeline) — drive - * from the kind tree and post-filter the author. Measured by - * `FsDriverSelectionBenchmark` at 30k events: 149 ms via `idx/kind/1/` - * vs 3.4 ms via `idx/author//` with kind post-filtered (~44×), and - * the gap grows with the kind tree, not the result. Comparing candidate - * directory entry counts before walking (kind dirs vs author dirs vs tag - * dirs) is enough; the slot shortcut already rescues the - * replaceable/addressable subset. + * Driver choice is cost-based: every legal driver (each `tagsAll` value + * alone — AND semantics make any single value a complete driver — each + * `tags` key's value union, the kind set, the author set) opens a lazy + * directory iterator, all are drained in lockstep, and the first to + * exhaust — the smallest listing — drives. A giant tree (`idx/kind/1/` + * with a million entries) is therefore never read past ~the smallest + * candidate's size. Before the pick, the fixed tags → kinds → authors + * order sent `authors + kinds + limit` — the most common CLI shape — + * through the kind tree: 149 ms vs 3.4 ms (~44×) at 30k events per + * `FsDriverSelectionBenchmark`. All FilterMatcher semantics (tag AND/OR, + * since/until, id, author, kind cross-checks) are enforced in the + * orchestrator, so any driver pick is correctness-safe. */ internal class FsQueryPlanner( private val layout: FsLayout, @@ -85,19 +81,100 @@ internal class FsQueryPlanner( return ftsDriver(search) } - firstTagKey(filter)?.let { (name, values) -> - return mergeDesc(values.map { v -> walkDir(layout.tagValueDir(name, v, hasher.hash(name, v))) }) + val candidates = driverCandidates(filter) + if (candidates.isEmpty()) return allKindsDriver() + + return mergeDesc(cheapestDriver(candidates).map { walkDir(it) }) + } + + /** + * Every set of index directories that, walked and post-filtered, yields + * a superset of the filter's matches: + * - each `tagsAll` value alone (AND semantics — every match carries it), + * - each `tags` key's full value union (OR within a key, AND across), + * - the kind set, and the author set. + * Listed in the old fixed-priority order so [cheapestDriver] keeps that + * order on cost ties. + */ + private fun driverCandidates(filter: Filter): List> { + val out = ArrayList>() + filter.tagsAll?.forEach { (name, values) -> + values.forEach { v -> out.add(listOf(layout.tagValueDir(name, v, hasher.hash(name, v)))) } + } + filter.tags?.forEach { (name, values) -> + if (values.isNotEmpty()) { + out.add(values.map { v -> layout.tagValueDir(name, v, hasher.hash(name, v)) }) + } + } + filter.kinds?.takeIf { it.isNotEmpty() }?.let { kinds -> + out.add(kinds.map { layout.kindDir(it) }) + } + filter.authors?.takeIf { it.isNotEmpty() }?.let { authors -> + out.add(authors.map { layout.authorDir(it) }) + } + return out + } + + /** + * Smallest candidate by lockstep listing drain: one lazy directory + * iterator per candidate, all advanced [COST_BATCH] entries per round — + * the first to exhaust its listing is the smallest, so a giant tree is + * never read past ~the smallest candidate's size (a candidate that + * exhausts on round one costs the others one batch each). A candidate + * whose dirs are all missing exhausts immediately: driving from an empty + * mandatory predicate correctly yields an empty result. If every + * candidate survives [COST_CAP] entries, all are huge and relative + * driver choice stops mattering — the first (old fixed-priority order) + * wins. + */ + private fun cheapestDriver(candidates: List>): List { + if (candidates.size == 1) return candidates[0] + val cursors = candidates.map { EntryCursor(it) } + try { + var advanced = 0L + while (advanced < COST_CAP) { + for (i in cursors.indices) { + if (!cursors[i].skip(COST_BATCH)) return candidates[i] + } + advanced += COST_BATCH + } + return candidates[0] + } finally { + cursors.forEach { it.close() } + } + } + + /** Lazy entry iterator over a candidate's directories, in order. */ + private class EntryCursor( + dirs: List, + ) : AutoCloseable { + private val remaining = ArrayDeque(dirs) + private var stream: DirectoryStream? = null + private var iter: Iterator = emptyList().iterator() + + /** Advances up to [n] entries; false when the listing ends first. */ + fun skip(n: Int): Boolean { + var left = n + while (left > 0) { + if (iter.hasNext()) { + iter.next() + left-- + continue + } + close() + val dir = remaining.removeFirstOrNull() ?: return false + if (!Files.isDirectory(dir)) continue + stream = Files.newDirectoryStream(dir) + iter = stream!!.iterator() + } + return true } - filter.kinds?.let { kinds -> - return mergeDesc(kinds.map { walkDir(layout.kindDir(it)) }) + override fun close() { + stream?.close() + stream = null + iter = emptyList().iterator() } - - filter.authors?.let { authors -> - return mergeDesc(authors.map { walkDir(layout.authorDir(it)) }) - } - - return allKindsDriver() } /** @@ -312,14 +389,15 @@ internal class FsQueryPlanner( var top: Candidate, ) - // ---- helpers ------------------------------------------------------ + private companion object { + /** Entries each candidate's cursor advances per lockstep round. */ + const val COST_BATCH = 64 - /** First tag filter with at least one value, preferring `tagsAll`. */ - private fun firstTagKey(filter: Filter): Pair>? { - filter.tagsAll?.firstNonEmpty()?.let { return it } - filter.tags?.firstNonEmpty()?.let { return it } - return null + /** + * Stop draining once every candidate has survived this many + * entries: past it they are all huge, relative choice stops + * mattering, and the first candidate in priority order wins. + */ + const val COST_CAP = 65_536L } - - private fun Map>.firstNonEmpty(): Pair>? = entries.firstOrNull { it.value.isNotEmpty() }?.let { it.key to it.value } } diff --git a/relayBench/src/main/kotlin/com/vitorpamplona/relaybench/Scenarios.kt b/relayBench/src/main/kotlin/com/vitorpamplona/relaybench/Scenarios.kt index 5c73ac5ed5..2c29711e45 100644 --- a/relayBench/src/main/kotlin/com/vitorpamplona/relaybench/Scenarios.kt +++ b/relayBench/src/main/kotlin/com/vitorpamplona/relaybench/Scenarios.kt @@ -70,11 +70,11 @@ object Scenarios { } val topAuthors = notesByAuthor.entries.sortedWith(compareByDescending> { it.value }.thenBy { it.key }).map { it.key } - val hottestThread = + val hotNotes = eTagRefs.entries .sortedWith(compareByDescending> { it.value }.thenBy { it.key }) - .firstOrNull() - ?.key + .map { it.key } + val hottestThread = hotNotes.firstOrNull() val mostMentioned = pTagRefs.entries .sortedWith(compareByDescending> { it.value }.thenBy { it.key }) @@ -86,6 +86,23 @@ object Scenarios { .firstOrNull() ?.key + // The author that most often tags the most-mentioned pubkey — a + // conversation pair for the tag ∩ author (DM-room) query shape. + val conversationPeer = + mostMentioned?.let { me -> + val byAuthor = HashMap() + for (e in events) { + if (e.kind != 1 || e.pubKey == me) continue + if (e.tags.any { it.size >= 2 && it[0] == "p" && it[1] == me }) { + byAuthor.merge(e.pubKey, 1, Int::plus) + } + } + byAuthor.entries + .sortedWith(compareByDescending> { it.value }.thenBy { it.key }) + .firstOrNull() + ?.key + } + // Evenly spread sample of note ids — a "fetch these 100 events" batch. val idSample = if (noteIds.size <= 100) { @@ -159,6 +176,33 @@ object Scenarios { ), ) } + if (mostMentioned != null && conversationPeer != null) { + // The tag ∩ author ∩ kind shape (65 client assembler call + // sites: NIP-04 DM rooms, reports-by-follows, follows-scoped + // community feeds). Modeled on kind 1 because public corpora + // carry no DMs; the index path exercised is identical. + add( + Scenario( + "conversation", + "notes by one author tagging the most-mentioned pubkey (DM-room shape)", + Filter(kinds = listOf(1), authors = listOf(conversationPeer), tags = mapOf("p" to listOf(mostMentioned)), limit = 500), + ), + ) + } + if (hotNotes.size > 1) { + // Large-IN tag watcher: per-value streams come sorted off the + // tag index but their union does not, exposing whether the + // store collects+sorts or merges. 150 values stays inside + // strfry's default 200-element filter cap. + val watched = hotNotes.take(150) + add( + Scenario( + "reactions-watch", + "reactions on the ${watched.size} hottest notes (visible-feed reaction watcher)", + Filter(kinds = listOf(7), tags = mapOf("e" to watched), limit = 500), + ), + ) + } topHashtag?.let { add( Scenario( From 57ffb3386d59c6693897c36c41f74a185da99353 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 15:07:12 +0000 Subject: [PATCH 04/34] feat(geode): enable the tag+kind+pubkey index, refresh measured docs TagAuthorIndexBenchmark at 1M events settles the flag: the DM-room shape (kinds + authors + #p, 65 client assembler call sites) drops 14.2 ms -> 0.66 ms (~21x, growing with corpus size) while batch-insert cost stays inside run noise (49.0 vs 47.4 us/event). Existing relay DBs build the index on next open via ensureOptionalIndexes. Also refreshes the docs the numbers made stale: IndexingStrategy KDoc now records the 200k and 1M measurements instead of a TODO, MergeQueryExecutor's tag-merge note points at the new relayBench reactions-watch scenario, FsQueryPlanner/FsDriverSelectionBenchmark reflect the landed cost-based pick (149 ms -> 4.0 ms at 30k events), and RELAY.md documents that strategy flag flips materialize indexes on the next open. Verified: quartz jvmTest store suites, geode test (126), desktopApp LocalRelayStore tests (5, incl. reopening a default-strategy DB with the new pubkey-alone flag), relayBench compiles. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01RzkoN3SJHCZWRAiadXXG4w --- .../geode/RelayIndexingStrategy.kt | 8 ++++ quartz/RELAY.md | 2 + .../store/sqlite/IndexingStrategy.kt | 13 +++-- .../store/sqlite/MergeQueryExecutor.kt | 11 +++-- .../nip01Core/store/fs/FsQueryPlanner.kt | 5 +- .../store/fs/FsDriverSelectionBenchmark.kt | 47 ++++++++++--------- 6 files changed, 51 insertions(+), 35 deletions(-) diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/RelayIndexingStrategy.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/RelayIndexingStrategy.kt index 33fa9f4441..97dde44807 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/RelayIndexingStrategy.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/RelayIndexingStrategy.kt @@ -57,6 +57,14 @@ fun relayIndexingStrategy( // index unconditionally; without it the filter walks the whole // time index. indexEventsByPubkeyAlone = true, + // The tag ∩ author ∩ kind shape (DM rooms, reports-by-follows, + // follows-scoped community feeds — 65 client assembler call sites) + // otherwise reads every row for the tag/kind before filtering the + // author. TagAuthorIndexBenchmark @ 1M events: 14.2 ms -> 0.66 ms + // (~21x, growing with corpus size) with insert cost inside run noise + // (49.0 vs 47.4 µs/event). Existing DBs build the index on next open + // via ensureOptionalIndexes. + indexTagsWithKindAndPubkey = true, indexFullTextSearch = fullTextSearch, // Tokenize off the commit path; NostrServer drives the catch-up // worker and search queries drain it first, so NIP-50 stays diff --git a/quartz/RELAY.md b/quartz/RELAY.md index 807b65caee..01bc5fc736 100644 --- a/quartz/RELAY.md +++ b/quartz/RELAY.md @@ -83,6 +83,8 @@ val store = EventStore( By default, all single-letter tags with values are indexed. Override `shouldIndex(kind, tag)` for custom behavior. More indexes = faster queries but larger database. +Flag flips are safe on existing databases: any flag-gated index the strategy wants but the on-disk schema lacks is created on the next open (idempotent `CREATE INDEX IF NOT EXISTS`, one-time build cost) — no schema version bump involved. Disabling a flag never drops an existing index. + `indexFullTextSearch` defaults to `true` and controls the NIP-50 full-text index (`event_fts`). Set it to `false` when search is served elsewhere (e.g. a Vespa backend, or a `SearchEventSource` as shown below): inserts skip the FTS tokenization cost, no `event_fts` table/trigger is created, and any filter carrying a non-empty `search` term returns no matches. ## Non-Storage Relays (search, redirector, computed) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt index 4f2e848c12..01362e03fb 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt @@ -81,11 +81,14 @@ interface IndexingStrategy { * `(tag_hash, kind)` and reads every row for that tag/kind before * filtering the author. * - * TODO: re-evaluate the off-by-default choice (especially for geode) - * with `TagAuthorIndexBenchmark` (jvmTest prodbench). At 200k events: - * DM-room query 9.4 ms → 0.6 ms (~15×) with the flag on, for a batch - * insert cost of 41.5 → 47.3 µs/event (+14%) — measure at target - * corpus size before flipping, since the index competes for page cache. + * Measured by `TagAuthorIndexBenchmark` (jvmTest prodbench): the + * DM-room query drops 9.4 ms → 0.6 ms (~15×) at 200k events and + * 14.2 ms → 0.66 ms (~21×) at 1M — the gap grows with corpus size — + * while batch-insert cost stays inside run noise (49.0 vs 47.4 + * µs/event at 1M). geode enables it; the client default stays off + * because a client store's per-tag row counts are bounded by one + * user's data. Flipping it on an existing DB is safe: the index is + * built on next open by `EventIndexesModule.ensureOptionalIndexes`. * * Keep in mind that activating too many indexes increases the size of the * DB so much that the indexes themselves won't fit in memory, requiring diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt index 24b62c56d5..34ada1e64c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt @@ -57,11 +57,12 @@ internal object MergeQueryExecutor { // reactions/replies watcher archetype) unions per-value streams that are // each sorted off `(tag_hash, kind, created_at)` and sorts the union. // `streamCount` currently rejects any filter with tags, so those queries - // never merge. Measured by `TagAuthorIndexBenchmark` at 200k events: - // `#e IN 300, limit 500` costs 12.8 ms cold / 6.0 ms since-bounded — - // tolerable client-side, but it scales with matching history like the - // follow-feed shape did; extend the merge to per-tag-value streams if - // relay-scale runs (relayBench) show it in the profile. + // never merge. Measured by `TagAuthorIndexBenchmark`: `#e IN 300, + // limit 500` costs 12.8 ms cold at 200k events and 14.2 ms at 1M + // (6.7 ms with indexTagsWithKindAndPubkey on) — tolerable, but it + // scales with matching history like the follow-feed shape did; extend + // the merge to per-tag-value streams if the relayBench + // `reactions-watch` scenario shows it in the profile vs strfry. const val COLS = "id, pubkey, created_at, kind, tags, content, sig" /** diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt index 431ccafe85..f650bfd613 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt @@ -49,8 +49,9 @@ import kotlin.io.path.exists * with a million entries) is therefore never read past ~the smallest * candidate's size. Before the pick, the fixed tags → kinds → authors * order sent `authors + kinds + limit` — the most common CLI shape — - * through the kind tree: 149 ms vs 3.4 ms (~44×) at 30k events per - * `FsDriverSelectionBenchmark`. All FilterMatcher semantics (tag AND/OR, + * through the kind tree: 149 ms fixed-order vs 4.0 ms cost-based at 30k + * events per `FsDriverSelectionBenchmark` (floor: author-only at + * 1.4 ms). All FilterMatcher semantics (tag AND/OR, * since/until, id, author, kind cross-checks) are enforced in the * orchestrator, so any driver pick is correctness-safe. */ diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDriverSelectionBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDriverSelectionBenchmark.kt index 54058191ba..320e3fcb9f 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDriverSelectionBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDriverSelectionBenchmark.kt @@ -30,23 +30,24 @@ import kotlin.io.path.exists import kotlin.test.Test /** - * Quantifies [FsQueryPlanner]'s "first available driver wins" ordering - * (tags → kinds → authors) on the `authors + kinds + limit` shape — the - * most common CLI query (27 assembler call sites; every `amy feed`-style - * author timeline over non-replaceable kinds). + * Guards [FsQueryPlanner]'s cost-based driver pick on the + * `authors + kinds + limit` shape — the most common CLI query (27 + * assembler call sites; every `amy feed`-style author timeline over + * non-replaceable kinds). * - * `Filter(authors=[pk], kinds=[1], limit=n)` drives from `idx/kind/1/` - * (the biggest tree in any real store) and post-filters the author, even - * though `idx/author//` holds exactly that author's events. The - * benchmark times: + * Under the pre-pick fixed order (tags → kinds → authors), + * `Filter(authors=[pk], kinds=[1], limit=n)` drove from `idx/kind/1/` + * (the biggest tree in any real store) and post-filtered the author: + * 149 ms at 30k events. The lockstep pick drives from the author tree + * and runs at ~4 ms. The benchmark times: * - * - **kind-driver (current)**: the filter as the planner runs it today. - * - **author-driver (proposed)**: same result set, but driven from the - * author tree with the kind check as a post-filter — what a cost-based - * picker (compare candidate directory sizes) would choose. - * - * Also reports the author-only shape (`authors + limit`) as the floor: the - * planner already picks the author tree there, so its time is the target. + * - **planner (cost-based pick)**: the filter as the planner runs it — + * should sit near the floor, far below a kind-tree walk. + * - **author-driver emulation**: the author tree walked via an + * authors-only query with the kind check applied by the caller — the + * reference the pick is expected to match or beat. + * - **author-only floor**: `authors + limit` with no kind, the cheapest + * possible walk of the same tree. * * Size the seed with `-DfsBenchScale=N` (default 1 ≈ ~30k events; each * event is a file + ~3 hardlinks, so seeding dominates wall time). @@ -115,14 +116,14 @@ class FsDriverSelectionBenchmark { val insertMs = (System.nanoTime() - t0) / 1e6 println("─ FsDriverSelectionBenchmark: ${bg.size} events (scale=$SCALE), seed %.0f ms ─".format(insertMs)) - // Current planner: kinds present → kind tree drives, author - // is a post-filter over the whole kind-1 listing. + // The planner's own pick — expected to choose the author + // tree over the ~30k-entry kind-1 tree. val kindDriven = Filter(authors = listOf(target), kinds = listOf(1), limit = 50) - time(store, "kind-driver (current planner)") { store.query(kindDriven).size } + time(store, "planner (cost-based pick)") { store.query(kindDriven).size } - // Proposed: drive from the author tree, post-filter kind — - // same semantics, what a cost-based picker would run. - time(store, "author-driver (proposed)") { + // Reference: author tree walked explicitly, kind checked + // by the caller — the pick should match or beat this. + time(store, "author-driver emulation") { store .query(Filter(authors = listOf(target), limit = 250)) .asSequence() @@ -131,9 +132,9 @@ class FsDriverSelectionBenchmark { .count() } - // Floor: author-only shape, planner already optimal here. + // Floor: author-only shape, the cheapest walk of the tree. val authorOnly = Filter(authors = listOf(target), limit = 50) - time(store, "author-only (planner floor)") { store.query(authorOnly).size } + time(store, "author-only floor") { store.query(authorOnly).size } } finally { store.close() if (root.exists()) { From 37923d91019840cd98cb7bada21cc4f821f1bf7a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 16:14:12 +0000 Subject: [PATCH 05/34] perf(relay): run the stored REQ replay undispatched SmallReqFloorBenchmark showed the per-REQ floor on small results is dominated by pipeline, not the store (raw query 0.125 ms vs 0.785 ms session REQ->EOSE in-process). Half of the dispatch slice was the scheduler hop between handleReq's launch and the query coroutine: starting the job with CoroutineStart.UNDISPATCHED runs the stored replay and EOSE inline on the receiving coroutine (the reader-pool acquire doesn't suspend when a connection is free), parking only at the live tail. Measured: dispatch+frames slice 0.397 -> 0.207 ms. Commands on a connection are processed sequentially, so nothing can target the subscription before the job lands in the registry at the first suspension point. Verified: quartz relay.server suite, SmallReqFloorBenchmark, geode full test suite. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01RzkoN3SJHCZWRAiadXXG4w --- .../quartz/nip01Core/relay/server/RelaySession.kt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt index 6acacc1705..f73493f19c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt @@ -49,6 +49,7 @@ import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quartz.utils.cache.LargeCache import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Job import kotlinx.coroutines.channels.ClosedSendChannelException import kotlinx.coroutines.launch @@ -291,8 +292,16 @@ class RelaySession( // Policy may rewrite filters to match the user's access level. val filters = (result as PolicyResult.Accepted).cmd.filters + // UNDISPATCHED: the stored replay runs inline on this coroutine — + // the reader-pool acquire doesn't suspend when a connection is + // free, so EVENT frames and EOSE go out without a scheduler hop + // (SmallReqFloorBenchmark: the hop was most of the dispatch + // slice on small REQs). The coroutine first parks at the live + // tail (awaitCancellation), which is when launch returns and the + // job lands in [subscriptions]; commands on this connection are + // processed sequentially, so nothing can target the sub earlier. val job = - scope.launch { + scope.launch(start = CoroutineStart.UNDISPATCHED) { try { if (policy.filtersOutgoingEvents) { // Screened path: every event is materialized so the From 6033957b1ca8d7fec23c50118bb5f1654afec686 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 16:22:13 +0000 Subject: [PATCH 06/34] perf(relay): persistent-map FilterIndex snapshots, benchmark the sub population MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FilterIndex registration runs on every REQ open/close but built each new snapshot by copying both full maps — O(S) work and allocation per REQ with S live subscriptions. Persistent (HAMT) maps keep the wait-free single-load reads and CAS write loop while making a write O(keys x log S) with structural sharing. SmallReqFloorBenchmark grows a B@1k stage (1000 idle parked subscriptions) to make the population cost visible, and its B stage now enters queryRaw undispatched like production does: @1000 subs the per-REQ cost drops 0.225 -> 0.151 ms and the measured population penalty falls below run noise (was +0.011 ms per REQ). With this and the undispatched replay, the in-process floor above the raw store query is ~0.11 ms (was ~0.66 ms as first measured): A 0.120, B 0.203, C 0.239 ms on a quiet machine. Verified: FilterIndex tests, quartz relay.server suite. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01RzkoN3SJHCZWRAiadXXG4w --- .../nip01Core/relay/filters/FilterIndex.kt | 53 ++++++++++++------- .../relay/prodbench/SmallReqFloorBenchmark.kt | 36 ++++++++++++- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt index 1cae5b471d..a5c058ec6d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt @@ -22,6 +22,11 @@ package com.vitorpamplona.quartz.nip01Core.relay.filters import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import kotlinx.collections.immutable.PersistentMap +import kotlinx.collections.immutable.PersistentSet +import kotlinx.collections.immutable.persistentHashMapOf +import kotlinx.collections.immutable.persistentHashSetOf +import kotlinx.collections.immutable.toPersistentHashSet import kotlin.concurrent.atomics.AtomicReference import kotlin.concurrent.atomics.ExperimentalAtomicApi @@ -62,9 +67,10 @@ import kotlin.concurrent.atomics.ExperimentalAtomicApi * copy-on-write CAS loops, mirroring the * `nip86RelayManagement.server.BanStore` pattern. Reads in * [candidatesFor] and [forEach] are wait-free single-load atomic. - * Writes (subscription register / unregister) copy the inner maps - * — fine for this workload because writes are subscription-rate - * (rare) while reads are event-rate (frequent). + * Writes (subscription register / unregister) build the next snapshot + * from persistent (HAMT) maps — O(keys × log S) with structural + * sharing rather than a full O(S) copy of both maps, since on a relay + * a write happens on every REQ open and close. * * ## What the index does NOT cover * @@ -113,10 +119,16 @@ class FilterIndex { * subscribers registered under it; [assignments] is the reverse * map used by [unregister] to find a subscriber's keys without * scanning every bucket. + * + * Persistent (HAMT) maps/sets: a register/unregister produces the + * next snapshot in O(keys × log S) with structural sharing, instead + * of copying both full maps — registration happens on every REQ + * open/close, so with S live subscriptions the full copy was + * O(S) work and O(S) allocation per REQ. */ private data class State( - val buckets: Map> = emptyMap(), - val assignments: Map> = emptyMap(), + val buckets: PersistentMap> = persistentHashMapOf(), + val assignments: PersistentMap> = persistentHashMapOf(), ) private val state: AtomicReference> = AtomicReference(State()) @@ -181,17 +193,18 @@ class FilterIndex { while (true) { val current = state.load() val keys = current.assignments[subscriber] ?: return - val newBuckets = current.buckets.toMutableMap() + var newBuckets = current.buckets for (key in keys) { val cur = newBuckets[key] ?: continue - val next = cur - subscriber - if (next.isEmpty()) { - newBuckets.remove(key) - } else { - newBuckets[key] = next - } + val next = cur.remove(subscriber) + newBuckets = + if (next.isEmpty()) { + newBuckets.remove(key) + } else { + newBuckets.put(key, next) + } } - val newAssignments = current.assignments - subscriber + val newAssignments = current.assignments.remove(subscriber) if (state.compareAndSet(current, State(newBuckets, newAssignments))) return } } @@ -235,18 +248,18 @@ class FilterIndex { keys: List, ) { if (keys.isEmpty()) return - val keySet = keys.toSet() + val keySet = keys.toPersistentHashSet() while (true) { val current = state.load() - val newBuckets = current.buckets.toMutableMap() + var newBuckets = current.buckets for (key in keySet) { - val cur = newBuckets[key] ?: emptySet() - if (subscriber in cur) continue - newBuckets[key] = cur + subscriber + val cur = newBuckets[key] ?: persistentHashSetOf() + val next = cur.add(subscriber) + if (next !== cur) newBuckets = newBuckets.put(key, next) } val existing = current.assignments[subscriber] - val merged = if (existing == null) keySet else existing + keySet - val newAssignments = current.assignments + (subscriber to merged) + val merged = existing?.addAll(keySet) ?: keySet + val newAssignments = current.assignments.put(subscriber, merged) if (state.compareAndSet(current, State(newBuckets, newAssignments))) return } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SmallReqFloorBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SmallReqFloorBenchmark.kt index 9fe3a6ec33..cd6eb4525b 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SmallReqFloorBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SmallReqFloorBenchmark.kt @@ -32,6 +32,7 @@ import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.utils.EventFactory import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel @@ -65,6 +66,7 @@ class SmallReqFloorBenchmark { const val AUTHORS = 2_500 // ~20 events per author, matching author-archive const val ROUNDS = 400 const val WARMUP = 100 + const val IDLE_SUBS = 1_000 } private fun hexId(seed: Int): String = seed.toString(16).padStart(64, '0') @@ -122,11 +124,14 @@ class SmallReqFloorBenchmark { } // --- B: backend queryRaw to EOSE (live machinery included) --- + // UNDISPATCHED mirrors the production path (RelaySession.handleReq + // starts the query coroutine undispatched), so B−A is the live + // machinery itself, not a benchmark-only scheduler hop. suspend fun timeBackend(round: Int): Long { val eose = CompletableDeferred() val t0 = System.nanoTime() val job = - scope.launch { + scope.launch(start = CoroutineStart.UNDISPATCHED) { live.queryRaw( ctx = ctx, filters = listOf(filterFor(round)), @@ -143,6 +148,33 @@ class SmallReqFloorBenchmark { val b = LongArray(ROUNDS) repeat(ROUNDS) { b[it] = timeBackend(it) } + // --- B@1k: same, with 1000 idle live subscriptions parked --- + // Register/unregister cost scales with the live population + // (FilterIndex mutates a shared snapshot per REQ open/close), + // which the single-sub stage can't see. Each idle sub filters + // on an author absent from the corpus: 0-row replay, then parks + // at the live tail and stays registered. + val idleJobs = + (0 until IDLE_SUBS).map { i -> + val ready = CompletableDeferred() + val job = + scope.launch(start = CoroutineStart.UNDISPATCHED) { + live.queryRaw( + ctx = ctx, + filters = listOf(Filter(authors = listOf(hexId(1_000_000 + i)), kinds = listOf(1), limit = 1)), + onEachStored = {}, + onEachLive = {}, + onEose = { ready.complete(Unit) }, + ) + } + ready.await() + job + } + repeat(WARMUP) { timeBackend(it) } + val b1k = LongArray(ROUNDS) + repeat(ROUNDS) { b1k[it] = timeBackend(it) } + idleJobs.forEach { it.cancel() } + // --- C: full session dispatch, REQ json in → EOSE frame out --- suspend fun timeSession(round: Int): Long { val eose = CompletableDeferred() @@ -164,10 +196,12 @@ class SmallReqFloorBenchmark { val mA = median(a) val mB = median(b) + val mB1k = median(b1k) val mC = median(c) println("SmallReqFloorBenchmark @ ${EVENTS / 1000}k events, ~${rowsA / ROUNDS} rows/req, medians of $ROUNDS") println(" A raw store query: ${"%6.3f".format(mA)} ms") println(" B backend queryRaw→EOSE: ${"%6.3f".format(mB)} ms (live machinery +${"%6.3f".format(mB - mA)})") + println(" B@${IDLE_SUBS} idle subs: ${"%6.3f".format(mB1k)} ms (population cost +${"%6.3f".format(mB1k - mB)})") println(" C session REQ→EOSE: ${"%6.3f".format(mC)} ms (dispatch+frames +${"%6.3f".format(mC - mB)})") server.close() From f67d242f7a9b116ab58e47c06136f1c25b783870 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 21 Jul 2026 13:21:35 -0400 Subject: [PATCH 07/34] fix: show Concord channels on Messages as soon as the control plane folds A Concord control-plane fold is what first reveals a community's channels and makes ConcordCommunitySession.state non-null, without which ChatroomListKnownFeedFilter emits nothing at all for that community. None of it flows through LocalCache.newEventBundles, so the additive feed path could not see it: a folded channel only reached the Messages tab if a message for it happened to arrive afterwards. Cold boot therefore showed a subset of a community's channels, or omitted a quiet community entirely, until some unrelated invalidation fired. Measured on device: the Concord hub reported 3 communities / 17 channels folded in memory while Messages rendered 3 rows and omitted one community completely. AccountFeedContentStates already forces a rebuild for the Marmot, NIP-29, geohash, view-mode and pin flows for exactly this reason; Concord was the one missing collector. Add it, sampled the same way Account.kt samples this flow to drive refreshConcordChannelIndex. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../loggedIn/AccountFeedContentStates.kt | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountFeedContentStates.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountFeedContentStates.kt index e1808649a8..0183867e41 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountFeedContentStates.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountFeedContentStates.kt @@ -77,7 +77,9 @@ import com.vitorpamplona.amethyst.ui.screen.loggedIn.webBookmarks.dal.WebBookmar import com.vitorpamplona.amethyst.ui.screen.loggedIn.workouts.dal.WorkoutFeedFilter import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.sample import kotlinx.coroutines.launch class AccountFeedContentStates( @@ -201,6 +203,26 @@ class AccountFeedContentStates( } } + // A Concord control-plane fold is what first reveals a community's channels (and what makes + // ConcordCommunitySession.state non-null, without which ChatroomListKnownFeedFilter emits + // nothing at all for that community). None of it flows through LocalCache.newEventBundles, + // so the additive path can't see it: a folded channel reaches the Messages tab only if a + // message for it happens to arrive afterwards. Cold boot therefore shows a *subset* of a + // community's channels, or omits a quiet community entirely, until some unrelated + // invalidation fires. Rebuild on every structural change instead. `revision` bumps only on + // fold/membership/rekey (never a plain message), and sample() coalesces the burst of folds + // that lands as each control plane catches up — the same pairing Account.kt uses to drive + // refreshConcordChannelIndex off this flow. + scope.launch(Dispatchers.IO) { + @OptIn(FlowPreview::class) + account.concordSessions.revision + .drop(1) + .sample(500) + .collect { + dmKnown.invalidateData() + } + } + // Same for the Concord view mode (inline channels vs one row per community). scope.launch(Dispatchers.IO) { account.settings.concordViewMode From 50025660a3a36a8f74c7ac5f96ce137393286e61 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 21 Jul 2026 13:21:54 -0400 Subject: [PATCH 08/34] perf: cut Concord revision churn and quadratic control-plane re-folds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold boot bumped the session revision ~292 times for 3 communities, driving 22 Messages rebuilds and re-deriving every plane subscription each time. Three compounding causes, all measured on device: 1. Every refold republished state even when the fold was identical. ConcordCommunityState and its components were plain classes, so StateFlow conflation never applied and a prior-epoch wrap that didn't move the anti-rollback floor still counted as a change. Make the fold result compare by value (AuthorityResolver holds only immutable value fields; a data class with a private constructor is fine). 2. A control wrap bumped twice — once from ingest() returning STRUCTURAL and once from the per-session state watcher reacting to the same refold. Add ConcordIngestOutcome.STRUCTURAL_FOLD for the two control-plane branches so the manager leaves those to the watcher, which (given 1) now fires only on genuine change. Guestbook and base-rekey keep STRUCTURAL: they mutate members/the rekey buffer, not state, so no watcher covers them. 3. refold() and controlFloorsLocked() re-opened the WHOLE wrap buffer on every control wrap, and opening a wrap is a NIP-44 decrypt + parse — making a backfill quadratic in decryptions (~8.6k opens to ingest 93 wraps for one community). Memoize editions by wrap id: one open per wrap, ingest() stays synchronous and results are unchanged. Measured over one cold boot: revision bumps 292 -> 87, Messages rebuilds 22 -> 7, and time from first fold to all 17 channels 43s -> 7.5s. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../model/concord/ConcordCommunitySession.kt | 57 ++++++++++++++++--- .../model/concord/ConcordSessionManager.kt | 3 + .../concord/ConcordCommunitySessionTest.kt | 2 +- .../concord/ConcordSessionRegistryTest.kt | 2 +- .../cord02Community/ConcordCommunityState.kt | 4 +- .../concord/cord04Roles/AuthorityResolver.kt | 2 +- .../concord/cord04Roles/ControlEntities.kt | 10 ++-- 7 files changed, 63 insertions(+), 17 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt index 9bb65aef67..49acfd9b1e 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt @@ -62,8 +62,17 @@ enum class ConcordIngestOutcome { * a chat/reaction/reply/delete message landing, or a duplicate wrap. Must NOT bump the revision. */ NON_STRUCTURAL, - /** Ours and changed structure: a Control-Plane fold (metadata/channels/membership/authority), a - * guestbook membership change, or a buffered base-rekey. Bumps the revision. */ + /** Ours and re-folded the Control Plane. The fold republishes [ConcordCommunitySession.state], + * so the session's own state watcher is what bumps the revision — and, because the folded state + * compares by value, only when the fold actually *changed* something. A control wrap that folds + * to an identical state (a prior-epoch wrap that doesn't move the anti-rollback floor, a role + * edition that touches nothing we subscribe on) therefore costs no bump at all. The manager must + * NOT bump on this outcome as well, or every control wrap counts twice. */ + STRUCTURAL_FOLD, + + /** Ours and changed structure *without* touching [ConcordCommunitySession.state]: a guestbook + * membership change (which republishes `members`) or a buffered base-rekey. No state watcher + * covers these, so the manager bumps the revision directly. */ STRUCTURAL, ; @@ -145,6 +154,22 @@ class ConcordCommunitySession( // Deduped inbound wraps. private val controlWraps = LinkedHashMap() + /** + * Decrypted control editions memoized by wrap id. + * + * Both [refold] and [controlFloorsLocked] fold their WHOLE buffer on every inbound control + * wrap, and turning a wrap into an edition is a NIP-44 open + parse. Re-deriving them each + * time made a cold-boot backfill quadratic in decryptions — one measured boot did ~8.6k opens + * to ingest 93 control wraps for a single community. Memoizing makes it one open per wrap. + * + * Wrap ids are unique and a wrap only ever belongs to one plane (it is routed by `pubKey`), so + * a single id-keyed map is safe across the current and prior-epoch Control Planes even though + * they open under different keys. A wrap that fails to open caches `null` so it is not retried + * on every subsequent fold. The wrap buffers are only ever added to, so this tracks their + * lifetime exactly and needs no separate eviction. + */ + private val editionByWrapId = HashMap() + // Prior-epoch Control Plane address -> (wrapId -> wrap). Kept apart from [controlWraps]: these // never join the live fold, they only produce the anti-rollback floor. private val historicalControlWraps = HashMap>() @@ -280,7 +305,7 @@ class ConcordCommunitySession( fun auxStreamKeys(): List = listOf(guestbookKey, nextBaseRekeyKey) /** The community's current Control Plane editions — the input a moderation edition chains onto. */ - fun controlEditions(): List = lock.withLock { ConcordActions.controlEditions(controlWraps.values.toList(), controlPlaneKey) } + fun controlEditions(): List = lock.withLock { editionsLocked(controlWraps.values.toList(), controlPlaneKey) } /** The raw Control Plane wraps buffered so far — the input a Refounding compacts (CORD-06 §3). */ fun controlPlaneWraps(): List = lock.withLock { controlWraps.values.toList() } @@ -314,7 +339,7 @@ class ConcordCommunitySession( if (controlWraps.put(wrap.id, wrap) != null) return ConcordIngestOutcome.NON_STRUCTURAL // dup } refold() - return ConcordIngestOutcome.STRUCTURAL + return ConcordIngestOutcome.STRUCTURAL_FOLD } guestbookAddress -> { lock.withLock { @@ -341,7 +366,7 @@ class ConcordCommunitySession( if (buffer.put(wrap.id, wrap) != null) return ConcordIngestOutcome.NON_STRUCTURAL // dup } refold() - return ConcordIngestOutcome.STRUCTURAL + return ConcordIngestOutcome.STRUCTURAL_FOLD } val current = lock.withLock { channelKeysByAddress[wrap.pubKey] } if (current != null) { @@ -422,7 +447,7 @@ class ConcordCommunitySession( val wraps = controlWraps.values.toList() val folded = ConcordCommunityState.fold( - ConcordActions.controlEditions(wraps, controlPlaneKey), + editionsLocked(wraps, controlPlaneKey), entry.owner, controlFloorsLocked(), ) @@ -455,6 +480,24 @@ class ConcordCommunitySession( for (channelIdHex in newChannels) reprojectChannel(channelIdHex) } + /** + * [wraps] opened into editions through [editionByWrapId], so a wrap is only ever decrypted + * once no matter how many folds it participates in. Caller must hold [lock]. + */ + private fun editionsLocked( + wraps: Collection, + planeKey: GroupKey, + ): List = + wraps.mapNotNull { wrap -> + if (editionByWrapId.containsKey(wrap.id)) { + editionByWrapId[wrap.id] + } else { + val edition = ConcordStreamEnvelope.openOrNull(wrap, planeKey)?.let { ControlEdition.fromRumor(it.rumor) } + editionByWrapId[wrap.id] = edition + edition + } + } + /** * The per-entity anti-rollback floor: the authority-gated heads of every prior epoch's * Control Plane we still hold a root for, folded **oldest epoch first** so each epoch is @@ -472,7 +515,7 @@ class ConcordCommunitySession( var floors = emptyMap() for ((address, keyAtEpoch) in historicalControlKeys.entries.sortedBy { it.value.second }) { val wraps = historicalControlWraps[address]?.values?.toList() ?: continue - val editions = ConcordActions.controlEditions(wraps, keyAtEpoch.first) + val editions = editionsLocked(wraps, keyAtEpoch.first) if (editions.isEmpty()) continue floors = ConcordCommunityState.authorizedHeads(editions, entry.owner, floors) } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionManager.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionManager.kt index bd2775eef2..0bb8faba0f 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionManager.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionManager.kt @@ -150,6 +150,9 @@ class ConcordSessionManager( seenOnRelays: Set = emptySet(), ): Boolean { val outcome = registry.ingest(wrap, seenOnRelays) + // Only the planes that change structure *without* republishing `state`. A control-plane + // fold returns STRUCTURAL_FOLD and is bumped by the per-session state watcher instead, which + // (since the folded state compares by value) fires only when the fold genuinely changed. if (outcome == ConcordIngestOutcome.STRUCTURAL) bumpRevision() return outcome.claimed } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt index 0c0afcfd84..7f7882a14d 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt @@ -108,7 +108,7 @@ class ConcordCommunitySessionTest { // Feed the genesis control wraps → state folds, channels + membership resolve. A fold is // STRUCTURAL (it moves the subscription set), so it's allowed to bump the revision. - community.genesisWraps.forEach { assertEquals(ConcordIngestOutcome.STRUCTURAL, session.ingest(it)) } + community.genesisWraps.forEach { assertEquals(ConcordIngestOutcome.STRUCTURAL_FOLD, session.ingest(it)) } val state = session.state.value assertEquals("Nostrichs", state?.metadata?.name) assertTrue(state!!.channels.containsKey(community.generalChannelIdHex)) diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt index e292d55451..0419520238 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt @@ -70,7 +70,7 @@ class ConcordSessionRegistryTest { assertTrue(registry.subscribeAddresses().contains(beta.controlPlane.publicKeyHex)) // A genesis control wrap routes to Alpha's session and folds it (STRUCTURAL). - alpha.genesisWraps.forEach { assertEquals(ConcordIngestOutcome.STRUCTURAL, registry.ingest(it)) } + alpha.genesisWraps.forEach { assertEquals(ConcordIngestOutcome.STRUCTURAL_FOLD, registry.ingest(it)) } val alphaState = registry.sessionFor(alpha.communityIdHex)!!.state.value assertEquals("Alpha", alphaState?.metadata?.name) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityState.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityState.kt index da4b6c817a..31c74caa7e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityState.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityState.kt @@ -33,7 +33,7 @@ import com.vitorpamplona.quartz.concord.cord04Roles.RoleEntity import com.vitorpamplona.quartz.concord.cord04Roles.asFloor /** A channel id paired with its current folded definition. */ -class ConcordChannel( +data class ConcordChannel( val channelIdHex: String, val definition: ChannelEntity, ) @@ -49,7 +49,7 @@ class ConcordChannel( * "Every member keeps the entire Control Plane in sync — it is small and must * stay complete." Recompute this whenever the known editions change. */ -class ConcordCommunityState( +data class ConcordCommunityState( val ownerPubKey: String, val metadata: MetadataEntity?, val channels: Map, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt index 8f67a24b5a..04fa0176c2 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt @@ -44,7 +44,7 @@ import com.vitorpamplona.quartz.nip01Core.core.toHexKey * assigned Role and holds [ConcordPermissions.MANAGE_ROLES]. Cycles that never * touch the owner can never bootstrap themselves. */ -class AuthorityResolver private constructor( +data class AuthorityResolver private constructor( private val ownerLower: String, private val roles: Map, private val memberRoles: Map>, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlEntities.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlEntities.kt index adacb499e6..71696ba8c9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlEntities.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlEntities.kt @@ -64,7 +64,7 @@ object ConcordJson { * drops the role, and with it every authority (grant) that depends on it. */ @Serializable -class RoleScope( +data class RoleScope( val kind: String = "server", @SerialName("channel_id") val channelId: String? = null, ) @@ -75,7 +75,7 @@ class RoleScope( * ranks higher; no role may claim position 0 (reserved for the owner). */ @Serializable -class RoleEntity( +data class RoleEntity( val name: String = "", val position: Long = 0, /** u64 permission bitfield as a decimal string. */ @@ -94,7 +94,7 @@ class RoleEntity( * terminates at the owner (see [AuthorityResolver]). */ @Serializable -class GrantEntity( +data class GrantEntity( val member: String = "", @SerialName("role_ids") val roleIds: List = emptyList(), ) @@ -105,7 +105,7 @@ class GrantEntity( * A [deleted] channel is terminal — its id is never reused. */ @Serializable -class ChannelEntity( +data class ChannelEntity( val name: String = "", val private: Boolean = false, val voice: Boolean = false, @@ -122,7 +122,7 @@ class ChannelEntity( * community name too. */ @Serializable -class MetadataEntity( +data class MetadataEntity( val name: String = "", val icon: ImagePointer? = null, val banner: ImagePointer? = null, From 1fee674350a1debb9c663f463dbb6ed7db5e53d3 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 21 Jul 2026 13:22:06 -0400 Subject: [PATCH 09/34] fix: keep event-less placeholder rooms when a deletion arrives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The additive feed path re-filters the existing list whenever an incoming batch contains a kind-5, dropping notes whose event has been deleted. Event-less notes fell into the else branch and returned false, so they were dropped too. An event-less row is a placeholder the filter synthesizes for a room with no message yet — a just-joined Concord channel, NIP-29 group, Marmot group or geohash cell. It carries no event, so it cannot have been deleted. Dropping it removed every such row from Messages the moment ANY unrelated deletion landed, and because this is the additive path the rows stayed gone until the next full rebuild. A community whose channels are all quiet looked like it had never loaded at all. Verified on device: surviving Concord placeholders in sort() went 0 -> 14, and a community that had been absent from Messages entirely now renders all of its channels. Not Concord-specific — the same placeholderNote() pattern backs NIP-29, Marmot and geohash rooms. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../amethyst/commons/ui/feeds/FeedContentState.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/ui/feeds/FeedContentState.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/ui/feeds/FeedContentState.kt index bc6eae3baf..9945c77fd0 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/ui/feeds/FeedContentState.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/ui/feeds/FeedContentState.kt @@ -161,7 +161,14 @@ class FeedContentState( if (noteEvent != null) { !cacheProvider.hasBeenDeleted(noteEvent) } else { - false + // An event-less row is a placeholder the filter synthesized for a room + // that has no message yet — a just-joined Concord channel, NIP-29 group, + // Marmot group or geohash cell. It carries no event, so it cannot have + // been deleted, and dropping it here deleted every such row from the + // Messages list the moment ANY kind-5 landed in an unrelated batch. The + // row then stayed gone until the next full rebuild, which is why a quiet + // community looked like it had never loaded at all. + true } }.toImmutableList() } From 9ddf571b1139d2ff9523f39beac9550b2bcde8d9 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 21 Jul 2026 14:09:59 -0400 Subject: [PATCH 10/34] fix: persist the Concord community list so cold boot doesn't refetch it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AccountSettings has held backupConcordList and saved it on change since the feature landed, but the field was never wired into LocalPreferences: it was neither written to nor read back from the encrypted prefs. So it only ever existed for the lifetime of the process, concordList() returned null on every cold boot, and the kind-13302 joined-communities list had to be refetched from relays before a single Concord plane could be subscribed. Every sibling list — channel, community, hashtag, geohash, ephemeral chat, relay group, trust provider — is persisted this way; Concord was the one that was missed. That made it the only chat type whose rooms could not appear until the network answered, which is the bulk of the cold-boot delay: the joined list gates the control-plane REQ, the control plane gates the fold, and the fold gates the channels. Measured on device, boot -> first Concord plane wrap: - without the backup: liveCommunities sat empty for ~56 s waiting on the 13302 fetch (first arrival from nostr.mom), first wrap at +45 s - with the backup restored: list decoded 1.6 s after boot (30 ms), first wrap at +6 s Wired the same five sites the other lists use (pref key, save, read, parse, restore). Prefs are encrypted and backupCashuWallet already sets the precedent for persisting a secret-bearing event, so the community roots in the 13302 content are stored no differently than the wallet's. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../java/com/vitorpamplona/amethyst/LocalPreferences.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/LocalPreferences.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/LocalPreferences.kt index 9603f78862..f489142661 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/LocalPreferences.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/LocalPreferences.kt @@ -38,6 +38,7 @@ import com.vitorpamplona.amethyst.model.UiSettings import com.vitorpamplona.amethyst.service.checkNotInMainThread import com.vitorpamplona.amethyst.ui.actions.mediaServers.DEFAULT_MEDIA_SERVERS import com.vitorpamplona.amethyst.ui.actions.mediaServers.ServerName +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent import com.vitorpamplona.quartz.experimental.ephemChat.list.EphemeralChatListEvent import com.vitorpamplona.quartz.experimental.nipA3.PaymentTargetsEvent import com.vitorpamplona.quartz.nip01Core.core.Event @@ -168,6 +169,7 @@ private object PrefKeys { const val LATEST_GEOHASH_LIST = "latestGeohashList" const val LATEST_EPHEMERAL_LIST = "latestEphemeralChatList" const val LATEST_RELAY_GROUP_LIST = "latestRelayGroupList" + const val LATEST_CONCORD_LIST = "latestConcordList" const val LATEST_TRUST_PROVIDER_LIST = "latestTrustProviderList" const val CALLS_ENABLED = "calls_enabled" const val HIDE_DELETE_REQUEST_DIALOG = "hide_delete_request_dialog" @@ -577,6 +579,7 @@ object LocalPreferences { putOrRemove(PrefKeys.LATEST_GEOHASH_LIST, settings.backupGeohashList) putOrRemove(PrefKeys.LATEST_EPHEMERAL_LIST, settings.backupEphemeralChatList) putOrRemove(PrefKeys.LATEST_RELAY_GROUP_LIST, settings.backupRelayGroupList) + putOrRemove(PrefKeys.LATEST_CONCORD_LIST, settings.backupConcordList) putOrRemove(PrefKeys.LATEST_TRUST_PROVIDER_LIST, settings.backupTrustProviderList) putOrRemove(PrefKeys.LATEST_PAYMENT_TARGETS, settings.backupNipA3PaymentTargets) putOrRemove(PrefKeys.LATEST_CASHU_WALLET, settings.backupCashuWallet) @@ -763,6 +766,7 @@ object LocalPreferences { val latestGeohashListStr = getString(PrefKeys.LATEST_GEOHASH_LIST, null) val latestEphemeralListStr = getString(PrefKeys.LATEST_EPHEMERAL_LIST, null) val latestRelayGroupListStr = getString(PrefKeys.LATEST_RELAY_GROUP_LIST, null) + val latestConcordListStr = getString(PrefKeys.LATEST_CONCORD_LIST, null) val latestTrustProviderListStr = getString(PrefKeys.LATEST_TRUST_PROVIDER_LIST, null) val latestPaymentTargetsStr = getString(PrefKeys.LATEST_PAYMENT_TARGETS, null) val latestCashuWalletStr = getString(PrefKeys.LATEST_CASHU_WALLET, null) @@ -823,6 +827,7 @@ object LocalPreferences { val latestGeohashList = async { parseEventOrNull(latestGeohashListStr) } val latestEphemeralList = async { parseEventOrNull(latestEphemeralListStr) } val latestRelayGroupList = async { parseEventOrNull(latestRelayGroupListStr) } + val latestConcordList = async { parseEventOrNull(latestConcordListStr) } val latestTrustProviderList = async { parseEventOrNull(latestTrustProviderListStr) } val latestPaymentTargets = async { parseEventOrNull(latestPaymentTargetsStr) } val latestCashuWallet = @@ -875,6 +880,7 @@ object LocalPreferences { val latestGeohashListResolved = latestGeohashList.await() val latestEphemeralListResolved = latestEphemeralList.await() val latestRelayGroupListResolved = latestRelayGroupList.await() + val latestConcordListResolved = latestConcordList.await() val latestTrustProviderListResolved = latestTrustProviderList.await() val latestPaymentTargetsResolved = latestPaymentTargets.await() val latestCashuWalletResolved = latestCashuWallet.await() @@ -969,6 +975,7 @@ object LocalPreferences { backupGeohashList = latestGeohashListResolved, backupEphemeralChatList = latestEphemeralListResolved, backupRelayGroupList = latestRelayGroupListResolved, + backupConcordList = latestConcordListResolved, backupTrustProviderList = latestTrustProviderListResolved, lastReadPerRoute = MutableStateFlow(lastReadPerRouteResolved), hasDonatedInVersion = MutableStateFlow(hasDonatedInVersion), From 0a228a935b29ec4a1b0a1b66c693c5d5dd208d1f Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 21 Jul 2026 14:23:38 -0400 Subject: [PATCH 11/34] fix: persist the MIP-00 key-package and favorite-algo-feeds lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AccountSettings declares 25 backup* fields and saves each on change, but two were never wired into LocalPreferences — neither written to nor read back from the encrypted prefs: - backupKeyPackageRelayList (MIP-00, Marmot/MLS) - backupFavoriteAlgoFeedsList (kind 10090) Both only ever existed for the lifetime of the process. Their consumers already implement the restore-from-backup path — KeyPackageRelayListState's normalizeKeyPackageRelayListWithBackup falls back to the field, and FavoriteAlgoFeedsListState's init seeds the cache from it — so that code was dead after every cold boot and the value read as empty until relays answered. For the key-package list that matters beyond latency: it feeds Account.publishRelaysFor(), which decides where this account's key packages are published so others can add it to groups, and Account.updateKeyPackageRelays() reads it as the *previous* list when computing an update. Found by auditing all 25 backup* fields against their five LocalPreferences wiring sites after the same gap turned up for the Concord community list; the other 23, NIP-29's relay-group list included, are correctly wired. Verified on device with a persist-then-cold-boot pair: the key-package list is absent on the first boot and restores 1.6 s after the second. The favorite algo feeds list could not be exercised on this account (it has no kind 10090, so there is nothing to persist); it is wired identically and its type arguments are compiler-checked, but it is not verified end to end. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../com/vitorpamplona/amethyst/LocalPreferences.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/LocalPreferences.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/LocalPreferences.kt index 9603f78862..6e0ae12b6e 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/LocalPreferences.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/LocalPreferences.kt @@ -40,6 +40,7 @@ import com.vitorpamplona.amethyst.ui.actions.mediaServers.DEFAULT_MEDIA_SERVERS import com.vitorpamplona.amethyst.ui.actions.mediaServers.ServerName import com.vitorpamplona.quartz.experimental.ephemChat.list.EphemeralChatListEvent import com.vitorpamplona.quartz.experimental.nipA3.PaymentTargetsEvent +import com.vitorpamplona.quartz.marmot.mip00KeyPackages.KeyPackageRelayListEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.JsonMapper @@ -55,6 +56,7 @@ import com.vitorpamplona.quartz.nip28PublicChat.list.ChannelListEvent import com.vitorpamplona.quartz.nip37Drafts.privateOutbox.PrivateOutboxRelayListEvent import com.vitorpamplona.quartz.nip47WalletConnect.Nip47WalletConnect import com.vitorpamplona.quartz.nip50Search.SearchRelayListEvent +import com.vitorpamplona.quartz.nip51Lists.favoriteAlgoFeedsList.FavoriteAlgoFeedsListEvent import com.vitorpamplona.quartz.nip51Lists.geohashList.GeohashListEvent import com.vitorpamplona.quartz.nip51Lists.hashtagList.HashtagListEvent import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent @@ -169,6 +171,8 @@ private object PrefKeys { const val LATEST_EPHEMERAL_LIST = "latestEphemeralChatList" const val LATEST_RELAY_GROUP_LIST = "latestRelayGroupList" const val LATEST_TRUST_PROVIDER_LIST = "latestTrustProviderList" + const val LATEST_KEY_PACKAGE_RELAY_LIST = "latestKeyPackageRelayList" + const val LATEST_FAVORITE_ALGO_FEEDS_LIST = "latestFavoriteAlgoFeedsList" const val CALLS_ENABLED = "calls_enabled" const val HIDE_DELETE_REQUEST_DIALOG = "hide_delete_request_dialog" const val HIDE_BLOCK_ALERT_DIALOG = "hide_block_alert_dialog" @@ -578,6 +582,8 @@ object LocalPreferences { putOrRemove(PrefKeys.LATEST_EPHEMERAL_LIST, settings.backupEphemeralChatList) putOrRemove(PrefKeys.LATEST_RELAY_GROUP_LIST, settings.backupRelayGroupList) putOrRemove(PrefKeys.LATEST_TRUST_PROVIDER_LIST, settings.backupTrustProviderList) + putOrRemove(PrefKeys.LATEST_KEY_PACKAGE_RELAY_LIST, settings.backupKeyPackageRelayList) + putOrRemove(PrefKeys.LATEST_FAVORITE_ALGO_FEEDS_LIST, settings.backupFavoriteAlgoFeedsList) putOrRemove(PrefKeys.LATEST_PAYMENT_TARGETS, settings.backupNipA3PaymentTargets) putOrRemove(PrefKeys.LATEST_CASHU_WALLET, settings.backupCashuWallet) putOrRemove(PrefKeys.LATEST_NUTZAP_INFO, settings.backupNutzapInfo) @@ -764,6 +770,8 @@ object LocalPreferences { val latestEphemeralListStr = getString(PrefKeys.LATEST_EPHEMERAL_LIST, null) val latestRelayGroupListStr = getString(PrefKeys.LATEST_RELAY_GROUP_LIST, null) val latestTrustProviderListStr = getString(PrefKeys.LATEST_TRUST_PROVIDER_LIST, null) + val latestKeyPackageRelayListStr = getString(PrefKeys.LATEST_KEY_PACKAGE_RELAY_LIST, null) + val latestFavoriteAlgoFeedsListStr = getString(PrefKeys.LATEST_FAVORITE_ALGO_FEEDS_LIST, null) val latestPaymentTargetsStr = getString(PrefKeys.LATEST_PAYMENT_TARGETS, null) val latestCashuWalletStr = getString(PrefKeys.LATEST_CASHU_WALLET, null) val latestNutzapInfoStr = getString(PrefKeys.LATEST_NUTZAP_INFO, null) @@ -824,6 +832,8 @@ object LocalPreferences { val latestEphemeralList = async { parseEventOrNull(latestEphemeralListStr) } val latestRelayGroupList = async { parseEventOrNull(latestRelayGroupListStr) } val latestTrustProviderList = async { parseEventOrNull(latestTrustProviderListStr) } + val latestKeyPackageRelayList = async { parseEventOrNull(latestKeyPackageRelayListStr) } + val latestFavoriteAlgoFeedsList = async { parseEventOrNull(latestFavoriteAlgoFeedsListStr) } val latestPaymentTargets = async { parseEventOrNull(latestPaymentTargetsStr) } val latestCashuWallet = async { @@ -876,6 +886,8 @@ object LocalPreferences { val latestEphemeralListResolved = latestEphemeralList.await() val latestRelayGroupListResolved = latestRelayGroupList.await() val latestTrustProviderListResolved = latestTrustProviderList.await() + val latestKeyPackageRelayListResolved = latestKeyPackageRelayList.await() + val latestFavoriteAlgoFeedsListResolved = latestFavoriteAlgoFeedsList.await() val latestPaymentTargetsResolved = latestPaymentTargets.await() val latestCashuWalletResolved = latestCashuWallet.await() val latestNutzapInfoResolved = latestNutzapInfo.await() @@ -970,6 +982,8 @@ object LocalPreferences { backupEphemeralChatList = latestEphemeralListResolved, backupRelayGroupList = latestRelayGroupListResolved, backupTrustProviderList = latestTrustProviderListResolved, + backupKeyPackageRelayList = latestKeyPackageRelayListResolved, + backupFavoriteAlgoFeedsList = latestFavoriteAlgoFeedsListResolved, lastReadPerRoute = MutableStateFlow(lastReadPerRouteResolved), hasDonatedInVersion = MutableStateFlow(hasDonatedInVersion), dismissedPollNoteIds = MutableStateFlow(dismissedPollNoteIds), From 387bfe99ee109bb920032b274c9211344868535b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 19:23:21 +0000 Subject: [PATCH 12/34] perf(relay): drop per-op allocations on the frame + search paths Three hot-path allocation cuts the SmallReqFloorBenchmark stages flagged: - strippingSearchExtensions: index-loop guard returns the same list with zero allocation when no filter carries a search term (every non-search REQ/COUNT/snapshot, the overwhelming majority). - EoseMessage/OkMessage: direct-buildString wire form on the escape-free fast path (EOSE per REQ, OK per publish), skipping the generic serializer's node tree; exotic subIds/reasons fall back. Shared isEscapeFreeAscii helper in WireJson.kt, mirroring NegMsgMessage. Verified: quartz relay.server + message-frame suites (110 tests). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01RzkoN3SJHCZWRAiadXXG4w --- .../relay/commands/toClient/EoseMessage.kt | 16 +++++++++ .../relay/commands/toClient/OkMessage.kt | 19 ++++++++++ .../relay/commands/toClient/WireJson.kt | 36 +++++++++++++++++++ .../quartz/nip50Search/SearchQuery.kt | 12 +++++++ 4 files changed, 83 insertions(+) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/WireJson.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/EoseMessage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/EoseMessage.kt index d6912571b0..f376e4bc82 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/EoseMessage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/EoseMessage.kt @@ -25,6 +25,22 @@ class EoseMessage( ) : Message { override fun label() = LABEL + /** + * Wire form is `["EOSE",""]` — sent once per REQ, so it is on + * the per-subscription floor. Splice it directly when [subId] needs no + * escaping (the common case: client-chosen sub ids are short ASCII), + * skipping the generic serializer's node tree. Byte-identical output; + * any exotic subId falls back. + */ + override fun toJson(): String { + if (!isEscapeFreeAscii(subId)) return super.toJson() + return buildString(subId.length + 12) { + append("[\"EOSE\",\"") + append(subId) + append("\"]") + } + } + companion object { const val LABEL = "EOSE" } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/OkMessage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/OkMessage.kt index 8a29794a44..47df9bedb0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/OkMessage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/OkMessage.kt @@ -29,6 +29,25 @@ class OkMessage( ) : Message { override fun label() = LABEL + /** + * Wire form is `["OK","",,""]` — sent + * once per published EVENT. [eventId] is validated hex (always + * escape-free); splice directly when [message] also needs no escaping, + * which covers the empty-string success ack and the plain-ASCII + * rejection reasons. Byte-identical output; a reason with quotes or + * non-ASCII falls back to the generic serializer. + */ + override fun toJson(): String { + if (!isEscapeFreeAscii(message)) return super.toJson() + return buildString(eventId.length + message.length + 20) { + append("[\"OK\",\"") + append(eventId) + append(if (success) "\",true,\"" else "\",false,\"") + append(message) + append("\"]") + } + } + companion object { const val LABEL = "OK" diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/WireJson.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/WireJson.kt new file mode 100644 index 0000000000..63e1a47e5e --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/WireJson.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.commands.toClient + +/** + * True when every char of [s] is printable ASCII (0x20–0x7e) and not a JSON + * metacharacter (`"` / `\`) — i.e. exactly the bytes a JSON string encoder + * would emit verbatim between the quotes. Frame builders use this to gate a + * direct-`buildString` fast path against the generic serializer: when it holds + * the spliced output is byte-identical, and any exotic value (control chars, + * quotes, non-ASCII) falls back to the escaping serializer. + */ +internal fun isEscapeFreeAscii(s: String): Boolean { + for (c in s) { + if (c < ' ' || c > '~' || c == '"' || c == '\\') return false + } + return true +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt index 12b3af995d..f96b620d5c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt @@ -202,8 +202,20 @@ fun Filter.strippingSearchExtensions(): Filter { /** * Applies [strippingSearchExtensions] to every filter, returning this * same list when no filter carried extension tokens. + * + * This runs on every REQ/COUNT/snapshot, and the overwhelming majority + * carry no `search` term at all, so the no-search case must not allocate: + * bail before building any list when nothing could be stripped. */ fun List.strippingSearchExtensions(): List { + var hasSearch = false + for (i in indices) { + if (!this[i].search.isNullOrEmpty()) { + hasSearch = true + break + } + } + if (!hasSearch) return this var changed = false val out = map { From 078758888a5829c09981a102e1b372114a5f3a8e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 19:23:38 +0000 Subject: [PATCH 13/34] perf(relay): remove live-path allocations in LiveEventStore + FilterIndex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remaining SmallReqFloorBenchmark waste, on the per-row replay, the per-event live fanout, and the per-accepted-event index probe: - LiveEventStore replay dedupe: a SeenIds holder with an inline lock replaces the local-fn-plus-lambda that allocated one closure per streamed row (and again per live delivery). Its HashSet is created empty so the JVM defers the backing table to the first add — a 0-row replay no longer allocates a 1024-slot table (was ~4 MB across the benchmark's 1000 idle subs). - Live fanout serializes the event body once and passes it through onEachLive(event, body); RelaySession splices it into the per-sub frame prefix. An event matching N live subscriptions paid N identical Jackson passes before; now one. queryRaw's onEachLive signature gains the body arg (EventSourceBackend default serializes inline, no cross-sub memo, no regression). Measured: fanout 1->200 live subs 0.50 ms (2.5 us/sub). - FilterIndex holds subscribers in one persistent map per dimension, so candidatesFor (once per accepted ingest event) probes with the event's own fields and allocates no IdKey/AuthorKey/KindKey/TagKey wrappers; BucketKey now lives only in the rare register/unregister bookkeeping. SmallReqFloorBenchmark grows a fanout stage (200 live subs, one submit) to anchor the fanout number; it drives `live` directly and guards the await with withTimeout so a future fanout regression fails fast. Verified: quartz relay.server + FilterIndex suites (110 tests), SmallReqFloorBenchmark, geode suite (126 tests). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01RzkoN3SJHCZWRAiadXXG4w --- .../nip01Core/relay/filters/FilterIndex.kt | 137 +++++++++++++----- .../nip01Core/relay/server/RelaySession.kt | 15 +- .../relay/server/backend/LiveEventStore.kt | 136 +++++++++-------- .../relay/server/backend/SessionBackend.kt | 4 +- .../relay/prodbench/SmallReqFloorBenchmark.kt | 57 +++++++- 5 files changed, 247 insertions(+), 102 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt index a5c058ec6d..bee08fc159 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt @@ -115,19 +115,25 @@ class FilterIndex { private object Unindexed : BucketKey /** - * Single immutable snapshot. [buckets] maps a key to the set of - * subscribers registered under it; [assignments] is the reverse - * map used by [unregister] to find a subscriber's keys without - * scanning every bucket. + * Single immutable snapshot. Subscribers are held in one map per + * indexable dimension so [candidatesFor] — called once per accepted + * ingest event, the hot read — can probe each dimension with the + * event's own field (`event.id`, `event.pubKey`, `tag[0]`/`tag[1]`, + * `event.kind`) and allocate no key-wrapper objects. [assignments] is + * the reverse map ([S] → the [BucketKey]s it occupies) used by + * [unregister]; the wrappers live only here, built on the rare + * register path. * * Persistent (HAMT) maps/sets: a register/unregister produces the * next snapshot in O(keys × log S) with structural sharing, instead - * of copying both full maps — registration happens on every REQ - * open/close, so with S live subscriptions the full copy was - * O(S) work and O(S) allocation per REQ. + * of copying full maps — registration happens on every REQ open/close. */ private data class State( - val buckets: PersistentMap> = persistentHashMapOf(), + val ids: PersistentMap> = persistentHashMapOf(), + val authors: PersistentMap> = persistentHashMapOf(), + val tags: PersistentMap>> = persistentHashMapOf(), + val kinds: PersistentMap> = persistentHashMapOf(), + val unindexed: PersistentSet = persistentHashSetOf(), val assignments: PersistentMap> = persistentHashMapOf(), ) @@ -193,19 +199,23 @@ class FilterIndex { while (true) { val current = state.load() val keys = current.assignments[subscriber] ?: return - var newBuckets = current.buckets + var ids = current.ids + var authors = current.authors + var tags = current.tags + var kinds = current.kinds + var unindexed = current.unindexed for (key in keys) { - val cur = newBuckets[key] ?: continue - val next = cur.remove(subscriber) - newBuckets = - if (next.isEmpty()) { - newBuckets.remove(key) - } else { - newBuckets.put(key, next) - } + when (key) { + is IdKey -> ids = ids.removeSub(key.id, subscriber) + is AuthorKey -> authors = authors.removeSub(key.author, subscriber) + is KindKey -> kinds = kinds.removeSub(key.kind, subscriber) + is TagKey -> tags = tags.removeTagSub(key.letter, key.value, subscriber) + Unindexed -> unindexed = unindexed.remove(subscriber) + } } - val newAssignments = current.assignments.remove(subscriber) - if (state.compareAndSet(current, State(newBuckets, newAssignments))) return + val next = + State(ids, authors, tags, kinds, unindexed, current.assignments.remove(subscriber)) + if (state.compareAndSet(current, next)) return } } @@ -215,19 +225,22 @@ class FilterIndex { * candidate to handle negative constraints. * * Iteration order is insertion-stable per call but otherwise - * unspecified. + * unspecified. Allocates only the result set — dimensions are + * probed with the event's own fields, no key wrappers. */ fun candidatesFor(event: Event): Set { val s = state.load() - if (s.buckets.isEmpty()) return emptySet() + if (s.assignments.isEmpty()) return emptySet() val result = LinkedHashSet() - s.buckets[Unindexed]?.let { result.addAll(it) } - s.buckets[IdKey(event.id)]?.let { result.addAll(it) } - s.buckets[AuthorKey(event.pubKey)]?.let { result.addAll(it) } - s.buckets[KindKey(event.kind)]?.let { result.addAll(it) } - for (tag in event.tags) { - if (tag.size >= 2 && tag[0].length == 1) { - s.buckets[TagKey(tag[0], tag[1])]?.let { result.addAll(it) } + if (s.unindexed.isNotEmpty()) result.addAll(s.unindexed) + s.ids[event.id]?.let { result.addAll(it) } + s.authors[event.pubKey]?.let { result.addAll(it) } + s.kinds[event.kind]?.let { result.addAll(it) } + if (s.tags.isNotEmpty()) { + for (tag in event.tags) { + if (tag.size >= 2 && tag[0].length == 1) { + s.tags[tag[0]]?.get(tag[1])?.let { result.addAll(it) } + } } } return result @@ -251,16 +264,72 @@ class FilterIndex { val keySet = keys.toPersistentHashSet() while (true) { val current = state.load() - var newBuckets = current.buckets + var ids = current.ids + var authors = current.authors + var tags = current.tags + var kinds = current.kinds + var unindexed = current.unindexed for (key in keySet) { - val cur = newBuckets[key] ?: persistentHashSetOf() - val next = cur.add(subscriber) - if (next !== cur) newBuckets = newBuckets.put(key, next) + when (key) { + is IdKey -> ids = ids.addSub(key.id, subscriber) + is AuthorKey -> authors = authors.addSub(key.author, subscriber) + is KindKey -> kinds = kinds.addSub(key.kind, subscriber) + is TagKey -> tags = tags.addTagSub(key.letter, key.value, subscriber) + Unindexed -> unindexed = unindexed.add(subscriber) + } } val existing = current.assignments[subscriber] val merged = existing?.addAll(keySet) ?: keySet - val newAssignments = current.assignments.put(subscriber, merged) - if (state.compareAndSet(current, State(newBuckets, newAssignments))) return + val next = State(ids, authors, tags, kinds, unindexed, current.assignments.put(subscriber, merged)) + if (state.compareAndSet(current, next)) return + } + } + + // Per-dimension add/remove of one subscriber, returning the same map + // instance when nothing changed so the CAS builds minimal new nodes. + private fun PersistentMap>.addSub( + key: K, + sub: S, + ): PersistentMap> { + val cur = this[key] ?: persistentHashSetOf() + val next = cur.add(sub) + return if (next === cur) this else put(key, next) + } + + private fun PersistentMap>.removeSub( + key: K, + sub: S, + ): PersistentMap> { + val cur = this[key] ?: return this + val next = cur.remove(sub) + return when { + next === cur -> this + next.isEmpty() -> remove(key) + else -> put(key, next) + } + } + + private fun PersistentMap>>.addTagSub( + letter: String, + value: String, + sub: S, + ): PersistentMap>> { + val inner = this[letter] ?: persistentHashMapOf() + val newInner = inner.addSub(value, sub) + return if (newInner === inner) this else put(letter, newInner) + } + + private fun PersistentMap>>.removeTagSub( + letter: String, + value: String, + sub: S, + ): PersistentMap>> { + val inner = this[letter] ?: return this + val newInner = inner.removeSub(value, sub) + return when { + newInner === inner -> this + newInner.isEmpty() -> remove(letter) + else -> put(letter, newInner) } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt index f73493f19c..7aa1e208d6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt @@ -340,7 +340,20 @@ class RelaySession( }, ) }, - onEachLive = { event -> send(EventMessage(cmd.subId, event)) }, + // Live events arrive with their wire body already + // serialized (once per event, shared across every + // matching subscription): splice it into the same + // per-sub frame prefix as the stored replay, no + // per-event EventMessage or re-serialize. + onEachLive = { _, body -> + sendRaw( + buildString(framePrefix.length + body.length + 1) { + append(framePrefix) + append(body) + append(']') + }, + ) + }, onEose = { send(EoseMessage(cmd.subId)) }, ) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt index 1d35362134..9f5f8c4ecc 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt @@ -74,13 +74,57 @@ class LiveEventStore( * One live REQ subscription. Carries the filters (for the * post-index `match` re-check needed for negative constraints * like `since` / `until` / `tagsAll`) and the delivery callback - * the index dispatches into. Identity-keyed inside [FilterIndex]. + * the index dispatches into. [deliver] receives the event and its + * pre-serialized wire body (memoized once per fanout across all + * matching subscribers). Identity-keyed inside [FilterIndex]. */ private class LiveSubscription( val filters: List, - val deliver: (Event) -> Unit, + val deliver: (Event, String) -> Unit, ) + /** + * Replay-dedupe set for one REQ. During the historical replay the + * store's ids are [record]ed here so the concurrent live path can + * drop an event the replay also emitted; after EOSE the set is + * [release]d and the live path forwards everything. + * + * Written from the replay coroutine and read from the [IngestQueue] + * drain coroutine (via `fanout`), so every access takes a tiny spin + * lock — [locked] is `inline`, so the per-row `record` / `isDuplicate` + * calls allocate no closure. The backing `HashSet` is created empty + * up front (so the register-before-replay race guarantee holds) but + * the JVM defers its table allocation to the first `add`, so a + * zero-row replay costs only the empty set object, not a sized table. + * It MUST stay a mutable set under a lock, never a copy-on-add + * immutable set — `set + id` per row made large replays O(n²). + */ + private class SeenIds { + private val lock = AtomicBoolean(false) + private var ids: HashSet? = HashSet() + + private inline fun locked(block: () -> R): R { + while (lock.exchange(true)) { + while (lock.load()) { } + } + try { + return block() + } finally { + lock.store(false) + } + } + + fun record(id: String) { + locked { ids?.add(id) } + } + + fun isDuplicate(id: String): Boolean = locked { ids?.contains(id) ?: false } + + fun release() { + locked { ids = null } + } + } + /** * Fire-and-forget enqueue: hand [event] to the [IngestQueue] and * fire [onComplete] once the writer's batch has a per-row @@ -149,9 +193,17 @@ class LiveEventStore( * batch writer. */ private fun fanout(event: Event) { - for (sub in index.candidatesFor(event)) { + val candidates = index.candidatesFor(event) + if (candidates.isEmpty()) return + // Serialize the wire body at most once for this event, no matter + // how many subscriptions match it — the old path re-serialized the + // whole event per matching subscriber, so a note landing in N live + // feeds paid N identical Jackson passes. Lazy so a fanout that + // matches nothing (index over-approximates) serializes nothing. + var body: String? = null + for (sub in candidates) { if (sub.filters.any { it.match(event) }) { - sub.deliver(event) + sub.deliver(event, body ?: event.toJson().also { body = it }) } } } @@ -178,61 +230,33 @@ class LiveEventStore( onEose: () -> Unit, ) { drainFtsIfSearching(filters) - // During the historical replay, record ids the store has - // emitted so the live path can dedupe. The index registers - // *before* the replay starts (otherwise an event accepted - // mid-replay would slip past the live path entirely — same - // race the previous SharedFlow-based implementation closed - // with `onSubscription`). - // - // The set is read from the [IngestQueue] drain coroutine (in - // `deliver`, called synchronously from `fanout`) and written - // from this coroutine (the historical-replay closure below), - // so access is guarded by a tiny spin lock (contains/add, - // never I/O). It MUST be a mutable set under a lock, not an - // immutable Set under an AtomicReference with copy-on-add: - // `set + id` copies the whole set per streamed event, which - // made large replays accidentally O(n²) — a 100k-event REQ - // crawled at ~700 events/s and the rate degraded as the - // response grew (see the plan doc's giant-REQ finding). - // - // Once cleared to null after EOSE, `deliver` short-circuits - // and every live event is forwarded. - val seenLock = AtomicBoolean(false) - var seenIds: HashSet? = HashSet(1024) - - fun seenLocked(block: () -> R): R { - while (seenLock.exchange(true)) { - while (seenLock.load()) { } - } - try { - return block() - } finally { - seenLock.store(false) - } - } + // The index registers *before* the replay starts (otherwise an + // event accepted mid-replay would slip past the live path entirely + // — same race the previous SharedFlow-based implementation closed + // with `onSubscription`), and [SeenIds] bridges the two coroutines: + // the replay records ids here, the live `deliver` drops duplicates, + // and after EOSE the set is released so every live event forwards. + val seen = SeenIds() val sub = LiveSubscription( filters = filters, - deliver = { event -> - val duplicate = seenLocked { seenIds?.contains(event.id) ?: false } - if (duplicate) return@LiveSubscription - onEach(event) + deliver = { event, _ -> + if (!seen.isDuplicate(event.id)) onEach(event) }, ) index.register(filters, sub) try { store.query(filters.strippingSearchExtensions()) { event -> - seenLocked { seenIds?.add(event.id) } + seen.record(event.id) onEach(event) } onEose() // Drop the dedupe set so the live path stops paying for // it. From this point the index drives delivery and // duplicates are no longer possible. - seenLocked { seenIds = null } + seen.release() // Suspend until the caller's coroutine is cancelled // (e.g. NIP-01 CLOSE or connection drop). The `finally` // unregisters from the index. @@ -255,42 +279,28 @@ class LiveEventStore( ctx: RequestContext, filters: List, onEachStored: (RawEvent) -> Unit, - onEachLive: (Event) -> Unit, + onEachLive: (Event, String) -> Unit, onEose: () -> Unit, ) { drainFtsIfSearching(filters) - val seenLock = AtomicBoolean(false) - var seenIds: HashSet? = HashSet(1024) - - fun seenLocked(block: () -> R): R { - while (seenLock.exchange(true)) { - while (seenLock.load()) { } - } - try { - return block() - } finally { - seenLock.store(false) - } - } + val seen = SeenIds() val sub = LiveSubscription( filters = filters, - deliver = { event -> - val duplicate = seenLocked { seenIds?.contains(event.id) ?: false } - if (duplicate) return@LiveSubscription - onEachLive(event) + deliver = { event, body -> + if (!seen.isDuplicate(event.id)) onEachLive(event, body) }, ) index.register(filters, sub) try { store.rawQuery(filters.strippingSearchExtensions()) { raw -> - seenLocked { seenIds?.add(raw.id) } + seen.record(raw.id) onEachStored(raw) } onEose() - seenLocked { seenIds = null } + seen.release() awaitCancellation() } finally { index.unregister(sub) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/SessionBackend.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/SessionBackend.kt index 1119d9e9be..f379364cc4 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/SessionBackend.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/SessionBackend.kt @@ -78,9 +78,9 @@ interface SessionBackend { ctx: RequestContext, filters: List, onEachStored: (RawEvent) -> Unit, - onEachLive: (Event) -> Unit, + onEachLive: (Event, String) -> Unit, onEose: () -> Unit, - ): Unit = query(ctx, filters, onEachLive, onEose) + ): Unit = query(ctx, filters, { onEachLive(it, it.toJson()) }, onEose) /** Answers a NIP-45 COUNT with an exact cardinality for the caller in [ctx]. */ suspend fun count( diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SmallReqFloorBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SmallReqFloorBenchmark.kt index cd6eb4525b..4c2c839965 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SmallReqFloorBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SmallReqFloorBenchmark.kt @@ -38,6 +38,8 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import java.util.concurrent.atomic.AtomicInteger import kotlin.test.Test import kotlin.test.assertEquals @@ -67,6 +69,7 @@ class SmallReqFloorBenchmark { const val ROUNDS = 400 const val WARMUP = 100 const val IDLE_SUBS = 1_000 + const val FANOUT_SUBS = 200 } private fun hexId(seed: Int): String = seed.toString(16).padStart(64, '0') @@ -136,7 +139,7 @@ class SmallReqFloorBenchmark { ctx = ctx, filters = listOf(filterFor(round)), onEachStored = {}, - onEachLive = {}, + onEachLive = { _, _ -> }, onEose = { eose.complete(System.nanoTime() - t0) }, ) } @@ -163,7 +166,7 @@ class SmallReqFloorBenchmark { ctx = ctx, filters = listOf(Filter(authors = listOf(hexId(1_000_000 + i)), kinds = listOf(1), limit = 1)), onEachStored = {}, - onEachLive = {}, + onEachLive = { _, _ -> }, onEose = { ready.complete(Unit) }, ) } @@ -192,6 +195,54 @@ class SmallReqFloorBenchmark { val c = LongArray(ROUNDS) repeat(ROUNDS) { c[it] = timeSession(it) } + // --- fanout: one live event → FANOUT_SUBS live subscriptions --- + // All subs register on `live` directly (via queryRaw, same backend + // we submit into) and filter an author with no stored events (0-row + // replay, then park live). Submitting one matching event fans out to + // every sub; the body is serialized once and spliced per sub, so + // this measures the shared-serialization path (#2). skipVerify so + // the synthetic sig is accepted. Fewer rounds than A–C: each round + // is FANOUT_SUBS deliveries and a real group-commit insert. + val fanAuthor = hexId(9_000_001) + val delivered = AtomicInteger(0) + var fanDone = CompletableDeferred() + var fanStart = 0L + val fanJobs = + (0 until FANOUT_SUBS).map { + val ready = CompletableDeferred() + val job = + scope.launch(start = CoroutineStart.UNDISPATCHED) { + live.queryRaw( + ctx = ctx, + filters = listOf(Filter(authors = listOf(fanAuthor), kinds = listOf(1))), + onEachStored = {}, + onEachLive = { _, _ -> + if (delivered.incrementAndGet() == FANOUT_SUBS) { + fanDone.complete(System.nanoTime() - fanStart) + } + }, + onEose = { ready.complete(Unit) }, + ) + } + ready.await() + job + } + val fanRounds = 60 + val fanWarmup = 15 + val fan = LongArray(fanRounds) + var fanSeq = 0 + repeat(fanWarmup + fanRounds) { r -> + delivered.set(0) + fanDone = CompletableDeferred() + val ev = EventFactory.create(hexId(9_500_000 + fanSeq), fanAuthor, 1_700_000_000L + fanSeq, 1, emptyArray(), "fanout $fanSeq", sig) + fanSeq++ + fanStart = System.nanoTime() + live.submit(ev, skipVerify = true) {} + val nanos = withTimeout(30_000) { fanDone.await() } + if (r >= fanWarmup) fan[r - fanWarmup] = nanos + } + fanJobs.forEach { it.cancel() } + assertEquals(true, rowsA > 0, "author filters must return rows") val mA = median(a) @@ -203,6 +254,8 @@ class SmallReqFloorBenchmark { println(" B backend queryRaw→EOSE: ${"%6.3f".format(mB)} ms (live machinery +${"%6.3f".format(mB - mA)})") println(" B@${IDLE_SUBS} idle subs: ${"%6.3f".format(mB1k)} ms (population cost +${"%6.3f".format(mB1k - mB)})") println(" C session REQ→EOSE: ${"%6.3f".format(mC)} ms (dispatch+frames +${"%6.3f".format(mC - mB)})") + val mFan = median(fan) + println(" fanout 1→$FANOUT_SUBS live subs: ${"%6.3f".format(mFan)} ms (${"%.2f".format(mFan * 1000 / FANOUT_SUBS)} µs/sub; body serialized once)") server.close() scope.cancel() From 6730100853fe8320578a8c5b7ebebb3345a6865d Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 21 Jul 2026 17:27:39 -0400 Subject: [PATCH 14/34] docs: fix license badge and refresh stale SKILL.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README license badge label read "Apache-2.0" while LICENSE, PRIVACY.md and every source header are MIT. Only the static label text was wrong (the shields.io endpoint auto-detects), but it is the license on the front page. SKILL.md had drifted from the codebase since the Kotlin DSL migration: - All Gradle references pointed at Groovy `build.gradle` / `settings.gradle`; the repo is `.gradle.kts` throughout. Converted the snippets to Kotlin DSL and matched the repo's existing `getByName("release")` style. - The plugins block listed `jetbrainsKotlinAndroid` (gone) and omitted `serialization` and `googleKsp`. - compileSdk is 37, not 35. Added a pointer to libs.versions.toml so the number has a source of truth rather than drifting again. - The client-tag section told readers to create `nip01Core/tags/clientTag/TagArrayBuilderExt.kt` and edit both `build()` functions in TextNoteEvent. That file already exists at `nip89AppHandlers/clientTag/`, and the tag is now applied centrally by the NostrSignerWithClientTag decorator — so rebranding is a one-constant edit to CLIENT_TAG_NAME. - Default relays pointed at `quartz/src/main/java/...`, a path that does not exist in the KMP layout; they live in commons `defaults/`. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 +- SKILL.md | 86 +++++++++++++++++++++++++++---------------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index f00faeabf3..e1ef3d88a0 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Join the social network you control. [![Maven Central](https://img.shields.io/maven-central/v/com.vitorpamplona.quartz/quartz?label=Quartz%20%28Maven%20Central%29&labelColor=27303D&color=0877d2)](https://central.sonatype.com/artifact/com.vitorpamplona.quartz/quartz) [![JitPack snapshots](https://img.shields.io/badge/Quartz%20snapshots-JitPack-27303D?labelColor=27303D&color=0877d2)](https://jitpack.io/#vitorpamplona/amethyst) [![CI](https://img.shields.io/github/actions/workflow/status/vitorpamplona/amethyst/build.yml?labelColor=27303D)](https://github.com/vitorpamplona/amethyst/actions/workflows/build.yml) -[![License: Apache-2.0](https://img.shields.io/github/license/vitorpamplona/amethyst?labelColor=27303D&color=0877d2)](/LICENSE) +[![License: MIT](https://img.shields.io/github/license/vitorpamplona/amethyst?labelColor=27303D&color=0877d2)](/LICENSE) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/vitorpamplona/amethyst) ## Download and Install diff --git a/SKILL.md b/SKILL.md index 124b4742b5..b1c2b7caa9 100644 --- a/SKILL.md +++ b/SKILL.md @@ -24,7 +24,9 @@ Build customized Amethyst Nostr clients for Android. Fork, rebrand, customize, a 2. **Android SDK** - Command-line tools from https://developer.android.com/studio#command-line-tools-only - - Required components: build-tools, platform-tools, platforms;android-35 + - Required components: build-tools, platform-tools, platforms;android-37 + - The exact SDK level is `android-compileSdk` in `gradle/libs.versions.toml` — + check there if this number has drifted. 3. **Git** for cloning the repository @@ -66,48 +68,54 @@ keyPassword=your-password ### 3. Configure Signing -Add to `amethyst/build.gradle` inside the `android {}` block: +Add to `amethyst/build.gradle.kts` inside the `android {}` block: -```gradle -def keystorePropertiesFile = rootProject.file("keystore.properties") -def keystoreProperties = new Properties() +```kotlin +val keystorePropertiesFile = rootProject.file("keystore.properties") +val keystoreProperties = Properties() if (keystorePropertiesFile.exists()) { - keystoreProperties.load(new FileInputStream(keystorePropertiesFile)) + keystorePropertiesFile.inputStream().use { keystoreProperties.load(it) } } signingConfigs { - release { + create("release") { if (keystorePropertiesFile.exists()) { - storeFile rootProject.file(keystoreProperties['storeFile']) - storePassword keystoreProperties['storePassword'] - keyAlias keystoreProperties['keyAlias'] - keyPassword keystoreProperties['keyPassword'] + storeFile = rootProject.file(keystoreProperties["storeFile"] as String) + storePassword = keystoreProperties["storePassword"] as String + keyAlias = keystoreProperties["keyAlias"] as String + keyPassword = keystoreProperties["keyPassword"] as String } } } ``` +This needs `import java.util.Properties` at the top of the file. + Update the release buildType to use the signing config: -```gradle +```kotlin buildTypes { - release { - signingConfig signingConfigs.release + getByName("release") { + signingConfig = signingConfigs.getByName("release") // ... existing config } } ``` +Verify with `./gradlew :amethyst:signingReport` — the release variants should +report your keystore rather than `~/.android/debug.keystore`. + ### 4. Disable Google Services (Required for F-Droid) **⚠️ CRITICAL:** The Google Services plugin fails when you change the package name. For F-Droid builds, disable it. -Edit `amethyst/build.gradle`, comment out the plugin: -```gradle +Edit `amethyst/build.gradle.kts`, comment out the plugin: +```kotlin plugins { alias(libs.plugins.androidApplication) - alias(libs.plugins.jetbrainsKotlinAndroid) // alias(libs.plugins.googleServices) // DISABLED for F-Droid alias(libs.plugins.jetbrainsComposeCompiler) + alias(libs.plugins.serialization) + alias(libs.plugins.googleKsp) } ``` @@ -141,8 +149,8 @@ Edit `amethyst/src/main/res/values/strings.xml`: ### Change Package ID -Edit `amethyst/build.gradle`: -```gradle +Edit `amethyst/build.gradle.kts`: +```kotlin android { defaultConfig { applicationId = "com.yourcompany.yourapp" @@ -152,8 +160,8 @@ android { ### Change Project Name -Edit `settings.gradle`: -```gradle +Edit `settings.gradle.kts`: +```kotlin rootProject.name = "YourAppName" ``` @@ -167,36 +175,28 @@ Replace icon files in: Make your app identify itself on posts with `["client", "YourAppName"]`. -**1. Create tag builder extension:** +You do **not** need to add the tag per event type. The client tag is applied +centrally by `NostrSignerWithClientTag`, a signer decorator that appends the tag +to everything it signs (and respects the user's "add client tag" privacy +setting). Changing the name is a one-constant edit: -Create `quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/tags/clientTag/TagArrayBuilderExt.kt`: +Edit `amethyst/src/main/java/com/vitorpamplona/amethyst/model/accountsCache/AccountCacheState.kt`: ```kotlin -package com.vitorpamplona.quartz.nip01Core.tags.clientTag - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.TagArrayBuilder - -fun TagArrayBuilder.client(clientName: String) = - addUnique(arrayOf(ClientTag.TAG_NAME, clientName)) +const val CLIENT_TAG_NAME = "YourAppName" ``` -**2. Add to TextNoteEvent:** +That constant is passed to `NostrSignerWithClientTag` when the account's signer +is built, so every signed event carries your name. -Edit `quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip10Notes/TextNoteEvent.kt`: - -Add import: -```kotlin -import com.vitorpamplona.quartz.nip01Core.tags.clientTag.client -``` - -In both `build()` functions, add after `alt(...)`: -```kotlin -client("YourAppName") -``` +The tag itself lives in +`quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip89AppHandlers/clientTag/` +(`ClientTag`, `TagArrayBuilderExt`, `NostrSignerWithClientTag`) — you only need to +touch it if you want the optional NIP-89 handler address / relay hint variants. ### Modify Default Relays -Edit relay configuration in `quartz/src/main/java/com/vitorpamplona/quartz/nip01Core/relay/` or the UI settings files. +Edit `commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/defaults/Constants.kt` +(see also `AmethystDefaults.kt` and `DefaultDmIndexerRelays.kt` in the same folder). ## Troubleshooting From c7670a52d4f548bb6b5d8c9303f1d4a02f82ed77 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 21:45:42 +0000 Subject: [PATCH 15/34] perf(store): scale NIP-50 search and tag-watcher queries with corpus size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scale-curve report showed the SQLite store degrading with corpus size on NIP-50 search (~18×) and the large-IN tag watcher, while point reads stayed flat. Three read/size changes (write path and index set unchanged): - FTS: rebuild event_fts as a contentless FTS5 table (content='', contentless_delete=1) keyed by rowid = event_headers.row_id. Drops the stored content copy (smaller index); external-content can't hold the derived indexable text, so contentless is the correct primitive. Adds an opt-in searchOrderByRowId strategy flag: ORDER BY event_fts.rowid DESC early-terminates (O(limit), corpus-independent) at the cost of ingestion-order results — flat ~0.22 ms vs created_at's 4.26 ms at 200k (~19×). reindexAll ends with 'optimize'; the periodic optimize() folds in a bounded segment 'merge'. DB version 4->5 with a drop-and-rebuild migration. - MergeQueryExecutor: extend the k-way merge to the tag path (kinds + #e IN [hundreds] + limit), one cursor per (value[,kind]) stream heap-merged to the limit, deduping events that carry several queried values. O(limit + streams) instead of collecting all matches and sorting. - StatementCachingConnection: pool multiple handles per SQL so the merge's many concurrent identical-SQL cursors all hit the cache (previously only the first did) and repeated polls reuse their per-stream statements. Tests: contentless migration (real v4 DB upgrade), rowid-order search, tag merge correctness (incl. cross-stream dedup), statement pool, FTS5 capability probe, and an FTS search-scaling benchmark. Plan in quartz/plans/2026-07-21-sqlite-query-scaling.md. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XqGuBuSUsRudGerPqDuoKA --- .../plans/2026-07-21-sqlite-query-scaling.md | 123 ++++++++ .../store/sqlite/FullTextSearchModule.kt | 150 +++++++-- .../store/sqlite/IndexingStrategy.kt | 27 ++ .../store/sqlite/MergeQueryExecutor.kt | 258 ++++++++++++--- .../nip01Core/store/sqlite/QueryBuilder.kt | 33 +- .../store/sqlite/SQLiteEventStore.kt | 14 +- .../sqlite/StatementCachingConnection.kt | 74 +++-- .../store/sqlite/QueryAssemblerTest.kt | 28 +- .../store/sqlite/SearchOrderByRowIdTest.kt | 109 +++++++ .../sqlite/StatementCachingConnectionTest.kt | 113 +++++++ .../store/sqlite/TagMergeCorrectnessTest.kt | 295 ++++++++++++++++++ .../prodbench/FollowFeedReadBenchmark.kt | 2 +- .../prodbench/FtsSearchScalingBenchmark.kt | 148 +++++++++ .../sqlite/ContentlessFtsMigrationTest.kt | 132 ++++++++ .../store/sqlite/Fts5CapabilityProbe.kt | 88 ++++++ 15 files changed, 1474 insertions(+), 120 deletions(-) create mode 100644 quartz/plans/2026-07-21-sqlite-query-scaling.md create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchOrderByRowIdTest.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnectionTest.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/TagMergeCorrectnessTest.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ContentlessFtsMigrationTest.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/Fts5CapabilityProbe.kt diff --git a/quartz/plans/2026-07-21-sqlite-query-scaling.md b/quartz/plans/2026-07-21-sqlite-query-scaling.md new file mode 100644 index 0000000000..d8a6860bf9 --- /dev/null +++ b/quartz/plans/2026-07-21-sqlite-query-scaling.md @@ -0,0 +1,123 @@ +# SQLite query scaling: contentless FTS + rowid search, tag-path merge, pooled statements + +**Status: shipped.** Follow-up to a scale-curve report (`SQLite vs Vespa`, +throughput vs corpus size) showing the app-side SQLite store degrading with +corpus size on several query shapes — NIP-50 search ~18× (25k→400k), batch +ingest 16×, author-timeline 13×, follow-feed 2.4× — while point reads stayed +flat. Two of those (author-timeline, follow-feed) were already addressed +(the pubkey-alone index and `MergeQueryExecutor`) and mostly reflect running +the report under the *client* `DefaultIndexingStrategy` rather than +`relayIndexingStrategy()`. This change takes the two structural read shapes +that were still corpus-bound: **NIP-50 search** and the **large-IN tag +watcher**, plus a statement-cache fix the merge paths needed. + +Everything here is read/size work; the write path and on-disk index set are +unchanged except the FTS table, which gets *smaller*. + +## 1. NIP-50 search — contentless FTS5 + optional rowid ordering + segment merge + +The old index was `fts5(event_header_row_id, content)`: it stored a second +copy of the tokenized text, and its rowid was auto-assigned (unrelated to the +event). Search ran `… event_fts MATCH ? ORDER BY event_headers.created_at DESC +LIMIT n`, which must materialize **every** document matching the term and sort +it — cost grows with the whole corpus. That is the 18× curve. + +Three changes (`FullTextSearchModule`, `IndexingStrategy`, `QueryBuilder`, +DB version 4→5): + +- **Contentless** (`fts5(content, content='', contentless_delete=1)`). The + indexed text is *derived* (`SearchableEvent.indexableContent()`, not a raw + column), so FTS5 **external-content** — which re-reads the source column from + the base table — cannot express it; **contentless** is the correct primitive. + It drops the stored content copy (index shrinks) and, with + `contentless_delete=1`, still supports the `fts_foreign_key` delete trigger. +- **rowid = `event_headers.row_id`.** Set explicitly on every insert. The + delete trigger and reindex/catch-up paths all key off it, so the join is + `event_headers.row_id = event_fts.rowid` with no stored linkage column. +- **`searchOrderByRowId`** (new `IndexingStrategy` flag, **default off**). When + on, simple search orders by `event_fts.rowid DESC LIMIT n`, which FTS5 + early-terminates off the doclist — **O(limit)**, corpus-independent. rowid is + *ingestion* order, so this ≈ recency only while events arrive in time order; + a relay that bulk-syncs history (NIP-77) diverges, which is why it is off by + default and left off for geode. Only the simple-search shape changes; tag∩ + search and the negentropy snapshot keep `created_at`. +- **Segment compaction.** `reindexAll` finishes with a full `'optimize'`, and + the periodic `SQLiteEventStore.optimize()` (geode's maintenance tick) folds + in a bounded `'merge'`, so incremental / deferred-catch-up inserts don't + leave the index as many small segments. + +Measured — `FtsSearchScalingBenchmark` (jvmTest prodbench), search a term in +~1% of events, `limit=50`, in-memory: + +| corpus | created_at, fragmented | created_at, optimized | **rowid, optimized** | +|---|---:|---:|---:| +| 50k | 1.92 ms | 1.16 ms | **0.26 ms** | +| 100k | 2.16 ms | 2.23 ms | **0.22 ms** | +| 200k | 4.33 ms | 4.26 ms | **0.22 ms** | + +- **rowid ordering is the corpus-independence lever**: flat ~0.22 ms vs + created_at's 1.16 → 4.26 ms (grows with the match set) — ~19× at 200k, and + the gap keeps widening. This is the direct answer to the search curve, for + deployments that accept ingestion-order results. +- **segment `optimize`** helps most at small sizes / right after bulk + incremental inserts (1.92 → 1.16 ms at 50k); FTS5 `automerge` already caps + fragmentation in steady state, so at 100k/200k here it is within noise. It is + a cheap, unconditional safety net (biggest for the deferred relay path, whose + catch-up commits in 1k batches), not the main lever. + +Migration (v4→v5): the old rowids can't be remapped, so `event_fts` is dropped +and rebuilt — synchronous stores rebuild in the upgrade transaction (client +corpora are small), deferred stores reset the catch-up watermark to 0 and let +the background worker repopulate (no long migration transaction). +`ContentlessFtsMigrationTest` fabricates a real v4 DB (old schema + garbage +rows + `user_version=4`) and asserts the reopen rebuilds search, wipes the +stale rows, maps rowid→row_id, and keeps the delete trigger working. + +## 2. Tag watcher — extend `MergeQueryExecutor` to the tag path + +`kinds=[7] AND #e=[hundreds of note ids] LIMIT n` (reactions/replies) had the +same shape the follow-feed fix already solved for authors: the per-value +streams come sorted off `(tag_hash[, kind], created_at)`, but their union does +not, so SQLite collected every matching row and TEMP-B-TREE-sorted to the limit +— O(matching history), growing with the corpus (`TagAuthorIndexBenchmark`: +12.8 ms cold at 200k → 14.2 ms at 1M). + +`MergeQueryExecutor` now opens one lazy newest-first cursor per `(value[, +kind])` off `query_by_tags_hash_kind` (or `query_by_tags_hash` when there is no +kind, gated by `indexTagsByCreatedAtAlone`) and heap-merges to the limit — +**O(limit + streams)**. Unlike the author path, one event can carry several +queried tag values, so the tag merge **dedups by event id** through a `seen` +set (the author path, one pubkey per event, skips it). Eligibility is narrow +(one non-`d` tag key, ≥2 values, a limit, no ids/authors/d-tag/search, no +AND-tags); everything else falls through to the single-SQL plan. Counts and +deletes are unchanged (still single-SQL). `TagMergeCorrectnessTest` pins it +against a Kotlin reference including cross-stream dedup, since/until windows, +the raw path, and tie handling (tag cursors order off `event_tags`, which has +no id column, so same-second ties are a valid newest-N but not id-exact). + +## 3. Pooled statement cache — so the merges actually cache + +`StatementCachingConnection` kept **one** handle per SQL string. The k-way +merge opens *many* identical-SQL cursors at once (one per author/tag stream), +so every stream past the first missed the cache and prepared uncached, and a +repeated follow-feed / reactions REQ re-prepared all of them each poll. The +cache now keeps a small **pool** per SQL (bounded by a global cap, default +raised 256→512), so concurrent same-SQL checkouts reuse cached handles and +repeated polls reuse their per-stream cursors. `StatementCachingConnectionTest` +covers sequential reuse, concurrent distinctness/independence, freed-handle +reuse, and cap overflow → uncached fallback. + +## Correctness / scope + +- DB version bump 4→5 with a real-upgrade test. +- New capability test (`Fts5CapabilityProbe`) pins the FTS5 features the + contentless index needs (contentless_delete, absent-rowid delete no-op, + rowid ordering, merge/optimize) to the bundled SQLite (3.50.1), so a driver + downgrade fails loudly. +- All existing `store.sqlite` tests pass unchanged except `QueryAssemblerTest`, + whose asserted EXPLAIN output updated for the new join column + (`event_fts.rowid`) and the contentless table's virtual-index marker + (`0:M2`→`0:M1`). +- Defaults preserved: `searchOrderByRowId` off, so existing deployments keep + exact `created_at DESC` search ordering; the size + optimize wins are + unconditional. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index 8693fba69d..c913e1f9eb 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -31,6 +31,22 @@ import com.vitorpamplona.quartz.utils.EventFactory /** * NIP-50 full-text search index over event content. * + * The index is a **contentless** FTS5 table (`content=''`, + * `contentless_delete=1`) whose `rowid` is the event's + * `event_headers.row_id`. Two consequences: + * + * - **Size.** A contentless table stores only the inverted index, not a + * second copy of the tokenized text (the old `fts5(event_header_row_id, + * content)` schema kept the content verbatim). The indexed text is + * *derived* ([SearchableEvent.indexableContent], not any raw column), so + * FTS5 external-content — which reads the source column from the base + * table — cannot express it; contentless is the correct primitive. + * - **Ordering.** With `rowid = event_headers.row_id`, + * `ORDER BY event_fts.rowid DESC LIMIT n` early-terminates off the FTS + * doclist ([IndexingStrategy.searchOrderByRowId]); the default + * `created_at DESC` ordering still works via the join back to + * `event_headers` but must sort all matches. + * * When [enabled] is `false` the module becomes an inert no-op: no * `event_fts` virtual table and no `fts_foreign_key` delete trigger are * created, inserts skip the per-event tokenization cost, and the reindex @@ -52,29 +68,53 @@ class FullTextSearchModule( ) : IModule { val tableName = "event_fts" val triggerName = "fts_foreign_key" - val eventHeaderRowIdName = "event_header_row_id" + + /** + * The FTS column that links back to `event_headers`. It is the implicit + * `rowid` of the (contentless) FTS table, which we set equal to + * `event_headers.row_id` on every insert — so the join is + * `event_headers.row_id = event_fts.rowid`, with no stored column. + */ + val rowIdColumn = "rowid" val contentName = "content" val stateTableName = "fts_catchup_state" + /** + * Whether the on-disk `event_fts` is an FTS5 table (vs the fts4/fts3 + * fallback). Only FTS5 supports the contentless schema and the + * `merge`/`optimize` maintenance commands; the fallback stores content + * and skips maintenance. Read only on the (single-threaded) writer. + * Cached lazily from `sqlite_master` so a reopened DB — where [create] + * never runs — still resolves it. + */ + private var isFts5: Boolean? = null + override fun create(db: SQLiteConnection) { if (!enabled) return val ftsVersion = versionFinder(db) - db.execSQL( - """ - CREATE VIRTUAL TABLE $tableName - USING fts$ftsVersion($eventHeaderRowIdName, $contentName) - """, - ) + isFts5 = ftsVersion >= 5 + // FTS5: contentless index (no stored content copy) with delete + // support. fts4/fts3 (bundled driver never selects them) fall back to + // a plain content-storing table — rowid-explicit insert and + // delete-by-rowid work there too, only the size win is FTS5-only. + val columns = + if (ftsVersion >= 5) { + "$contentName, content='', contentless_delete=1" + } else { + contentName + } + db.execSQL("CREATE VIRTUAL TABLE $tableName USING fts$ftsVersion($columns)") - // Foreign key cleanup for full text search + // Foreign key cleanup for full text search. Deletes by the FTS rowid + // (= event_headers.row_id); a header with no FTS row (non-searchable + // kind) deletes nothing, which is a harmless no-op. db.execSQL( """ CREATE TRIGGER $triggerName AFTER DELETE ON event_headers FOR EACH ROW BEGIN - DELETE FROM $tableName - WHERE old.row_id = $tableName.$eventHeaderRowIdName; + DELETE FROM $tableName WHERE $tableName.rowid = old.row_id; END; """, ) @@ -82,6 +122,16 @@ class FullTextSearchModule( createStateTable(db) } + private fun resolveIsFts5(db: SQLiteConnection): Boolean { + isFts5?.let { return it } + val sql = + db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").use { stmt -> + stmt.bindText(1, tableName) + if (stmt.step()) stmt.getText(0) else "" + } + return sql.contains("fts5", ignoreCase = true).also { isFts5 = it } + } + /** * Watermark for the deferred path: everything with * `row_id <= last_row_id` is guaranteed indexed. Idempotent — also @@ -124,13 +174,13 @@ class FullTextSearchModule( val insertFTS = """ - INSERT OR ROLLBACK INTO $tableName ($eventHeaderRowIdName, $contentName) + INSERT OR ROLLBACK INTO $tableName (rowid, $contentName) VALUES (?, ?) """.trimIndent() val deleteFTSByRowId = """ - DELETE FROM $tableName WHERE $eventHeaderRowIdName = ? + DELETE FROM $tableName WHERE rowid = ? """.trimIndent() fun insert( @@ -200,7 +250,19 @@ class FullTextSearchModule( dropTrigger(db) drop(db) create(db) + populateAll(db) + // The rebuild just wrote every row as its own tiny segment; compact + // them into one so the first search after a reindex isn't a scan over + // hundreds of segments. + optimize(db) + } + /** + * Scan every stored searchable event and insert its derived content into + * the (already created, empty) FTS index, keyed by `event_headers.row_id`. + * The caller owns the transaction and the create/drop lifecycle. + */ + private fun populateAll(db: SQLiteConnection) { val kinds = searchableKindsPresent(db) if (kinds.isEmpty()) return @@ -230,6 +292,58 @@ class FullTextSearchModule( } } + /** + * v4 → v5 migration: the pre-v5 index was `fts5(event_header_row_id, + * content)` with an auto-assigned rowid unrelated to `event_headers`, and + * it stored a second copy of the content. v5 is the contentless, + * rowid = row_id schema. The old rowids can't be remapped in place, so the + * table is dropped and repopulated. Runs inside the migration transaction. + * + * - **synchronous** stores rebuild now (client corpora are small). + * - **deferred** stores reset the catch-up watermark to 0 so the relay's + * background worker repopulates without a long migration transaction. + */ + fun migrateV4ToContentless(db: SQLiteConnection) { + if (!enabled) return + dropTrigger(db) + drop(db) + create(db) + if (deferIndexing) { + db.prepare("UPDATE $stateTableName SET last_row_id = 0 WHERE id = 1").use { it.step() } + } else { + populateAll(db) + optimize(db) + } + } + + /** + * Full FTS5 segment compaction — merges the b-tree segments left by + * incremental inserts into one, so a `MATCH` touches a single segment + * instead of dozens. Expensive (rewrites the whole index); call it once + * after a rebuild, not per batch. No-op on the fts4/fts3 fallback. + */ + fun optimize(db: SQLiteConnection) { + if (!enabled || !resolveIsFts5(db)) return + db.prepare("INSERT INTO $tableName($tableName) VALUES ('optimize')").use { it.step() } + } + + /** + * Bounded incremental segment merge — does at most [pages] pages of merge + * work, so it stays cheap enough to run on a periodic maintenance tick + * while the index keeps growing from deferred catch-up. No-op on the + * fts4/fts3 fallback. + */ + fun mergeSegments( + db: SQLiteConnection, + pages: Int = 16, + ) { + if (!enabled || !resolveIsFts5(db)) return + db.prepare("INSERT INTO $tableName($tableName, rank) VALUES ('merge', ?)").use { stmt -> + stmt.bindLong(1, pages.toLong()) + stmt.step() + } + } + /** * Process one batch of a resumable rebuild: re-derive the FTS rows * for up to [batchSize] events whose `row_id > ` [afterRowId] and @@ -329,13 +443,11 @@ class FullTextSearchModule( // Unlike [reindexBatch] there is NO per-row delete here: rows past // the watermark were never indexed (deferred mode skips insert()), // and the watermark advances atomically with the FTS rows it - // covers, so a crash replay is impossible. The delete would also - // be ruinous — `event_header_row_id` is a plain FTS5 column, so - // deleting by it scans the whole FTS table per row, which turned - // the first catch-up implementation O(n²). Consequence: switching - // a database back and forth between deferred and synchronous - // strategies requires a [reindexAll] in between (same rule as a - // searchable-kinds change). + // covers, so a crash replay is impossible. (Delete-by-rowid is now + // O(log n) on the contentless index, so this is purely about not + // doing redundant work.) Consequence: switching a database back and + // forth between deferred and synchronous strategies requires a + // [reindexAll] in between (same rule as a searchable-kinds change). val limit = batchSize.coerceAtLeast(1) val kinds = searchableKindsPresent(db) var last = watermark diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt index 01362e03fb..ea76cf181d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt @@ -131,6 +131,32 @@ interface IndexingStrategy { */ val indexFullTextSearch: Boolean + /** + * Order NIP-50 search results by the FTS index rowid (= event row_id, + * i.e. **ingestion order**) instead of `created_at DESC`. + * + * The `event_fts` rowid is the event's `event_headers.row_id`, so + * `ORDER BY event_fts.rowid DESC LIMIT n` walks the FTS doclist + * newest-rowid-first and stops at the limit — **O(limit)** — where the + * default `ORDER BY created_at DESC` must materialize *every* document + * matching the term and sort it (cost grows with the whole corpus; the + * NIP-50 curve that falls ~18× from 25k→400k events). The unconditional + * contentless-index + segment-merge changes already shrink and speed the + * default path; this flag is the one that makes search cost independent + * of corpus size. + * + * The trade-off is semantic: ingestion order ≈ `created_at` order only + * while events arrive roughly in time order. A relay that bulk-syncs + * historical events (NIP-77) ingests old events late, so "newest rowid" + * and "newest created_at" diverge during/after a sync. Leave it **off** + * (the default) where strict recency ordering matters; turn it **on** + * for stores that want corpus-independent search latency and accept + * recently-ingested-first ordering. Only affects the simple-search query + * shape (`search [+ kinds/authors] + limit`); tag∩search and the + * negentropy snapshot path keep `created_at` ordering. + */ + val searchOrderByRowId: Boolean get() = false + /** * Maintain an always-current in-memory `(created_at, id)` set (a * [com.vitorpamplona.quartz.nip77Negentropy.LiveNegentropyIndex]) so @@ -160,6 +186,7 @@ class DefaultIndexingStrategy( override val useAndIndexIdOnOrderBy: Boolean = false, override val indexFullTextSearch: Boolean = true, override val deferFullTextSearchIndexing: Boolean = false, + override val searchOrderByRowId: Boolean = false, override val maintainLiveNegentropyIndex: Boolean = false, ) : IndexingStrategy { override fun shouldIndex( diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt index 34ada1e64c..4b75bb4cda 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt @@ -24,63 +24,94 @@ import androidx.sqlite.SQLiteConnection import androidx.sqlite.SQLiteStatement /** - * k-way merge executor for the **home-feed** query shape: - * `authors=[…] (+ kinds=[…]) [+ since/until] limit=N` ordered newest-first. + * k-way merge executor for the two "wide fan-out, newest-N" query shapes + * whose single-SQL plan reads O(matching history) rather than O(limit): * - * SQLite serves this by seeking every `(kind, pubkey)` combo and feeding - * *all* matching rows through a LIMIT-bounded sorter — so it reads O(the - * followed set's whole matching history). For prolific follows on a cold - * on-disk DB that's the `follow-feed` regression (relayBench: 97 ms vs - * strfry 17 ms). See `quartz/plans/2026-07-04-follow-feed-read-tradeoff.md`. + * 1. **home-feed** — `authors=[…] (+ kinds=[…]) [+ since/until] limit=N`. + * SQLite seeks every `(kind, pubkey)` combo and feeds *all* matching rows + * through a LIMIT-bounded sorter — O(the followed set's whole matching + * history). For prolific follows on a cold on-disk DB that was the + * `follow-feed` regression (relayBench: 97 ms vs strfry 17 ms). See + * `quartz/plans/2026-07-04-follow-feed-read-tradeoff.md`. + * 2. **tag watcher** — `#=[hundreds of values] (+ kinds=[…]) + * [+ since/until] limit=N`, the reactions/replies archetype + * (`kinds=[7] AND #e=[note ids]`). The per-value streams come sorted off + * `(tag_hash[, kind], created_at)`, but their union does not, so SQLite + * collects every matching row and TEMP-B-TREE sorts to the limit — the + * tag-index analogue of the follow-feed shape. Measured by + * `TagAuthorIndexBenchmark` (jvmTest prodbench): `#e IN 300, limit 500` + * cost 12.8 ms cold at 200k events and 14.2 ms at 1M, growing with + * matching history. * - * Each `(kind, pubkey)` is already a newest-first stream off the - * `query_by_kind_pubkey_created (kind, pubkey, created_at DESC)` index - * (or `query_by_pubkey_created` for authors-only). This opens one lazy - * cursor per stream and merges their heads, stopping at the limit — so it - * reads only **O(limit + streams)** rows regardless of how much history the - * authors have, and it reuses the existing indexes (no write/size cost). + * Each stream is already a newest-first cursor off an existing index: + * - authors: `query_by_kind_pubkey_created (kind, pubkey, created_at DESC)` + * (or `query_by_pubkey_created` for authors-only); + * - tags: `query_by_tags_hash_kind (tag_hash, kind, created_at DESC)` + * (or `query_by_tags_hash` for the no-kind case, gated by + * [IndexingStrategy.indexTagsByCreatedAtAlone]). + * + * The merge opens one lazy cursor per stream, merges their heads newest-first, + * and stops at the limit — reading **O(limit + streams)** rows regardless of + * how much history the authors/tags have, and reusing the existing indexes + * (no write/size cost). With the pooled statement cache + * ([StatementCachingConnection]) the per-stream cursors are prepared once and + * reused across repeated polls of the same REQ. * * Merge order is `created_at DESC`, tie-broken by `id ASC`. NIP-01 leaves - * same-`created_at` ties unspecified, so the returned set is a valid - * newest-N either way. The `id ASC` tie-break is exact — byte-for-byte the - * same events the single-SQL path returns — only when the store indexes id - * ([IndexingStrategy.useAndIndexIdOnOrderBy]): then each per-stream cursor - * streams in `(created_at DESC, id ASC)` straight off the index, so a - * stream's same-second head really is its id-minimum. Without that index - * the per-stream cursor yields same-second rows in rowid order, so the - * result is still a valid newest-N but may differ from the single-SQL path - * exactly at a same-second boundary. + * same-`created_at` ties unspecified, so the returned set is a valid newest-N + * either way. The `id ASC` tie-break is exact — byte-for-byte the same events + * the single-SQL path returns — only when the store indexes id + * ([IndexingStrategy.useAndIndexIdOnOrderBy]) **and** the stream cursor can + * order by id off the index. The author streams can (id is on + * `event_headers`); the tag streams cannot (the cursor orders off + * `event_tags`, which has no id column), so a tag stream yields same-second + * rows in rowid order — still a valid newest-N, but same-second ties may + * differ from an id-ordered reference. + * + * **Cross-stream duplicates.** An author appears in exactly one author stream + * (one pubkey per event), so the home-feed merge never double-counts. A single + * event can carry several of the queried tag values (or a repeated tag), so it + * can surface in several tag streams — the single-SQL path dedups with + * `SELECT DISTINCT`. The tag merge therefore dedups by event id through a + * `seen` set; the author merge skips that set entirely. */ internal object MergeQueryExecutor { - // TODO: the same collect-all + TEMP-B-TREE-sort pattern exists one index - // over, on the tag path: `kinds + tags(#e IN [hundreds]) + limit` (the - // reactions/replies watcher archetype) unions per-value streams that are - // each sorted off `(tag_hash, kind, created_at)` and sorts the union. - // `streamCount` currently rejects any filter with tags, so those queries - // never merge. Measured by `TagAuthorIndexBenchmark`: `#e IN 300, - // limit 500` costs 12.8 ms cold at 200k events and 14.2 ms at 1M - // (6.7 ms with indexTagsWithKindAndPubkey on) — tolerable, but it - // scales with matching history like the follow-feed shape did; extend - // the merge to per-tag-value streams if the relayBench - // `reactions-watch` scenario shows it in the profile vs strfry. + /** Author-stream projection: a single `event_headers` scan, unqualified. */ const val COLS = "id, pubkey, created_at, kind, tags, content, sig" + /** Tag-stream projection: `event_tags` joins `event_headers`, so qualify. */ + private const val EH_COLS = + "event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig" + /** * Above this many streams, fall back to the single-SQL plan: the - * per-stream cursor setup stops paying off, and huge author lists are - * collecting a lot no matter what. `kinds.size × authors.size`. + * per-stream cursor setup stops paying off, and huge fan-outs are + * collecting a lot no matter what. `kinds.size × (authors|values).size`. */ const val MAX_STREAMS = 2048 /** - * Stream count if [filter] is merge-eligible, else `-1`. Eligible = a - * simple (no tag/search/id/d-tag) query with authors + a limit, whose - * per-stream index exists. `kinds` optional: with it, one stream per - * `(kind, author)`; without, one per author (needs the pubkey index). + * Stream count if [filter] is merge-eligible under *either* shape, else + * `-1`. Routing check for [QueryBuilder]; [run] re-derives which shape. */ fun streamCount( filter: QueryBuilder.FilterWithDTags, indexStrategy: IndexingStrategy, + ): Int { + val authorStreams = authorStreamCount(filter, indexStrategy) + if (authorStreams > 0) return authorStreams + return tagStreamCount(filter, indexStrategy) + } + + /** + * Author-shape stream count, or `-1`. Eligible = a simple (no tag/search/ + * id/d-tag) query with authors + a limit, whose per-stream index exists. + * `kinds` optional: with it, one stream per `(kind, author)`; without, one + * per author (needs the pubkey index). + */ + fun authorStreamCount( + filter: QueryBuilder.FilterWithDTags, + indexStrategy: IndexingStrategy, ): Int { if (!filter.isSimpleQuery()) return -1 if (filter.ids != null) return -1 @@ -107,14 +138,54 @@ internal object MergeQueryExecutor { return if (streams in 2..MAX_STREAMS) streams else -1 } - /** Prepares one bound, newest-first cursor per stream. */ - private fun prepareStreams( + /** + * Tag-shape stream count, or `-1`. Eligible = a single non-`d` tag key + * with `IN` (any-of) semantics and ≥2 distinct values, plus a limit, no + * ids/authors/d-tag/search, and no `AND`-tags (`tagsAll`) — the large-IN + * watcher shape. `kinds` optional: with it, one stream per + * `(value, kind)` off `query_by_tags_hash_kind`; without, one per value + * off `query_by_tags_hash` (needs [IndexingStrategy.indexTagsByCreatedAtAlone]). + * + * Authors are excluded on purpose: `tag ∩ author ∩ kind` is a covered + * single seek under [IndexingStrategy.indexTagsWithKindAndPubkey], not a + * fan-out, and mixing an author predicate into per-tag streams would not + * reduce the read. + */ + fun tagStreamCount( + filter: QueryBuilder.FilterWithDTags, + indexStrategy: IndexingStrategy, + ): Int { + if (filter.ids != null) return -1 + if (filter.authors != null) return -1 + if (filter.dTags != null) return -1 + if (filter.search != null && filter.search.isNotEmpty()) return -1 + if (filter.limit == null || filter.limit <= 0) return -1 + // AND-tags can't be expressed as a union of per-value streams. + if (filter.nonDTagsAll != null && filter.nonDTagsAll.isNotEmpty()) return -1 + val inTags = filter.nonDTagsIn ?: return -1 + // A second tag key would AND across keys — not a single union. + if (inTags.size != 1) return -1 + val values = inTags.values.first().distinct() + if (values.size < 2) return -1 + val kinds = filter.kinds?.distinct()?.takeIf { it.isNotEmpty() } + val streams = + if (kinds != null) { + values.size * kinds.size + } else { + if (!indexStrategy.indexTagsByCreatedAtAlone) return -1 + values.size + } + return if (streams in 2..MAX_STREAMS) streams else -1 + } + + /** Prepares one bound, newest-first cursor per author stream. */ + private fun prepareAuthorStreams( db: SQLiteConnection, filter: QueryBuilder.FilterWithDTags, indexStrategy: IndexingStrategy, ): List { // Dedup so a repeated pubkey/kind can't open two identical cursors and - // double-emit (see streamCount). + // double-emit (see authorStreamCount). val authors = filter.authors!!.distinct() val kinds = filter.kinds?.distinct()?.takeIf { it.isNotEmpty() } val since = filter.since @@ -172,18 +243,104 @@ internal object MergeQueryExecutor { return stmts } + /** Prepares one bound, newest-first cursor per tag-value stream. */ + private fun prepareTagStreams( + db: SQLiteConnection, + filter: QueryBuilder.FilterWithDTags, + hasher: TagNameValueHasher, + ): List { + val entry = filter.nonDTagsIn!!.entries.first() + val tagName = entry.key + val values = entry.value.distinct() + val kinds = filter.kinds?.distinct()?.takeIf { it.isNotEmpty() } + val since = filter.since + val until = filter.until + + // The tag cursors stream off event_tags (which has no id column), so + // the tie order can only be created_at DESC — see the class doc. + val stmts = ArrayList((kinds?.size ?: 1) * values.size) + if (kinds != null) { + val sql = + buildString { + append("SELECT ").append(EH_COLS) + append(" FROM event_tags INDEXED BY query_by_tags_hash_kind") + append(" JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id") + append(" WHERE event_tags.tag_hash = ? AND event_tags.kind = ?") + if (until != null) append(" AND event_tags.created_at <= ?") + if (since != null) append(" AND event_tags.created_at >= ?") + append(" ORDER BY event_tags.created_at DESC") + } + for (value in values) { + val tagHash = hasher.hash(tagName, value) + for (kind in kinds) { + val stmt = db.prepare(sql) + var p = 1 + stmt.bindLong(p++, tagHash) + stmt.bindLong(p++, kind.toLong()) + if (until != null) stmt.bindLong(p++, until) + if (since != null) stmt.bindLong(p++, since) + stmts.add(stmt) + } + } + } else { + val sql = + buildString { + append("SELECT ").append(EH_COLS) + append(" FROM event_tags INDEXED BY query_by_tags_hash") + append(" JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id") + append(" WHERE event_tags.tag_hash = ?") + if (until != null) append(" AND event_tags.created_at <= ?") + if (since != null) append(" AND event_tags.created_at >= ?") + append(" ORDER BY event_tags.created_at DESC") + } + for (value in values) { + val tagHash = hasher.hash(tagName, value) + val stmt = db.prepare(sql) + var p = 1 + stmt.bindLong(p++, tagHash) + if (until != null) stmt.bindLong(p++, until) + if (since != null) stmt.bindLong(p++, since) + stmts.add(stmt) + } + } + return stmts + } + /** - * Runs the merge, calling [onRow] with each winning cursor positioned on - * the row to emit, newest-first, up to `limit`. [onRow] must read the - * current row (it stays valid until the next step). + * Runs the merge for whichever shape [filter] matches, calling [onRow] + * with each winning cursor positioned on the row to emit, newest-first, + * up to `limit`. [onRow] must read the current row (it stays valid until + * the next step). [hasher] is only consulted for the tag shape. */ fun run( db: SQLiteConnection, filter: QueryBuilder.FilterWithDTags, indexStrategy: IndexingStrategy, + hasher: (SQLiteConnection) -> TagNameValueHasher, + onRow: (SQLiteStatement) -> Unit, + ) { + if (authorStreamCount(filter, indexStrategy) > 0) { + // One pubkey per event ⇒ author streams never overlap: no dedup. + mergeStreams(prepareAuthorStreams(db, filter, indexStrategy), filter.limit!!, dedup = false, onRow) + } else { + // A single event can match several tag values ⇒ dedup by id. + mergeStreams(prepareTagStreams(db, filter, hasher(db)), filter.limit!!, dedup = true, onRow) + } + } + + /** + * Heap-free k-way merge over the prepared [stmts]: repeatedly emits the + * newest live head (`created_at DESC`, tie `id ASC`) until [limit] rows + * are emitted or every stream is drained. When [dedup] is set an event id + * already emitted is skipped (its cursor still advances), so a row that + * surfaces in several streams is emitted once. + */ + private fun mergeStreams( + stmts: List, + limit: Int, + dedup: Boolean, onRow: (SQLiteStatement) -> Unit, ) { - val stmts = prepareStreams(db, filter, indexStrategy) try { val k = stmts.size val headCreatedAt = LongArray(k) @@ -199,8 +356,8 @@ internal object MergeQueryExecutor { } } + val seen = if (dedup) HashSet() else null var emitted = 0 - val limit = filter.limit!! while (emitted < limit) { // Pick the newest live head: created_at DESC, then id ASC. var best = -1 @@ -215,8 +372,11 @@ internal object MergeQueryExecutor { } if (best == -1) break - onRow(stmts[best]) // cursor is still on the head row - emitted++ + // Emit unless this id was already emitted by another stream. + if (seen == null || seen.add(headId[best]!!)) { + onRow(stmts[best]) // cursor is still on the head row + emitted++ + } // Advance the winner to its next row. if (stmts[best].step()) { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 5e42969ca9..5af41c7684 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -49,7 +49,7 @@ class QueryBuilder( val merge = filter.toFilterWithDTags() if (MergeQueryExecutor.streamCount(merge, indexStrategy) > 0) { val out = ArrayList(merge.limit!!) - MergeQueryExecutor.run(db, merge, indexStrategy) { out.add(it.toEvent()) } + MergeQueryExecutor.run(db, merge, indexStrategy, hasher) { out.add(it.toEvent()) } return out } return db.runQuery(toSql(filter, hasher(db))) @@ -62,7 +62,7 @@ class QueryBuilder( ) { val merge = filter.toFilterWithDTags() if (MergeQueryExecutor.streamCount(merge, indexStrategy) > 0) { - MergeQueryExecutor.run(db, merge, indexStrategy) { onEach(it.toEvent()) } + MergeQueryExecutor.run(db, merge, indexStrategy, hasher) { onEach(it.toEvent()) } return } db.runQuery(toSql(filter, hasher(db)), onEach) @@ -102,7 +102,7 @@ class QueryBuilder( val merge = filter.toFilterWithDTags() if (MergeQueryExecutor.streamCount(merge, indexStrategy) > 0) { val out = ArrayList(merge.limit!!) - MergeQueryExecutor.run(db, merge, indexStrategy) { out.add(it.toRawEvent()) } + MergeQueryExecutor.run(db, merge, indexStrategy, hasher) { out.add(it.toRawEvent()) } return out } return db.runRawQuery(toSql(filter, hasher(db))) @@ -115,7 +115,7 @@ class QueryBuilder( ) { val merge = filter.toFilterWithDTags() if (MergeQueryExecutor.streamCount(merge, indexStrategy) > 0) { - MergeQueryExecutor.run(db, merge, indexStrategy) { onEach(it.toRawEvent()) } + MergeQueryExecutor.run(db, merge, indexStrategy, hasher) { onEach(it.toRawEvent()) } return } db.runRawQuery(toSql(filter, hasher(db)), onEach) @@ -427,7 +427,7 @@ class QueryBuilder( val sql = buildString { append("SELECT event_headers.id, event_headers.created_at FROM event_headers") - append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.rowid") if (clause.conditions.isNotEmpty()) { append("\nWHERE ${clause.conditions}") } @@ -815,13 +815,13 @@ class QueryBuilder( } if (mustJoinSearch) { - append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id") + append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.rowid = event_tags.event_header_row_id") } } else if (mustJoinSearch) { - append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName}") + append("SELECT ${fts.tableName}.rowid as row_id FROM ${fts.tableName}") if (hasHeaders) { - append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.rowid") } } else { // no tags and no search. @@ -1001,13 +1001,22 @@ class QueryBuilder( val sql = buildString { append("SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers") - append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.rowid") if (clause.conditions.isNotEmpty()) { append("\nWHERE ${clause.conditions}") } - append("\nORDER BY event_headers.created_at DESC") - if (indexStrategy.useAndIndexIdOnOrderBy) { - append(", event_headers.id ASC") + if (indexStrategy.searchOrderByRowId) { + // The FTS rowid is event_headers.row_id (ingestion order). + // Ordering by it lets FTS5 walk the doclist newest-first + // and stop at LIMIT — O(limit) — instead of materializing + // and sorting every match by created_at. See + // IndexingStrategy.searchOrderByRowId for the trade-off. + append("\nORDER BY ${fts.tableName}.rowid DESC") + } else { + append("\nORDER BY event_headers.created_at DESC") + if (indexStrategy.useAndIndexIdOnOrderBy) { + append(", event_headers.id ASC") + } } if (limit != null) { append("\nLIMIT ") diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 3150b7625b..aee8f0c40f 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -60,7 +60,7 @@ class SQLiteEventStore( val extraPragmas: List = emptyList(), ) { companion object { - const val DATABASE_VERSION = 4 + const val DATABASE_VERSION = 5 } val seedModule = SeedModule() @@ -213,6 +213,12 @@ class SQLiteEventStore( // watermark seeds at the current MAX(row_id). fullTextSearchModule.createStateTable(db) } + 4 -> { + // Upgrade from version 4 to 5: the FTS index became a + // contentless table keyed by event_headers.row_id. The old + // rowids can't be remapped, so drop and repopulate. + fullTextSearchModule.migrateV4ToContentless(db) + } } } } @@ -274,6 +280,12 @@ class SQLiteEventStore( pool.useWriter { db -> db.execSQL("PRAGMA analysis_limit = 400;") db.execSQL("PRAGMA optimize;") + // Fold a bounded FTS segment merge into the same periodic + // maintenance tick: incremental (and deferred catch-up) inserts + // leave the NIP-50 index as many small segments, and a MATCH + // queries every one. Bounded so the tick stays cheap; a no-op when + // there is nothing to merge or FTS is off. + fullTextSearchModule.mergeSegments(db) } /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnection.kt index 62ed50f833..110ae9c351 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnection.kt @@ -34,46 +34,72 @@ import androidx.sqlite.SQLiteStatement * real reset + clearBindings happens on the next checkout). Statements are * only truly finalized when the connection itself closes. * + * Each SQL string caches a small **pool** of handles rather than a single + * one, so overlapping checkouts of the *same* SQL all reuse cached handles. + * That is exactly the k-way-merge query shape ([MergeQueryExecutor]): it + * opens one identical-SQL cursor per author/tag stream — dozens to hundreds + * live at once — which a single-handle cache could not serve (every stream + * past the first fell back to an uncached prepare). The pool lets a repeated + * follow-feed / reactions-watcher REQ reuse its per-stream cursors instead + * of re-preparing them each poll. + * * Constraints, by design of the call sites: * - **Not thread-safe** — same contract as the underlying connection, - * which the pool already serializes (single writer under a mutex). - * - **No overlapping use of the same SQL** — checking out one SQL string - * twice without closing the first use would alias one native handle. - * Insert/query paths never nest the same statement; a checkout while - * the previous one is still open falls back to an uncached statement. + * which the pool already serializes (single writer under a mutex; each + * reader held by one coroutine at a time). */ class StatementCachingConnection( private val delegate: SQLiteConnection, /** - * Ceiling on retained statements. Query SQL embeds one `?` per filter - * element, so shape variety is client-controlled — without a cap a - * long-lived relay connection would accumulate native handles without - * bound. Once full, unseen SQL just prepares uncached. 256 comfortably - * covers the write path's fixed set plus the recurring filter shapes. + * Ceiling on retained statements across all SQL strings. Query SQL + * embeds one `?` per filter element, so shape variety is + * client-controlled — without a cap a long-lived relay connection would + * accumulate native handles without bound. Once full, unseen SQL (or an + * extra concurrent copy of a cached SQL) just prepares uncached. 512 + * covers the write path's fixed set, the recurring single-shot filter + * shapes, and a few hundred concurrent per-stream merge cursors. */ - private val maxCachedStatements: Int = 256, + private val maxCachedStatements: Int = 512, ) : SQLiteConnection by delegate { - private val cache = HashMap() + // One reusable pool per SQL string. Several entries of the same pool may + // be checked out simultaneously (the merge path); a `prepare` reuses the + // first free entry, grows the pool while under the global cap, and only + // then falls back to an uncached statement. + private val cache = HashMap>() + private var cachedCount = 0 override fun prepare(sql: String): SQLiteStatement { - val cached = - cache[sql] ?: run { - if (cache.size >= maxCachedStatements) return delegate.prepare(sql) - CachedStatement(delegate.prepare(sql)).also { cache[sql] = it } + val pool = cache[sql] + if (pool != null) { + for (i in pool.indices) { + val stmt = pool[i] + if (!stmt.checkedOut) { + stmt.checkedOut = true + stmt.clearBindings() + return stmt + } + } + // Every pooled handle for this SQL is in use — grow if the global + // budget allows, else serve an uncached statement. + if (cachedCount >= maxCachedStatements) return delegate.prepare(sql) + return CachedStatement(delegate.prepare(sql)).also { + it.checkedOut = true + pool.add(it) + cachedCount++ } - if (cached.checkedOut) { - // Same SQL prepared while the previous handle is still in use — - // stay correct with a plain uncached statement. - return delegate.prepare(sql) } - cached.checkedOut = true - cached.clearBindings() - return cached + if (cachedCount >= maxCachedStatements) return delegate.prepare(sql) + return CachedStatement(delegate.prepare(sql)).also { + it.checkedOut = true + cache[sql] = arrayListOf(it) + cachedCount++ + } } override fun close() { - cache.values.forEach { runCatching { it.finalize() } } + cache.values.forEach { pool -> pool.forEach { runCatching { it.finalize() } } } cache.clear() + cachedCount = 0 delegate.close() } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt index 32621025e3..f3995f7b8f 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -238,9 +238,9 @@ class QueryAssemblerTest : BaseDBTest() { INNER JOIN ( SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY $orderBy @@ -252,13 +252,13 @@ class QueryAssemblerTest : BaseDBTest() { │ │ └── SCAN (subquery-1) │ ├── UNION USING TEMP B-TREE │ │ ├── CO-ROUTINE (subquery-3) - │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ │ └── USE TEMP B-TREE FOR ORDER BY │ │ └── SCAN (subquery-3) │ └── UNION USING TEMP B-TREE │ ├── CO-ROUTINE (subquery-5) - │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ └── USE TEMP B-TREE FOR ORDER BY │ └── SCAN (subquery-5) @@ -275,9 +275,9 @@ class QueryAssemblerTest : BaseDBTest() { INNER JOIN ( SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY $orderBy @@ -290,13 +290,13 @@ class QueryAssemblerTest : BaseDBTest() { │ │ └── SCAN (subquery-1) │ ├── UNION USING TEMP B-TREE │ │ ├── CO-ROUTINE (subquery-3) - │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ │ └── USE TEMP B-TREE FOR ORDER BY │ │ └── SCAN (subquery-3) │ └── UNION USING TEMP B-TREE │ ├── CO-ROUTINE (subquery-5) - │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ └── USE TEMP B-TREE FOR ORDER BY │ └── SCAN (subquery-5) @@ -712,10 +712,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) ORDER BY event_headers.created_at DESC, event_headers.id ASC - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(), @@ -725,10 +725,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) ORDER BY event_headers.created_at DESC - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(), @@ -745,10 +745,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.kind IN ("1", "1111", "10000")) ORDER BY $orderBy - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(), diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchOrderByRowIdTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchOrderByRowIdTest.kt new file mode 100644 index 0000000000..104630782c --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchOrderByRowIdTest.kt @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Contrast the two NIP-50 result orderings the contentless FTS index enables. + * The FTS rowid is `event_headers.row_id` (ingestion order), so: + * - default: `ORDER BY created_at DESC` — strict recency. + * - [IndexingStrategy.searchOrderByRowId]: `ORDER BY event_fts.rowid DESC` — + * ingestion order (early-terminating, corpus-independent). + * + * Events are inserted in an order that deliberately disagrees with their + * `created_at`, so the two orderings produce visibly different results. + */ +class SearchOrderByRowIdTest { + private val signer = NostrSignerSync() + + private fun note(createdAt: Long) = signer.sign(TextNoteEvent.build("uniqorder token body", createdAt = createdAt)) + + private fun store(rowIdOrder: Boolean) = + EventStore( + dbName = null, + indexStrategy = DefaultIndexingStrategy(searchOrderByRowId = rowIdOrder), + ) + + @Test + fun defaultOrdersByCreatedAt_rowIdFlagOrdersByIngestion() = + runBlocking { + // created_at: A newest, B oldest, C middle. Inserted A, B, C — so + // ingestion (row_id) order is A < B < C. + val byCreatedAt = store(rowIdOrder = false) + val byRowId = store(rowIdOrder = true) + try { + val a = note(300) + val b = note(100) + val c = note(200) + for (store in listOf(byCreatedAt, byRowId)) { + store.insert(a) + store.insert(b) + store.insert(c) + } + + val filter = Filter(search = "uniqorder", limit = 10) + + // created_at DESC: A(300), C(200), B(100) + assertEquals( + listOf(a.id, c.id, b.id), + byCreatedAt.query(filter).map { it.id }, + "default search must order by created_at DESC", + ) + + // rowid DESC = ingestion order reversed: C, B, A + assertEquals( + listOf(c.id, b.id, a.id), + byRowId.query(filter).map { it.id }, + "searchOrderByRowId must order by ingestion (row_id) DESC", + ) + } finally { + byCreatedAt.close() + byRowId.close() + } + } + + @Test + fun rowIdOrderStillHonorsLimitAndSecondaryFilters() = + runBlocking { + val store = store(rowIdOrder = true) + try { + val notes = (0 until 6).map { note(1_700_000_000L + it) } + notes.forEach { store.insert(it) } + + // limit slices the newest-ingested 3. + val top3 = store.query(Filter(search = "uniqorder", limit = 3)).map { it.id } + assertEquals(notes.takeLast(3).reversed().map { it.id }, top3) + + // kind filter still applies alongside rowid ordering. + val wrongKind = store.query(Filter(kinds = listOf(30023), search = "uniqorder", limit = 10)) + assertEquals(0, wrongKind.size) + } finally { + store.close() + } + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnectionTest.kt new file mode 100644 index 0000000000..cbda32511e --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnectionTest.kt @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import androidx.sqlite.SQLiteStatement +import androidx.sqlite.driver.bundled.BundledSQLiteDriver +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotSame +import kotlin.test.assertSame + +class StatementCachingConnectionTest { + private lateinit var conn: StatementCachingConnection + + private val sql = "SELECT ? AS v" + + @BeforeTest + fun setup() { + conn = StatementCachingConnection(BundledSQLiteDriver().open(":memory:")) + } + + @AfterTest + fun tearDown() { + conn.close() + } + + private fun readOne(stmt: SQLiteStatement): Long { + stmt.step() + return stmt.getLong(0) + } + + @Test + fun sequentialSameSqlReusesTheSameHandle() { + var first: SQLiteStatement? = null + conn.prepare(sql).use { stmt -> + stmt.bindLong(1, 7) + assertEquals(7, readOne(stmt)) + first = stmt + } + // Closed (returned to pool) — the next prepare of the same SQL must + // hand back the very same cached handle, not a fresh prepare. + conn.prepare(sql).use { stmt -> + assertSame(first, stmt) + stmt.bindLong(1, 9) + assertEquals(9, readOne(stmt)) + } + } + + @Test + fun concurrentSameSqlHandlesAreDistinctAndIndependent() { + // The k-way-merge shape: many identical-SQL cursors live at once. + val a = conn.prepare(sql) + val b = conn.prepare(sql) + val c = conn.prepare(sql) + assertNotSame(a, b) + assertNotSame(b, c) + assertNotSame(a, c) + + a.bindLong(1, 1) + b.bindLong(1, 2) + c.bindLong(1, 3) + // Each cursor keeps its own bindings/position even while the others + // are open — no aliasing of one native handle. + assertEquals(1, readOne(a)) + assertEquals(2, readOne(b)) + assertEquals(3, readOne(c)) + a.close() + b.close() + c.close() + + // After release, a fresh concurrent burst reuses the pooled handles. + val reused = conn.prepare(sql) + assertSame(a, reused, "pool should hand back a freed handle before preparing anew") + reused.close() + } + + @Test + fun overflowingTheGlobalCapFallsBackToUncached() { + val small = StatementCachingConnection(BundledSQLiteDriver().open(":memory:"), maxCachedStatements = 2) + try { + val live = (0 until 5).map { small.prepare(sql) } + // All five must be usable even though only two can be cached; the + // extra three are plain uncached statements. + live.forEachIndexed { i, stmt -> + stmt.bindLong(1, i.toLong()) + assertEquals(i.toLong(), readOne(stmt)) + } + live.forEach { it.close() } + } finally { + small.close() + } + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/TagMergeCorrectnessTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/TagMergeCorrectnessTest.kt new file mode 100644 index 0000000000..87db429ca2 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/TagMergeCorrectnessTest.kt @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.utils.EventFactory +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * Correctness guard for the tag-stream path of [MergeQueryExecutor]: the + * `#=[values] (+ kinds) [+ since/until] limit=N` watcher shape + * (reactions/replies). The merge must return the same newest-N a + * `SELECT DISTINCT … ORDER BY created_at DESC LIMIT N` would, deduping events + * that carry several of the queried tag values. + * + * Where `created_at` is distinct the order is fully determined and asserted + * against a Kotlin reference. Where it ties, the tag cursors can only order by + * `created_at` (event_tags has no id column), so the result is a valid + * newest-N but not id-exact — those cases assert set + size instead. + */ +class TagMergeCorrectnessTest { + private val hex = "0123456789abcdef" + + private fun mix(seed: Long): Long { + var z = seed + -0x61c8864680b583ebL + z = (z xor (z ushr 30)) * -0x40a7b892e31b1a47L + z = (z xor (z ushr 27)) * -0x6b2fb644ecceee15L + return z xor (z ushr 31) + } + + private fun hex64( + salt: Long, + index: Int, + ): String { + val out = CharArray(64) + for (w in 0 until 4) { + val v = mix(salt * 1_000_003 + index.toLong() * 4 + w) + for (b in 0 until 8) { + val byte = ((v ushr (b * 8)) and 0xFF).toInt() + out[(w * 8 + b) * 2] = hex[byte ushr 4] + out[(w * 8 + b) * 2 + 1] = hex[byte and 0xF] + } + } + return out.concatToString() + } + + private val sig = "0".repeat(128) + private var idSeq = 0 + + private fun ev( + createdAt: Long, + kind: Int, + eTags: List, + ): Event = + EventFactory.create( + hex64(7, idSeq++), + hex64(1, idSeq), + createdAt, + kind, + eTags.map { arrayOf("e", it) }.toTypedArray(), + "", + sig, + ) + + private val newestFirst = + Comparator { a, b -> + if (a.createdAt != b.createdAt) b.createdAt.compareTo(a.createdAt) else a.id.compareTo(b.id) + } + + private fun reference( + all: List, + values: Set, + kinds: Set?, + since: Long?, + until: Long?, + limit: Int, + ): List = + all + .asSequence() + .filter { e -> e.tags.any { it.size >= 2 && it[0] == "e" && it[1] in values } } + .filter { kinds == null || it.kind in kinds } + .filter { since == null || it.createdAt >= since } + .filter { until == null || it.createdAt <= until } + .sortedWith(newestFirst) + .map { it.id } + .distinct() + .take(limit) + .toList() + + private fun mergeEligible( + store: EventStore, + filter: Filter, + ): Boolean = + MergeQueryExecutor.streamCount( + with(store.store.queryBuilder) { filter.toFilterWithDTags() }, + store.store.queryBuilder.indexStrategy, + ) > 0 + + private fun newStore() = + EventStore( + dbName = null, + indexStrategy = + DefaultIndexingStrategy( + indexTagsByCreatedAtAlone = true, + useAndIndexIdOnOrderBy = true, + indexFullTextSearch = false, + ), + ) + + @Test + fun distinctCreatedAt_withKinds_matchesReference() = + runBlocking { + val store = newStore() + val notes = (0 until 6).map { hex64(2, it) } + val all = ArrayList() + var t = 1_700_000_000L + for (round in 0 until 40) { + // reactions (kind 7) and replies (kind 1) to a rotating note + all.add(ev(t++, 7, listOf(notes[round % notes.size]))) + all.add(ev(t++, 1, listOf(notes[(round + 1) % notes.size]))) + } + // Noise: kind-7 to notes NOT in the query set, and other tags. + for (i in 0 until 100) all.add(ev(t++, 7, listOf(hex64(9, i)))) + store.batchInsert(all) + + val queried = notes.take(3) + val filter = Filter(kinds = listOf(7), tags = mapOf("e" to queried), limit = 25) + assertTrue(mergeEligible(store, filter), "tag watcher must be merge-eligible") + + val merged = store.query(filter).map { it.id } + assertEquals(reference(all, queried.toSet(), setOf(7), null, null, 25), merged) + store.close() + } + + @Test + fun crossStreamDuplicate_emittedOnce() = + runBlocking { + val store = newStore() + val a = hex64(3, 0) + val b = hex64(3, 1) + val all = ArrayList() + var t = 1_700_000_000L + // Events tagging BOTH a and b — they appear in both value streams + // and must be emitted exactly once. + repeat(5) { all.add(ev(t++, 1, listOf(a, b))) } + // Events tagging only one of them. + repeat(5) { all.add(ev(t++, 1, listOf(a))) } + repeat(5) { all.add(ev(t++, 1, listOf(b))) } + store.batchInsert(all) + + val filter = Filter(kinds = listOf(1), tags = mapOf("e" to listOf(a, b)), limit = 500) + assertTrue(mergeEligible(store, filter)) + + val merged = store.query(filter).map { it.id } + assertEquals(merged.size, merged.toSet().size, "no event may be emitted twice") + assertEquals(15, merged.size, "5 both + 5 a-only + 5 b-only = 15 distinct") + assertEquals(reference(all, setOf(a, b), setOf(1), null, null, 500), merged) + store.close() + } + + @Test + fun noKinds_usesTagCreatedAtIndex() = + runBlocking { + val store = newStore() + val notes = (0 until 5).map { hex64(4, it) } + val all = ArrayList() + var t = 1_700_000_000L + for (round in 0 until 30) all.add(ev(t++, (round % 3) + 1, listOf(notes[round % notes.size]))) + for (i in 0 until 60) all.add(ev(t++, 1, listOf(hex64(8, i)))) + store.batchInsert(all) + + val queried = notes.take(3) + val filter = Filter(tags = mapOf("e" to queried), limit = 20) + assertTrue(mergeEligible(store, filter), "no-kind tag watcher must be merge-eligible with the tag index") + + val merged = store.query(filter).map { it.id } + assertEquals(reference(all, queried.toSet(), null, null, null, 20), merged) + store.close() + } + + @Test + fun withSinceAndUntil_boundsTheWindow() = + runBlocking { + val store = newStore() + val notes = (0 until 4).map { hex64(5, it) } + val all = ArrayList() + val base = 1_700_000_000L + for (i in 0 until 300) all.add(ev(base + i.toLong(), 7, listOf(notes[i % notes.size]))) + store.batchInsert(all) + + val since = base + 50 + val until = base + 250 + val filter = Filter(kinds = listOf(7), tags = mapOf("e" to notes), since = since, until = until, limit = 500) + assertTrue(mergeEligible(store, filter)) + + val merged = store.query(filter).map { it.id } + val ref = reference(all, notes.toSet(), setOf(7), since, until, 500) + assertEquals(ref, merged) + assertTrue(merged.isNotEmpty() && ref.size < 300) + store.close() + } + + @Test + fun rawPathMatchesDecoded() = + runBlocking { + val store = newStore() + val notes = (0 until 6).map { hex64(6, it) } + val all = ArrayList() + var t = 1_700_000_000L + for (round in 0 until 25) all.add(ev(t++, 7, listOf(notes[round % notes.size]))) + store.batchInsert(all) + + val filter = Filter(kinds = listOf(7), tags = mapOf("e" to notes.take(3)), limit = 15) + val decoded = store.query(filter).map { it.id } + val raw = store.store.rawQuery(filter).map { it.id } + assertEquals(decoded, raw, "the zero-decode raw path must match the decoded query") + store.close() + } + + @Test + fun tiedCreatedAt_matchesReferenceAsSet() = + runBlocking { + val store = newStore() + val notes = (0 until 4).map { hex64(10, it) } + val all = ArrayList() + // Many events share a created_at — the tag cursor can't id-order + // within a second, so assert a valid newest-N by set + size. + var t = 1_700_000_000L + for (block in 0 until 15) { + val ts = t + for (n in notes) all.add(ev(ts, 7, listOf(n))) + t += 1 + } + store.batchInsert(all) + + val filter = Filter(kinds = listOf(7), tags = mapOf("e" to notes), limit = 22) + assertTrue(mergeEligible(store, filter)) + + val merged = store.query(filter).map { it.id } + assertEquals(22, merged.size) + assertEquals(merged.size, merged.toSet().size) + // The whole result must sit within the newest slice the reference + // would return once ties are resolved either way: everything in + // `merged` must be at or above the created_at cutoff. + val ids = merged.toSet() + val chosen = all.filter { it.id in ids } + val cutoff = chosen.minOf { it.createdAt } + val eligibleAboveCutoff = all.filter { it.createdAt > cutoff }.map { it.id }.toSet() + assertTrue(eligibleAboveCutoff.all { it in ids }, "every event newer than the cutoff must be included") + store.close() + } + + @Test + fun ineligibleShapesFallThrough() = + runBlocking { + val store = newStore() + store.batchInsert(listOf(ev(1_700_000_000L, 7, listOf(hex64(2, 0))))) + + // Single value → single seek, not a merge. + assertFalse(mergeEligible(store, Filter(kinds = listOf(7), tags = mapOf("e" to listOf(hex64(2, 0))), limit = 10))) + // No limit → not merge-eligible. + assertFalse(mergeEligible(store, Filter(kinds = listOf(7), tags = mapOf("e" to listOf(hex64(2, 0), hex64(2, 1)))))) + // Authors present → covered-index seek shape, not a tag merge. + assertFalse( + mergeEligible( + store, + Filter(kinds = listOf(7), authors = listOf(hex64(1, 1)), tags = mapOf("e" to listOf(hex64(2, 0), hex64(2, 1))), limit = 10), + ), + ) + store.close() + } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FollowFeedReadBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FollowFeedReadBenchmark.kt index 15fb244d82..40bd402cba 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FollowFeedReadBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FollowFeedReadBenchmark.kt @@ -202,7 +202,7 @@ class FollowFeedReadBenchmark { runBlocking { store.store.pool.useReader { c -> var n = 0 - MergeQueryExecutor.run(c, filter, store.store.queryBuilder.indexStrategy) { n++ } + MergeQueryExecutor.run(c, filter, store.store.queryBuilder.indexStrategy, store.store.seedModule::hasher) { n++ } n } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt new file mode 100644 index 0000000000..e8398925a3 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.prodbench + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.utils.EventFactory +import kotlinx.coroutines.runBlocking +import kotlin.test.Test + +/** + * Isolates the NIP-50 search-scaling curve the scale-curve report flagged + * (FTS5 falling ~18× from 25k→400k events) and the two levers against it: + * + * 1. **segment compaction** (`INSERT INTO event_fts(event_fts) VALUES + * ('optimize')`). Incremental inserts leave the index as many small + * segments and a MATCH queries every one; measured fragmented vs + * optimized. + * 2. **rowid ordering** ([DefaultIndexingStrategy.searchOrderByRowId]). + * `ORDER BY created_at DESC` must materialize + sort *every* document + * matching the term (cost grows with the corpus); `ORDER BY + * event_fts.rowid DESC LIMIT n` early-terminates — O(limit). + * + * The seed injects a common term into ~1% of events, so the match set — and + * thus the created_at sort — grows with the corpus while the limit stays 50. + * + * Size with `-DftsBenchScale=N` (default 1). Not an assertion test; run + * explicitly and read stdout. + */ +class FtsSearchScalingBenchmark { + companion object { + val SCALE = System.getProperty("ftsBenchScale")?.toInt() ?: 1 + val SIZES = listOf(50_000, 100_000, 200_000).map { it * SCALE } + const val NEEDLE = "zzneedle" + } + + private val hex = "0123456789abcdef" + + private fun mix(seed: Long): Long { + var z = seed + -0x61c8864680b583ebL + z = (z xor (z ushr 30)) * -0x40a7b892e31b1a47L + z = (z xor (z ushr 27)) * -0x6b2fb644ecceee15L + return z xor (z ushr 31) + } + + private fun hex64(index: Int): String { + val out = CharArray(64) + for (w in 0 until 4) { + val v = mix(index.toLong() * 4 + w + 99) + for (b in 0 until 8) { + val byte = ((v ushr (b * 8)) and 0xFF).toInt() + out[(w * 8 + b) * 2] = hex[byte ushr 4] + out[(w * 8 + b) * 2 + 1] = hex[byte and 0xF] + } + } + return String(out) + } + + private val vocab = (0 until 400).map { "word$it" } + private val sig = "0".repeat(128) + + private fun seed(n: Int): List { + val base = 1_700_000_000L + val events = ArrayList(n) + for (i in 0 until n) { + val r = mix(i.toLong()) + val content = + buildString { + for (w in 0 until 8) append(vocab[((r ushr (w * 3)) and 0x1FF).toInt() % vocab.size]).append(' ') + // ~1% carry the searched term. + if (i % 100 == 0) append(NEEDLE) + } + events.add(EventFactory.create(hex64(i), hex64(i % 5000), base + i.toLong(), 1, emptyArray(), content, sig)) + } + return events + } + + @Test + fun searchScaling() = + runBlocking { + println("─ FtsSearchScalingBenchmark (scale=$SCALE, limit=50) ─") + println(" %-9s %14s %14s %14s".format("corpus", "createdAt/frag", "createdAt/opt", "rowid/opt")) + for (size in SIZES) { + val events = seed(size) + + val createdAt = EventStore(dbName = null, indexStrategy = DefaultIndexingStrategy()) + val rowId = EventStore(dbName = null, indexStrategy = DefaultIndexingStrategy(searchOrderByRowId = true)) + try { + // Insert in 10k chunks → many FTS segments (fragmented). + events.chunked(10_000).forEach { + createdAt.batchInsert(it) + rowId.batchInsert(it) + } + + val f = Filter(search = NEEDLE, limit = 50) + val frag = time(createdAt, f) + + createdAt.store.reindexFullTextSearch() // rebuild + optimize() + rowId.store.reindexFullTextSearch() + val optCreatedAt = time(createdAt, f) + val optRowId = time(rowId, f) + + println( + " %-9s %12.2f ms %12.2f ms %12.2f ms".format( + if (size >= 1000) "${size / 1000}k" else "$size", + frag, + optCreatedAt, + optRowId, + ), + ) + } finally { + createdAt.close() + rowId.close() + } + } + } + + private suspend fun time( + store: EventStore, + filter: Filter, + ): Double { + repeat(3) { store.query(filter) } + val runs = 20 + val start = System.nanoTime() + repeat(runs) { store.query(filter) } + return (System.nanoTime() - start) / 1e6 / runs + } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ContentlessFtsMigrationTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ContentlessFtsMigrationTest.kt new file mode 100644 index 0000000000..27de19bfde --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ContentlessFtsMigrationTest.kt @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import androidx.sqlite.SQLiteConnection +import androidx.sqlite.driver.bundled.BundledSQLiteDriver +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.deleteIfExists +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Verifies the real v4 → v5 FTS upgrade path: a database written with the old + * `fts5(event_header_row_id, content)` index (auto-assigned rowid, content + * stored) must, on next open, drop that table and rebuild a contentless index + * keyed by `event_headers.row_id`, with search intact. + * + * The v4 state is fabricated by opening a fresh v5 store, then rewriting its + * `event_fts` to the old schema (with deliberately wrong content, to prove the + * rebuild wipes it) and stamping `user_version = 4`. + */ +class ContentlessFtsMigrationTest { + private val signer = NostrSignerSync() + private lateinit var dbFile: Path + + private fun path() = dbFile.toAbsolutePath().toString() + + @BeforeTest + fun setup() { + Secp256k1Instance + dbFile = Files.createTempFile("contentless-fts-migration-", ".db") + Files.deleteIfExists(dbFile) + } + + @AfterTest + fun tearDown() { + listOf("", "-wal", "-shm", "-journal").forEach { Path.of(dbFile.toString() + it).deleteIfExists() } + } + + @Test + fun upgradesOldFtsSchemaAndRebuildsSearch() = + runBlocking { + val alpha = signer.sign(TextNoteEvent.build("uniqalpha searchable body", createdAt = 1_700_000_000L)) + val beta = signer.sign(TextNoteEvent.build("uniqbeta searchable body", createdAt = 1_700_000_100L)) + + // 1. Fresh v5 store, seed events, close. + EventStore(dbName = path(), relay = null).also { + it.insert(alpha) + it.insert(beta) + it.close() + } + + // 2. Rewrite event_fts to the pre-v5 schema with WRONG content and + // downgrade user_version to 4 — the state a v4 database is in. + BundledSQLiteDriver().open(path()).use { db -> + db.exec("DROP TRIGGER IF EXISTS fts_foreign_key") + db.exec("DROP TABLE IF EXISTS event_fts") + db.exec("CREATE VIRTUAL TABLE event_fts USING fts5(event_header_row_id, content)") + db.exec( + """ + CREATE TRIGGER fts_foreign_key AFTER DELETE ON event_headers FOR EACH ROW + BEGIN DELETE FROM event_fts WHERE old.row_id = event_fts.event_header_row_id; END + """.trimIndent(), + ) + // Stale/garbage rows: a real v4 index would hold correct data, + // but seeding garbage proves the migration rebuilds from + // event_headers rather than trusting the old table. + db.exec("INSERT INTO event_fts(event_header_row_id, content) VALUES (1, 'uniqstale garbage')") + db.exec("PRAGMA user_version = 4") + } + + // 3. Reopen with current code → onUpgrade(4→5) → migrateV4ToContentless. + val store = EventStore(dbName = path(), relay = null) + try { + // Rebuilt from event_headers: real content is searchable... + assertEquals(alpha.id, store.query(Filter(search = "uniqalpha")).single().id) + assertEquals(beta.id, store.query(Filter(search = "uniqbeta")).single().id) + // ...and the old garbage is gone. + assertTrue(store.query(Filter(search = "uniqstale")).isEmpty()) + + // The new rowid IS event_headers.row_id: the join returns the + // right event for each FTS rowid. + store.store.pool.useReader { db -> + db + .prepare( + "SELECT h.id FROM event_fts f JOIN event_headers h ON h.row_id = f.rowid ORDER BY f.rowid", + ).use { stmt -> + val ids = ArrayList() + while (stmt.step()) ids.add(stmt.getText(0)) + assertEquals(listOf(alpha.id, beta.id), ids, "FTS rowid must map to event_headers.row_id") + } + } + + // The rebuilt delete trigger still cleans up FTS on delete. + store.store.delete(beta.id) + assertTrue(store.query(Filter(search = "uniqbeta")).isEmpty()) + assertEquals(alpha.id, store.query(Filter(search = "uniqalpha")).single().id) + } finally { + store.close() + } + } + + private fun SQLiteConnection.exec(sql: String) = prepare(sql).use { it.step() } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/Fts5CapabilityProbe.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/Fts5CapabilityProbe.kt new file mode 100644 index 0000000000..23f9e54125 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/Fts5CapabilityProbe.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import androidx.sqlite.driver.bundled.BundledSQLiteDriver +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Pins the FTS5 features the [FullTextSearchModule] contentless index depends + * on to the bundled SQLite. All three shipped between SQLite 3.43 and 3.44 + * (2023); if the bundled driver is ever downgraded below that, this fails + * loudly instead of the store silently breaking search on delete. + * + * - `content=''` **contentless** table with `contentless_delete=1`: lets the + * index drop the duplicated content column yet still delete rows (the + * `fts_foreign_key` trigger needs it). + * - explicit `rowid` on insert + `ORDER BY rowid DESC LIMIT n`: the + * early-terminating recency search path. + * - `'merge'` / `'optimize'` maintenance commands: segment compaction. + */ +class Fts5CapabilityProbe { + @Test + fun contentlessDeleteAndRowidOrderingAreSupported() { + val db = BundledSQLiteDriver().open(":memory:") + try { + db.execSQL("CREATE VIRTUAL TABLE cl USING fts5(content, content='', contentless_delete=1)") + db.execSQL("INSERT INTO cl(rowid, content) VALUES (100, 'hello world')") + db.execSQL("INSERT INTO cl(rowid, content) VALUES (50, 'hello there')") + db.execSQL("INSERT INTO cl(rowid, content) VALUES (200, 'hello again')") + db.execSQL("DELETE FROM cl WHERE rowid = 50") + + // Deleting an absent rowid must be a harmless no-op: the store's + // fts_foreign_key trigger fires on EVERY event_headers delete, but + // only searchable events ever got an FTS row. + db.execSQL("DELETE FROM cl WHERE rowid = 999999") + + val order = ArrayList() + db.prepare("SELECT rowid FROM cl WHERE cl MATCH 'hello' ORDER BY rowid DESC LIMIT 5").use { + while (it.step()) order.add(it.getLong(0)) + } + // Deleted 50 is gone; the rest come back newest-rowid first. + assertEquals(listOf(200L, 100L), order) + } finally { + db.close() + } + } + + @Test + fun segmentMergeAndOptimizeAreSupported() { + val db = BundledSQLiteDriver().open(":memory:") + try { + db.execSQL("CREATE VIRTUAL TABLE m USING fts5(content)") + db.execSQL("INSERT INTO m(rowid, content) VALUES (1, 'a b c')") + db.execSQL("INSERT INTO m(rowid, content) VALUES (2, 'd e f')") + // Bounded incremental merge, then a full optimize — both must parse + // and run without error on the bundled build. + db.execSQL("INSERT INTO m(m, rank) VALUES ('merge', -16)") + db.execSQL("INSERT INTO m(m) VALUES ('optimize')") + + val hits = ArrayList() + db.prepare("SELECT rowid FROM m WHERE m MATCH 'e' ORDER BY rowid").use { + while (it.step()) hits.add(it.getLong(0)) + } + assertEquals(listOf(2L), hits) + } finally { + db.close() + } + } +} From e6a8f80826c486e8752bb4ac34eedec0d622c8dd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 22:05:30 +0000 Subject: [PATCH 16/34] fix(store): drop non-compliant rowid search ordering, keep FTS delete/size wins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit searchOrderByRowId ordered NIP-50 search by the FTS rowid (ingestion order) to get O(limit) search, but NIP-01's limit requires the newest events by created_at. Once ingestion diverges from created_at (any historical sync) that returns the wrong events under a limit — a spec violation — so the flag, its QueryBuilder branch, and its test are removed. Search stays created_at-ordered; corpus-independent search is an external-engine job, not this index. The contentless + rowid=row_id schema stays for the reasons that don't touch ordering: the delete trigger now seeks by rowid (O(log n)) instead of scanning by an FTS column (O(n)) — measured ~78× faster at 8k rows and widening, on a path every deletion hits — and the index is smaller. Benchmark reframed around the delete win and the honest (unchanged) search cost; plan doc updated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XqGuBuSUsRudGerPqDuoKA --- .../plans/2026-07-21-sqlite-query-scaling.md | 98 +++++++------- .../store/sqlite/FullTextSearchModule.kt | 29 ++-- .../store/sqlite/IndexingStrategy.kt | 27 ---- .../nip01Core/store/sqlite/QueryBuilder.kt | 15 +-- .../store/sqlite/SearchOrderByRowIdTest.kt | 109 --------------- .../prodbench/FtsSearchScalingBenchmark.kt | 126 ++++++++++++------ 6 files changed, 156 insertions(+), 248 deletions(-) delete mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchOrderByRowIdTest.kt diff --git a/quartz/plans/2026-07-21-sqlite-query-scaling.md b/quartz/plans/2026-07-21-sqlite-query-scaling.md index d8a6860bf9..704843cb92 100644 --- a/quartz/plans/2026-07-21-sqlite-query-scaling.md +++ b/quartz/plans/2026-07-21-sqlite-query-scaling.md @@ -7,63 +7,69 @@ ingest 16×, author-timeline 13×, follow-feed 2.4× — while point reads staye flat. Two of those (author-timeline, follow-feed) were already addressed (the pubkey-alone index and `MergeQueryExecutor`) and mostly reflect running the report under the *client* `DefaultIndexingStrategy` rather than -`relayIndexingStrategy()`. This change takes the two structural read shapes -that were still corpus-bound: **NIP-50 search** and the **large-IN tag -watcher**, plus a statement-cache fix the merge paths needed. +`relayIndexingStrategy()`. This change adds the **large-IN tag watcher** to the +merge executor, fixes the **FTS delete path** (which degraded with corpus +size) and shrinks the FTS index, and fixes a **statement-cache** miss the merge +paths hit. + +It does **not** fix NIP-50 *search* latency: that is bounded by the +`created_at`-ordered sort over all matches, which FTS5 can't early-terminate +while staying NIP-01-compliant (see §1). Corpus-independent search is an +external-engine job. Everything here is read/size work; the write path and on-disk index set are -unchanged except the FTS table, which gets *smaller*. +unchanged except the FTS table, which gets *smaller* and deletes faster. -## 1. NIP-50 search — contentless FTS5 + optional rowid ordering + segment merge +## 1. FTS — contentless index (fast deletes + smaller), NOT a search-scaling fix The old index was `fts5(event_header_row_id, content)`: it stored a second -copy of the tokenized text, and its rowid was auto-assigned (unrelated to the -event). Search ran `… event_fts MATCH ? ORDER BY event_headers.created_at DESC -LIMIT n`, which must materialize **every** document matching the term and sort -it — cost grows with the whole corpus. That is the 18× curve. +copy of the tokenized text, its rowid was auto-assigned (unrelated to the +event), and — the real problem — the `fts_foreign_key` delete trigger deleted +by the `event_header_row_id` *column*, which FTS5 cannot seek, so it **scanned +the whole index per delete**. -Three changes (`FullTextSearchModule`, `IndexingStrategy`, `QueryBuilder`, -DB version 4→5): +Changes (`FullTextSearchModule`, `QueryBuilder`, DB version 4→5): +- **rowid = `event_headers.row_id`.** Set explicitly on every insert; the + delete trigger, reindex, and catch-up paths key off it. Deletes become an + O(log n) primary-key seek instead of an O(n) column scan. This is the win: + every event removal fires the trigger — replaceable rotation, kind-5, + expiration, right-to-vanish — so on the old schema deletion throughput + degraded with corpus size. - **Contentless** (`fts5(content, content='', contentless_delete=1)`). The indexed text is *derived* (`SearchableEvent.indexableContent()`, not a raw column), so FTS5 **external-content** — which re-reads the source column from the base table — cannot express it; **contentless** is the correct primitive. - It drops the stored content copy (index shrinks) and, with - `contentless_delete=1`, still supports the `fts_foreign_key` delete trigger. -- **rowid = `event_headers.row_id`.** Set explicitly on every insert. The - delete trigger and reindex/catch-up paths all key off it, so the join is - `event_headers.row_id = event_fts.rowid` with no stored linkage column. -- **`searchOrderByRowId`** (new `IndexingStrategy` flag, **default off**). When - on, simple search orders by `event_fts.rowid DESC LIMIT n`, which FTS5 - early-terminates off the doclist — **O(limit)**, corpus-independent. rowid is - *ingestion* order, so this ≈ recency only while events arrive in time order; - a relay that bulk-syncs history (NIP-77) diverges, which is why it is off by - default and left off for geode. Only the simple-search shape changes; tag∩ - search and the negentropy snapshot keep `created_at`. -- **Segment compaction.** `reindexAll` finishes with a full `'optimize'`, and - the periodic `SQLiteEventStore.optimize()` (geode's maintenance tick) folds - in a bounded `'merge'`, so incremental / deferred-catch-up inserts don't - leave the index as many small segments. + It drops the stored content copy (index shrinks) and `contentless_delete=1` + keeps the delete trigger working. +- **Segment compaction.** `reindexAll` finishes with `'optimize'`, and the + periodic `SQLiteEventStore.optimize()` (geode's maintenance tick) folds in a + bounded `'merge'`, so incremental / deferred-catch-up inserts don't leave the + index as many small segments. -Measured — `FtsSearchScalingBenchmark` (jvmTest prodbench), search a term in -~1% of events, `limit=50`, in-memory: +Delete cost — `FtsSearchScalingBenchmark.deleteByColumnVsByRowid`, 500 deletes, +in-memory: -| corpus | created_at, fragmented | created_at, optimized | **rowid, optimized** | -|---|---:|---:|---:| -| 50k | 1.92 ms | 1.16 ms | **0.26 ms** | -| 100k | 2.16 ms | 2.23 ms | **0.22 ms** | -| 200k | 4.33 ms | 4.26 ms | **0.22 ms** | +| rows | by column (old) | by rowid (new) | +|---|---:|---:| +| 2k | 91.8 ms | 6.6 ms | +| 8k | 362.0 ms | 4.7 ms | -- **rowid ordering is the corpus-independence lever**: flat ~0.22 ms vs - created_at's 1.16 → 4.26 ms (grows with the match set) — ~19× at 200k, and - the gap keeps widening. This is the direct answer to the search curve, for - deployments that accept ingestion-order results. -- **segment `optimize`** helps most at small sizes / right after bulk - incremental inserts (1.92 → 1.16 ms at 50k); FTS5 `automerge` already caps - fragmentation in steady state, so at 100k/200k here it is within noise. It is - a cheap, unconditional safety net (biggest for the deferred relay path, whose - catch-up commits in 1k batches), not the main lever. +By-column grows ~linearly with the table (O(n)/delete); by-rowid is flat — +~78× at 8k rows and widening. + +**Search is deliberately unchanged and still `created_at`-ordered.** NIP-01's +`limit` requires the newest events *by `created_at`*, and FTS5 only +early-terminates on its own rowid — so `MATCH … ORDER BY created_at DESC LIMIT +n` still materializes and sorts every match, and search latency still grows +with the match set (the report's 18× curve). An earlier draft added a +`searchOrderByRowId` flag that ordered by the FTS rowid to get O(limit) search; +that is *ingestion* order, which returns the wrong events under a limit once +ingestion diverges from `created_at` (any historical sync) — a NIP-01 +violation — so it was removed. `optimize` compacts the index but does not +change the asymptotics (measured within noise at 100k/200k). Corpus-independent +search is genuinely an external-engine job (the Vespa side of the report), not +this index. Migration (v4→v5): the old rowids can't be remapped, so `event_fts` is dropped and rebuilt — synchronous stores rebuild in the upgrade transaction (client @@ -118,6 +124,6 @@ reuse, and cap overflow → uncached fallback. whose asserted EXPLAIN output updated for the new join column (`event_fts.rowid`) and the contentless table's virtual-index marker (`0:M2`→`0:M1`). -- Defaults preserved: `searchOrderByRowId` off, so existing deployments keep - exact `created_at DESC` search ordering; the size + optimize wins are - unconditional. +- Search ordering unchanged: still exact `created_at DESC` (NIP-01 `limit` + semantics); the delete + size + optimize wins are unconditional and + spec-neutral. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index c913e1f9eb..e61f04e1ca 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -35,17 +35,26 @@ import com.vitorpamplona.quartz.utils.EventFactory * `contentless_delete=1`) whose `rowid` is the event's * `event_headers.row_id`. Two consequences: * + * - **Deletes.** The `fts_foreign_key` trigger deletes by `rowid` (an FTS5 + * primary-key seek, O(log n)). The old `fts5(event_header_row_id, + * content)` schema deleted by a *regular* column, which FTS5 cannot seek — + * it scans the whole index per delete (O(n)), so deletion throughput + * degraded with corpus size. Every event removal fires this trigger + * (replaceable rotation, kind-5, expiration, right-to-vanish), so the + * seek matters. Measured `Fts5CapabilityProbe`/`FtsSearchScalingBenchmark`: + * ~78× at 8k rows and widening with the table. * - **Size.** A contentless table stores only the inverted index, not a - * second copy of the tokenized text (the old `fts5(event_header_row_id, - * content)` schema kept the content verbatim). The indexed text is - * *derived* ([SearchableEvent.indexableContent], not any raw column), so - * FTS5 external-content — which reads the source column from the base - * table — cannot express it; contentless is the correct primitive. - * - **Ordering.** With `rowid = event_headers.row_id`, - * `ORDER BY event_fts.rowid DESC LIMIT n` early-terminates off the FTS - * doclist ([IndexingStrategy.searchOrderByRowId]); the default - * `created_at DESC` ordering still works via the join back to - * `event_headers` but must sort all matches. + * second copy of the tokenized text. The indexed text is *derived* + * ([SearchableEvent.indexableContent], not any raw column), so FTS5 + * external-content — which reads the source column from the base table — + * cannot express it; contentless is the correct primitive. + * + * Search still orders by `event_headers.created_at DESC` (NIP-01 `limit` + * semantics: the newest events *by created_at*), joining back to + * `event_headers` on `row_id = event_fts.rowid`. FTS5 cannot early-terminate + * that — it materializes and sorts all matches — so search latency still + * grows with the match set; corpus-independent search needs an external + * engine, not this index. * * When [enabled] is `false` the module becomes an inert no-op: no * `event_fts` virtual table and no `fts_foreign_key` delete trigger are diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt index ea76cf181d..01362e03fb 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt @@ -131,32 +131,6 @@ interface IndexingStrategy { */ val indexFullTextSearch: Boolean - /** - * Order NIP-50 search results by the FTS index rowid (= event row_id, - * i.e. **ingestion order**) instead of `created_at DESC`. - * - * The `event_fts` rowid is the event's `event_headers.row_id`, so - * `ORDER BY event_fts.rowid DESC LIMIT n` walks the FTS doclist - * newest-rowid-first and stops at the limit — **O(limit)** — where the - * default `ORDER BY created_at DESC` must materialize *every* document - * matching the term and sort it (cost grows with the whole corpus; the - * NIP-50 curve that falls ~18× from 25k→400k events). The unconditional - * contentless-index + segment-merge changes already shrink and speed the - * default path; this flag is the one that makes search cost independent - * of corpus size. - * - * The trade-off is semantic: ingestion order ≈ `created_at` order only - * while events arrive roughly in time order. A relay that bulk-syncs - * historical events (NIP-77) ingests old events late, so "newest rowid" - * and "newest created_at" diverge during/after a sync. Leave it **off** - * (the default) where strict recency ordering matters; turn it **on** - * for stores that want corpus-independent search latency and accept - * recently-ingested-first ordering. Only affects the simple-search query - * shape (`search [+ kinds/authors] + limit`); tag∩search and the - * negentropy snapshot path keep `created_at` ordering. - */ - val searchOrderByRowId: Boolean get() = false - /** * Maintain an always-current in-memory `(created_at, id)` set (a * [com.vitorpamplona.quartz.nip77Negentropy.LiveNegentropyIndex]) so @@ -186,7 +160,6 @@ class DefaultIndexingStrategy( override val useAndIndexIdOnOrderBy: Boolean = false, override val indexFullTextSearch: Boolean = true, override val deferFullTextSearchIndexing: Boolean = false, - override val searchOrderByRowId: Boolean = false, override val maintainLiveNegentropyIndex: Boolean = false, ) : IndexingStrategy { override fun shouldIndex( diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 5af41c7684..8ad16c5d4a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -1005,18 +1005,9 @@ class QueryBuilder( if (clause.conditions.isNotEmpty()) { append("\nWHERE ${clause.conditions}") } - if (indexStrategy.searchOrderByRowId) { - // The FTS rowid is event_headers.row_id (ingestion order). - // Ordering by it lets FTS5 walk the doclist newest-first - // and stop at LIMIT — O(limit) — instead of materializing - // and sorting every match by created_at. See - // IndexingStrategy.searchOrderByRowId for the trade-off. - append("\nORDER BY ${fts.tableName}.rowid DESC") - } else { - append("\nORDER BY event_headers.created_at DESC") - if (indexStrategy.useAndIndexIdOnOrderBy) { - append(", event_headers.id ASC") - } + append("\nORDER BY event_headers.created_at DESC") + if (indexStrategy.useAndIndexIdOnOrderBy) { + append(", event_headers.id ASC") } if (limit != null) { append("\nLIMIT ") diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchOrderByRowIdTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchOrderByRowIdTest.kt deleted file mode 100644 index 104630782c..0000000000 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchOrderByRowIdTest.kt +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store.sqlite - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync -import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent -import kotlinx.coroutines.runBlocking -import kotlin.test.Test -import kotlin.test.assertEquals - -/** - * Contrast the two NIP-50 result orderings the contentless FTS index enables. - * The FTS rowid is `event_headers.row_id` (ingestion order), so: - * - default: `ORDER BY created_at DESC` — strict recency. - * - [IndexingStrategy.searchOrderByRowId]: `ORDER BY event_fts.rowid DESC` — - * ingestion order (early-terminating, corpus-independent). - * - * Events are inserted in an order that deliberately disagrees with their - * `created_at`, so the two orderings produce visibly different results. - */ -class SearchOrderByRowIdTest { - private val signer = NostrSignerSync() - - private fun note(createdAt: Long) = signer.sign(TextNoteEvent.build("uniqorder token body", createdAt = createdAt)) - - private fun store(rowIdOrder: Boolean) = - EventStore( - dbName = null, - indexStrategy = DefaultIndexingStrategy(searchOrderByRowId = rowIdOrder), - ) - - @Test - fun defaultOrdersByCreatedAt_rowIdFlagOrdersByIngestion() = - runBlocking { - // created_at: A newest, B oldest, C middle. Inserted A, B, C — so - // ingestion (row_id) order is A < B < C. - val byCreatedAt = store(rowIdOrder = false) - val byRowId = store(rowIdOrder = true) - try { - val a = note(300) - val b = note(100) - val c = note(200) - for (store in listOf(byCreatedAt, byRowId)) { - store.insert(a) - store.insert(b) - store.insert(c) - } - - val filter = Filter(search = "uniqorder", limit = 10) - - // created_at DESC: A(300), C(200), B(100) - assertEquals( - listOf(a.id, c.id, b.id), - byCreatedAt.query(filter).map { it.id }, - "default search must order by created_at DESC", - ) - - // rowid DESC = ingestion order reversed: C, B, A - assertEquals( - listOf(c.id, b.id, a.id), - byRowId.query(filter).map { it.id }, - "searchOrderByRowId must order by ingestion (row_id) DESC", - ) - } finally { - byCreatedAt.close() - byRowId.close() - } - } - - @Test - fun rowIdOrderStillHonorsLimitAndSecondaryFilters() = - runBlocking { - val store = store(rowIdOrder = true) - try { - val notes = (0 until 6).map { note(1_700_000_000L + it) } - notes.forEach { store.insert(it) } - - // limit slices the newest-ingested 3. - val top3 = store.query(Filter(search = "uniqorder", limit = 3)).map { it.id } - assertEquals(notes.takeLast(3).reversed().map { it.id }, top3) - - // kind filter still applies alongside rowid ordering. - val wrongKind = store.query(Filter(kinds = listOf(30023), search = "uniqorder", limit = 10)) - assertEquals(0, wrongKind.size) - } finally { - store.close() - } - } -} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt index e8398925a3..8a97974d04 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt @@ -20,6 +20,8 @@ */ package com.vitorpamplona.quartz.nip01Core.relay.prodbench +import androidx.sqlite.SQLiteConnection +import androidx.sqlite.driver.bundled.BundledSQLiteDriver import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy @@ -29,23 +31,24 @@ import kotlinx.coroutines.runBlocking import kotlin.test.Test /** - * Isolates the NIP-50 search-scaling curve the scale-curve report flagged - * (FTS5 falling ~18× from 25k→400k events) and the two levers against it: + * Two measurements around the contentless FTS index, motivating the v4→v5 + * schema change and being honest about what it does and does not fix. * - * 1. **segment compaction** (`INSERT INTO event_fts(event_fts) VALUES - * ('optimize')`). Incremental inserts leave the index as many small - * segments and a MATCH queries every one; measured fragmented vs - * optimized. - * 2. **rowid ordering** ([DefaultIndexingStrategy.searchOrderByRowId]). - * `ORDER BY created_at DESC` must materialize + sort *every* document - * matching the term (cost grows with the corpus); `ORDER BY - * event_fts.rowid DESC LIMIT n` early-terminates — O(limit). + * 1. **Delete scaling — the win.** The old `fts5(event_header_row_id, + * content)` schema had the `fts_foreign_key` trigger delete by a regular + * FTS column, which FTS5 cannot seek (it scans, O(n) per delete). The + * contentless schema keys deletes off the rowid (= event_headers.row_id), + * an O(log n) primary-key seek. Every event removal fires this trigger + * (replaceable rotation, kind-5, expiration, right-to-vanish). + * 2. **Search scaling — the limit.** `MATCH … ORDER BY created_at DESC + * LIMIT n` must materialize + sort *every* matching document (FTS5 only + * early-terminates on its own rowid, and NIP-01's `limit` requires newest + * *by created_at*), so search cost grows with the match set. Segment + * `optimize` compacts the index but does not change that; corpus- + * independent search needs an external engine. Shown fragmented vs + * optimized to size the (secondary) compaction effect. * - * The seed injects a common term into ~1% of events, so the match set — and - * thus the created_at sort — grows with the corpus while the limit stays 50. - * - * Size with `-DftsBenchScale=N` (default 1). Not an assertion test; run - * explicitly and read stdout. + * Size search with `-DftsBenchScale=N` (default 1). Not an assertion test. */ class FtsSearchScalingBenchmark { companion object { @@ -95,42 +98,77 @@ class FtsSearchScalingBenchmark { return events } + private inline fun timeMs(block: () -> Unit): Double { + val start = System.nanoTime() + block() + return (System.nanoTime() - start) / 1e6 + } + + private fun SQLiteConnection.exec(sql: String) = prepare(sql).use { it.step() } + + @Test + fun deleteByColumnVsByRowid() { + // Old-schema (delete by FTS column) vs contentless (delete by rowid), + // 500 deletes at two table sizes. By-column should grow with the + // table; by-rowid should stay flat. + println("─ FtsSearchScalingBenchmark.delete (500 deletes) ─") + println(" %-9s %14s %14s".format("rows", "byColumn", "byRowid")) + for (n in listOf(2_000 * SCALE, 8_000 * SCALE)) { + val db = BundledSQLiteDriver().open(":memory:") + try { + db.exec("CREATE VIRTUAL TABLE col USING fts5(event_header_row_id, content)") + db.exec("CREATE VIRTUAL TABLE row USING fts5(content, content='', contentless_delete=1)") + for (i in 1..n) { + db.prepare("INSERT INTO col(event_header_row_id, content) VALUES (?, 'alpha beta gamma')").use { + it.bindLong(1, i.toLong()) + it.step() + } + db.prepare("INSERT INTO row(rowid, content) VALUES (?, 'alpha beta gamma')").use { + it.bindLong(1, i.toLong()) + it.step() + } + } + val byColumn = + timeMs { + for (i in 1..500) { + db.prepare("DELETE FROM col WHERE event_header_row_id = ?").use { + it.bindLong(1, i.toLong()) + it.step() + } + } + } + val byRowid = + timeMs { + for (i in 1..500) { + db.prepare("DELETE FROM row WHERE rowid = ?").use { + it.bindLong(1, i.toLong()) + it.step() + } + } + } + println(" %-9s %11.2f ms %11.2f ms".format("${n / 1000}k", byColumn, byRowid)) + } finally { + db.close() + } + } + } + @Test fun searchScaling() = runBlocking { - println("─ FtsSearchScalingBenchmark (scale=$SCALE, limit=50) ─") - println(" %-9s %14s %14s %14s".format("corpus", "createdAt/frag", "createdAt/opt", "rowid/opt")) + println("─ FtsSearchScalingBenchmark.search (created_at DESC, limit=50) ─") + println(" %-9s %16s %16s".format("corpus", "fragmented", "optimized")) for (size in SIZES) { - val events = seed(size) - - val createdAt = EventStore(dbName = null, indexStrategy = DefaultIndexingStrategy()) - val rowId = EventStore(dbName = null, indexStrategy = DefaultIndexingStrategy(searchOrderByRowId = true)) + val store = EventStore(dbName = null, indexStrategy = DefaultIndexingStrategy()) try { - // Insert in 10k chunks → many FTS segments (fragmented). - events.chunked(10_000).forEach { - createdAt.batchInsert(it) - rowId.batchInsert(it) - } - + seed(size).chunked(10_000).forEach { store.batchInsert(it) } val f = Filter(search = NEEDLE, limit = 50) - val frag = time(createdAt, f) - - createdAt.store.reindexFullTextSearch() // rebuild + optimize() - rowId.store.reindexFullTextSearch() - val optCreatedAt = time(createdAt, f) - val optRowId = time(rowId, f) - - println( - " %-9s %12.2f ms %12.2f ms %12.2f ms".format( - if (size >= 1000) "${size / 1000}k" else "$size", - frag, - optCreatedAt, - optRowId, - ), - ) + val frag = time(store, f) + store.store.reindexFullTextSearch() // rebuild + optimize() + val opt = time(store, f) + println(" %-9s %13.2f ms %13.2f ms".format("${size / 1000}k", frag, opt)) } finally { - createdAt.close() - rowId.close() + store.close() } } } From 8e7b1b63be7e3eda8ce5da220c6e9bb6a311bdf1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 22:21:54 +0000 Subject: [PATCH 17/34] fix(store): order NIP-50 search by relevance (bm25), not created_at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NIP-50: results are returned "in descending order by quality of search result ... not by the usual .created_at", with the limit applied after the score. The store sorted search by created_at DESC (pre-existing), so it returned the newest matches rather than the best ones. makeSimpleSearch (the search [+ kinds/authors/since/until] + limit shape) now orders by FTS5 bm25 (ORDER BY event_fts.rank, created_at DESC as a tie-break). Verified bm25 rank works on the contentless table through the join, and that a stronger-but-older match outranks a weaker-but-newer one. The rarer search+specific-tag shape and the negentropy snapshot still sort by created_at (the row-id subquery can't carry rank; negentropy is a sync set) — documented. This is a correctness fix, not a scaling one: bm25 scores every match, so search latency still grows with the match set. Tests: SearchRelevanceOrderTest, Fts5CapabilityProbe.bm25RankWorksOnContentlessTableInAJoin; QueryAssemblerTest search plans updated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XqGuBuSUsRudGerPqDuoKA --- .../plans/2026-07-21-sqlite-query-scaling.md | 46 +++++---- .../store/sqlite/FullTextSearchModule.kt | 16 ++-- .../nip01Core/store/sqlite/QueryBuilder.kt | 8 +- .../store/sqlite/QueryAssemblerTest.kt | 6 +- .../store/sqlite/SearchRelevanceOrderTest.kt | 93 +++++++++++++++++++ .../prodbench/FtsSearchScalingBenchmark.kt | 13 ++- .../store/sqlite/Fts5CapabilityProbe.kt | 43 ++++++++- 7 files changed, 186 insertions(+), 39 deletions(-) create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt diff --git a/quartz/plans/2026-07-21-sqlite-query-scaling.md b/quartz/plans/2026-07-21-sqlite-query-scaling.md index 704843cb92..a6b9c59afc 100644 --- a/quartz/plans/2026-07-21-sqlite-query-scaling.md +++ b/quartz/plans/2026-07-21-sqlite-query-scaling.md @@ -9,13 +9,13 @@ flat. Two of those (author-timeline, follow-feed) were already addressed the report under the *client* `DefaultIndexingStrategy` rather than `relayIndexingStrategy()`. This change adds the **large-IN tag watcher** to the merge executor, fixes the **FTS delete path** (which degraded with corpus -size) and shrinks the FTS index, and fixes a **statement-cache** miss the merge -paths hit. +size) and shrinks the FTS index, fixes **NIP-50 search ordering** (it was +sorting by `created_at`, not relevance), and fixes a **statement-cache** miss +the merge paths hit. -It does **not** fix NIP-50 *search* latency: that is bounded by the -`created_at`-ordered sort over all matches, which FTS5 can't early-terminate -while staying NIP-01-compliant (see §1). Corpus-independent search is an -external-engine job. +It does **not** fix NIP-50 *search* latency: bm25 relevance scoring (like the +old `created_at` sort) must visit every match, so search cost still grows with +the match set (§1). Corpus-independent search is an external-engine job. Everything here is read/size work; the write path and on-disk index set are unchanged except the FTS table, which gets *smaller* and deletes faster. @@ -58,18 +58,28 @@ in-memory: By-column grows ~linearly with the table (O(n)/delete); by-rowid is flat — ~78× at 8k rows and widening. -**Search is deliberately unchanged and still `created_at`-ordered.** NIP-01's -`limit` requires the newest events *by `created_at`*, and FTS5 only -early-terminates on its own rowid — so `MATCH … ORDER BY created_at DESC LIMIT -n` still materializes and sorts every match, and search latency still grows -with the match set (the report's 18× curve). An earlier draft added a -`searchOrderByRowId` flag that ordered by the FTS rowid to get O(limit) search; -that is *ingestion* order, which returns the wrong events under a limit once -ingestion diverges from `created_at` (any historical sync) — a NIP-01 -violation — so it was removed. `optimize` compacts the index but does not -change the asymptotics (measured within noise at 100k/200k). Corpus-independent -search is genuinely an external-engine job (the Vespa side of the report), not -this index. +**Search ordering fixed to relevance (NIP-50), which the store was getting +wrong.** NIP-50 says results are returned "in descending order by quality of +search result ... not by the usual `.created_at`", with the limit applied after +the score — but the store sorted search by `created_at DESC` (pre-existing). +`makeSimpleSearch` (the `search [+ kinds/authors/since/until] + limit` shape) +now orders by FTS5 bm25 (`ORDER BY event_fts.rank`, `created_at DESC` as a +tie-break), verified against a stronger-but-older match outranking a +weaker-but-newer one (`SearchRelevanceOrderTest`, `Fts5CapabilityProbe`). The +rarer `search + specific tag` shape and the negentropy snapshot still sort by +`created_at` (the row-id subquery can't carry rank through; negentropy is a +sync set, not a ranked result) — a documented follow-up. + +An earlier draft of *this* change instead added a `searchOrderByRowId` flag +(order by the FTS rowid for O(limit) search); that is *ingestion* order — wrong +events under a limit once ingestion diverges from time order, and not relevance +either — so it was removed. + +This is a **correctness** fix, not a scaling one: bm25 (like the created_at +sort) must score every match, so search latency still grows with the match set +(the report's 18× curve) and `optimize` doesn't change the asymptotics +(measured within noise at 100k/200k). Corpus-independent search is genuinely an +external-engine job (the Vespa side of the report), not this index. Migration (v4→v5): the old rowids can't be remapped, so `event_fts` is dropped and rebuilt — synchronous stores rebuild in the upgrade transaction (client diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index e61f04e1ca..789827df1a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -49,12 +49,16 @@ import com.vitorpamplona.quartz.utils.EventFactory * external-content — which reads the source column from the base table — * cannot express it; contentless is the correct primitive. * - * Search still orders by `event_headers.created_at DESC` (NIP-01 `limit` - * semantics: the newest events *by created_at*), joining back to - * `event_headers` on `row_id = event_fts.rowid`. FTS5 cannot early-terminate - * that — it materializes and sorts all matches — so search latency still - * grows with the match set; corpus-independent search needs an external - * engine, not this index. + * Search results are ordered by **relevance**, per NIP-50 ("descending order + * by quality of search result ... not by the usual `.created_at`", limit + * applied after the score) — via FTS5 bm25 (`ORDER BY event_fts.rank`), with + * `created_at DESC` only as a tie-break. This is [QueryBuilder.makeSimpleSearch] + * (the `search [+ kinds/authors/since/until] + limit` shape); the rarer + * `search + specific tag` combination still sorts by `created_at` (its row-id + * subquery can't carry the rank through), and the negentropy snapshot keeps + * `created_at` (a sync set, not a ranked result). bm25 must score every match, + * so search latency still grows with the match set regardless of ordering; + * corpus-independent search needs an external engine, not this index. * * When [enabled] is `false` the module becomes an inert no-op: no * `event_fts` virtual table and no `fts_foreign_key` delete trigger are diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 8ad16c5d4a..2a1eb4dadd 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -1005,7 +1005,13 @@ class QueryBuilder( if (clause.conditions.isNotEmpty()) { append("\nWHERE ${clause.conditions}") } - append("\nORDER BY event_headers.created_at DESC") + // NIP-50: search results are ordered by relevance ("quality of + // search result"), not created_at, and the limit is applied + // after the score. FTS5 exposes bm25 as the `rank` column (more + // negative = more relevant), so ORDER BY rank ascending is + // best-match-first. created_at DESC is only a tie-break so + // equally-relevant matches come newest-first deterministically. + append("\nORDER BY ${fts.tableName}.rank, event_headers.created_at DESC") if (indexStrategy.useAndIndexIdOnOrderBy) { append(", event_headers.id ASC") } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt index f3995f7b8f..ac034e6af3 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -714,7 +714,7 @@ class QueryAssemblerTest : BaseDBTest() { SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) - ORDER BY event_headers.created_at DESC, event_headers.id ASC + ORDER BY event_fts.rank, event_headers.created_at DESC, event_headers.id ASC ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY @@ -727,7 +727,7 @@ class QueryAssemblerTest : BaseDBTest() { SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) - ORDER BY event_headers.created_at DESC + ORDER BY event_fts.rank, event_headers.created_at DESC ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY @@ -741,7 +741,7 @@ class QueryAssemblerTest : BaseDBTest() { fun testKindAndSearch() = forEachDB { db -> val filter = Filter(kinds = listOf(1, 1111, 10000), search = "keywords") - val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "event_headers.created_at DESC, event_headers.id ASC" else "event_headers.created_at DESC" + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "event_fts.rank, event_headers.created_at DESC, event_headers.id ASC" else "event_fts.rank, event_headers.created_at DESC" assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt new file mode 100644 index 0000000000..1077b30e3d --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * NIP-50: search results are ordered by "quality of search result" (relevance), + * **not** by `created_at`, and the limit is applied after the score. The store + * uses FTS5 bm25 (`ORDER BY event_fts.rank`), so a stronger match outranks a + * newer one. + */ +class SearchRelevanceOrderTest { + private val signer = NostrSignerSync() + + private fun note( + content: String, + createdAt: Long, + ) = signer.sign(TextNoteEvent.build(content, createdAt = createdAt)) + + @Test + fun strongerMatchOutranksNewer() = + runBlocking { + val store = EventStore(dbName = null) + try { + // Older, but the term appears 3× in a short doc → most relevant. + val strong = note("bitcoin bitcoin bitcoin", createdAt = 1_000) + // Newer, term once buried in a long doc → least relevant. + val weak = note("bitcoin is one topic among many other unrelated words here padding", createdAt = 9_000) + // Newer still, but does not match at all. + val nonMatch = note("completely different subject entirely", createdAt = 9_999) + + store.insert(weak) + store.insert(strong) + store.insert(nonMatch) + + val results = store.query(Filter(search = "bitcoin", limit = 10)).map { it.id } + // Relevance order (strong before weak) — the opposite of + // created_at DESC (which would put weak first) — and the + // non-matching note is absent. + assertEquals(listOf(strong.id, weak.id), results) + } finally { + store.close() + } + } + + @Test + fun limitAppliesAfterRelevanceScore() = + runBlocking { + val store = EventStore(dbName = null) + try { + // Three docs of decreasing relevance but increasing created_at, + // so created_at order and relevance order are exact opposites. + val best = note("apple apple apple apple", createdAt = 1) + val mid = note("apple apple filler words", createdAt = 2) + val worst = note("apple among lots of other unrelated filler words here", createdAt = 3) + store.insert(best) + store.insert(mid) + store.insert(worst) + + // limit=2 after scoring keeps the two MOST RELEVANT, not the + // two newest. + val top2 = store.query(Filter(search = "apple", limit = 2)).map { it.id } + assertEquals(listOf(best.id, mid.id), top2) + } finally { + store.close() + } + } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt index 8a97974d04..2d07db5dcb 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FtsSearchScalingBenchmark.kt @@ -40,13 +40,12 @@ import kotlin.test.Test * contentless schema keys deletes off the rowid (= event_headers.row_id), * an O(log n) primary-key seek. Every event removal fires this trigger * (replaceable rotation, kind-5, expiration, right-to-vanish). - * 2. **Search scaling — the limit.** `MATCH … ORDER BY created_at DESC - * LIMIT n` must materialize + sort *every* matching document (FTS5 only - * early-terminates on its own rowid, and NIP-01's `limit` requires newest - * *by created_at*), so search cost grows with the match set. Segment - * `optimize` compacts the index but does not change that; corpus- - * independent search needs an external engine. Shown fragmented vs - * optimized to size the (secondary) compaction effect. + * 2. **Search scaling — the limit.** `MATCH … ORDER BY rank LIMIT n` (NIP-50 + * relevance ordering, bm25) must score *every* matching document, so + * search cost grows with the match set regardless of ordering (created_at + * has the same shape). Segment `optimize` compacts the index but does not + * change that; corpus-independent search needs an external engine. Shown + * fragmented vs optimized to size the (secondary) compaction effect. * * Size search with `-DftsBenchScale=N` (default 1). Not an assertion test. */ diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/Fts5CapabilityProbe.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/Fts5CapabilityProbe.kt index 23f9e54125..e982234e66 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/Fts5CapabilityProbe.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/Fts5CapabilityProbe.kt @@ -31,10 +31,11 @@ import kotlin.test.assertEquals * loudly instead of the store silently breaking search on delete. * * - `content=''` **contentless** table with `contentless_delete=1`: lets the - * index drop the duplicated content column yet still delete rows (the - * `fts_foreign_key` trigger needs it). - * - explicit `rowid` on insert + `ORDER BY rowid DESC LIMIT n`: the - * early-terminating recency search path. + * index drop the duplicated content column yet still delete rows by rowid + * (the `fts_foreign_key` trigger needs it). + * - explicit `rowid` on insert (= `event_headers.row_id`): the join key and + * the O(log n) delete key. + * - bm25 `rank` on a contentless table, through a join: NIP-50 relevance order. * - `'merge'` / `'optimize'` maintenance commands: segment compaction. */ class Fts5CapabilityProbe { @@ -64,6 +65,40 @@ class Fts5CapabilityProbe { } } + @Test + fun bm25RankWorksOnContentlessTableInAJoin() { + // NIP-50 orders by relevance, not created_at. Verify FTS5 bm25 `rank` + // works on a contentless table and is reachable through the same + // join-back-to-base-table shape the store's search query uses. + val db = BundledSQLiteDriver().open(":memory:") + try { + db.execSQL("CREATE TABLE headers (row_id INTEGER PRIMARY KEY, created_at INTEGER, tag TEXT)") + db.execSQL("CREATE VIRTUAL TABLE fts USING fts5(content, content='', contentless_delete=1)") + // row 10: term appears 3× in a short doc (most relevant). + // row 20: term once in a long doc (least relevant) but NEWER. + db.execSQL("INSERT INTO headers VALUES (10, 100, 'A')") + db.execSQL("INSERT INTO fts(rowid, content) VALUES (10, 'needle needle needle')") + db.execSQL("INSERT INTO headers VALUES (20, 999, 'B')") + db.execSQL("INSERT INTO fts(rowid, content) VALUES (20, 'needle alpha beta gamma delta epsilon zeta eta')") + + // created_at DESC would return B (999) first; relevance returns A. + val byRank = ArrayList() + db + .prepare( + """ + SELECT headers.tag FROM headers + INNER JOIN fts ON headers.row_id = fts.rowid + WHERE fts MATCH 'needle' + ORDER BY fts.rank + LIMIT 10 + """.trimIndent(), + ).use { while (it.step()) byRank.add(it.getText(0)) } + assertEquals(listOf("A", "B"), byRank, "bm25 rank must put the more relevant (shorter, higher-tf) doc first") + } finally { + db.close() + } + } + @Test fun segmentMergeAndOptimizeAreSupported() { val db = BundledSQLiteDriver().open(":memory:") From de114d1621a2acb5a462c6e409f4b2b66b7f876c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 22:54:47 +0000 Subject: [PATCH 18/34] fix(store): relevance-order search+tag queries too, not just tag-free search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Any filter with a search term must be relevance-ranked (NIP-50), but only the tag-free shape (makeSimpleSearch) was — search + a tag fell through to the row-id-subquery path (prepareRowIDSubQueries/makeQueryIn), which ordered by created_at. prepareRowIDSubQueries gains projectRank: when a search filter joins event_fts, it also projects the bm25 score as a `rank` column and cuts its LIMIT by rank (most relevant, not newest); makeQueryIn(orderByRank) then presents the joined result by that rank, created_at DESC as tie-break. Off by default, so count/delete/union/negentropy (which must stay single-column and unranked) are untouched. toSql wires it on whenever the filter carries a search term. SearchRelevanceOrderTest adds a tag-scoped case (stronger-but-older outranks weaker-but-newer, wrong-tag and non-matching excluded, limit cuts by score). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XqGuBuSUsRudGerPqDuoKA --- .../plans/2026-07-21-sqlite-query-scaling.md | 15 +++--- .../store/sqlite/FullTextSearchModule.kt | 14 ++--- .../nip01Core/store/sqlite/QueryBuilder.kt | 53 ++++++++++++++----- .../store/sqlite/SearchRelevanceOrderTest.kt | 52 ++++++++++++++++++ 4 files changed, 108 insertions(+), 26 deletions(-) diff --git a/quartz/plans/2026-07-21-sqlite-query-scaling.md b/quartz/plans/2026-07-21-sqlite-query-scaling.md index a6b9c59afc..489000b133 100644 --- a/quartz/plans/2026-07-21-sqlite-query-scaling.md +++ b/quartz/plans/2026-07-21-sqlite-query-scaling.md @@ -62,13 +62,14 @@ By-column grows ~linearly with the table (O(n)/delete); by-rowid is flat — wrong.** NIP-50 says results are returned "in descending order by quality of search result ... not by the usual `.created_at`", with the limit applied after the score — but the store sorted search by `created_at DESC` (pre-existing). -`makeSimpleSearch` (the `search [+ kinds/authors/since/until] + limit` shape) -now orders by FTS5 bm25 (`ORDER BY event_fts.rank`, `created_at DESC` as a -tie-break), verified against a stronger-but-older match outranking a -weaker-but-newer one (`SearchRelevanceOrderTest`, `Fts5CapabilityProbe`). The -rarer `search + specific tag` shape and the negentropy snapshot still sort by -`created_at` (the row-id subquery can't carry rank through; negentropy is a -sync set, not a ranked result) — a documented follow-up. +*Every* search filter now orders by FTS5 bm25 (`ORDER BY event_fts.rank`, +`created_at DESC` as a tie-break): the tag-free shape (`makeSimpleSearch`) and +`search + tag` (the `prepareRowIDSubQueries`/`makeQueryIn` path, which carries +the rank column through the row-id subquery via `projectRank` and applies the +LIMIT by rank). Only the negentropy snapshot keeps `created_at` — a sync set, +not a ranked result. Verified against stronger-but-older matches outranking +weaker-but-newer ones, tag-scoped, with the limit cutting by score +(`SearchRelevanceOrderTest`, `Fts5CapabilityProbe`). An earlier draft of *this* change instead added a `searchOrderByRowId` flag (order by the FTS rowid for O(limit) search); that is *ingestion* order — wrong diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index 789827df1a..8a0f156ff0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -52,13 +52,13 @@ import com.vitorpamplona.quartz.utils.EventFactory * Search results are ordered by **relevance**, per NIP-50 ("descending order * by quality of search result ... not by the usual `.created_at`", limit * applied after the score) — via FTS5 bm25 (`ORDER BY event_fts.rank`), with - * `created_at DESC` only as a tie-break. This is [QueryBuilder.makeSimpleSearch] - * (the `search [+ kinds/authors/since/until] + limit` shape); the rarer - * `search + specific tag` combination still sorts by `created_at` (its row-id - * subquery can't carry the rank through), and the negentropy snapshot keeps - * `created_at` (a sync set, not a ranked result). bm25 must score every match, - * so search latency still grows with the match set regardless of ordering; - * corpus-independent search needs an external engine, not this index. + * `created_at DESC` only as a tie-break. This holds for *every* search filter: + * the tag-free shape ([QueryBuilder.makeSimpleSearch]) and `search + tag` + * (whose row-id subquery carries the rank through via `projectRank`). Only the + * negentropy snapshot keeps `created_at` — it is a sync set, not a ranked + * result. bm25 must score every match, so search latency still grows with the + * match set regardless of ordering; corpus-independent search needs an + * external engine, not this index. * * When [enabled] is `false` the module becomes an inert no-op: no * `event_fts` virtual table and no `fts_foreign_key` delete trigger are diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 2a1eb4dadd..7ce69ed9f7 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -202,13 +202,17 @@ class QueryBuilder( ) } - val rowIdSubqueries = prepareRowIDSubQueries(filter, hasher) + // A search term that survived the simple-search branch above always + // combines with tags here (search + #t etc.). NIP-50 still orders by + // relevance, so carry the FTS rank through the subquery and out. + val rankSearch = newFilter.search != null && newFilter.search.isNotEmpty() + val rowIdSubqueries = prepareRowIDSubQueries(filter, hasher, projectRank = rankSearch) return if (rowIdSubqueries == null) { QuerySpec(makeEverythingQuery()) } else { QuerySpec( - makeQueryIn(rowIdSubqueries.sql), + makeQueryIn(rowIdSubqueries.sql, orderByRank = rankSearch), rowIdSubqueries.args, ) } @@ -470,14 +474,20 @@ class QueryBuilder( private fun makeEverythingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC${if (indexStrategy.useAndIndexIdOnOrderBy) ", id ASC" else ""}" - private fun makeQueryIn(rowIdQuery: String) = - """ + // [orderByRank] presents the joined result in NIP-50 relevance order: the + // subquery (built with `projectRank`) exposes the FTS bm25 score as a + // `rank` column, and `created_at DESC` is only a tie-break. Off, it keeps + // the default newest-first ordering for every non-search shape. + private fun makeQueryIn( + rowIdQuery: String, + orderByRank: Boolean = false, + ) = """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers INNER JOIN ( $rowIdQuery ) AS filtered ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC${if (indexStrategy.useAndIndexIdOnOrderBy) ", id ASC" else ""} + ORDER BY ${if (orderByRank) "filtered.rank, " else ""}created_at DESC${if (indexStrategy.useAndIndexIdOnOrderBy) ", id ASC" else ""} """.trimIndent() private fun SQLiteConnection.runQuery(query: QuerySpec): List = @@ -748,6 +758,13 @@ class QueryBuilder( fun prepareRowIDSubQueries( filter: Filter, hasher: TagNameValueHasher, + // When set on a search filter, the subquery also projects the FTS + // bm25 score as a `rank` column and orders its own LIMIT by relevance, + // so the caller ([makeQueryIn] with `orderByRank`) can present results + // NIP-50-ranked. Off (the default) for count/delete/union/negentropy, + // which never expose a second column (a two-column subquery breaks + // `row_id IN (…)`) and don't rank. + projectRank: Boolean = false, ): QuerySpec? { if (filter.isEmpty()) return null @@ -761,6 +778,10 @@ class QueryBuilder( val mustJoinSearch = filter.search != null && fts.enabled + // Only emit the rank column when there is actually an FTS join to take + // it from; a `projectRank` request on a tag-only filter is ignored. + val emitRank = projectRank && mustJoinSearch + val nonDTagsIn = filter.tags?.filter { it.key != "d" } ?: emptyMap() val nonDTagsAll = filter.tagsAll?.filter { it.key != "d" } ?: emptyMap() @@ -789,7 +810,11 @@ class QueryBuilder( buildString { // always do tags if there are any if (reverseLookup) { - append("SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags") + append("SELECT DISTINCT(event_tags.event_header_row_id) as row_id") + // rank is functionally determined by the row_id (one FTS + // row per event), so it doesn't change what DISTINCT folds. + if (emitRank) append(", ${fts.tableName}.rank as rank") + append(" FROM event_tags") // it's quite rare to have 2 tags in the filter, but possible nonDTagsIn.keys.forEachIndexed { index, tagName -> @@ -818,7 +843,9 @@ class QueryBuilder( append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.rowid = event_tags.event_header_row_id") } } else if (mustJoinSearch) { - append("SELECT ${fts.tableName}.rowid as row_id FROM ${fts.tableName}") + append("SELECT ${fts.tableName}.rowid as row_id") + if (emitRank) append(", ${fts.tableName}.rank as rank") + append(" FROM ${fts.tableName}") if (hasHeaders) { append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.rowid") @@ -934,15 +961,17 @@ class QueryBuilder( append(" WHERE ${clause.conditions}") } if (filter.limit != null) { - if (reverseLookup) { + if (emitRank) { + // NIP-50: the LIMIT keeps the most RELEVANT rows, not + // the newest, so the inner cut is by rank too. + append(" ORDER BY rank") + } else if (reverseLookup) { append(" ORDER BY event_tags.created_at DESC") - append(" LIMIT ") - append(filter.limit) } else { append(" ORDER BY event_headers.created_at DESC") - append(" LIMIT ") - append(filter.limit) } + append(" LIMIT ") + append(filter.limit) } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt index 1077b30e3d..023fac27ee 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt @@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.utils.EventFactory import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertEquals @@ -90,4 +91,55 @@ class SearchRelevanceOrderTest { store.close() } } + + /** A `#t` tag makes this a `search + tag` filter — the combined path. */ + private var idSeq = 0 + + private fun hexId(n: Int): String { + val s = n.toString(16) + return "0".repeat(64 - s.length) + s + } + + private fun tagged( + content: String, + createdAt: Long, + topic: String, + ): Event = + EventFactory.create( + hexId(++idSeq), + "00".repeat(32), + createdAt, + 1, + arrayOf(arrayOf("t", topic)), + content, + "0".repeat(128), + ) + + @Test + fun searchWithATagIsAlsoRelevanceOrdered() = + runBlocking { + val store = EventStore(dbName = null) + try { + // All tagged #t=nostr; relevance decreases as created_at rises, + // so created_at order would be the exact reverse of relevance. + val strong = tagged("nostr nostr nostr", createdAt = 1, topic = "nostr") + val weak = tagged("nostr among many other unrelated filler words here padding", createdAt = 2, topic = "nostr") + // Matches the term but wrong tag → excluded by the tag filter. + val wrongTag = tagged("nostr nostr nostr nostr", createdAt = 3, topic = "other") + // Right tag but doesn't match the term → excluded by search. + val noMatch = tagged("bitcoin only", createdAt = 4, topic = "nostr") + + store.batchInsert(listOf(strong, weak, wrongTag, noMatch)) + + val filter = Filter(search = "nostr", tags = mapOf("t" to listOf("nostr")), limit = 10) + val ids = store.query(filter).map { it.id } + assertEquals(listOf(strong.id, weak.id), ids, "search + tag must be relevance-ordered, tag-scoped") + + // limit after score keeps the most relevant one. + val top1 = store.query(filter.copy(limit = 1)).map { it.id } + assertEquals(listOf(strong.id), top1) + } finally { + store.close() + } + } } From 52ba19d3242da83e9334b2e3d885a6f84a46d31c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 23:12:59 +0000 Subject: [PATCH 19/34] test(store): adversarial search-rank cases (two-tag+search, count parity, non-searchable delete) From the branch audit: verifies the search-relevance path holds under a two-tag-key filter (requires both tags, ranks by bm25, no duplicate rows), that count(filter) matches query size under a limit smaller than the match set (NIP-45), and that deleting a non-searchable event (no FTS row) fires the contentless delete trigger against an absent rowid harmlessly. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XqGuBuSUsRudGerPqDuoKA --- .../store/sqlite/AdversarialSearchRankTest.kt | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AdversarialSearchRankTest.kt diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AdversarialSearchRankTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AdversarialSearchRankTest.kt new file mode 100644 index 0000000000..d3359151be --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AdversarialSearchRankTest.kt @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.utils.EventFactory +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class AdversarialSearchRankTest { + private var idSeq = 0 + + private fun hexId(n: Int): String { + val s = n.toString(16) + return "0".repeat(64 - s.length) + s + } + + private fun ev( + content: String, + createdAt: Long, + tags: Array>, + kind: Int = 1, + ): Event = + EventFactory.create( + hexId(++idSeq), + "00".repeat(32), + createdAt, + kind, + tags, + content, + "0".repeat(128), + ) + + // A filter with TWO tag keys + search → self-join on event_tags + rank projection. + @Test + fun twoTagKeysPlusSearchRanksAndDoesNotDuplicate() = + runBlocking { + val store = EventStore(dbName = null) + try { + // has both t=nostr and e=, strong match + val strong = ev("nostr nostr nostr", 1, arrayOf(arrayOf("t", "nostr"), arrayOf("e", "aa".repeat(32)))) + // has both tags, weak match (newer) + val weak = ev("nostr among many other filler words here padding", 2, arrayOf(arrayOf("t", "nostr"), arrayOf("e", "aa".repeat(32)))) + // matches term + t but MISSING e tag → excluded by tagsAll-like AND of two keys + val missingE = ev("nostr nostr nostr nostr", 3, arrayOf(arrayOf("t", "nostr"))) + store.batchInsert(listOf(strong, weak, missingE)) + + val filter = + Filter( + search = "nostr", + tags = mapOf("t" to listOf("nostr"), "e" to listOf("aa".repeat(32))), + limit = 10, + ) + val ids = store.query(filter).map { it.id } + // relevance order, both tags required, no duplicate rows + assertEquals(listOf(strong.id, weak.id), ids, "two-tag + search must rank and require both tags with no dupes") + + // count parity with query for the same filter + assertEquals(ids.size, store.count(filter), "count must equal query size for two-tag search") + } finally { + store.close() + } + } + + // count(filter) vs query(filter).size for a search+tag filter WITH a limit + // smaller than the match set — NIP-45 parity. + @Test + fun countEqualsQuerySizeForSearchTagWithLimit() = + runBlocking { + val store = EventStore(dbName = null) + try { + val a = ev("apple apple apple", 1, arrayOf(arrayOf("t", "fruit"))) + val b = ev("apple apple filler", 2, arrayOf(arrayOf("t", "fruit"))) + val c = ev("apple filler filler words here", 3, arrayOf(arrayOf("t", "fruit"))) + store.batchInsert(listOf(a, b, c)) + + val filter = Filter(search = "apple", tags = mapOf("t" to listOf("fruit")), limit = 2) + val qSize = store.query(filter).size + val cnt = store.count(filter) + assertEquals(2, qSize, "query must honor limit") + assertEquals(qSize, cnt, "count must match query size under a limit (NIP-45)") + } finally { + store.close() + } + } + + // Deleting a NON-searchable event (never had an FTS row) fires the + // contentless delete trigger against a non-existent rowid. + @Test + fun deletingNonSearchableEventDoesNotCrash() = + runBlocking { + val store = EventStore(dbName = null) + try { + // kind 7 reaction is not a SearchableEvent → no FTS row inserted. + val reaction = ev("+", 1, arrayOf(arrayOf("e", "bb".repeat(32))), kind = 7) + val note = ev("hello searchable world", 2, arrayOf()) + store.batchInsert(listOf(reaction, note)) + + // Trigger fires DELETE FROM event_fts WHERE rowid = + // which never existed. Must be a harmless no-op. + store.store.delete(reaction.id) + + // Searchable note still findable; store intact. + assertEquals(note.id, store.query(Filter(search = "searchable")).single().id) + assertTrue(store.query(Filter(ids = listOf(reaction.id))).isEmpty()) + } finally { + store.close() + } + } +} From d571bee3943d660ca018302c981ccba26b0ecde6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 23:33:33 +0000 Subject: [PATCH 20/34] fix(store): rank multi-filter search REQs + harden merge from branch audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-reviewer adversarial audit of the branch found no correctness/data-loss/ crash bugs. This closes one gap and applies three robustness fixes: - Multi-filter search ordering: a REQ whose filters all carry a search term (e.g. the client's search-across-kinds) was created_at-ordered via the union path. Now relevance-ordered — unionSubqueriesIfNeeded(projectRank) projects rank per branch, UNION ALL + GROUP BY row_id MIN(rank) dedups across branches keeping the best score. Only when every branch is a search branch; mixed search/non-search REQs and count/delete unions stay as before. - prepareAuthorStreams/prepareTagStreams build cursors via buildStreams, which closes already-prepared statements if a later prepare throws (was: stranded checked-out, un-reset handles holding read locks in the pooled connection). - Stream counts computed as Long so authors×kinds / values×kinds can't overflow Int back into the eligible band and route a huge fan-out into the merge. - Renamed CachedStatement.finalize() -> finalizeStatement(): a no-arg finalize() is the JVM Object.finalize, risking a GC double-close of the native handle. Tests: multi-filter search relevance + cross-branch dedup + count parity; existing merge/cache/search suites green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XqGuBuSUsRudGerPqDuoKA --- .../plans/2026-07-21-sqlite-query-scaling.md | 34 ++++- .../store/sqlite/FullTextSearchModule.kt | 10 +- .../store/sqlite/MergeQueryExecutor.kt | 98 +++++++------ .../nip01Core/store/sqlite/QueryBuilder.kt | 34 ++++- .../sqlite/StatementCachingConnection.kt | 8 +- .../store/sqlite/SearchRelevanceOrderTest.kt | 57 ++++++++ .../store/sqlite/AdversarialSearchRankTest.kt | 131 ------------------ 7 files changed, 187 insertions(+), 185 deletions(-) delete mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AdversarialSearchRankTest.kt diff --git a/quartz/plans/2026-07-21-sqlite-query-scaling.md b/quartz/plans/2026-07-21-sqlite-query-scaling.md index 489000b133..b57b858d2d 100644 --- a/quartz/plans/2026-07-21-sqlite-query-scaling.md +++ b/quartz/plans/2026-07-21-sqlite-query-scaling.md @@ -135,6 +135,34 @@ reuse, and cap overflow → uncached fallback. whose asserted EXPLAIN output updated for the new join column (`event_fts.rowid`) and the contentless table's virtual-index marker (`0:M2`→`0:M1`). -- Search ordering unchanged: still exact `created_at DESC` (NIP-01 `limit` - semantics); the delete + size + optimize wins are unconditional and - spec-neutral. +- Search ordering: bm25 relevance for every search REQ shape — tag-free, + `search + tag`, and multi-filter all-search (e.g. the client's + search-across-kinds, unioned then deduped by event keeping the best score). + Mixed search/non-search multi-filter REQs stay `created_at` (a non-search + branch has no defined relevance). + +## Audit follow-ups (post-review hardening) + +A two-reviewer adversarial audit of the branch found no correctness/data-loss/ +crash bugs; it produced these robustness fixes and one gap-closure: + +- **Multi-filter search ordering** (gap): a REQ of several filters that all + carry a search term was `created_at`-ordered (the union path). Now + relevance-ordered via `unionSubqueriesIfNeeded(projectRank)` — each branch + projects `rank`, `UNION ALL` + `GROUP BY row_id MIN(rank)` dedups across + branches keeping the best score. Only when every branch is a search branch + (and FTS is on); count/delete unions stay single-column. +- **Merge stream-prep leak** (F1): `prepareAuthorStreams`/`prepareTagStreams` + now build cursors through `buildStreams`, which closes any already-prepared + statements if a later prepare throws — otherwise a mid-loop failure stranded + checked-out, un-reset handles (read locks) in the pooled connection. +- **Stream-count overflow** (F3): `authors×kinds` / `values×kinds` computed as + `Long` so a pathological product can't wrap `Int` back into the eligible + band and route a huge fan-out into the merge. +- **Finalizer footgun** (F2): the pooled statement's `finalize()` was renamed + `finalizeStatement()` — a no-arg `finalize()` is the JVM's `Object.finalize`, + so the GC could double-close the native handle after explicit close. + +Migration cost noted (low/operational, not correctness): the synchronous +v4→v5 rebuild runs in the upgrade transaction; atomic and kill-safe (rolls +back to v4), but a very large client store pays a one-time first-open stall. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index 8a0f156ff0..b098e30d02 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -53,10 +53,12 @@ import com.vitorpamplona.quartz.utils.EventFactory * by quality of search result ... not by the usual `.created_at`", limit * applied after the score) — via FTS5 bm25 (`ORDER BY event_fts.rank`), with * `created_at DESC` only as a tie-break. This holds for *every* search filter: - * the tag-free shape ([QueryBuilder.makeSimpleSearch]) and `search + tag` - * (whose row-id subquery carries the rank through via `projectRank`). Only the - * negentropy snapshot keeps `created_at` — it is a sync set, not a ranked - * result. bm25 must score every match, so search latency still grows with the + * the tag-free shape ([QueryBuilder.makeSimpleSearch]), `search + tag` (whose + * row-id subquery carries the rank through via `projectRank`), and a + * multi-filter all-search REQ (unioned, deduped by event keeping the best + * score). Only the negentropy snapshot — and a multi-filter REQ mixing search + * and non-search branches — keep `created_at` (a sync set / a branch with no + * defined relevance). bm25 must score every match, so search latency still grows with the * match set regardless of ordering; corpus-independent search needs an * external engine, not this index. * diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt index 4b75bb4cda..bd1b174f5b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryExecutor.kt @@ -125,17 +125,19 @@ internal object MergeQueryExecutor { // distinct set. val distinctAuthors = authors.distinct().size val kinds = filter.kinds + // Long product: a pathological authors×kinds could overflow Int and + // wrap back into the eligible band, routing a huge fan-out here. val streams = if (kinds != null && kinds.isNotEmpty()) { - distinctAuthors * kinds.distinct().size + distinctAuthors.toLong() * kinds.distinct().size } else { // authors-only needs the (pubkey, created_at) index to stream. if (!indexStrategy.indexEventsByPubkeyAlone) return -1 - distinctAuthors + distinctAuthors.toLong() } // A single stream is already the optimal single index seek — let the // normal path handle it; only merge when there's something to merge. - return if (streams in 2..MAX_STREAMS) streams else -1 + return if (streams in 2..MAX_STREAMS.toLong()) streams.toInt() else -1 } /** @@ -170,12 +172,12 @@ internal object MergeQueryExecutor { val kinds = filter.kinds?.distinct()?.takeIf { it.isNotEmpty() } val streams = if (kinds != null) { - values.size * kinds.size + values.size.toLong() * kinds.size } else { if (!indexStrategy.indexTagsByCreatedAtAlone) return -1 - values.size + values.size.toLong() } - return if (streams in 2..MAX_STREAMS) streams else -1 + return if (streams in 2..MAX_STREAMS.toLong()) streams.toInt() else -1 } /** Prepares one bound, newest-first cursor per author stream. */ @@ -199,8 +201,7 @@ internal object MergeQueryExecutor { val orderBy = if (indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" - val stmts = ArrayList((kinds?.size ?: 1) * authors.size) - if (kinds != null) { + return if (kinds != null) { val sql = buildString { append("SELECT ").append(COLS) @@ -210,16 +211,14 @@ internal object MergeQueryExecutor { if (since != null) append(" AND created_at >= ?") append(" ORDER BY ").append(orderBy) } - for (kind in kinds) { - for (author in authors) { - val stmt = db.prepare(sql) - var p = 1 - stmt.bindLong(p++, kind.toLong()) - stmt.bindText(p++, author) - if (until != null) stmt.bindLong(p++, until) - if (since != null) stmt.bindLong(p++, since) - stmts.add(stmt) - } + buildStreams(kinds.size * authors.size) { i -> + val stmt = db.prepare(sql) + var p = 1 + stmt.bindLong(p++, kinds[i / authors.size].toLong()) + stmt.bindText(p++, authors[i % authors.size]) + if (until != null) stmt.bindLong(p++, until) + if (since != null) stmt.bindLong(p++, since) + stmt } } else { val sql = @@ -231,16 +230,15 @@ internal object MergeQueryExecutor { if (since != null) append(" AND created_at >= ?") append(" ORDER BY ").append(orderBy) } - for (author in authors) { + buildStreams(authors.size) { i -> val stmt = db.prepare(sql) var p = 1 - stmt.bindText(p++, author) + stmt.bindText(p++, authors[i]) if (until != null) stmt.bindLong(p++, until) if (since != null) stmt.bindLong(p++, since) - stmts.add(stmt) + stmt } } - return stmts } /** Prepares one bound, newest-first cursor per tag-value stream. */ @@ -258,8 +256,7 @@ internal object MergeQueryExecutor { // The tag cursors stream off event_tags (which has no id column), so // the tie order can only be created_at DESC — see the class doc. - val stmts = ArrayList((kinds?.size ?: 1) * values.size) - if (kinds != null) { + return if (kinds != null) { val sql = buildString { append("SELECT ").append(EH_COLS) @@ -270,17 +267,14 @@ internal object MergeQueryExecutor { if (since != null) append(" AND event_tags.created_at >= ?") append(" ORDER BY event_tags.created_at DESC") } - for (value in values) { - val tagHash = hasher.hash(tagName, value) - for (kind in kinds) { - val stmt = db.prepare(sql) - var p = 1 - stmt.bindLong(p++, tagHash) - stmt.bindLong(p++, kind.toLong()) - if (until != null) stmt.bindLong(p++, until) - if (since != null) stmt.bindLong(p++, since) - stmts.add(stmt) - } + buildStreams(values.size * kinds.size) { i -> + val stmt = db.prepare(sql) + var p = 1 + stmt.bindLong(p++, hasher.hash(tagName, values[i / kinds.size])) + stmt.bindLong(p++, kinds[i % kinds.size].toLong()) + if (until != null) stmt.bindLong(p++, until) + if (since != null) stmt.bindLong(p++, since) + stmt } } else { val sql = @@ -293,17 +287,15 @@ internal object MergeQueryExecutor { if (since != null) append(" AND event_tags.created_at >= ?") append(" ORDER BY event_tags.created_at DESC") } - for (value in values) { - val tagHash = hasher.hash(tagName, value) + buildStreams(values.size) { i -> val stmt = db.prepare(sql) var p = 1 - stmt.bindLong(p++, tagHash) + stmt.bindLong(p++, hasher.hash(tagName, values[i])) if (until != null) stmt.bindLong(p++, until) if (since != null) stmt.bindLong(p++, since) - stmts.add(stmt) + stmt } } - return stmts } /** @@ -328,6 +320,32 @@ internal object MergeQueryExecutor { } } + /** + * Prepares [count] cursors via [prepareOne], closing any already-prepared + * statements if a later prepare throws — otherwise a mid-loop failure would + * strand checked-out, un-reset handles in the pooled connection (dead + * slots holding read locks). On success the caller ([mergeStreams]) owns + * closing them. + */ + private inline fun buildStreams( + count: Int, + prepareOne: (Int) -> SQLiteStatement, + ): List { + val stmts = ArrayList(count) + try { + for (i in 0 until count) stmts.add(prepareOne(i)) + } catch (e: Throwable) { + for (s in stmts) { + try { + s.close() + } catch (_: Throwable) { + } + } + throw e + } + return stmts + } + /** * Heap-free k-way merge over the prepared [stmts]: repeatedly emits the * newest live head (`created_at DESC`, tie `id ASC`) until [limit] rows diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 7ce69ed9f7..15c69596f7 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -224,7 +224,13 @@ class QueryBuilder( ): QuerySpec { if (filters.size == 1) return toSql(filters.first(), hasher) - val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher) + // A multi-filter search REQ (e.g. the client's search-across-kinds, + // all filters sharing one term) must still be NIP-50 relevance-ordered. + // Only when EVERY branch is a search branch (and FTS is on, so each has + // a rank column) — a non-search branch has no defined relevance, so a + // mixed REQ falls back to created_at. + val rankSearch = fts.enabled && filters.all { !it.search.isNullOrEmpty() } + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher, projectRank = rankSearch) return if (rowIdSubqueries == null) { QuerySpec( @@ -233,7 +239,7 @@ class QueryBuilder( ) } else { QuerySpec( - makeQueryIn(rowIdSubqueries.sql), + makeQueryIn(rowIdSubqueries.sql, orderByRank = rankSearch), rowIdSubqueries.args, ) } @@ -723,16 +729,34 @@ class QueryBuilder( fun unionSubqueriesIfNeeded( filters: List, hasher: TagNameValueHasher, + // See [prepareRowIDSubQueries]. When set, every branch is a search + // branch that also projects a `rank` column; the union keeps one row + // per event with its BEST (min) bm25 score so the caller can present + // the whole multi-filter search REQ NIP-50-ranked. Callers must only + // pass true when all filters carry a search term (else a branch has no + // rank column). Off for count/delete, which stay single-column. + projectRank: Boolean = false, ): QuerySpec? { val inner = filters.mapNotNull { filter -> - prepareRowIDSubQueries(filter, hasher) + prepareRowIDSubQueries(filter, hasher, projectRank) } if (inner.isEmpty()) return null - return if (inner.size == 1) { - inner.first() + if (inner.size == 1) return inner.first() + + return if (projectRank) { + // UNION ALL keeps every (row_id, rank) so an event matching two + // branches under different terms isn't dropped before MIN; GROUP BY + // then dedups by event keeping the best score. + QuerySpec( + sql = + "SELECT row_id, MIN(rank) as rank FROM (\n " + + inner.joinToString("\n UNION ALL\n ") { "SELECT row_id, rank FROM (${it.sql})" } + + "\n ) GROUP BY row_id", + args = inner.flatMap { it.args }, + ) } else { QuerySpec( sql = inner.joinToString("\n UNION\n ") { "SELECT row_id FROM (${it.sql})" }, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnection.kt index 110ae9c351..a21c2714a1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StatementCachingConnection.kt @@ -97,7 +97,7 @@ class StatementCachingConnection( } override fun close() { - cache.values.forEach { pool -> pool.forEach { runCatching { it.finalize() } } } + cache.values.forEach { pool -> pool.forEach { runCatching { it.finalizeStatement() } } } cache.clear() cachedCount = 0 delegate.close() @@ -120,6 +120,10 @@ class StatementCachingConnection( checkedOut = false } - fun finalize() = delegate.close() + // Not named `finalize`: a no-arg `finalize()` is treated by the JVM as + // Object.finalize(), so the GC would call it and double-close the + // native handle after our explicit close(). This is only ever invoked + // explicitly from the connection's close(). + fun finalizeStatement() = delegate.close() } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt index 023fac27ee..9d40843648 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchRelevanceOrderTest.kt @@ -115,6 +115,63 @@ class SearchRelevanceOrderTest { "0".repeat(128), ) + private fun evk( + content: String, + createdAt: Long, + kind: Int, + ): Event = EventFactory.create(hexId(++idSeq), "00".repeat(32), createdAt, kind, arrayOf(), content, "0".repeat(128)) + + @Test + fun multiFilterSearchIsRelevanceOrderedAcrossBranches() = + runBlocking { + val store = EventStore(dbName = null) + try { + // Relevance A > B > C, created_at A < B < C (reverse), and the + // branches split by kind: A,C are kind 1 (TextNote), B is kind + // 1111 (Comment) — both searchable kinds. + val a = evk("apple apple apple apple", 1, 1) + val b = evk("apple apple apple", 2, 1111) + val c = evk("apple filler filler filler filler filler", 3, 1) + store.batchInsert(listOf(a, b, c)) + + // The client's search-across-kinds shape: one term, two filters. + val filters = + listOf( + Filter(search = "apple", kinds = listOf(1), limit = 100), + Filter(search = "apple", kinds = listOf(1111), limit = 100), + ) + val ids = store.query(filters).map { it.id } + assertEquals(listOf(a.id, b.id, c.id), ids, "multi-filter search must be relevance-ordered across branches") + } finally { + store.close() + } + } + + @Test + fun multiFilterSearchDedupsEventsMatchingSeveralBranches() = + runBlocking { + val store = EventStore(dbName = null) + try { + val x = evk("banana banana banana", 1, 1) + val y = evk("banana one", 2, 1) + store.batchInsert(listOf(x, y)) + + // Overlapping branches: both kind-1 events match BOTH filters. + // GROUP BY row_id must fold each to a single ranked row. + val filters = + listOf( + Filter(search = "banana", kinds = listOf(1, 6), limit = 100), + Filter(search = "banana", kinds = listOf(1, 2), limit = 100), + ) + val ids = store.query(filters).map { it.id } + assertEquals(ids.size, ids.toSet().size, "no event may appear twice across branches") + assertEquals(listOf(x.id, y.id), ids, "deduped, relevance-ordered") + assertEquals(2, store.count(filters), "count parity across the union") + } finally { + store.close() + } + } + @Test fun searchWithATagIsAlsoRelevanceOrdered() = runBlocking { diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AdversarialSearchRankTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AdversarialSearchRankTest.kt deleted file mode 100644 index d3359151be..0000000000 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AdversarialSearchRankTest.kt +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store.sqlite - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.utils.EventFactory -import kotlinx.coroutines.runBlocking -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertTrue - -class AdversarialSearchRankTest { - private var idSeq = 0 - - private fun hexId(n: Int): String { - val s = n.toString(16) - return "0".repeat(64 - s.length) + s - } - - private fun ev( - content: String, - createdAt: Long, - tags: Array>, - kind: Int = 1, - ): Event = - EventFactory.create( - hexId(++idSeq), - "00".repeat(32), - createdAt, - kind, - tags, - content, - "0".repeat(128), - ) - - // A filter with TWO tag keys + search → self-join on event_tags + rank projection. - @Test - fun twoTagKeysPlusSearchRanksAndDoesNotDuplicate() = - runBlocking { - val store = EventStore(dbName = null) - try { - // has both t=nostr and e=, strong match - val strong = ev("nostr nostr nostr", 1, arrayOf(arrayOf("t", "nostr"), arrayOf("e", "aa".repeat(32)))) - // has both tags, weak match (newer) - val weak = ev("nostr among many other filler words here padding", 2, arrayOf(arrayOf("t", "nostr"), arrayOf("e", "aa".repeat(32)))) - // matches term + t but MISSING e tag → excluded by tagsAll-like AND of two keys - val missingE = ev("nostr nostr nostr nostr", 3, arrayOf(arrayOf("t", "nostr"))) - store.batchInsert(listOf(strong, weak, missingE)) - - val filter = - Filter( - search = "nostr", - tags = mapOf("t" to listOf("nostr"), "e" to listOf("aa".repeat(32))), - limit = 10, - ) - val ids = store.query(filter).map { it.id } - // relevance order, both tags required, no duplicate rows - assertEquals(listOf(strong.id, weak.id), ids, "two-tag + search must rank and require both tags with no dupes") - - // count parity with query for the same filter - assertEquals(ids.size, store.count(filter), "count must equal query size for two-tag search") - } finally { - store.close() - } - } - - // count(filter) vs query(filter).size for a search+tag filter WITH a limit - // smaller than the match set — NIP-45 parity. - @Test - fun countEqualsQuerySizeForSearchTagWithLimit() = - runBlocking { - val store = EventStore(dbName = null) - try { - val a = ev("apple apple apple", 1, arrayOf(arrayOf("t", "fruit"))) - val b = ev("apple apple filler", 2, arrayOf(arrayOf("t", "fruit"))) - val c = ev("apple filler filler words here", 3, arrayOf(arrayOf("t", "fruit"))) - store.batchInsert(listOf(a, b, c)) - - val filter = Filter(search = "apple", tags = mapOf("t" to listOf("fruit")), limit = 2) - val qSize = store.query(filter).size - val cnt = store.count(filter) - assertEquals(2, qSize, "query must honor limit") - assertEquals(qSize, cnt, "count must match query size under a limit (NIP-45)") - } finally { - store.close() - } - } - - // Deleting a NON-searchable event (never had an FTS row) fires the - // contentless delete trigger against a non-existent rowid. - @Test - fun deletingNonSearchableEventDoesNotCrash() = - runBlocking { - val store = EventStore(dbName = null) - try { - // kind 7 reaction is not a SearchableEvent → no FTS row inserted. - val reaction = ev("+", 1, arrayOf(arrayOf("e", "bb".repeat(32))), kind = 7) - val note = ev("hello searchable world", 2, arrayOf()) - store.batchInsert(listOf(reaction, note)) - - // Trigger fires DELETE FROM event_fts WHERE rowid = - // which never existed. Must be a harmless no-op. - store.store.delete(reaction.id) - - // Searchable note still findable; store intact. - assertEquals(note.id, store.query(Filter(search = "searchable")).single().id) - assertTrue(store.query(Filter(ids = listOf(reaction.id))).isEmpty()) - } finally { - store.close() - } - } -} From b2adb157ff615e8e22b5f44c52c61a087af0e5b4 Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Tue, 21 Jul 2026 16:53:43 +0300 Subject: [PATCH 21/34] feat(desktop): NIP-88 polls (render, vote, create) + "Polls" search facet Adds full NIP-88 poll support to Amethyst Desktop and a search content-type filter for polls. Polls (DesktopPollCard): - Render kind-1068 polls in feed + thread (and reposted/boosted polls) as an interactive card via NoteCard's bottomContent slot. - Vote (single-choice radio / multi-choice checkbox), re-vote ("Change vote") seeded with the prior selection; hide-until-voted with a "View results" opt-in. - Tallies reuse commons PollResponsesCache; responses are fetched from the poll's OWN declared relays (NIP-88 relay tags) unioned with connected relays, so the full tally loads regardless of the viewer's relay set. Votes are likewise published to the poll's relays (not just broadcastToAll). - Result row marks the viewer's own choice (border + check), tap a row to see its voters, footer shows distinct-voter count + deadline/ended state, and the voter gallery draws the viewer front-most with a ring. - Create polls from the composer (options, single/multi, optional deadline); the dialog content scrolls with a pinned Cancel/Publish row; a poll requires a question and >=2 options. Wiring: - DesktopLocalCache.consume for kind 1068/1018 (response links into pollState). - DesktopFeedFilters + FilterBuilders surface polls; feed/thread interaction subscriptions fetch kind-1018 responses. - Thread + profile pass myPubKeyHex so the viewer's vote-state renders. Search "Polls" facet: - KindRegistry preset + alias for kind 1068 (auto-renders the filter chip and a NIP-50 kind filter); SearchResultsList renders poll results interactively and SearchScreen fetches their responses. Also: - Read-only accounts see results instead of dead vote controls. - Cold-start: the response subscription re-evaluates as relays connect. - Pull the upstream fix for the pre-existing RelayLatencyTracker.sweep ConcurrentModificationException (synchronized(pending)) so relay-health reclassify no longer crashes the UI during search. Ripple/shaping: clickable elements clip to their shape for bounded ripple. Tests: commons PollResponsesCache (dedup/tally/WoT sort) + DesktopLocalCache response-linking. Deferred (noted in review): wall-clock re-check of a poll expiring mid-view; mention-dropdown now inside the composer scroll. Co-Authored-By: Claude Opus 4.8 --- .../amethyst/commons/search/KindRegistry.kt | 3 + .../nip88Polls/PollResponsesCacheTest.kt | 145 ++++ ...7-16-desktop-polls-manual-testing-sheet.md | 106 +++ .../desktop/cache/DesktopLocalCache.kt | 69 +- .../desktop/feeds/DesktopFeedFilters.kt | 2 + .../DesktopRelaySubscriptionsCoordinator.kt | 5 + .../desktop/subscriptions/FilterBuilders.kt | 2 +- .../amethyst/desktop/ui/ComposeNoteDialog.kt | 687 ++++++++++++------ .../amethyst/desktop/ui/FeedScreen.kt | 44 ++ .../amethyst/desktop/ui/NoteActions.kt | 37 + .../amethyst/desktop/ui/SearchScreen.kt | 22 + .../amethyst/desktop/ui/ThreadScreen.kt | 1 + .../amethyst/desktop/ui/UserProfileScreen.kt | 2 + .../desktop/ui/note/DesktopPollCard.kt | 650 +++++++++++++++++ .../desktop/ui/search/SearchResultsList.kt | 104 ++- .../cache/DesktopLocalCachePollTest.kt | 113 +++ .../desktop/filters/FilterBuildersTest.kt | 14 +- 17 files changed, 1754 insertions(+), 252 deletions(-) create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/nip88Polls/PollResponsesCacheTest.kt create mode 100644 desktopApp/plans/2026-07-16-desktop-polls-manual-testing-sheet.md create mode 100644 desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/note/DesktopPollCard.kt create mode 100644 desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCachePollTest.kt diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/search/KindRegistry.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/search/KindRegistry.kt index eb6bf79b79..5e3dc5f45c 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/search/KindRegistry.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/search/KindRegistry.kt @@ -32,6 +32,7 @@ import com.vitorpamplona.quartz.nip53LiveActivities.streaming.LiveActivitiesEven import com.vitorpamplona.quartz.nip54Wiki.WikiNoteEvent import com.vitorpamplona.quartz.nip72ModCommunities.definition.CommunityDefinitionEvent import com.vitorpamplona.quartz.nip84Highlights.HighlightEvent +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent import com.vitorpamplona.quartz.nip99Classifieds.ClassifiedsEvent data class ContentPreset( @@ -63,6 +64,7 @@ object KindRegistry { "wiki" to listOf(WikiNoteEvent.KIND), "classified" to listOf(ClassifiedsEvent.KIND), "highlight" to listOf(HighlightEvent.KIND), + "poll" to listOf(PollEvent.KIND), ) val pseudoKinds: Set = setOf("reply", "media") @@ -75,6 +77,7 @@ object KindRegistry { "Channels" to ContentPreset(kinds = listOf(ChannelCreateEvent.KIND, ChannelMetadataEvent.KIND)), "Communities" to ContentPreset(kinds = listOf(CommunityDefinitionEvent.KIND)), "Wiki" to ContentPreset(kinds = listOf(WikiNoteEvent.KIND)), + "Polls" to ContentPreset(kinds = listOf(PollEvent.KIND)), ) fun resolve(alias: String): List? = aliases[alias.lowercase()] diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/nip88Polls/PollResponsesCacheTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/nip88Polls/PollResponsesCacheTest.kt new file mode 100644 index 0000000000..9db5dce702 --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/nip88Polls/PollResponsesCacheTest.kt @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.model.nip88Polls + +import com.vitorpamplona.amethyst.commons.model.Note +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class PollResponsesCacheTest { + private val pollId = "a".repeat(64) + + // The tally keys votes by `User` identity, and the real cache returns one `User` + // instance per pubkey (getOrCreateUser). Mirror that here so same-pubkey re-votes + // and hasPubKeyVoted() lookups resolve to the same object. + private val userCache = mutableMapOf() + + private fun user(pubKey: HexKey): User = userCache.getOrPut(pubKey) { User(pubKey) { addr -> Note(addr.toValue()) } } + + /** Builds a kind-1018 response Note authored by [pubKey] choosing [option] at [createdAt]. */ + private fun responseNote( + id: HexKey, + pubKey: HexKey, + option: String, + createdAt: Long, + ): Note { + val event = + PollResponseEvent( + id = id, + pubKey = pubKey, + createdAt = createdAt, + tags = + arrayOf( + arrayOf("e", pollId), + arrayOf("response", option), + ), + content = "", + sig = "0".repeat(128), + ) + val note = Note(id) + note.loadEvent(event, user(pubKey), emptyList()) + return note + } + + @Test + fun latestVoteWinsDedup() { + val cache = PollResponsesCache() + val voter = "b".repeat(64) + + // Same voter votes twice; the later-timestamp response must win. + cache.addResponse(responseNote("1".repeat(64), voter, option = "yes", createdAt = 100)) + cache.addResponse(responseNote("2".repeat(64), voter, option = "no", createdAt = 200)) + + val tally = cache.responses.value + + // Exactly one vote counted for this user. + assertEquals(1, tally.totalVotes()) + // The winning option is the newer one. + assertEquals("no", tally.winning()) + // Old option carries no voters. + assertTrue(tally.tally["yes"].isNullOrEmpty()) + } + + @Test + fun tallyPercentReflectsVoteShare() { + val cache = PollResponsesCache() + val forKey = "0".repeat(64) + + cache.addResponse(responseNote("1".repeat(64), "b".repeat(64), option = "yes", createdAt = 10)) + cache.addResponse(responseNote("2".repeat(64), "c".repeat(64), option = "yes", createdAt = 10)) + cache.addResponse(responseNote("3".repeat(64), "d".repeat(64), option = "no", createdAt = 10)) + + val yes = cache.currentTally("yes", forKey, emptySet()) + val no = cache.currentTally("no", forKey, emptySet()) + + assertEquals(2f / 3f, yes.percent) + assertEquals(1f / 3f, no.percent) + assertTrue(yes.isWinning) + assertFalse(no.isWinning) + } + + @Test + fun wotPrioritySortOrdersUsers() { + val cache = PollResponsesCache() + val forKey = "f".repeat(64) // the logged-in user + val followed = "e".repeat(64) + val stranger = "d".repeat(64) + + // Three voters all pick "yes": self, a followed user, and a stranger. + cache.addResponse(responseNote("1".repeat(64), forKey, option = "yes", createdAt = 10)) + cache.addResponse(responseNote("2".repeat(64), stranger, option = "yes", createdAt = 10)) + cache.addResponse(responseNote("3".repeat(64), followed, option = "yes", createdAt = 10)) + + val tally = cache.currentTally("yes", forKey, priorityAccounts = setOf(followed)) + val order = tally.users.map { it.pubkeyHex } + + // Self first, then followed (WoT priority), then the stranger. + assertEquals(listOf(forKey, followed, stranger), order) + } + + @Test + fun hasPubKeyVotedTracksVoter() { + val cache = PollResponsesCache() + val voter = "b".repeat(64) + val other = "c".repeat(64) + + cache.addResponse(responseNote("1".repeat(64), voter, option = "yes", createdAt = 10)) + + assertTrue(cache.hasPubKeyVoted(user(voter))) + assertFalse(cache.hasPubKeyVoted(user(other))) + } + + @Test + fun addResponseIsIdempotentForSameNote() { + val cache = PollResponsesCache() + val note = responseNote("1".repeat(64), "b".repeat(64), option = "yes", createdAt = 10) + + cache.addResponse(note) + cache.addResponse(note) // relay echo of the same note must not double-count + + assertEquals(1, cache.responses.value.totalVotes()) + } +} diff --git a/desktopApp/plans/2026-07-16-desktop-polls-manual-testing-sheet.md b/desktopApp/plans/2026-07-16-desktop-polls-manual-testing-sheet.md new file mode 100644 index 0000000000..e70d4f4157 --- /dev/null +++ b/desktopApp/plans/2026-07-16-desktop-polls-manual-testing-sheet.md @@ -0,0 +1,106 @@ +# Desktop Polls (NIP-88) — Manual Testing Sheet + +**Feature:** render + vote + create polls on Amethyst Desktop +**Branch:** `worktree-feat+desktop-polls` (worktree `.claude/worktrees/feat+desktop-polls`) +**Plan:** `docs/plans/2026-07-16-feat-desktop-polls-nip88-plan.md` +**Status when written:** code-complete; compile + unit tests + spotless GREEN; **manual run not yet done.** + +## Automated gates already passing +- [x] `./gradlew :commons:compileKotlinJvm :desktopApp:compileKotlin` — clean +- [x] `./gradlew :commons:jvmTest --tests "*nip88Polls*"` — 5 pass (dedup, tally %, WoT sort, hasVoted, idempotency) +- [x] `./gradlew :desktopApp:test --tests "*Poll*"` — 2 pass (response links into tally; relay-echo dedup) +- [x] `./gradlew :commons:spotlessKotlinCheck :desktopApp:spotlessKotlinCheck` — clean + +## How to run +```bash +cd .claude/worktrees/feat+desktop-polls +./gradlew :desktopApp:run +``` +Log in (existing account, or NIP-46 bunker). Use a relay set that carries polls — good sources: relays where clients post NIP-88 polls, or create one yourself (Test C) and read it back. A **second client** (Amethyst Android, or `amy`) is useful to cross-verify events on the wire. + +--- + +## A. Rendering (feed + thread) +- [ ] A1. A kind-1068 poll appears in the **Home/Global feed as a poll card** (description + options), NOT as plain text. *(If polls never show: verify `DesktopFeedFilters.isFeedNote` includes `PollEvent` and `FEED_KINDS` has 1068.)* +- [ ] A2. Open the poll in a **thread column** → renders as a poll card there too. +- [ ] A3. Single-choice poll shows **radio**-style option rows; multi-choice shows **checkbox**-style rows with a **Submit** button. +- [ ] A4. Before voting, **no percentages/tally are shown** — only actionable options + a **"View results"** button. +- [ ] A5. Tapping **"View results"** reveals the tally without casting a vote; a way back to voting exists (unless ended/author). +- [ ] A6. A poll authored by **you**, seen in your own feed, shows **results-only** (you cannot vote). +- [ ] A7. An **ended** poll (deadline in the past) shows results-only, no vote controls. +- [ ] A8. Media/description of the poll render via the normal note card (links, images in the description behave as usual). + +## B. Voting +- [ ] B1. Cast a **single-choice** vote → card immediately flips to results (optimistic), your option marked as your vote. +- [ ] B2. Results show a **% bar per option**, a **winning** highlight, and **voter avatars** (up to ~4) + "+N". +- [ ] B3. Voter avatars are **ordered with people you follow first** (WoT). Verify by having a followed account vote — their avatar should sort ahead of strangers. +- [ ] B4. The bar does **not** do a distracting 0→N sweep when opening an already-tallied poll (first-frame animation guard). +- [ ] B5. **Multi-choice**: select 2 options → Submit → both recorded; results reflect both. +- [ ] B6. **Multi-choice empty submit is rejected** — with nothing selected, Submit does nothing / is disabled (no empty response event sent). +- [ ] B7. **Change vote**: after voting, use **"Change vote"** → re-open options → pick a different option → tally updates so the **new** choice wins for you (newest response wins). +- [ ] B8. Cross-check on a second client (Android/amy): your vote is a **kind-1018** event referencing the poll via a lowercase `e` tag. +- [ ] B9. **Scroll-away during send** (stress the scope fix): cast a vote and immediately scroll the poll out of view. Re-find it / check a second client — the vote should have **broadcast to relays**, not just shown locally. *(This validates `voteOnPoll` runs on the long-lived `appScope`, not the card scope.)* + +## C. Creating a poll +- [ ] C1. Open the composer; toggle **Poll** on → poll option editor appears; image attachment is disabled while Poll is on. +- [ ] C2. Add/remove options; **minimum 2 non-blank** options enforced before send is allowed. +- [ ] C3. Toggle **Single vs Multiple** choice. +- [ ] C4. Set a **duration** (Never / 1d / 3d / 7d). "Never" = open-ended (no deadline). +- [ ] C5. Send → a **kind-1068 PollEvent** is published (verify on a second client): correct options, `polltype`, and `endsAt` (absent for "Never"). +- [ ] C6. The poll you created appears in your feed and is votable from **another** account/client; its tally updates as votes arrive. + +## D. Edge cases +- [ ] D1. Poll with an unusually **long option label** wraps/renders without breaking layout. +- [ ] D2. A poll received with **0 options** (malformed) does not crash the feed (renders degraded / skipped). +- [ ] D3. Receiving **many responses from multiple relays** converges to a stable, non-inflated tally (no double counting of the same response). +- [ ] D4. Late votes arriving **after** a poll's deadline: they may still count in the tally, but the card stays results-only (no re-vote UI). + +## E. Regression (nothing else broke) +- [ ] E1. Normal text notes, reposts, and reactions still render + behave in the feed. +- [ ] E2. Composing a normal note (Poll toggle OFF) works exactly as before, including image attachment. +- [ ] E3. Thread view still loads reactions/zaps/reposts for non-poll notes. + +--- + +--- + +## F. Search "Polls" content-type filter (added 2026-07-20) +*Feature: filter search to only polls + interact with them. Plan: `docs/plans/2026-07-20-feat-desktop-search-polls-facet-plan.md`.* + +- [ ] F1. Open the **Search** column → advanced filter panel shows a **"Polls"** chip alongside Notes/Articles/Media/Channels/Communities/Wiki. +- [ ] F2. Enter a query, select **Polls** → results contain **only** poll notes (kind 1068); other content types are excluded. +- [ ] F3. Results appear under a dedicated **"Polls" section** (poll icon) and render as **interactive `DesktopPollCard`** — options visible, not plain text. +- [ ] F4. **Vote from search** (dedicated Search screen): cast a vote on a poll in results → flips to results/optimistic tally, and a kind-1018 event is published (cross-check on a 2nd client). +- [ ] F5. Deselect the Polls chip → results return to mixed content; other facets still work. +- [ ] F6. Section collapse/expand + "Show all N more" work like the other search sections. +- [ ] F7. **Feed header quick-search** (the search box in the feed header): polls render as cards **and are now votable** (account threaded 2026-07-20). + +## G. Cross-context consistency fixes (2026-07-20) +*Fixes for reported bugs: "can't always tap depending on how it's opened" + "only see my own answer, no other tallies".* + +- [ ] G1. **Thread view:** open a poll into a thread → you can vote, and **after voting the card correctly shows your choice** as selected (previously your vote-state didn't register — missing `myPubKeyHex`). +- [ ] G2. **Thread tallies:** a poll opened in a thread shows **other people's votes**, not just yours. +- [ ] G3. **Profile tabs (Notes/Replies):** polls on a user's profile are votable and reflect your vote correctly. +- [ ] G4. **Dedicated Search tallies:** filter to Polls → results now show **existing tallies from others** (search fetches kind-1018 responses via `requestInteractions`), not just your own vote. +- [ ] G5. **Feed header quick-search:** polls there are now **votable** (account threaded). +- [ ] G6. **Consistency:** the SAME poll shows consistent vote-state + tallies whether opened in feed, thread, profile, or search. + +**Remaining known gaps (expected):** +- **Notifications tab** renders polls as the compact notification card (not interactive) — out of scope. +- **Poll posted as a thread *reply*** (not root) renders via the thread's custom reply card (not interactive) — edge case, deferred. +- Feed-header quick-search fetches tallies only after the poll is also seen in a context that requests interactions; the **dedicated Search** column always fetches them. + +## Known caveats (expected, not bugs) +- **Same-second re-vote:** if you change your vote **within the same 1-second** as the first, the tally may not flip until a later-second vote (tie-break on `createdAt` uses strict `>` with no id fallback). Wait ~1s between re-votes to see B7 flip reliably. +- **Option labels are plain text (v1):** links/custom-emoji inside an option label are shown literally, not hyperlinked/rendered (no desktop rich-text path for option labels yet). +- **Deadline = preset chips (v1):** Never/1d/3d/7d instead of a full date/time picker. +- **Not wired this PR (deferred):** poll rendering in profile/bookmarks/search/notifications tabs (still show as plain notes there); poll-draft round-trip; zap-weighted polls. + +## If something fails — where to look +| Symptom | Check | +|---|---| +| Polls never appear in feed | `feeds/DesktopFeedFilters.kt` `isFeedNote` (PollEvent), `subscriptions/FilterBuilders.kt` `FEED_KINDS`=…,1068 | +| Poll shows but tally always empty | kind-1018 sub: `ui/FeedScreen.kt` fetch-interactions filter + `DesktopRelaySubscriptionsCoordinator.requestInteractions` (`e` tag) | +| Vote shows locally but never reaches relays | `DesktopPollCard.castVote` must launch on `localCache.appScope`; `voteOnPoll` in `ui/NoteActions.kt` | +| Double-counted votes | `DesktopLocalCache.consumePollResponse` new-event gate (line ~385) | +| Created poll malformed | `ComposeNoteDialog.publishPoll` → `PollEvent.build` options/type/endsAt | diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt index d3ea5999fc..af026c5746 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt @@ -58,13 +58,16 @@ import com.vitorpamplona.quartz.nip51Lists.followList.FollowListEvent import com.vitorpamplona.quartz.nip57Zaps.LnZapEvent import com.vitorpamplona.quartz.nip57Zaps.LnZapRequestEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent +import com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent import com.vitorpamplona.quartz.nipB7Blossom.BlossomServersEvent import com.vitorpamplona.quartz.utils.DualCase import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow @@ -129,6 +132,17 @@ class DesktopLocalCache : ICacheProvider { val paymentTracker = NwcPaymentTracker() + /** + * Long-lived, cache-scoped coroutine scope for fire-and-forget work that must + * outlive any single composition — e.g. the optimistic-consume → relay-broadcast + * pair of a poll vote (see [com.vitorpamplona.amethyst.desktop.ui.voteOnPoll]). + * Using a card's [androidx.compose.runtime.rememberCoroutineScope] there would let + * scrolling the card out of composition cancel the broadcast after the local consume, + * leaving the vote visible locally but never sent. Uses a [SupervisorJob] so one + * failed job doesn't tear down the rest. + */ + val appScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private fun trackNoteAuthor( note: Note, authorPubkey: HexKey, @@ -337,6 +351,14 @@ class DesktopLocalCache : ICacheProvider { consumeBlossomServerList(event, relay) } + is PollEvent -> { + consumePoll(event, relay) + } + + is PollResponseEvent -> { + consumePollResponse(event, relay) + } + else -> { false } @@ -455,6 +477,49 @@ class DesktopLocalCache : ICacheProvider { return true } + /** + * Consumes a kind 1068 poll event (NIP-88). + * Creates a Note in the cache like a text note, minus reply-linking — a poll is + * always a root post. The [Note.pollState] tally is populated by the responses. + */ + private fun consumePoll( + event: PollEvent, + relay: NormalizedRelayUrl?, + ): Boolean { + val note = getOrCreateNote(event.id) + if (note.event != null) return false + val author = getOrCreateUser(event.pubKey) + note.loadEvent(event, author, emptyList()) + trackNoteAuthor(note, event.pubKey) + relay?.let { note.addRelay(it) } + return true + } + + /** + * Consumes a kind 1018 poll response event (NIP-88). + * Resolves the referenced poll, loads the response note, and links it into the + * poll's tally. Mirrors Android `LocalCache.consume(PollResponseEvent)`: the + * [com.vitorpamplona.amethyst.commons.model.nip88Polls.PollResponsesCache.addResponse] + * call and the `true` return happen only on a genuinely new event, so a relay echo + * of the user's own optimistically-consumed vote can't double-count (id-dedup here + * plus `addResponse`'s own containment guard). + */ + private fun consumePollResponse( + event: PollResponseEvent, + relay: NormalizedRelayUrl?, + ): Boolean { + val pollId = event.poll()?.eventId ?: return false + val pollNote = getOrCreateNote(pollId) + val responseNote = getOrCreateNote(event.id) + if (responseNote.event != null) return false + val author = getOrCreateUser(event.pubKey) + responseNote.loadEvent(event, author, emptyList()) + trackNoteAuthor(responseNote, event.pubKey) + relay?.let { responseNote.addRelay(it) } + pollNote.pollState().addResponse(responseNote) + return true + } + /** * NIP-18 quote reposts: a note carrying a `q` tag is a quote-repost of the quoted * note, so it counts as a boost in the quoted note's repost counter alongside @@ -821,7 +886,7 @@ class DesktopLocalCache : ICacheProvider { requestNote?.let { req -> pending.zappedNote?.addZapPayment(req, note) } // Invoke callback on IO dispatcher - GlobalScope.launch(Dispatchers.IO) { + appScope.launch { pending.onResponse(event) } diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/feeds/DesktopFeedFilters.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/feeds/DesktopFeedFilters.kt index 13fb4ff300..ebcfb2e98d 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/feeds/DesktopFeedFilters.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/feeds/DesktopFeedFilters.kt @@ -37,9 +37,11 @@ import com.vitorpamplona.quartz.nip22Comments.CommentEvent import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent import com.vitorpamplona.quartz.nip25Reactions.ReactionEvent import com.vitorpamplona.quartz.nip57Zaps.LnZapEvent +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent private fun isFeedNote(event: Event?): Boolean = event is TextNoteEvent || + event is PollEvent || event.isRenderableRepost() private fun List.deduplicateReposts(): List = diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt index 9aff62678a..b70c4e2bea 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt @@ -294,6 +294,11 @@ class DesktopRelaySubscriptionsCoordinator( kinds = listOf(com.vitorpamplona.quartz.nip10Notes.TextNoteEvent.KIND), tags = mapOf("e" to noteIds), ), + // Poll responses (kind 1018) targeting these notes (NIP-88, lowercase `e`) + Filter( + kinds = listOf(com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent.KIND), + tags = mapOf("e" to noteIds), + ), ) val listener = diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/FilterBuilders.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/FilterBuilders.kt index 64eae951a9..9b2c043d53 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/FilterBuilders.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/FilterBuilders.kt @@ -29,7 +29,7 @@ import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent * Provides convenience functions for creating relay subscription filters. */ object FilterBuilders { - private val FEED_KINDS = listOf(1, 6, 16) // TextNoteEvent, RepostEvent, GenericRepostEvent + private val FEED_KINDS = listOf(1, 6, 16, 1068) // TextNoteEvent, RepostEvent, GenericRepostEvent, PollEvent /** * Creates a filter for text notes (kind 1) from all authors. diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ComposeNoteDialog.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ComposeNoteDialog.kt index 9eedb59385..fe7a29c773 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ComposeNoteDialog.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ComposeNoteDialog.kt @@ -43,6 +43,9 @@ import androidx.compose.material3.Button import androidx.compose.material3.Card import androidx.compose.material3.CardDefaults import androidx.compose.material3.Checkbox +import androidx.compose.material3.FilterChip +import androidx.compose.material3.FilterChipDefaults +import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton import androidx.compose.material3.OutlinedTextField @@ -56,6 +59,7 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue +import androidx.compose.runtime.snapshots.SnapshotStateList import androidx.compose.ui.Alignment import androidx.compose.ui.ExperimentalComposeUiApi import androidx.compose.ui.Modifier @@ -66,6 +70,8 @@ import androidx.compose.ui.text.input.TextFieldValue import androidx.compose.ui.unit.dp import androidx.compose.ui.window.Dialog import androidx.compose.ui.window.DialogProperties +import com.vitorpamplona.amethyst.commons.icons.symbols.Icon +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols import com.vitorpamplona.amethyst.commons.model.User import com.vitorpamplona.amethyst.commons.scheduledposts.ScheduledPost import com.vitorpamplona.amethyst.commons.scheduledposts.ScheduledPostStore @@ -114,6 +120,9 @@ import com.vitorpamplona.quartz.nip18Reposts.quotes.quote import com.vitorpamplona.quartz.nip19Bech32.entities.NEvent import com.vitorpamplona.quartz.nip22Comments.CommentEvent import com.vitorpamplona.quartz.nip37Drafts.DraftWrapEvent +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent +import com.vitorpamplona.quartz.nip88Polls.poll.tags.OptionTag +import com.vitorpamplona.quartz.nip88Polls.poll.tags.PollType import com.vitorpamplona.quartz.nip89AppHandlers.clientTag.isClient import com.vitorpamplona.quartz.nip92IMeta.IMetaTag import com.vitorpamplona.quartz.utils.TimeUtils @@ -228,6 +237,14 @@ fun ComposeNoteDialog( var syncDraft by remember { mutableStateOf(false) } var isSavingDraft by remember { mutableStateOf(false) } + // Poll (NIP-88) composer state. `wantsPoll` gates the poll UI; options start with + // two blank fields (a poll needs ≥2 non-blank options to publish). + var wantsPoll by remember { mutableStateOf(false) } + val pollOptions = remember { mutableStateListOf("", "") } + var pollType by remember { mutableStateOf(PollType.SINGLE_CHOICE) } + // Optional poll deadline, expressed as seconds-from-now (null = open-ended). + var pollDurationDays by remember { mutableStateOf(null) } + // Image compression: global default + optional per-post override. // Override resets after every successful send so the next post // starts from the saved default again. @@ -377,7 +394,21 @@ fun ComposeNoteDialog( } val scheduleAt = scheduledForSec - if (postAsPicture) { + if (wantsPoll) { + val endsAt = + pollDurationDays?.let { days -> + TimeUtils.now() + days * 24L * 60L * 60L + } + publishPoll( + description = content, + options = pollOptions.map { it.trim() }.filter { it.isNotEmpty() }, + pollType = pollType, + endsAt = endsAt, + account = account, + relayManager = relayManager, + relays = selectedRelays, + ) + } else if (postAsPicture) { val pictureMetas = buildPictureMetas(uploadResults) publishPicture( description = content, @@ -434,8 +465,9 @@ fun ComposeNoteDialog( Modifier .width(780.dp) // Cap the dialog height so a tall composer (e.g. the schedule - // picker expanded) can't push the Cancel/Schedule buttons off - // screen — the body scrolls instead (see the content Column). + // picker expanded, or the poll composer with many options) can't + // push the Cancel/Publish buttons off screen — the body scrolls + // instead (see the content Column). .heightIn(max = 760.dp) .padding(16.dp) .dragAndDropTarget(shouldStartDragAndDrop = { true }, target = dropTarget) @@ -458,206 +490,246 @@ fun ComposeNoteDialog( color = MaterialTheme.colorScheme.onSurface, ) - replyTo?.let { reply -> - Spacer(Modifier.height(8.dp)) - Text( - "Replying to: ${reply.content.take(50)}${if (reply.content.length > 50) "..." else ""}", - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } + // Scrollable content area so the pinned Cancel/Publish row below stays + // reachable even when the poll section grows with many options. + Column( + modifier = + Modifier + .weight(1f, fill = false) + .verticalScroll(rememberScrollState()), + ) { + replyTo?.let { reply -> + Spacer(Modifier.height(8.dp)) + Text( + "Replying to: ${reply.content.take(50)}${if (reply.content.length > 50) "..." else ""}", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } - quoteOf?.let { quoted -> - Spacer(Modifier.height(8.dp)) - Text( - "Quoting: ${quoted.content.take(50)}${if (quoted.content.length > 50) "..." else ""}", - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } + quoteOf?.let { quoted -> + Spacer(Modifier.height(8.dp)) + Text( + "Quoting: ${quoted.content.take(50)}${if (quoted.content.length > 50) "..." else ""}", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } - Spacer(Modifier.height(16.dp)) + Spacer(Modifier.height(16.dp)) - Box { - OutlinedTextField( - value = if (postAsPicture) TextFieldValue("") else contentField, - onValueChange = { - contentField = it - errorMessage = null - }, - modifier = Modifier.fillMaxWidth().height(if (postAsPicture) 60.dp else 200.dp), - label = { - Text( - if (postAsPicture) "Text disabled for picture posts" else "What's on your mind?", - ) - }, - placeholder = { Text(if (postAsPicture) "" else "Write your note... (type @ to mention)") }, - enabled = !isPosting && !postAsPicture, - maxLines = if (postAsPicture) 1 else 10, - ) + Box { + OutlinedTextField( + value = if (postAsPicture) TextFieldValue("") else contentField, + onValueChange = { + contentField = it + errorMessage = null + }, + modifier = Modifier.fillMaxWidth().height(if (postAsPicture) 60.dp else 200.dp), + label = { + Text( + if (postAsPicture) "Text disabled for picture posts" else "What's on your mind?", + ) + }, + placeholder = { Text(if (postAsPicture) "" else "Write your note... (type @ to mention)") }, + enabled = !isPosting && !postAsPicture, + maxLines = if (postAsPicture) 1 else 10, + ) - // Mention autocomplete dropdown - if (mentionSuggestions.isNotEmpty()) { - Card( - modifier = Modifier.fillMaxWidth().padding(top = 4.dp), - elevation = CardDefaults.cardElevation(defaultElevation = 4.dp), - ) { - LazyColumn(modifier = Modifier.heightIn(max = 200.dp)) { - items(mentionSuggestions, key = { it.pubkeyHex }) { user -> - MentionSuggestionRow( - user = user, - onClick = { - val npub = user.pubkeyNpub() - val replacement = "nostr:$npub " - val cursorEnd = contentField.selection.end - val newText = - contentField.text.replaceRange( - mentionWordStart, - cursorEnd, - replacement, - ) - val newCursor = mentionWordStart + replacement.length - contentField = TextFieldValue(newText, TextRange(newCursor)) - mentionSuggestions = emptyList() - mentionQuery = null - }, - ) + // Mention autocomplete dropdown + if (mentionSuggestions.isNotEmpty()) { + Card( + modifier = Modifier.fillMaxWidth().padding(top = 4.dp), + elevation = CardDefaults.cardElevation(defaultElevation = 4.dp), + ) { + LazyColumn(modifier = Modifier.heightIn(max = 200.dp)) { + items(mentionSuggestions, key = { it.pubkeyHex }) { user -> + MentionSuggestionRow( + user = user, + onClick = { + val npub = user.pubkeyNpub() + val replacement = "nostr:$npub " + val cursorEnd = contentField.selection.end + val newText = + contentField.text.replaceRange( + mentionWordStart, + cursorEnd, + replacement, + ) + val newCursor = mentionWordStart + replacement.length + contentField = TextFieldValue(newText, TextRange(newCursor)) + mentionSuggestions = emptyList() + mentionQuery = null + }, + ) + } } } } } - } - Spacer(Modifier.height(8.dp)) - - Row( - modifier = Modifier.fillMaxWidth(), - verticalAlignment = Alignment.CenterVertically, - ) { - // MediaAttachmentRow fills its width, so give it a weighted slot; - // otherwise it consumes the whole Row and pushes the schedule - // button off the right edge (making it invisible). - Box(Modifier.weight(1f)) { - MediaAttachmentRow( - attachedFiles = attachedFiles, - isUploading = uploadState.isUploading, - onAttach = { - val files = DesktopFilePicker.pickMediaFiles() - attachedFiles.addAll(files) - }, - onPaste = { - val files = ClipboardPasteHandler.getClipboardFiles() - attachedFiles.addAll(files) - }, - onRemove = { attachedFiles.remove(it) }, - ) - } - - // Picture posts aren't schedulable in v1 — hide the toggle then. - if (!postAsPicture) { - DesktopScheduleAtButton( - isActive = scheduledForSec != null, - onClick = { - scheduledForSec = - if (scheduledForSec != null) { - null - } else { - sanitizeScheduleTime(presetInOneHour()) - } - }, - ) - } - } - - if (scheduledForSec != null && !postAsPicture) { Spacer(Modifier.height(8.dp)) - DesktopScheduleAtPicker( - scheduledForSec = scheduledForSec ?: 0L, - onChanged = { scheduledForSec = it }, + + // Poll toggle — mutually exclusive with image posting (a poll carries no + // media attachments). Disabled while there are attached files. + Row(verticalAlignment = Alignment.CenterVertically) { + FilterChip( + selected = wantsPoll, + onClick = { wantsPoll = !wantsPoll }, + enabled = attachedFiles.isEmpty(), + label = { Text("Poll") }, + leadingIcon = + if (wantsPoll) { + { Icon(MaterialSymbols.Check, contentDescription = null, modifier = Modifier.size(18.dp)) } + } else { + null + }, + ) + } + + if (wantsPoll) { + PollComposerSection( + options = pollOptions, + pollType = pollType, + onPollTypeChange = { pollType = it }, + pollDurationDays = pollDurationDays, + onDurationChange = { pollDurationDays = it }, + ) + } + + // Media attachment + scheduling — hidden for polls (a poll carries no + // media attachments and isn't schedulable in v1). + if (!wantsPoll) { + Row( + modifier = Modifier.fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically, + ) { + // MediaAttachmentRow fills its width, so give it a weighted slot; + // otherwise it consumes the whole Row and pushes the schedule + // button off the right edge (making it invisible). + Box(Modifier.weight(1f)) { + MediaAttachmentRow( + attachedFiles = attachedFiles, + isUploading = uploadState.isUploading, + onAttach = { + val files = DesktopFilePicker.pickMediaFiles() + attachedFiles.addAll(files) + }, + onPaste = { + val files = ClipboardPasteHandler.getClipboardFiles() + attachedFiles.addAll(files) + }, + onRemove = { attachedFiles.remove(it) }, + ) + } + + // Picture posts aren't schedulable in v1 — hide the toggle then. + if (!postAsPicture) { + DesktopScheduleAtButton( + isActive = scheduledForSec != null, + onClick = { + scheduledForSec = + if (scheduledForSec != null) { + null + } else { + sanitizeScheduleTime(presetInOneHour()) + } + }, + ) + } + } + + if (scheduledForSec != null && !postAsPicture) { + Spacer(Modifier.height(8.dp)) + DesktopScheduleAtPicker( + scheduledForSec = scheduledForSec ?: 0L, + onChanged = { scheduledForSec = it }, + ) + } + } + + // Server selector + per-post quality + post type — shown when files are attached + if (attachedFiles.isNotEmpty()) { + Spacer(Modifier.height(4.dp)) + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = androidx.compose.ui.Alignment.CenterVertically, + ) { + ServerSelector( + servers = effectiveServers, + selectedServer = selectedServer, + onServerSelected = { selectedServer = it }, + ) + + // Quality override chip — only when images are attached + // (no point picking a JPEG preset for a video upload). + if (hasImages) { + QualitySelectorChip( + activeQuality = activeQuality, + isOverride = perPostQualityOverride != null, + onSelect = { perPostQualityOverride = it }, + onReset = { perPostQualityOverride = null }, + ) + } + + // Post type toggle — only when images are attached + if (hasImages) { + PostTypeSelector( + isPicture = postAsPicture, + onToggle = { postAsPicture = it }, + ) + } + } + } + + Spacer(Modifier.height(4.dp)) + + // Character count + Text( + "${content.length} characters", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + + errorMessage?.let { error -> + Spacer(Modifier.height(8.dp)) + SelectionContainer { + Text( + error, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + ) + } + } + + uploadState.error?.let { error -> + Spacer(Modifier.height(4.dp)) + SelectionContainer { + Text( + "Upload error: $error", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + ) + } + } + + Spacer(Modifier.height(8.dp)) + + ComposeRelayPicker( + pickerState = pickerState, + selectedRelays = selectedRelays, + onToggleRelay = { url -> + selectedRelays = + if (url in selectedRelays) { + selectedRelays - url + } else { + selectedRelays + url + } + }, ) } - // Server selector + per-post quality + post type — shown when files are attached - if (attachedFiles.isNotEmpty()) { - Spacer(Modifier.height(4.dp)) - Row( - modifier = Modifier.fillMaxWidth(), - horizontalArrangement = Arrangement.SpaceBetween, - verticalAlignment = androidx.compose.ui.Alignment.CenterVertically, - ) { - ServerSelector( - servers = effectiveServers, - selectedServer = selectedServer, - onServerSelected = { selectedServer = it }, - ) - - // Quality override chip — only when images are attached - // (no point picking a JPEG preset for a video upload). - if (hasImages) { - QualitySelectorChip( - activeQuality = activeQuality, - isOverride = perPostQualityOverride != null, - onSelect = { perPostQualityOverride = it }, - onReset = { perPostQualityOverride = null }, - ) - } - - // Post type toggle — only when images are attached - if (hasImages) { - PostTypeSelector( - isPicture = postAsPicture, - onToggle = { postAsPicture = it }, - ) - } - } - } - - Spacer(Modifier.height(4.dp)) - - // Character count - Text( - "${content.length} characters", - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - - errorMessage?.let { error -> - Spacer(Modifier.height(8.dp)) - SelectionContainer { - Text( - error, - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.error, - ) - } - } - - uploadState.error?.let { error -> - Spacer(Modifier.height(4.dp)) - SelectionContainer { - Text( - "Upload error: $error", - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.error, - ) - } - } - - Spacer(Modifier.height(8.dp)) - - ComposeRelayPicker( - pickerState = pickerState, - selectedRelays = selectedRelays, - onToggleRelay = { url -> - selectedRelays = - if (url in selectedRelays) { - selectedRelays - url - } else { - selectedRelays + url - } - }, - ) - Spacer(Modifier.height(8.dp)) // NIP-37 draft-sync opt-in — only meaningful for plain notes. @@ -704,61 +776,74 @@ fun ComposeNoteDialog( // Save-as-draft: always writes a local row; optionally publishes a // NIP-37 encrypted event. Local save still succeeds if sync fails. - OutlinedButton( - onClick = { - if (content.isBlank()) { - errorMessage = "Draft cannot be empty" - return@OutlinedButton - } - scope.launch { - isSavingDraft = true - errorMessage = null - var syncError: String? = null - try { - if (syncDraft) { - syncError = - syncDraftToRelays( - content = content, - dTag = draftTag, - account = account, - relayManager = relayManager, - replyTo = replyTo, - quoteOf = quoteOf, - relays = selectedRelays, - ) - } - - noteDraftStore.save( - NoteDraft( - dTag = draftTag, - content = content, - updatedAt = TimeUtils.now(), - synced = syncDraft && syncError == null, - accountPubkey = account.pubKeyHex, - ), - ) - - if (syncError != null) { - errorMessage = "Draft saved locally, but sync failed: $syncError" - } else { - onDismiss() - } - } catch (e: Exception) { - errorMessage = "Failed to save draft: ${e.message}" - } finally { - isSavingDraft = false + // Not shown while composing a poll — polls aren't draftable in v1. + if (!wantsPoll) { + OutlinedButton( + onClick = { + if (content.isBlank()) { + errorMessage = "Draft cannot be empty" + return@OutlinedButton } - } - }, - enabled = !isPosting && !isSavingDraft && content.isNotBlank(), - ) { - Text(if (isSavingDraft) "Saving..." else "Save as draft") + scope.launch { + isSavingDraft = true + errorMessage = null + var syncError: String? = null + try { + if (syncDraft) { + syncError = + syncDraftToRelays( + content = content, + dTag = draftTag, + account = account, + relayManager = relayManager, + replyTo = replyTo, + quoteOf = quoteOf, + relays = selectedRelays, + ) + } + + noteDraftStore.save( + NoteDraft( + dTag = draftTag, + content = content, + updatedAt = TimeUtils.now(), + synced = syncDraft && syncError == null, + accountPubkey = account.pubKeyHex, + ), + ) + + if (syncError != null) { + errorMessage = "Draft saved locally, but sync failed: $syncError" + } else { + onDismiss() + } + } catch (e: Exception) { + errorMessage = "Failed to save draft: ${e.message}" + } finally { + isSavingDraft = false + } + } + }, + enabled = !isPosting && !isSavingDraft && content.isNotBlank(), + ) { + Text(if (isSavingDraft) "Saving..." else "Save as draft") + } + + Spacer(Modifier.width(8.dp)) } - Spacer(Modifier.width(8.dp)) - + // A poll needs a question (description) and at least two options. + val pollValid = content.isNotBlank() && pollOptions.count { it.trim().isNotEmpty() } >= 2 Button( onClick = { + if (wantsPoll) { + if (!pollValid) { + errorMessage = "A poll needs a question and at least two options" + return@Button + } + runPublish(null, emptySet()) + return@Button + } if (content.isBlank() && attachedFiles.isEmpty()) { errorMessage = "Note cannot be empty" return@Button @@ -786,7 +871,9 @@ fun ComposeNoteDialog( } runPublish(null, emptySet()) }, - enabled = !isPosting && !isSavingDraft && (content.isNotBlank() || attachedFiles.isNotEmpty()), + enabled = + !isPosting && !isSavingDraft && + if (wantsPoll) pollValid else (content.isNotBlank() || attachedFiles.isNotEmpty()), ) { Text( when { @@ -982,6 +1069,40 @@ private suspend fun publishPicture( } } +private suspend fun publishPoll( + description: String, + options: List, + pollType: PollType, + endsAt: Long?, + account: AccountState.LoggedIn, + relayManager: DesktopRelayConnectionManager, + relays: Set, +) { + withContext(Dispatchers.IO) { + if (account.isReadOnly) { + throw IllegalStateException("Cannot post in read-only mode") + } + require(options.size >= 2) { "A poll needs at least two options" } + + // Deterministic per-position codes; labels come straight from the fields. + val optionTags = options.mapIndexed { index, label -> OptionTag(index.toString(), label) } + + val template = + PollEvent.build( + description = description, + options = optionTags, + endsAt = endsAt, + relays = relays.toList(), + pollType = pollType, + ) { + hashtags(findHashtags(description)) + } + + val signedEvent = account.signer.sign(template) + relayManager.publish(signedEvent, relays) + } +} + private suspend fun publishNote( content: String, account: AccountState.LoggedIn, @@ -1174,6 +1295,90 @@ private suspend fun syncDraftToRelays( } } +/** + * Poll composer body: N option fields (add/remove, ≥2), a single/multi choice chip pair, + * and an optional duration (deadline) chip row. The description is the main note text field. + */ +@Composable +private fun PollComposerSection( + options: SnapshotStateList, + pollType: PollType, + onPollTypeChange: (PollType) -> Unit, + pollDurationDays: Int?, + onDurationChange: (Int?) -> Unit, +) { + Column( + modifier = Modifier.fillMaxWidth().padding(vertical = 8.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + options.forEachIndexed { index, value -> + Row( + modifier = Modifier.fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(4.dp), + ) { + OutlinedTextField( + value = value, + onValueChange = { options[index] = it }, + modifier = Modifier.weight(1f), + singleLine = true, + placeholder = { Text("Option ${index + 1}") }, + ) + IconButton( + onClick = { if (options.size > 2) options.removeAt(index) }, + enabled = options.size > 2, + ) { + Icon(MaterialSymbols.Close, contentDescription = "Remove option", modifier = Modifier.size(20.dp)) + } + } + } + + OutlinedButton(onClick = { options.add("") }) { + Icon(MaterialSymbols.Add, contentDescription = null, modifier = Modifier.size(18.dp)) + Spacer(Modifier.width(4.dp)) + Text("Add option") + } + + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + FilterChip( + selected = pollType == PollType.SINGLE_CHOICE, + onClick = { onPollTypeChange(PollType.SINGLE_CHOICE) }, + label = { Text("Single choice") }, + ) + FilterChip( + selected = pollType == PollType.MULTI_CHOICE, + onClick = { onPollTypeChange(PollType.MULTI_CHOICE) }, + label = { Text("Multiple choice") }, + ) + } + + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Text("Ends:", style = MaterialTheme.typography.bodySmall) + listOf>( + "Never" to null, + "1d" to 1, + "3d" to 3, + "7d" to 7, + ).forEach { (label, days) -> + FilterChip( + selected = pollDurationDays == days, + onClick = { onDurationChange(days) }, + label = { Text(label) }, + leadingIcon = + if (pollDurationDays == days) { + { Icon(MaterialSymbols.Check, contentDescription = null, modifier = Modifier.size(FilterChipDefaults.IconSize)) } + } else { + null + }, + ) + } + } + } +} + @Composable private fun MentionSuggestionRow( user: User, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/FeedScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/FeedScreen.kt index 1eace52ec5..ac737368e4 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/FeedScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/FeedScreen.kt @@ -132,6 +132,7 @@ import com.vitorpamplona.amethyst.desktop.subscriptions.createThreadRepliesSubsc import com.vitorpamplona.amethyst.desktop.subscriptions.generateSubId import com.vitorpamplona.amethyst.desktop.subscriptions.rememberSubscription import com.vitorpamplona.amethyst.desktop.ui.media.LightboxOverlay +import com.vitorpamplona.amethyst.desktop.ui.note.DesktopPollCard import com.vitorpamplona.amethyst.desktop.ui.note.NoteCard import com.vitorpamplona.amethyst.desktop.ui.note.SpamCheckedNoteRender import com.vitorpamplona.amethyst.desktop.ui.note.WoTBadgedAvatar @@ -159,6 +160,8 @@ import com.vitorpamplona.quartz.nip19Bech32.Nip19Parser import com.vitorpamplona.quartz.nip19Bech32.entities.NEvent import com.vitorpamplona.quartz.nip19Bech32.entities.NNote import com.vitorpamplona.quartz.nip22Comments.CommentEvent +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent +import com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.GlobalScope @@ -247,6 +250,21 @@ private fun FeedNoteCardBody( myPubKeyHex: String? = null, onFollow: ((String) -> Unit)? = null, ) { + if (event is PollEvent) { + DesktopPollCard( + note = note, + event = event, + relayManager = relayManager, + localCache = localCache, + account = account, + myPubKeyHex = myPubKeyHex, + onNavigateToThread = onNavigateToThread, + onNavigateToProfile = onNavigateToProfile, + onHashtagClick = onHashtagClick, + ) + return + } + val isRepost = event is RepostEvent || event is GenericRepostEvent if (isRepost) { @@ -273,6 +291,22 @@ private fun FeedNoteCardBody( return } + // A boosted poll must still render as an interactive poll card, not a plain note. + if (originalEvent is PollEvent) { + DesktopPollCard( + note = originalNote, + event = originalEvent, + relayManager = relayManager, + localCache = localCache, + account = account, + myPubKeyHex = myPubKeyHex, + onNavigateToThread = onNavigateToThread, + onNavigateToProfile = onNavigateToProfile, + onHashtagClick = onHashtagClick, + ) + return + } + val reactionCount = remember(reactionsState) { originalNote.countReactions() } val replyCount = remember(repliesState) { originalNote.replies.size } val repostCount = remember(metadataState) { originalNote.boosts.size } @@ -859,6 +893,11 @@ fun FeedScreen( kinds = listOf(com.vitorpamplona.quartz.nip10Notes.TextNoteEvent.KIND), tags = mapOf("e" to interactionNoteIds), ), + // Poll responses (kind 1018) referencing these notes (NIP-88, lowercase `e`). + Filter( + kinds = listOf(PollResponseEvent.KIND), + tags = mapOf("e" to interactionNoteIds), + ), ), relays = allRelayUrls, onEvent = { event, _, relay, _ -> @@ -1117,6 +1156,7 @@ fun FeedScreen( onSearchClick = openFullSearch, relayManager = relayManager, localCache = localCache, + account = account, onNavigateToProfile = onNavigateToProfile, onNavigateToThread = onNavigateToThread, ) @@ -1156,6 +1196,7 @@ private fun FeedTabsHeader( onSearchClick: () -> Unit = {}, relayManager: DesktopRelayConnectionManager? = null, localCache: DesktopLocalCache? = null, + account: AccountState.LoggedIn? = null, onNavigateToProfile: (String) -> Unit = {}, onNavigateToThread: (String) -> Unit = {}, ) { @@ -1508,6 +1549,9 @@ private fun FeedTabsHeader( onNavigateToThread(noteId) }, localCache = localCache, + relayManager = relayManager, + account = account, + myPubKeyHex = account?.pubKeyHex, modifier = Modifier.heightIn(max = 400.dp).fillMaxWidth(), ) } else if (isSearching) { diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NoteActions.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NoteActions.kt index 7a52fb1d08..9c91070f36 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NoteActions.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NoteActions.kt @@ -103,6 +103,8 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip47WalletConnect.Nip47WalletConnect import com.vitorpamplona.quartz.nip51Lists.bookmarkList.BookmarkListEvent import com.vitorpamplona.quartz.nip57Zaps.LnZapEvent +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent +import com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -1396,6 +1398,41 @@ private suspend fun reactToNote( } } +/** + * Casts a NIP-88 poll vote: builds a kind-1018 [PollResponseEvent] referencing [poll], + * signs it, optimistically consumes it locally (so the tally + hasVoted gate flip + * immediately), then broadcasts to all relays. The relay echo of the same signed event + * is deduped by id, so no double count. + * + * MUST be launched on a long-lived scope (e.g. `localCache.appScope`) — never a card's + * `rememberCoroutineScope()` — so scrolling the poll out of composition between the + * local consume and the broadcast can't cancel the send. + */ +suspend fun voteOnPoll( + poll: PollEvent, + responses: Set, + account: AccountState.LoggedIn, + relayManager: DesktopRelayConnectionManager, + localCache: DesktopLocalCache, +) { + if (responses.isEmpty()) return + withContext(Dispatchers.IO) { + val template = PollResponseEvent.build(EventHintBundle(poll), responses) + val signed = account.signer.sign(template) + localCache.consume(signed, null, wasVerified = true) + // Publish to the poll's OWN declared relays (NIP-88 `relay` tags) as well as our + // connected relays — the poll author and other viewers read votes from the poll's + // relays, which we may not be connected to. broadcastToAll alone would lose the vote + // for everyone but us (mirrors the read path in DesktopPollCard.responseRelays). + val targetRelays = (poll.relays() + relayManager.connectedRelays.value).toSet() + if (targetRelays.isNotEmpty()) { + relayManager.publish(signed, targetRelays) + } else { + relayManager.broadcastToAll(signed) + } + } +} + /** * Adds an event to bookmarks (public or private). * Returns the new bookmark list event, or null if operation failed. diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/SearchScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/SearchScreen.kt index 29f56c5047..ba976472ea 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/SearchScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/SearchScreen.kt @@ -51,6 +51,7 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue @@ -109,6 +110,7 @@ import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip05DnsIdentifiers.namecoin.NamecoinNameResolver import com.vitorpamplona.quartz.nip05DnsIdentifiers.namecoin.NamecoinResolveOutcome import com.vitorpamplona.quartz.nip19Bech32.decodePublicKeyAsHexOrNull +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent import kotlinx.coroutines.launch @Composable @@ -361,6 +363,23 @@ fun SearchScreen( } } + // Fetch interactions (incl. kind-1018 poll responses) for poll results so their + // tallies populate — NIP-50 search returns the polls but not their responses. + val pollResultIds = + remember(noteResults) { + noteResults.filter { it.kind == PollEvent.KIND }.map { it.id } + } + DisposableEffect(pollResultIds, subscriptionsCoordinator, searchRelays) { + val coordinator = subscriptionsCoordinator + val subId = + if (coordinator != null && pollResultIds.isNotEmpty() && searchRelays.isNotEmpty()) { + coordinator.requestInteractions(pollResultIds, searchRelays) + } else { + null + } + onDispose { subId?.let { coordinator?.releaseInteractions(it) } } + } + // History state val historyItems by SearchHistoryStore.history.collectAsState() val savedSearches by SearchHistoryStore.savedSearches.collectAsState() @@ -720,6 +739,9 @@ fun SearchScreen( onNavigateToProfile = onNavigateToProfile, onNavigateToThread = onNavigateToThread, localCache = localCache, + relayManager = relayManager, + account = account, + myPubKeyHex = account?.pubKeyHex, modifier = Modifier.padding(horizontal = sidePadding), ) } else if (!debouncedQuery.isEmpty && !isSearching) { diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ThreadScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ThreadScreen.kt index f637ae4523..ba8a07a4b4 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ThreadScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ThreadScreen.kt @@ -304,6 +304,7 @@ fun ThreadScreen( relayManager = relayManager, localCache = localCache, account = account, + myPubKeyHex = account?.pubKeyHex, nwcConnection = nwcConnection, onReply = { rootNote.event?.let { onReply(it) } }, onZapFeedback = onZapFeedback, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/UserProfileScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/UserProfileScreen.kt index 8540b85045..896b9d930f 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/UserProfileScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/UserProfileScreen.kt @@ -944,6 +944,7 @@ fun UserProfileScreen( relayManager = relayManager, localCache = localCache, account = account, + myPubKeyHex = account?.pubKeyHex, nwcConnection = nwcConnection, onReply = onCompose, onZapFeedback = onZapFeedback, @@ -1034,6 +1035,7 @@ fun UserProfileScreen( relayManager = relayManager, localCache = localCache, account = account, + myPubKeyHex = account?.pubKeyHex, nwcConnection = nwcConnection, onReply = onCompose, onZapFeedback = onZapFeedback, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/note/DesktopPollCard.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/note/DesktopPollCard.kt new file mode 100644 index 0000000000..6fa5219d06 --- /dev/null +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/note/DesktopPollCard.kt @@ -0,0 +1,650 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.desktop.ui.note + +import androidx.compose.animation.core.Animatable +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.Button +import androidx.compose.material3.ElevatedCard +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.key +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.drawWithContent +import androidx.compose.ui.graphics.drawscope.clipRect +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.IntOffset +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.window.Popup +import androidx.compose.ui.window.PopupProperties +import androidx.compose.ui.zIndex +import com.vitorpamplona.amethyst.commons.icons.symbols.Icon +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.commons.model.Note +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.nip88Polls.TallyResults +import com.vitorpamplona.amethyst.commons.ui.components.UserAvatar +import com.vitorpamplona.amethyst.desktop.account.AccountState +import com.vitorpamplona.amethyst.desktop.cache.DesktopLocalCache +import com.vitorpamplona.amethyst.desktop.network.DesktopRelayConnectionManager +import com.vitorpamplona.amethyst.desktop.subscriptions.SubscriptionConfig +import com.vitorpamplona.amethyst.desktop.subscriptions.generateSubId +import com.vitorpamplona.amethyst.desktop.subscriptions.rememberSubscription +import com.vitorpamplona.amethyst.desktop.ui.toNoteDisplayData +import com.vitorpamplona.amethyst.desktop.ui.voteOnPoll +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent +import com.vitorpamplona.quartz.nip88Polls.poll.tags.OptionTag +import com.vitorpamplona.quartz.nip88Polls.poll.tags.PollType +import com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.launch +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter + +/** + * A single poll option paired with its (stable, per-event) tally flow so each option + * row is its own leaf collector — no combining all options into one flow (which would + * cause a recomposition storm). + */ +private class PollOptionFlow( + val option: OptionTag, + val results: Flow, + val currentResults: () -> TallyResults, +) + +/** + * Desktop NIP-88 poll card. Reuses [NoteCard] for the author/description/media header + * (via [bottomContent] slot for the interactive options) and renders the tally itself. + * + * Hide-until-voted (decision #2): controls are shown unless the viewer is the author, + * has voted, the poll ended, or opted into "View results". Re-vote allowed (decision #6): + * results view offers "Change vote". + */ +@Composable +fun DesktopPollCard( + note: Note, + event: PollEvent, + relayManager: DesktopRelayConnectionManager, + localCache: DesktopLocalCache, + account: AccountState.LoggedIn?, + myPubKeyHex: String?, + onNavigateToThread: (String) -> Unit = {}, + onNavigateToProfile: (String) -> Unit = {}, + onHashtagClick: ((String) -> Unit)? = null, +) { + val options = remember(event) { event.options() } + if (options.isEmpty()) return + + val pollState = remember(note) { note.pollState() } + val forKey = myPubKeyHex ?: "" + + // Load this poll's responses from the poll's OWN declared relays (NIP-88 `relay` tags) + // unioned with the viewer's connected relays. Votes are published to the poll's relays, + // which the viewer usually isn't subscribed to — so the feed/thread/search interaction + // fetches (which only query the viewer's relays) miss them and the tally shows just the + // viewer's own vote. Querying the poll's declared relays makes the full tally load in + // any context that renders this card. + val connectedRelays by relayManager.connectedRelays.collectAsState() + val responseRelays = + remember(event, connectedRelays) { + (event.relays() + connectedRelays).toSet() + } + rememberSubscription(responseRelays, relayManager = relayManager) { + if (responseRelays.isEmpty()) return@rememberSubscription null + SubscriptionConfig( + subId = generateSubId("poll-resp-${event.id.take(8)}"), + filters = + listOf( + Filter( + kinds = listOf(PollResponseEvent.KIND), + tags = mapOf("e" to listOf(event.id)), + ), + ), + relays = responseRelays, + onEvent = { ev, _, relay, _ -> localCache.consume(ev, relay, wasVerified = false) }, + ) + } + + // One stable flow per option, built once per event (Delta #6). + val optionFlows = + remember(event) { + options.map { option -> + PollOptionFlow( + option = option, + results = pollState.tallyFlow(option.code, forKey, localCache.followedUsers), + currentResults = { pollState.currentTally(option.code, forKey, localCache.followedUsers.value) }, + ) + } + } + + val pollType = remember(event) { event.pollType() } + val hasEnded = remember(event) { event.hasEnded() } + val isMyPoll = myPubKeyHex != null && event.pubKey == myPubKeyHex + // A read-only (watch-only) account can't sign — show results instead of dead controls. + val canVote = account != null && !account.isReadOnly + + // Seed the voted-gate synchronously to avoid a first-frame flash (Delta #7). + val myUser = remember(note, myPubKeyHex) { myPubKeyHex?.let { localCache.getOrCreateUser(it) } } + val hasVotedSeed = remember(pollState, myUser) { myUser?.let { pollState.hasPubKeyVoted(it) } ?: false } + val hasVoted by + remember(pollState, myUser) { + myUser?.let { pollState.hasPubKeyVotedFlow(it) } ?: flowOf(false) + }.collectAsState(hasVotedSeed) + + // Local UI state keyed by note id so LazyColumn slot recycling can't leak one + // poll's selection into another (Delta #9). `viewingResults` = opted into results + // before voting; `revoting` = tapped "Change vote" to reopen controls after voting. + var viewingResults by remember(note.idHex) { mutableStateOf(false) } + var revoting by remember(note.idHex) { mutableStateOf(false) } + + // Tap a result row to see who voted for that option. + var voterPopup by remember(note.idHex) { mutableStateOf>?>(null) } + + // Total votes + deadline label for the footer. + val tallyState by pollState.responses.collectAsState() + // Distinct voters (not total selections) so a multi-choice voter counts once. + val totalVotes = tallyState.votes.size + // Pre-seed a multi-choice re-vote with the viewer's existing selection. + val myCurrentVote = + remember(tallyState, myUser) { + myUser?.let { tallyState.votes[it]?.responses()?.toSet() } ?: emptySet() + } + val endsAtSec = remember(event) { event.endsAt() } + val deadlineLabel = + remember(endsAtSec, hasEnded) { + endsAtSec?.let { (if (hasEnded) "Ended " else "Ends ") + formatPollTimestamp(it) } + } + + val displayData = remember(event) { event.toNoteDisplayData(localCache) } + + NoteCard( + note = displayData, + modifier = Modifier.fillMaxWidth(), + localCache = localCache, + onClick = { onNavigateToThread(event.id) }, + onAuthorClick = onNavigateToProfile, + onMentionClick = onNavigateToProfile, + onHashtagClick = onHashtagClick, + onNavigateToThread = onNavigateToThread, + bottomContent = { + Column( + modifier = Modifier.fillMaxWidth().padding(horizontal = 16.dp, vertical = 4.dp), + verticalArrangement = Arrangement.spacedBy(6.dp), + ) { + // Results gate (decision #2): author / ended / already-voted / opted-in + // see results — unless the viewer explicitly reopened controls to re-vote. + val showResults = !revoting && (isMyPoll || hasVoted || hasEnded || viewingResults || !canVote) + if (showResults) { + optionFlows.forEach { of -> + key(of.option.code) { + PollResultRow(of, forKey) { label, voters -> + voterPopup = label to voters + } + } + } + if (!hasEnded && !isMyPoll && canVote && hasVoted) { + Text( + text = "Change vote", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.primary, + modifier = + Modifier + .clip(RoundedCornerShape(4.dp)) + .clickable { revoting = true } + .padding(horizontal = 6.dp, vertical = 4.dp), + ) + } + } else { + when (pollType) { + PollType.SINGLE_CHOICE -> + SingleChoiceOptions(options, account) { code -> + revoting = false + castVote(event, setOf(code), account, relayManager, localCache) + } + PollType.MULTI_CHOICE -> + MultiChoiceOptions(note, options, account, myCurrentVote) { codes -> + revoting = false + castVote(event, codes, account, relayManager, localCache) + } + } + Text( + text = if (revoting) "Back to results" else "View results", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.primary, + modifier = + Modifier + .clip(RoundedCornerShape(4.dp)) + .clickable { + if (revoting) revoting = false else viewingResults = true + }.padding(horizontal = 6.dp, vertical = 4.dp), + ) + } + + if (totalVotes > 0 || deadlineLabel != null) { + Row( + modifier = Modifier.fillMaxWidth().padding(top = 2.dp), + horizontalArrangement = Arrangement.SpaceBetween, + ) { + Text( + text = "$totalVotes ${if (totalVotes == 1) "vote" else "votes"}", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + deadlineLabel?.let { + Text( + text = it, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } + } + } + }, + ) + + voterPopup?.let { (label, voters) -> + VoterListPopup( + optionLabel = label, + voters = voters, + forKey = forKey, + onDismiss = { voterPopup = null }, + onNavigateToProfile = onNavigateToProfile, + ) + } +} + +private fun castVote( + event: PollEvent, + codes: Set, + account: AccountState.LoggedIn?, + relayManager: DesktopRelayConnectionManager, + localCache: DesktopLocalCache, +) { + if (account == null || account.isReadOnly || codes.isEmpty()) return + // Launch on the cache-scoped scope, NOT the card's scope, so the consume→broadcast + // pair can't be half-cancelled when the card leaves composition (Delta #2). + localCache.appScope.launch { + voteOnPoll(event, codes, account, relayManager, localCache) + } +} + +@Composable +private fun SingleChoiceOptions( + options: List, + account: AccountState.LoggedIn?, + onRespond: (String) -> Unit, +) { + options.forEach { option -> + Box( + modifier = + Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(8.dp)) + .border(1.dp, MaterialTheme.colorScheme.outline, RoundedCornerShape(8.dp)) + .then( + if (account != null) { + Modifier.clickable { onRespond(option.code) } + } else { + Modifier + }, + ), + ) { + Row( + modifier = Modifier.fillMaxWidth().padding(12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Icon( + symbol = MaterialSymbols.RadioButtonUnchecked, + contentDescription = null, + modifier = Modifier.size(20.dp), + tint = MaterialTheme.colorScheme.primary, + ) + Text(text = option.label, style = MaterialTheme.typography.bodyMedium) + } + } + } +} + +@Composable +private fun MultiChoiceOptions( + note: Note, + options: List, + account: AccountState.LoggedIn?, + initialSelection: Set, + onRespond: (Set) -> Unit, +) { + // Keyed by note id so recycling doesn't leak selection across polls (Delta #9); + // seeded with the viewer's existing vote so a re-vote starts from prior choices. + var selected by remember(note.idHex) { mutableStateOf(initialSelection) } + + options.forEach { option -> + val isChecked = option.code in selected + Box( + modifier = + Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(8.dp)) + .border(1.dp, MaterialTheme.colorScheme.outline, RoundedCornerShape(8.dp)) + .clickable { + selected = if (isChecked) selected - option.code else selected + option.code + }, + ) { + Row( + modifier = Modifier.fillMaxWidth().padding(12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + // No CheckBox glyph in the subset font — a bordered box with a Check + // glyph when selected (avoids a new codepoint / font regen). + Box( + modifier = + Modifier + .size(20.dp) + .clip(RoundedCornerShape(4.dp)) + .then( + if (isChecked) { + Modifier.background(MaterialTheme.colorScheme.primary) + } else { + Modifier.border( + 1.dp, + MaterialTheme.colorScheme.outline, + RoundedCornerShape(4.dp), + ) + }, + ), + contentAlignment = Alignment.Center, + ) { + if (isChecked) { + Icon( + symbol = MaterialSymbols.Check, + contentDescription = null, + modifier = Modifier.size(16.dp), + tint = MaterialTheme.colorScheme.onPrimary, + ) + } + } + Text(text = option.label, style = MaterialTheme.typography.bodyMedium) + } + } + } + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.End, + ) { + Button( + onClick = { onRespond(selected) }, + enabled = account != null && selected.isNotEmpty(), + ) { + Text("Submit") + } + } +} + +@Composable +private fun PollResultRow( + of: PollOptionFlow, + forKey: String, + onShowVoters: (String, List) -> Unit, +) { + val tally by of.results.collectAsState(of.currentResults()) + + // First-frame bar guard: snap on first emission, animate afterwards (Delta #10). + val animated = remember { Animatable(tally.percent) } + LaunchedEffect(tally.percent) { + animated.animateTo(tally.percent) + } + + val isMyVote = forKey.isNotEmpty() && tally.users.any { it.pubkeyHex == forKey } + val winning = tally.isWinning + val barColor = if (winning) MaterialTheme.colorScheme.tertiary else MaterialTheme.colorScheme.primary + // Border marks YOUR choice (primary); the winner is conveyed by the bar fill color. + val borderColor = + when { + isMyVote -> MaterialTheme.colorScheme.primary + winning -> MaterialTheme.colorScheme.tertiary + else -> MaterialTheme.colorScheme.outline + } + val borderWidth = if (isMyVote) 2.dp else 1.dp + + Box( + modifier = + Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(8.dp)) + .border(borderWidth, borderColor, RoundedCornerShape(8.dp)) + .background(MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.4f)) + .clickable { onShowVoters(of.option.label, tally.users) }, + ) { + Box( + modifier = + Modifier + .matchParentSize() + .alpha(0.32f) + .drawWithContent { + clipRect(right = size.width * animated.value) { + drawRect(barColor) + } + drawContent() + }, + ) + + Row( + modifier = Modifier.fillMaxWidth().padding(12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.SpaceBetween, + ) { + Row( + modifier = Modifier.weight(1f), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(6.dp), + ) { + if (isMyVote) { + Icon( + symbol = MaterialSymbols.Check, + contentDescription = "Your vote", + modifier = Modifier.size(16.dp), + tint = MaterialTheme.colorScheme.primary, + ) + } + Text( + text = of.option.label, + style = MaterialTheme.typography.bodyMedium, + fontWeight = if (isMyVote) FontWeight.SemiBold else FontWeight.Normal, + ) + } + Spacer(Modifier.width(12.dp)) + Row(verticalAlignment = Alignment.CenterVertically) { + VoterGallery(tally.users, forKey) + Spacer(Modifier.width(8.dp)) + Text( + text = "${(tally.percent * 100).toInt()}%", + style = MaterialTheme.typography.bodyMedium, + fontWeight = FontWeight.Bold, + textAlign = TextAlign.End, + ) + } + } + } +} + +@Composable +private fun VoterGallery( + users: List, + forKey: String, +) { + if (users.isEmpty()) return + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy((-10).dp), + ) { + users.take(4).forEachIndexed { index, user -> + key(user.pubkeyHex) { + val isMe = forKey.isNotEmpty() && user.pubkeyHex == forKey + UserAvatar( + userHex = user.pubkeyHex, + pictureUrl = user.profilePicture(), + size = 24.dp, + // Earlier avatars draw on top so the leftmost (you, sorted first) is + // front-most instead of buried under the next ones; ring your own. + modifier = + Modifier + .zIndex((users.size - index).toFloat()) + .then( + if (isMe) { + Modifier.border(2.dp, MaterialTheme.colorScheme.primary, CircleShape) + } else { + Modifier + }, + ), + ) + } + } + if (users.size > 4) { + Box( + contentAlignment = Alignment.Center, + modifier = + Modifier + .size(24.dp) + .clip(CircleShape) + .background(MaterialTheme.colorScheme.secondaryContainer), + ) { + Text( + text = "+${users.size - 4}", + fontSize = 10.sp, + color = MaterialTheme.colorScheme.onSecondaryContainer, + ) + } + } + } +} + +@Composable +private fun VoterListPopup( + optionLabel: String, + voters: List, + forKey: String, + onDismiss: () -> Unit, + onNavigateToProfile: (String) -> Unit, +) { + Popup( + alignment = Alignment.Center, + offset = IntOffset(0, 0), + onDismissRequest = onDismiss, + properties = PopupProperties(focusable = true), + ) { + ElevatedCard(modifier = Modifier.widthIn(max = 320.dp)) { + Column( + modifier = + Modifier + .verticalScroll(rememberScrollState()) + .heightIn(max = 360.dp) + .padding(12.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + Text( + text = "${voters.size} ${if (voters.size == 1) "vote" else "votes"} · $optionLabel", + style = MaterialTheme.typography.titleSmall, + fontWeight = FontWeight.SemiBold, + ) + HorizontalDivider() + if (voters.isEmpty()) { + Text( + text = "No votes yet", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } else { + voters.forEach { user -> + key(user.pubkeyHex) { + val isMe = forKey.isNotEmpty() && user.pubkeyHex == forKey + Row( + modifier = + Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(6.dp)) + .clickable { + onNavigateToProfile(user.pubkeyHex) + onDismiss() + }.padding(vertical = 4.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + UserAvatar( + userHex = user.pubkeyHex, + pictureUrl = user.profilePicture(), + size = 28.dp, + ) + Text( + text = user.toBestDisplayName() + if (isMe) " (you)" else "", + style = MaterialTheme.typography.bodyMedium, + ) + } + } + } + } + } + } + } +} + +private val POLL_TIME_FORMAT: DateTimeFormatter = DateTimeFormatter.ofPattern("MMM d, HH:mm") + +private fun formatPollTimestamp(epochSeconds: Long): String = + Instant + .ofEpochSecond(epochSeconds) + .atZone(ZoneId.systemDefault()) + .format(POLL_TIME_FORMAT) diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/search/SearchResultsList.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/search/SearchResultsList.kt index 68bd2e8fbc..c225256ee4 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/search/SearchResultsList.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/search/SearchResultsList.kt @@ -61,12 +61,16 @@ import com.vitorpamplona.amethyst.commons.search.SearchSortOrder import com.vitorpamplona.amethyst.commons.ui.components.UserSearchCard import com.vitorpamplona.amethyst.commons.wot.LocalWoTReady import com.vitorpamplona.amethyst.commons.wot.LocalWoTService +import com.vitorpamplona.amethyst.desktop.account.AccountState import com.vitorpamplona.amethyst.desktop.cache.DesktopLocalCache +import com.vitorpamplona.amethyst.desktop.network.DesktopRelayConnectionManager +import com.vitorpamplona.amethyst.desktop.ui.note.DesktopPollCard import com.vitorpamplona.amethyst.desktop.ui.note.NoteCard import com.vitorpamplona.amethyst.desktop.ui.note.SpamCheckedNoteRender import com.vitorpamplona.amethyst.desktop.ui.note.WoTBadge import com.vitorpamplona.amethyst.desktop.ui.rememberDisplayData import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent @Composable fun SearchResultsList( @@ -74,6 +78,10 @@ fun SearchResultsList( onNavigateToProfile: (String) -> Unit, onNavigateToThread: (String) -> Unit, localCache: DesktopLocalCache? = null, + relayManager: DesktopRelayConnectionManager? = null, + account: AccountState.LoggedIn? = null, + myPubKeyHex: String? = null, + onHashtagClick: ((String) -> Unit)? = null, modifier: Modifier = Modifier, listState: LazyListState = rememberLazyListState(), ) { @@ -89,7 +97,8 @@ fun SearchResultsList( // Group notes by kind val textNotes = notes.filter { it.kind == 1 } val articles = notes.filter { it.kind == LongTextNoteEvent.KIND } - val otherNotes = notes.filter { it.kind != 1 && it.kind != LongTextNoteEvent.KIND } + val polls = notes.filter { it.kind == PollEvent.KIND } + val otherNotes = notes.filter { it.kind != 1 && it.kind != LongTextNoteEvent.KIND && it.kind != PollEvent.KIND } // Per-section collapsed state (absent = expanded) val collapsedSections = remember { mutableStateMapOf() } @@ -252,6 +261,56 @@ fun SearchResultsList( } } + // Polls section (interactive cards — read tallies + vote) + if (polls.isNotEmpty()) { + item(key = "divider-polls") { HorizontalDivider(Modifier.padding(vertical = 4.dp)) } + val collapsed = collapsedSections["polls"] == true + stickyHeader(key = "header-polls") { + SortableHeader( + title = "Polls", + count = polls.size, + icon = MaterialSymbols.Poll, + options = SearchSortOrder.EVENT_OPTIONS, + selected = eventSortOrder, + onSelect = { state.updateEventSortOrder(it) }, + collapsed = collapsed, + onToggleCollapse = { collapsedSections["polls"] = !collapsed }, + ) + } + if (!collapsed) { + items(polls.take(5), key = { "poll-${it.id}" }) { event -> + PollSearchItem( + event = event as PollEvent, + localCache = localCache, + relayManager = relayManager, + account = account, + myPubKeyHex = myPubKeyHex, + onNavigateToThread = onNavigateToThread, + onNavigateToProfile = onNavigateToProfile, + onHashtagClick = onHashtagClick, + ) + } + if (polls.size > 5) { + item(key = "polls-expand") { + ExpandableSection( + remaining = polls.drop(5), + ) { event -> + PollSearchItem( + event = event as PollEvent, + localCache = localCache, + relayManager = relayManager, + account = account, + myPubKeyHex = myPubKeyHex, + onNavigateToThread = onNavigateToThread, + onNavigateToProfile = onNavigateToProfile, + onHashtagClick = onHashtagClick, + ) + } + } + } + } + } + // Other section if (otherNotes.isNotEmpty()) { item(key = "divider-other") { HorizontalDivider(Modifier.padding(vertical = 4.dp)) } @@ -320,6 +379,49 @@ private fun wotBadgeFor(userHex: String): (@Composable androidx.compose.foundati } } +@Composable +private fun PollSearchItem( + event: PollEvent, + localCache: DesktopLocalCache?, + relayManager: DesktopRelayConnectionManager?, + account: AccountState.LoggedIn?, + myPubKeyHex: String?, + onNavigateToThread: (String) -> Unit, + onNavigateToProfile: (String) -> Unit, + onHashtagClick: ((String) -> Unit)?, +) { + SpamCheckedNoteRender( + displayedEvent = event, + noteIdHex = event.id, + localCache = localCache, + ) { + if (localCache != null && relayManager != null) { + // Interactive: read tallies + vote. Note is resolved from the cache; option + // rendering comes from the event, so an empty (unconsumed) note still renders. + DesktopPollCard( + note = localCache.getOrCreateNote(event.id), + event = event, + relayManager = relayManager, + localCache = localCache, + account = account, + myPubKeyHex = myPubKeyHex, + onNavigateToThread = onNavigateToThread, + onNavigateToProfile = onNavigateToProfile, + onHashtagClick = onHashtagClick, + ) + } else { + // Read-only fallback when the live cache/relay manager isn't available. + NoteCard( + note = event.rememberDisplayData(localCache), + localCache = localCache, + onClick = { onNavigateToThread(event.id) }, + onAuthorClick = onNavigateToProfile, + onMentionClick = onNavigateToProfile, + ) + } + } +} + @Composable private fun SortableHeader( title: String, diff --git a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCachePollTest.kt b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCachePollTest.kt new file mode 100644 index 0000000000..88c32fb660 --- /dev/null +++ b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCachePollTest.kt @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.desktop.cache + +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent +import com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * NIP-88 poll consumption: a kind-1068 poll becomes a renderable Note, and a kind-1018 + * response is linked into that poll Note's `pollState()` tally. Second identical response + * (a relay echo) must not double-count. + */ +class DesktopLocalCachePollTest { + private val relayUrl = NormalizedRelayUrl("wss://relay.test/") + + private fun signedPoll( + signer: NostrSignerSync, + createdAt: Long, + ): PollEvent = + signer.sign( + createdAt = createdAt, + kind = PollEvent.KIND, + tags = + arrayOf( + arrayOf("option", "0", "Yes"), + arrayOf("option", "1", "No"), + arrayOf("polltype", "singlechoice"), + ), + content = "Pick one", + ) + + private fun signedResponse( + signer: NostrSignerSync, + pollId: String, + option: String, + createdAt: Long, + ): PollResponseEvent = + signer.sign( + createdAt = createdAt, + kind = PollResponseEvent.KIND, + tags = + arrayOf( + arrayOf("e", pollId), + arrayOf("response", option), + ), + content = "", + ) + + @Test + fun `a poll response is linked into the poll's tally`() { + val cache = DesktopLocalCache() + val author = NostrSignerSync(KeyPair()) + val voter = NostrSignerSync(KeyPair()) + + val poll = signedPoll(author, createdAt = 1_700_000_000) + assertTrue(cache.consume(poll, relayUrl, wasVerified = true), "poll should be consumed") + + val response = signedResponse(voter, poll.id, option = "0", createdAt = 1_700_000_100) + assertTrue(cache.consume(response, relayUrl, wasVerified = true), "response should be consumed") + + val pollNote = cache.getNoteIfExists(poll.id) + assertTrue(pollNote != null, "poll note must exist") + val tally = pollNote.pollState().responses.value + assertEquals(1, tally.totalVotes()) + assertEquals("0", tally.winning()) + } + + @Test + fun `a duplicate response is not counted twice`() { + val cache = DesktopLocalCache() + val author = NostrSignerSync(KeyPair()) + val voter = NostrSignerSync(KeyPair()) + + val poll = signedPoll(author, createdAt = 1_700_000_000) + cache.consume(poll, relayUrl, wasVerified = true) + + val response = signedResponse(voter, poll.id, option = "1", createdAt = 1_700_000_100) + assertTrue(cache.consume(response, relayUrl, wasVerified = true)) + // Same signed event echoed back by another relay — id-dedup must reject it. + assertTrue(!cache.consume(response, relayUrl, wasVerified = true)) + + val tally = + cache + .getNoteIfExists(poll.id)!! + .pollState() + .responses.value + assertEquals(1, tally.totalVotes()) + } +} diff --git a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/filters/FilterBuildersTest.kt b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/filters/FilterBuildersTest.kt index c32cb76657..9ce9d808bb 100644 --- a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/filters/FilterBuildersTest.kt +++ b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/filters/FilterBuildersTest.kt @@ -37,7 +37,7 @@ class FilterBuildersTest { fun testTextNotesGlobal() { val filter = FilterBuilders.textNotesGlobal(limit = 50) - assertEquals(listOf(1, 6, 16), filter.kinds) + assertEquals(listOf(1, 6, 16, 1068), filter.kinds) assertEquals(50, filter.limit) assertNull(filter.authors) assertNull(filter.tags) @@ -51,7 +51,7 @@ class FilterBuildersTest { val until = 1640995200L // 2022-01-01 val filter = FilterBuilders.textNotesGlobal(limit = 100, since = since, until = until) - assertEquals(listOf(1, 6, 16), filter.kinds) + assertEquals(listOf(1, 6, 16, 1068), filter.kinds) assertEquals(100, filter.limit) assertEquals(since, filter.since) assertEquals(until, filter.until) @@ -62,7 +62,7 @@ class FilterBuildersTest { val authors = listOf(testPubKey, testPubKey2) val filter = FilterBuilders.textNotesFromAuthors(authors, limit = 25) - assertEquals(listOf(1, 6, 16), filter.kinds) + assertEquals(listOf(1, 6, 16, 1068), filter.kinds) assertEquals(authors, filter.authors) assertEquals(25, filter.limit) assertNull(filter.tags) @@ -74,7 +74,7 @@ class FilterBuildersTest { val since = 1609459200L val filter = FilterBuilders.textNotesFromAuthors(authors, limit = 10, since = since) - assertEquals(listOf(1, 6, 16), filter.kinds) + assertEquals(listOf(1, 6, 16, 1068), filter.kinds) assertEquals(authors, filter.authors) assertEquals(10, filter.limit) assertEquals(since, filter.since) @@ -432,7 +432,7 @@ class FilterBuildersTest { val filter = FilterBuilders.textNotesGlobal(limit = 50) assertTrue(!filter.isEmpty()) - assertEquals(listOf(1, 6, 16), filter.kinds) + assertEquals(listOf(1, 6, 16, 1068), filter.kinds) assertEquals(50, filter.limit) } @@ -442,7 +442,7 @@ class FilterBuildersTest { val filter = FilterBuilders.textNotesFromAuthors(followedUsers, limit = 50) assertTrue(!filter.isEmpty()) - assertEquals(listOf(1, 6, 16), filter.kinds) + assertEquals(listOf(1, 6, 16, 1068), filter.kinds) assertEquals(followedUsers, filter.authors) assertEquals(50, filter.limit) } @@ -458,7 +458,7 @@ class FilterBuildersTest { assertTrue(!contactListFilter.isEmpty()) assertEquals(listOf(0), metadataFilter.kinds) - assertEquals(listOf(1, 6, 16), postsFilter.kinds) + assertEquals(listOf(1, 6, 16, 1068), postsFilter.kinds) assertEquals(listOf(3), contactListFilter.kinds) } From 1c9f8f8d54dec6f5db78a0b200ec3f3f4ac596da Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:24:31 +0000 Subject: [PATCH 22/34] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-hi-rIN/strings.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/amethyst/src/main/res/values-hi-rIN/strings.xml b/amethyst/src/main/res/values-hi-rIN/strings.xml index f08afc5551..aca8fc8d3a 100644 --- a/amethyst/src/main/res/values-hi-rIN/strings.xml +++ b/amethyst/src/main/res/values-hi-rIN/strings.xml @@ -503,6 +503,15 @@ चित्र योजक के रूप में बाँटें पूर्वीक्षण चित्र उत्पादन चालू… अमेथिस्ट द्वारा बाँटा गया + क्यूआर॰ चित्र के रूप में बाँटें + जाल योजक + नोस्टर योजक + किसी भी संचारयन्त्र चित्रग्राहक के साथ परखें + नोस्टर क्रमक के साथ परखें + चित्र + क्यूआर॰ चित्र जिसमें इस टीका का एक जाल योजक समाविष्ट है + क्यूआर॰ चित्र जिसमें इस टीका का एक नोस्टर योजक समाविष्ट है + अंगुलचित्र छिपाया गया संवेदनशिल विषयवस्तु के कारण लेखक विभेदक टीका विभेदक लेख की अनुकृति करें From 2e4afbf75ee285fba86b7d423b734af1365c96a8 Mon Sep 17 00:00:00 2001 From: davotoula Date: Wed, 22 Jul 2026 13:41:55 +0100 Subject: [PATCH 23/34] fix: stop ConcurrentModificationException in AddressableAuthorRelayLoaderSubAssembler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `activeSubscriptions` was a plain LinkedHashSet iterated (`SetsKt.minus`) and mutated (`clear`/`addAll`) with no synchronization, while `invalidateFilters()` ran synchronously on whatever thread called subscribe/unsubscribe. Those callers are genuinely concurrent: `ComposeSubscriptionManager` invokes `invalidateKeys()` after releasing its own lock, and `LifecycleAwareSubscription`'s 30s grace-period unsubscribe fires on a `Dispatchers.Default` worker while composition subscribes from elsewhere. Hence the reported `ConcurrentModificationException` on `DefaultDispatcher-worker-70`. This is the only member of `EventFinderFilterAssembler.group` that implements `IEoseManager` directly; its two siblings extend `BaseEoseManager`, whose `invalidateFilters` hands off to `BundledUpdate` and is therefore never run on the caller's thread nor concurrently with itself. Fix, matching that existing pattern and avoiding locks: - Route through `BundledUpdate`. `BasicBundledUpdate` holds `isProcessing` under a Mutex, so only one body runs at a time — the concurrent iterate-vs-mutate window is gone by construction. It also moves the `allKeys()` scan and per-stub `getOrCreateUser` off the caller thread, which `ComposeSubscriptionManager` documents as "called by main. Keep it really fast." - Hold the state in an `AtomicReference>` of immutable snapshots swapped with `exchange()`, plus an `AtomicBoolean` teardown flag, mirroring the `AtomicReference` + CAS idiom in `FilterIndex`/`BanStore`. - `bundler.cancel()` cannot stop a body already executing (no suspension points), so `destroy()` flags first and an in-flight body compensates by releasing what it just acquired. Double-unsubscribe is a no-op. No locks are introduced; the hot path is strictly cheaper than before. Tested: the new concurrency test reproduces the exact production failure against the pre-fix code (`ConcurrentModificationException` alongside the overlap detector) and passes after. Full :amethyst suite green (941 tests). Note the pre-existing `UserFinderQueryState` identity-equality churn is deliberately NOT addressed here: the set-diff never converges because each run allocates fresh wrappers. It widens this race window but is an independent defect needing its own design decision. --- ...ddressableAuthorRelayLoaderSubAssembler.kt | 47 ++++- ...ssableAuthorRelayLoaderSubAssemblerTest.kt | 196 ++++++++++++++++++ 2 files changed, 234 insertions(+), 9 deletions(-) create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt index 5f81cb8d22..fc72c295f9 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt @@ -21,11 +21,17 @@ package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.IEoseManager +import com.vitorpamplona.amethyst.commons.service.BundledUpdate import com.vitorpamplona.amethyst.model.AddressableNote import com.vitorpamplona.amethyst.model.LocalCache import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.EventFinderQueryState import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderFilterAssembler import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlin.concurrent.atomics.AtomicBoolean +import kotlin.concurrent.atomics.AtomicReference +import kotlin.concurrent.atomics.ExperimentalAtomicApi /** * Bridges missing-addressable-note authors into [UserFinderFilterAssembler]. @@ -37,14 +43,29 @@ import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinder * kind-0 / kind-10002 and resolve outbox relays via [UserOutboxFinderSubAssembler]. Once the * relay list arrives, [EventFinderFilterAssembler] is invalidated and can query the correct relay. */ +@OptIn(ExperimentalAtomicApi::class) class AddressableAuthorRelayLoaderSubAssembler( val cache: LocalCache, val allKeys: () -> Set, val userFinder: UserFinderFilterAssembler, ) : IEoseManager { - private val activeSubscriptions = mutableSetOf() + // Immutable snapshots swapped atomically, so a diff can never observe a half-written set. + // Mutual exclusion between runs comes from the bundler (one body at a time), not from these + // atomics — they exist to hand state over to destroy(), the one caller the bundler cannot + // serialize. + private val activeSubscriptions = AtomicReference>(emptySet()) + private val destroyed = AtomicBoolean(false) + + // Keeps the scan off the caller's thread. invalidateFilters() is reached synchronously from + // ComposeSubscriptionManager.subscribe/unsubscribe on every note composable mount/unmount, + // and those are documented "called by main. Keep it really fast." + private val bundler = BundledUpdate(500, Dispatchers.IO) override fun invalidateFilters(ignoreIfDoing: Boolean) { + bundler.invalidate(ignoreIfDoing, ::forceInvalidate) + } + + private fun forceInvalidate() { val needed = mutableSetOf() allKeys().forEach { key -> @@ -57,18 +78,26 @@ class AddressableAuthorRelayLoaderSubAssembler( } } - val toAdd = needed - activeSubscriptions - val toRemove = activeSubscriptions - needed + if (destroyed.load()) return - userFinder.subscribe(toAdd.toList()) - userFinder.unsubscribe(toRemove.toList()) + val previous = activeSubscriptions.exchange(needed) - activeSubscriptions.clear() - activeSubscriptions.addAll(needed) + userFinder.subscribe((needed - previous).toList()) + userFinder.unsubscribe((previous - needed).toList()) + + // destroy() landed while we were subscribing. bundler.cancel() cannot stop a body that is + // already running — it has no suspension points — so the body releases what it just + // acquired. destroy() may unsubscribe the same states concurrently; that is a no-op. + if (destroyed.load()) { + activeSubscriptions.store(emptySet()) + userFinder.unsubscribe(needed.toList()) + } } override fun destroy() { - userFinder.unsubscribe(activeSubscriptions.toList()) - activeSubscriptions.clear() + // Flag before cancelling so an in-flight body is guaranteed to see the teardown. + destroyed.store(true) + bundler.cancel() + userFinder.unsubscribe(activeSubscriptions.exchange(emptySet()).toList()) } } diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt new file mode 100644 index 0000000000..db81e6c618 --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders + +import com.vitorpamplona.amethyst.model.Account +import com.vitorpamplona.amethyst.model.LocalCache +import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.EventFinderQueryState +import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderFilterAssembler +import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState +import com.vitorpamplona.quartz.nip01Core.core.Address +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread + +class AddressableAuthorRelayLoaderSubAssemblerTest { + /** + * Unique per-test-class identities so the shared [LocalCache] singleton isn't polluted with + * notes another test class also claims. The prefix keeps the key a valid 64-char hex pubkey. + */ + private fun stubKeys(count: Int): Set { + val account = mockk() + return (1..count).mapTo(mutableSetOf()) { i -> + val address = Address(30023, "ad04%060x".format(i), "d$i") + EventFinderQueryState(LocalCache.getOrCreateAddressableNoteInternal(address), account) + } + } + + /** + * Regression test for the `ConcurrentModificationException` in + * `SetsKt.minus` reported from `DefaultDispatcher-worker-70`. + * + * `ComposeSubscriptionManager.subscribe`/`unsubscribe` call `invalidateKeys()` *after* + * releasing their own lock, and `LifecycleAwareSubscription`'s 30s grace-period unsubscribe + * fires on a `Dispatchers.Default` worker — so this manager is genuinely re-entered from + * several threads at once. + * + * The invariant asserted here is the one that makes the crash impossible: **the body that + * reads and swaps the subscription state never runs concurrently with itself.** It's checked + * via the injected `allKeys()` lambda (called exactly once per body) rather than by catching + * the exception, because once the body is bundled its throwables are swallowed by + * `BundledUpdate`'s `CoroutineExceptionHandler` and would never reach the test thread. + */ + @Test + fun concurrentInvalidateFiltersNeverOverlap() { + val errors = CopyOnWriteArrayList() + val userFinder = mockk(relaxed = true) + val keys = stubKeys(50) + + val inFlight = AtomicInteger(0) + val assembler = + AddressableAuthorRelayLoaderSubAssembler( + LocalCache, + { + if (inFlight.incrementAndGet() > 1) { + errors.add(IllegalStateException("forceInvalidate bodies overlapped")) + } + try { + keys + } finally { + inFlight.decrementAndGet() + } + }, + userFinder, + ) + + val start = CountDownLatch(1) + try { + val threads = + (1..8).map { + thread(start = false) { + start.await() + repeat(500) { + try { + assembler.invalidateFilters() + } catch (t: Throwable) { + errors.add(t) + } + } + } + } + threads.forEach { it.start() } + start.countDown() + threads.forEach { it.join() } + } finally { + assembler.destroy() + } + + assertTrue( + "invalidateFilters raced under concurrency: " + + errors.map { "${it::class.simpleName}: ${it.message}" }.distinct(), + errors.isEmpty(), + ) + } + + /** The manager still does its job: unresolved stub authors reach the user finder. */ + @Test + fun bridgesMissingAuthorsIntoUserFinder() { + val userFinder = mockk(relaxed = true) + val keys = stubKeys(3) + val assembler = AddressableAuthorRelayLoaderSubAssembler(LocalCache, { keys }, userFinder) + + try { + assembler.invalidateFilters() + + verify(timeout = 3000) { + userFinder.subscribe( + match> { it.size == 3 }, + ) + } + } finally { + assembler.destroy() + } + } + + /** + * `bundler.cancel()` cannot stop a body that is already executing — the body has no + * suspension points, so it runs to completion after `destroy()` returns. Without the + * `destroyed` handshake the body re-subscribes authors that `destroy()` just released, + * leaving live kind-0/10002 REQs (and retained `User`/`Account` references) for a dead + * account after logout. + * + * The body is gated inside the injected `allKeys()` lambda so `destroy()` provably runs + * underneath an in-flight run rather than racing it by luck. + */ + @Test + fun destroyDuringInFlightInvalidateDoesNotLeakSubscriptions() { + val userFinder = mockk(relaxed = true) + val subscribed = CopyOnWriteArrayList() + val unsubscribed = CopyOnWriteArrayList() + val subscribeHappened = CountDownLatch(1) + every { userFinder.subscribe(any>()) } answers { + subscribed.addAll(firstArg>()) + subscribeHappened.countDown() + } + every { userFinder.unsubscribe(any>()) } answers { + unsubscribed.addAll(firstArg>()) + } + + val keys = stubKeys(3) + val invalidateEntered = CountDownLatch(1) + val destroyFinished = CountDownLatch(1) + val assembler = + AddressableAuthorRelayLoaderSubAssembler( + LocalCache, + { + invalidateEntered.countDown() + destroyFinished.await(5, TimeUnit.SECONDS) + keys + }, + userFinder, + ) + + try { + assembler.invalidateFilters() + assertTrue("bundled run never started", invalidateEntered.await(5, TimeUnit.SECONDS)) + } finally { + assembler.destroy() + destroyFinished.countDown() + } + + // Let the in-flight run finish (it either subscribes — the leak — or + // observes the teardown and skips; both settle within the timeout). + subscribeHappened.await(2, TimeUnit.SECONDS) + + val leaked = subscribed - unsubscribed.toSet() + assertTrue( + "destroy() left ${leaked.size} subscriptions alive in userFinder", + leaked.isEmpty(), + ) + } +} From 53a823e1b2a7c603e1ee10cd9f83274be5be4fcd Mon Sep 17 00:00:00 2001 From: davotoula Date: Wed, 22 Jul 2026 14:00:57 +0100 Subject: [PATCH 24/34] Code review: - private monitor, and correct the commit() KDoc - replace atomics handshake with a single monitor - flag userFinder re-entrancy assumption in commit() KDoc --- ...ddressableAuthorRelayLoaderSubAssembler.kt | 58 ++++++++------- ...ssableAuthorRelayLoaderSubAssemblerTest.kt | 72 ++++++------------- .../relayClient/eoseManagers/IEoseManager.kt | 6 ++ 3 files changed, 63 insertions(+), 73 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt index fc72c295f9..9dba6ed2bd 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt @@ -29,9 +29,6 @@ import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinder import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO -import kotlin.concurrent.atomics.AtomicBoolean -import kotlin.concurrent.atomics.AtomicReference -import kotlin.concurrent.atomics.ExperimentalAtomicApi /** * Bridges missing-addressable-note authors into [UserFinderFilterAssembler]. @@ -43,18 +40,18 @@ import kotlin.concurrent.atomics.ExperimentalAtomicApi * kind-0 / kind-10002 and resolve outbox relays via [UserOutboxFinderSubAssembler]. Once the * relay list arrives, [EventFinderFilterAssembler] is invalidated and can query the correct relay. */ -@OptIn(ExperimentalAtomicApi::class) class AddressableAuthorRelayLoaderSubAssembler( val cache: LocalCache, val allKeys: () -> Set, val userFinder: UserFinderFilterAssembler, ) : IEoseManager { - // Immutable snapshots swapped atomically, so a diff can never observe a half-written set. - // Mutual exclusion between runs comes from the bundler (one body at a time), not from these - // atomics — they exist to hand state over to destroy(), the one caller the bundler cannot - // serialize. - private val activeSubscriptions = AtomicReference>(emptySet()) - private val destroyed = AtomicBoolean(false) + // Private monitor: @Synchronized locks on `this`, which leaves the instance's monitor + // reachable to anything holding a reference to this assembler. + private val lock = Any() + + // Only ever touched while holding [lock]. See commit() and destroy(). + private var activeSubscriptions: Set = emptySet() + private var destroyed = false // Keeps the scan off the caller's thread. invalidateFilters() is reached synchronously from // ComposeSubscriptionManager.subscribe/unsubscribe on every note composable mount/unmount, @@ -78,26 +75,39 @@ class AddressableAuthorRelayLoaderSubAssembler( } } - if (destroyed.load()) return + commit(needed) + } - val previous = activeSubscriptions.exchange(needed) + /** + * Serializes against [destroy] — the one caller the bundler cannot order, because + * `bundler.cancel()` cannot stop a body that is already running (it has no suspension points). + * + * The scan in [forceInvalidate] stays outside [lock], so [destroy] never waits on a + * [LocalCache] sweep. It can still wait on the two calls below, which are bounded: a pair of + * map updates inside [UserFinderFilterAssembler] plus the coroutine launches its + * `invalidateKeys()` fans out to. + * + * Calling [userFinder] while holding [lock] relies on subscribe/unsubscribe only taking + * ComposeSubscriptionManager's own lock and deferring real work to bundled coroutines — they + * never call back into this class. Revisit if that changes. + */ + private fun commit(needed: Set) { + synchronized(lock) { + if (destroyed) return - userFinder.subscribe((needed - previous).toList()) - userFinder.unsubscribe((previous - needed).toList()) + userFinder.subscribe((needed - activeSubscriptions).toList()) + userFinder.unsubscribe((activeSubscriptions - needed).toList()) - // destroy() landed while we were subscribing. bundler.cancel() cannot stop a body that is - // already running — it has no suspension points — so the body releases what it just - // acquired. destroy() may unsubscribe the same states concurrently; that is a no-op. - if (destroyed.load()) { - activeSubscriptions.store(emptySet()) - userFinder.unsubscribe(needed.toList()) + activeSubscriptions = needed } } override fun destroy() { - // Flag before cancelling so an in-flight body is guaranteed to see the teardown. - destroyed.store(true) - bundler.cancel() - userFinder.unsubscribe(activeSubscriptions.exchange(emptySet()).toList()) + synchronized(lock) { + destroyed = true + bundler.cancel() + userFinder.unsubscribe(activeSubscriptions.toList()) + activeSubscriptions = emptySet() + } } } diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt index db81e6c618..842471e6a4 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt @@ -29,9 +29,10 @@ import com.vitorpamplona.quartz.nip01Core.core.Address import io.mockk.every import io.mockk.mockk import io.mockk.verify +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test -import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger @@ -40,7 +41,7 @@ import kotlin.concurrent.thread class AddressableAuthorRelayLoaderSubAssemblerTest { /** * Unique per-test-class identities so the shared [LocalCache] singleton isn't polluted with - * notes another test class also claims. The prefix keeps the key a valid 64-char hex pubkey. + * notes another test class also claims. */ private fun stubKeys(count: Int): Set { val account = mockk() @@ -51,34 +52,28 @@ class AddressableAuthorRelayLoaderSubAssemblerTest { } /** - * Regression test for the `ConcurrentModificationException` in - * `SetsKt.minus` reported from `DefaultDispatcher-worker-70`. + * Regression test for the `ConcurrentModificationException` in `SetsKt.minus` reported from + * `DefaultDispatcher-worker-70`: this manager is genuinely re-entered from several threads at + * once, because `ComposeSubscriptionManager.subscribe`/`unsubscribe` call `invalidateKeys()` + * *after* releasing their own lock, and `LifecycleAwareSubscription`'s 30s grace-period + * unsubscribe fires on a `Dispatchers.Default` worker. * - * `ComposeSubscriptionManager.subscribe`/`unsubscribe` call `invalidateKeys()` *after* - * releasing their own lock, and `LifecycleAwareSubscription`'s 30s grace-period unsubscribe - * fires on a `Dispatchers.Default` worker — so this manager is genuinely re-entered from - * several threads at once. - * - * The invariant asserted here is the one that makes the crash impossible: **the body that - * reads and swaps the subscription state never runs concurrently with itself.** It's checked - * via the injected `allKeys()` lambda (called exactly once per body) rather than by catching - * the exception, because once the body is bundled its throwables are swallowed by - * `BundledUpdate`'s `CoroutineExceptionHandler` and would never reach the test thread. + * Overlap is detected through the injected `allKeys()` lambda rather than by catching — once + * the body is bundled its throwables are swallowed by `BundledUpdate`'s + * `CoroutineExceptionHandler` and would never reach the test thread. */ @Test fun concurrentInvalidateFiltersNeverOverlap() { - val errors = CopyOnWriteArrayList() val userFinder = mockk(relaxed = true) val keys = stubKeys(50) val inFlight = AtomicInteger(0) + val overlaps = AtomicInteger(0) val assembler = AddressableAuthorRelayLoaderSubAssembler( LocalCache, { - if (inFlight.incrementAndGet() > 1) { - errors.add(IllegalStateException("forceInvalidate bodies overlapped")) - } + if (inFlight.incrementAndGet() > 1) overlaps.incrementAndGet() try { keys } finally { @@ -94,13 +89,7 @@ class AddressableAuthorRelayLoaderSubAssemblerTest { (1..8).map { thread(start = false) { start.await() - repeat(500) { - try { - assembler.invalidateFilters() - } catch (t: Throwable) { - errors.add(t) - } - } + repeat(500) { assembler.invalidateFilters() } } } threads.forEach { it.start() } @@ -110,11 +99,7 @@ class AddressableAuthorRelayLoaderSubAssemblerTest { assembler.destroy() } - assertTrue( - "invalidateFilters raced under concurrency: " + - errors.map { "${it::class.simpleName}: ${it.message}" }.distinct(), - errors.isEmpty(), - ) + assertEquals("forceInvalidate bodies overlapped", 0, overlaps.get()) } /** The manager still does its job: unresolved stub authors reach the user finder. */ @@ -138,11 +123,9 @@ class AddressableAuthorRelayLoaderSubAssemblerTest { } /** - * `bundler.cancel()` cannot stop a body that is already executing — the body has no - * suspension points, so it runs to completion after `destroy()` returns. Without the - * `destroyed` handshake the body re-subscribes authors that `destroy()` just released, - * leaving live kind-0/10002 REQs (and retained `User`/`Account` references) for a dead - * account after logout. + * `destroy()` must win against a body that is already past its `allKeys()` scan: otherwise the + * body re-subscribes authors `destroy()` just released, leaving live kind-0/10002 REQs (and + * retained `User`/`Account` references) for a dead account after logout. * * The body is gated inside the injected `allKeys()` lambda so `destroy()` provably runs * underneath an in-flight run rather than racing it by luck. @@ -150,16 +133,10 @@ class AddressableAuthorRelayLoaderSubAssemblerTest { @Test fun destroyDuringInFlightInvalidateDoesNotLeakSubscriptions() { val userFinder = mockk(relaxed = true) - val subscribed = CopyOnWriteArrayList() - val unsubscribed = CopyOnWriteArrayList() val subscribeHappened = CountDownLatch(1) every { userFinder.subscribe(any>()) } answers { - subscribed.addAll(firstArg>()) subscribeHappened.countDown() } - every { userFinder.unsubscribe(any>()) } answers { - unsubscribed.addAll(firstArg>()) - } val keys = stubKeys(3) val invalidateEntered = CountDownLatch(1) @@ -183,14 +160,11 @@ class AddressableAuthorRelayLoaderSubAssemblerTest { destroyFinished.countDown() } - // Let the in-flight run finish (it either subscribes — the leak — or - // observes the teardown and skips; both settle within the timeout). - subscribeHappened.await(2, TimeUnit.SECONDS) - - val leaked = subscribed - unsubscribed.toSet() - assertTrue( - "destroy() left ${leaked.size} subscriptions alive in userFinder", - leaked.isEmpty(), + // The gated body resumes the instant destroyFinished counts down, so a leak shows up + // immediately; the wait only has to outlast that hand-off. + assertFalse( + "in-flight body subscribed after destroy()", + subscribeHappened.await(500, TimeUnit.MILLISECONDS), ) } } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/IEoseManager.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/IEoseManager.kt index 9d4e940c7d..6373a64cf8 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/IEoseManager.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/IEoseManager.kt @@ -21,6 +21,12 @@ package com.vitorpamplona.amethyst.commons.relayClient.eoseManagers interface IEoseManager { + /** + * May be called from any thread, concurrently with itself and with [destroy], and is reached + * synchronously from main on every composable mount/unmount. Implementations must return fast + * and must serialize their own state — see [BaseEoseManager], which does both by routing the + * work through a [com.vitorpamplona.amethyst.commons.service.BundledUpdate]. + */ fun invalidateFilters(ignoreIfDoing: Boolean = false) fun destroy() From 71620380fb7ec7985ab236ec2bd160ce9f5f7d16 Mon Sep 17 00:00:00 2001 From: davotoula Date: Wed, 22 Jul 2026 14:24:02 +0100 Subject: [PATCH 25/34] update KotlinJpsPluginSettings version --- .idea/kotlinc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.idea/kotlinc.xml b/.idea/kotlinc.xml index 5877ff2fad..ceeca8c125 100644 --- a/.idea/kotlinc.xml +++ b/.idea/kotlinc.xml @@ -8,6 +8,6 @@ \ No newline at end of file From 041a4c1c714418a52f26fc20a99cf9fe01ebb699 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 22 Jul 2026 11:12:19 -0400 Subject: [PATCH 26/34] fix(playback): enforce the decoder budget when acquiring players MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MediaCodec instances are a per-process resource with a hard per-device ceiling — the Android emulator's c2.goldfish.h264.decoder declares `concurrent-instances max="4"`. Past that, MediaCodec.start() fails with NO_MEMORY, MediaCodecRenderer reports "Failed to initialize decoder", and the video surfaces to the user as "can't load". Opening a live stream after scrolling a few feed videos reproduced this reliably; killing the process made the same stream play, since that released every held codec. The device ceiling was already computed by SimultaneousPlaybackCalculator, but only reached ExoPlayerPool as `poolSize`, which governs how many idle players are *retained*. The acquire path was uncapped (`coldPool.poll() ?: builder.build(context)`), and MediaSessionPool held a hardcoded LruCache(10) of sessions, each pinning a checked-out player. So a 4-decoder device would happily hold 10. Enforce the budget where players are handed out: - Track live decoders process-wide, counting checked-out and warm players (cold ones have been stop()'d and hold none). The counter and the pool registry are global because PlaybackService builds one pool for direct traffic and another for Tor-proxied traffic; a per-pool budget let the app hold twice the ceiling. - Before a cold or fresh player is handed out, reclaim headroom by demoting warm players to cold — own pool first, then siblings. Warm entries are a scroll-back cache, so they are the right thing to give up under pressure. - Size the session cache from the same device budget, keeping the previous 10 as an upper bound so capable devices are unaffected. Verified on the emulator: 9 codec allocations across a session with zero NO_MEMORY and zero decoder-init failures, where allocation #5 previously died. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../playback/playerPool/ExoPlayerPool.kt | 61 +++++++++++++++++++ .../playback/playerPool/MediaSessionPool.kt | 10 ++- .../playback/service/PlaybackService.kt | 8 ++- 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/playerPool/ExoPlayerPool.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/playerPool/ExoPlayerPool.kt index e38aa51c0a..24800b9010 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/playerPool/ExoPlayerPool.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/playerPool/ExoPlayerPool.kt @@ -37,6 +37,7 @@ import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.yield import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger @OptIn(UnstableApi::class) class ExoPlayerPool( @@ -71,6 +72,10 @@ class ExoPlayerPool( private val warmPool = ArrayDeque(warmSlotsCap.coerceAtLeast(1)) private val warmPoolLock = Any() + init { + livePools.add(this) + } + // Exists to avoid exceptions stopping the coroutine val exceptionHandler = CoroutineExceptionHandler { _, throwable -> @@ -127,15 +132,53 @@ class ExoPlayerPool( Log.d("PlaybackService") { "ExoPlayerPool discarding errored warm player: $preferredMediaId (${error.errorCodeName})" } PcmTapRegistry.unregisterPlayer(warm) warm.release() + liveDecoders.decrementAndGet() } else { Log.d("PlaybackService") { "ExoPlayerPool warm hit: $preferredMediaId" } + // Already counted against the decoder budget for as long as it sat warm. return warm } } } + ensureDecoderHeadroom() + liveDecoders.incrementAndGet() return coldPool.poll() ?: builder.build(context) } + /** + * Frees decoder headroom before a cold or freshly built player is handed out. + * + * Every player that still holds a prepared MediaItem — checked out or merely warm — owns a + * MediaCodec instance, and devices advertise a hard ceiling on those (the emulator's + * c2.goldfish.h264.decoder declares `concurrent-instances max="4"`). Past that ceiling + * MediaCodec.start() fails with NO_MEMORY and the video surfaces as "can't load", so the + * budget has to be enforced at acquisition rather than only at retention. + * + * Warm players are a scroll-back cache, so they are what gives way: demoting one to cold + * stop()s it and releases its codec. This pool's own entries go first, then any other pool's + * — [PlaybackService] keeps a separate pool for direct and for Tor-proxied traffic, and both + * draw on the one per-process pile of decoders. + */ + private fun ensureDecoderHeadroom() { + while (liveDecoders.get() >= poolSize) { + if (!evictOldestWarm() && !evictOldestWarmElsewhere()) return + } + } + + private fun evictOldestWarm(): Boolean { + val oldest = synchronized(warmPoolLock) { warmPool.removeFirstOrNull() } ?: return false + Log.d("PlaybackService") { "ExoPlayerPool decoder-budget evict: ${oldest.mediaId}" } + demoteToCold(oldest.player) + return true + } + + private fun evictOldestWarmElsewhere(): Boolean { + livePools.forEach { pool -> + if (pool !== this && pool.evictOldestWarm()) return true + } + return false + } + private fun takeWarm(mediaId: String): ExoPlayer? = synchronized(warmPoolLock) { // Iterate from the newest end so a duplicated URI returns the freshest player. @@ -170,6 +213,7 @@ class ExoPlayerPool( Log.d("PlaybackService") { "ExoPlayerPool dropping errored player: ${player.currentMediaItem?.mediaId} (${error.errorCodeName})" } PcmTapRegistry.unregisterPlayer(player) player.release() + liveDecoders.decrementAndGet() return@withLock } @@ -214,7 +258,10 @@ class ExoPlayerPool( private fun demoteToCold(player: ExoPlayer) { if (player.isReleased) return player.pause() + // stop() tears the renderers down, which is what actually hands the MediaCodec instance + // back to the system — so this is the point where the player stops costing budget. player.stop() + liveDecoders.decrementAndGet() player.clearVideoSurface() player.clearMediaItems() @@ -260,6 +307,7 @@ class ExoPlayerPool( } fun destroy() { + livePools.remove(this) scope .launch { mutex.withLock { @@ -272,6 +320,7 @@ class ExoPlayerPool( warmSnapshot.forEach { PcmTapRegistry.unregisterPlayer(it.player) it.player.release() + liveDecoders.decrementAndGet() } coldPool.forEach { PcmTapRegistry.unregisterPlayer(it) @@ -286,5 +335,17 @@ class ExoPlayerPool( companion object { private const val DEFAULT_WARM_SLOTS = 3 + + // MediaCodec instances are a per-process resource, but PlaybackService builds one pool for + // direct traffic and another for Tor-proxied traffic, so a per-pool budget would let the + // app hold twice the device's decoder ceiling. Both counters below are therefore global. + + // Players currently holding a decoder: checked out, or warm (paused but still prepared). + // Cold players have been stop()'d and own none. + private val liveDecoders = AtomicInteger(0) + + // Every pool that hasn't been destroy()'d, so a pool starved of headroom can reclaim a + // warm player from a sibling instead of overshooting the shared ceiling. + private val livePools = ConcurrentLinkedQueue() } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/playerPool/MediaSessionPool.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/playerPool/MediaSessionPool.kt index 92b6fa2300..193fd82b7f 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/playerPool/MediaSessionPool.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/playerPool/MediaSessionPool.kt @@ -64,6 +64,10 @@ class MediaSessionPool( val exoPlayerPool: ExoPlayerPool, val dataSourceFactory: DataSource.Factory, val appContext: Context, + // Ceiling on cached sessions. Each one holds a checked-out ExoPlayer, so on a device whose + // decoder ceiling is lower than [MAX_CACHED_SESSIONS] this is what keeps the session cache + // from pinning more MediaCodec instances than the hardware will grant. + maxSessions: Int = MAX_CACHED_SESSIONS, val reset: (MediaSession, Boolean) -> Unit, ) { private val exceptionHandler = @@ -123,7 +127,7 @@ class MediaSessionPool( private val playingMap = mutableMapOf() private val cache = - object : LruCache(10) { // up to 10 videos in the screen at the same time + object : LruCache(maxSessions.coerceIn(1, MAX_CACHED_SESSIONS)) { override fun entryRemoved( evicted: Boolean, key: String?, @@ -296,6 +300,10 @@ class MediaSessionPool( companion object { private val CLEANUP_INTERVAL_NS = TimeUnit.MINUTES.toNanos(1) + // Roughly how many videos can share a screen at once. Acts as the upper bound only — + // a device that advertises fewer concurrent decoders than this caps lower. + const val MAX_CACHED_SESSIONS = 10 + // AOSP default for config_mediaMetadataBitmapMaxSize, used when the framework resource // can't be resolved by name on a given ROM. private const val DEFAULT_METADATA_BITMAP_DP = 320 diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/service/PlaybackService.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/service/PlaybackService.kt index 59fca51fdd..4640126b16 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/service/PlaybackService.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/service/PlaybackService.kt @@ -80,14 +80,20 @@ class PlaybackService : MediaSessionService() { }, ) + // The device's concurrent-decoder ceiling bounds both how many players may be checked out + // at once (the session cache) and how many the pool may retain, since a session and a warm + // pool entry each pin one MediaCodec instance. + val decoderBudget = SimultaneousPlaybackCalculator.max(applicationContext) + return MediaSessionPool( exoPlayerPool = ExoPlayerPool( ExoPlayerBuilder(videoCache, resolvingDataSourceFactory), - poolSize = SimultaneousPlaybackCalculator.max(applicationContext), + poolSize = decoderBudget, ), dataSourceFactory = resolvingDataSourceFactory, appContext = applicationContext, + maxSessions = decoderBudget, reset = { session, keepPlaying -> (session.player as ExoPlayer).apply { repeatMode = if (keepPlaying) Player.REPEAT_MODE_ONE else Player.REPEAT_MODE_OFF From edf30489d03a5e5cf801db5f4b7ea8a96b901001 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 22 Jul 2026 11:12:39 -0400 Subject: [PATCH 27/34] fix(video): size the player box so live streams stop rendering black bars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live stream opened for the first time drew ~90px of black above and below the picture. The video surface itself was correct 16:9; the box enclosing it was not. Measured on a Pixel 9 emulator: container [0,274][1080,1062] (788px = StreamingHeaderModifier's 300.dp cap) holding a TextureView of [0,364][1080,972] (608px = 16:9 at 1080 wide), centred, so (788-608)/2 = 90px per side. Two independent causes, both needed fixing: ContentWarningGate takes a `modifier` but drops it for anything not flagged sensitive — the non-sensitive path emits `content()` bare. ZoomableContentView was routing mediaSizingModifier() through exactly that parameter, so for ordinary media the sizing never reached the layout at all. With no height constraint the player stretched to whatever ceiling enclosed it and letterboxed the frame inside. Apply the sizing to the inner Box, which is always emitted. Even applied, the ratio was unknown on a first play: a NIP-53 stream carries no imeta `dim`, and MediaAspectRatioCache is only filled once the decoder reports a size. The miss was frozen for the whole visit because the cache was a plain LruCache read during composition, which triggers no recomposition when it later fills — hence the bars vanishing only on a *second* visit to the same stream. Back cache entries with snapshot state so a composition-time read updates, and default an unknown video to 16:9 so the first layout already lands in the right place. VideoView keeps reading the cache inside remember() on purpose, with a comment explaining why: making it observable there flips the ratio mid-playback, which both adds an aspectRatio and emits an extra Spacer, and restructuring children around a live AndroidView strands the player on a stale surface — the video redraws at native size in the corner while layout bounds still look correct. Verified on a cold cache: container and TextureView are both [0,274][1080,882], against a header ending at 274 — zero gap. Feed image and video layouts unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../amethyst/model/MediaAspectRatioCache.kt | 25 ++++++++++++++++--- .../service/playback/composable/VideoView.kt | 5 ++++ .../ui/components/ZoomableContentView.kt | 18 +++++++++++-- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/MediaAspectRatioCache.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/MediaAspectRatioCache.kt index 0616a6efe4..c5b9acfee8 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/MediaAspectRatioCache.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/MediaAspectRatioCache.kt @@ -21,6 +21,8 @@ package com.vitorpamplona.amethyst.model import androidx.collection.LruCache +import androidx.compose.runtime.MutableState +import androidx.compose.runtime.mutableStateOf interface MutableMediaAspectRatioCache { fun get(url: String): Float? @@ -32,10 +34,27 @@ interface MutableMediaAspectRatioCache { ) } +/** + * Aspect ratios keyed by media URL, learned from imeta `dim` tags up front or from the decoder once + * a first frame lands. + * + * Entries are snapshot state, so a composable that calls [get] **during composition** recomposes + * when the real dimensions arrive later. That matters because players and image loaders only report + * size after the first frame decodes: a caller that sized itself off a plain cache miss would stay + * wrong for the whole visit and only look right the *next* time the media is opened. Note this only + * works for reads made in composition — a read from inside `remember { }` is cached by `remember` + * itself and won't pick the update up. + */ object MediaAspectRatioCache : MutableMediaAspectRatioCache { - val mediaAspectRatioCacheByUrl = LruCache(1000) + private val cache = LruCache>(1000) - override fun get(url: String): Float? = mediaAspectRatioCacheByUrl.get(url) + // get-then-put has to be atomic, so the compound op is guarded even though LruCache is itself + // thread-safe. A miss still stores a slot: that empty slot is what the caller observes until + // add() fills it in. + @Synchronized + private fun entry(url: String): MutableState = cache.get(url) ?: mutableStateOf(null).also { cache.put(url, it) } + + override fun get(url: String): Float? = entry(url).value override fun add( url: String, @@ -43,7 +62,7 @@ object MediaAspectRatioCache : MutableMediaAspectRatioCache { height: Int, ) { if (height > 1) { - mediaAspectRatioCacheByUrl.put(url, width.toFloat() / height.toFloat()) + entry(url).value = width.toFloat() / height.toFloat() } } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/VideoView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/VideoView.kt index 45f72cf3fe..798166ddb3 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/VideoView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/VideoView.kt @@ -124,6 +124,11 @@ fun VideoView( // DimensionTag uses reference equality, not structural. val dimW = dimensions?.width val dimH = dimensions?.height + // Deliberately snapshotted in a remember rather than observing MediaAspectRatioCache: when the + // ratio flips null -> known mid-playback this branch both adds an aspectRatio and emits an + // extra Spacer, and restructuring the children around a live AndroidView leaves the player's + // TextureView on a stale surface (the video redraws at native size in the corner). The + // enclosing box in ZoomableContentView is what sizes the player, and that one does observe. val ratio = remember(videoUri, dimW, dimH) { if (dimW != null && dimH != null && dimW > 0 && dimH > 0) { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt index a499c08102..e918d88d80 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt @@ -136,6 +136,14 @@ import java.io.IOException // Allows time for receiving app to copy the file after user confirms share. private const val SHARED_VIDEO_CLEANUP_DELAY_MS = 120_000L +// Assumed shape of a video whose dimensions nobody has reported yet — no imeta `dim` and nothing +// cached, which is the norm for a NIP-53 live stream on its first play. Without a ratio the sizing +// modifier leaves height unconstrained, so the player stretches to whatever ceiling encloses it +// (300.dp on the live-stream screen) and letterboxes the real frame inside, leaving black bars top +// and bottom. Guessing the overwhelmingly common video shape puts the first layout in the right +// place; [MediaAspectRatioCache] then corrects anything unusual once the decoder reports its size. +private const val DEFAULT_VIDEO_ASPECT_RATIO = 16f / 9f + @Composable fun ZoomableContentView( content: BaseMediaContent, @@ -195,7 +203,7 @@ fun ZoomableContentView( } is MediaUrlVideo -> { - val ratio = content.dim?.aspectRatio() ?: MediaAspectRatioCache.get(content.url) + val ratio = content.dim?.aspectRatio() ?: MediaAspectRatioCache.get(content.url) ?: DEFAULT_VIDEO_ASPECT_RATIO val bridgedUrl = remember(content.url, useLocalBlossomBridge) { content.toCoilModel(useLocalBlossomBridge) @@ -209,7 +217,13 @@ fun ZoomableContentView( backdrop = (content.thumbhash ?: content.blurhash)?.let { { BlurhashBackdrop(content.blurhash, content.description, content.thumbhash) } }, ) { Box( - modifier = Modifier.fillMaxWidth().then(boundsTrackingModifier), + // The sizing modifier is repeated here because ContentWarningGate only applies + // the one it is handed when the content is actually sensitive — the common + // non-sensitive path emits content() bare. Without a height constraint of its + // own this box stretches to whatever ceiling encloses it and the player + // letterboxes the frame inside, which is what put black bars above and below + // live streams (their enclosure is StreamingHeaderModifier's 300.dp cap). + modifier = mediaSizingModifier(ratio, contentScale).then(boundsTrackingModifier), contentAlignment = Alignment.Center, ) { VideoView( From 5f954c0e0409147c130b6a10dc59044dd89002f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 15:42:02 +0000 Subject: [PATCH 28/34] build(quartz): target JVM 17 instead of JVM 21 Lower the quartz module's Kotlin jvmTarget for both the JVM and Android compilations from JVM_21 to JVM_17, broadening the range of runtimes that can consume the published library. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01V1Hn61gQJ1joUznESzUHU4 --- quartz/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quartz/build.gradle.kts b/quartz/build.gradle.kts index 513bdd3cd5..8a93c5758d 100644 --- a/quartz/build.gradle.kts +++ b/quartz/build.gradle.kts @@ -17,7 +17,7 @@ kotlin { } jvm { compilerOptions { - jvmTarget.set(JvmTarget.JVM_21) + jvmTarget.set(JvmTarget.JVM_17) } } @@ -33,7 +33,7 @@ kotlin { .toInt() compilerOptions { - jvmTarget.set(JvmTarget.JVM_21) + jvmTarget.set(JvmTarget.JVM_17) } optimization { From 6ed955f0cf064497a3b73b8fa8bc5e306fc166e9 Mon Sep 17 00:00:00 2001 From: davotoula Date: Wed, 22 Jul 2026 17:23:04 +0100 Subject: [PATCH 29/34] docs: language updates --- .../skills/find-missing-translations/SKILL.md | 33 +++++++++++++++---- amethyst/src/main/res/CLAUDE.md | 3 ++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.claude/skills/find-missing-translations/SKILL.md b/.claude/skills/find-missing-translations/SKILL.md index 26faf041fd..ef7d1c5e40 100644 --- a/.claude/skills/find-missing-translations/SKILL.md +++ b/.claude/skills/find-missing-translations/SKILL.md @@ -212,9 +212,25 @@ Before presenting results, **scan the missing English strings** for two red-flag Also **audit existing `` resources** for two anti-patterns: 1. **`quantity="one"` items that hardcode the literal `1`** (instead of using a `%d` / `%1$d` placeholder) — broken for languages where the `one` CLDR category covers more than just `n=1` (Russian, Ukrainian, Croatian, etc.). -2. **`quantity="zero"` items in any locale that doesn't natively use the `zero` CLDR category** — i.e. **everything except Arabic (`ar`) and Welsh (`cy`)**. ICU/CLDR maps `count=0` to `other` for English and all the locales we ship to (cs, de, pt-BR, sv, etc.), so `` is **dead code** there: `getQuantityString(id, 0)` will pick `other`, never the zero entry, and the visible runtime string ends up `"…0 items"` instead of the intended `"…no items"`. +2. **`quantity="zero"` items in any locale that doesn't natively use the `zero` CLDR category** — i.e. everything except **Arabic (`ar`)**, **Latvian (`lv`)** and **Welsh (`cy`)**. ICU/CLDR maps `count=0` to `other` for English and most of the locales we ship to (cs, de, pt-BR, sv, etc.), so `` is **dead code** there: `getQuantityString(id, 0)` will pick `other`, never the zero entry, and the visible runtime string ends up `"…0 items"` instead of the intended `"…no items"`. -If a UX genuinely wants special "no items" wording at count=0, that has to be a call-site `if (count == 0)` branch to a separate ``, **not** a `quantity="zero"` plural item. +> ⚠️ **Latvian is the trap here — do NOT strip its `zero` items** (we nearly did, 2026-07-22). `lv` has an integer-bearing `zero` category that covers far more than 0: `select(0)`, `select(10)` and `select(11)` all return `zero` (the rule is `n % 10 = 0` or `n % 100 = 11..19`). So a Latvian `` is *live code on the majority of counts*, and it must read as a normal plural form ("%1$d minūšu"), **not** as "no items" wording. An earlier version of this skill claimed only `ar` and `cy` had `zero`, which flagged all ~40 correct Latvian entries as dead and would have deleted working translations. + +If a UX genuinely wants special "no items" wording at count=0, that has to be a call-site `if (count == 0)` branch to a separate ``, **not** a `quantity="zero"` plural item. (This is why `zero` is the wrong tool even where it exists: in `lv` it does not mean "zero".) + +**Verify, don't recall.** Before asserting any locale's category set, check it against CLDR rather than memory: + +```bash +python3 -m venv /tmp/cldr && /tmp/cldr/bin/pip -q install babel +/tmp/cldr/bin/python -c " +from babel import Locale +for c in ['en','lv','ar','cy','cs','de','sv','pt_BR','ru','pl']: + r = Locale.parse(c).plural_form + print(c, sorted({r(n) for n in range(0,10001)}), 'select(0)=', r(0), 'select(10)=', r(10)) +" +``` + +Across the 56 locale dirs this repo ships, **only `ar-rSA` and `lv-rLV`** have an integer-bearing `zero`. Flag and offer to fix: @@ -240,15 +256,16 @@ for f in amethyst/src/main/res/values/strings.xml amethyst/src/main/res/values-* done ``` -Then scan for dead `quantity="zero"` entries. CLDR's `zero` category is integer-bearing only in **Arabic (`ar`)** and **Welsh (`cy`)**. In every other locale, count=0 falls through to `other`, so a `` entry is dead and likely a translator/author bug (or it silently never fires): +Then scan for dead `quantity="zero"` entries. CLDR's `zero` category is integer-bearing only in **Arabic (`ar`)**, **Latvian (`lv`)** and **Welsh (`cy`)** — those three are skipped below, so a hit is a genuine bug. In every other locale, count=0 falls through to `other`, so a `` entry is dead and likely a translator/author bug (or it silently never fires): ```bash for f in amethyst/src/main/res/values/strings.xml amethyst/src/main/res/values-*/strings.xml \ commons/src/commonMain/composeResources/values/strings.xml \ commons/src/commonMain/composeResources/values-*/strings.xml; do - # Skip Arabic and Welsh — they natively use the zero category. + # Skip Arabic, Latvian and Welsh — they natively use the zero category. + # (Latvian's zero covers 0, 10, 11-19, 20, 30, … — stripping it breaks most counts.) case "$f" in - *values-ar*|*values-cy*) continue ;; + *values-ar*|*values-cy*|*values-lv*) continue ;; esac awk -v file="$f" ' /`** entries, follow these rules: - Polish (`pl`): `one`, `few`, `many`, `other` - Russian (`ru`): `one`, `few`, `many`, `other` - Arabic (`ar`): `zero`, `one`, `two`, `few`, `many`, `other` + - Latvian (`lv`): `zero`, `one`, `other` — its `zero` is **not** "no items"; it covers 0, 10, 11–19, 20, 30, … - German / Swedish / Brazilian Portuguese: `one`, `other` - When a missing string contains a count placeholder and is conceptually a singular/plural pair, **flag it before translating** — it may belong as a `` resource rather than a single ``. Surface this to the user before proposing translations. -- **Do not use `quantity="zero"` outside Arabic (`ar`) and Welsh (`cy`).** CLDR's `zero` category is integer-bearing only in those two languages. Android calls `PluralRules.select(0)` for the device locale; in English/German/Czech/Polish/Russian/Swedish/Portuguese/etc. it returns `other`, so the explicit `` is never picked at runtime and the user sees `"…0 items"` instead of the intended wording. If the design calls for "no items" at count=0, model it as a separate `` and an `if (count == 0)` branch at the call site: +- **Do not use `quantity="zero"` outside Arabic (`ar`), Latvian (`lv`) and Welsh (`cy`).** CLDR's `zero` category is integer-bearing only in those three languages. Android calls `PluralRules.select(0)` for the device locale; in English/German/Czech/Polish/Russian/Swedish/Portuguese/etc. it returns `other`, so the explicit `` is never picked at runtime and the user sees `"…0 items"` instead of the intended wording. Conversely, **never delete an existing `zero` item from `ar`/`lv`/`cy`** — there it is live. If the design calls for "no items" at count=0, model it as a separate `` and an `if (count == 0)` branch at the call site: ```kotlin val label = if (count == 0) { stringRes(R.string.foo_no_items, dateLabel) @@ -377,4 +395,5 @@ When adding translated strings to locale files: - **Inserting strings in a specific position** — always append at the bottom; ordering is handled separately - **Hardcoding `"1"` in a `` `quantity="one"` item** — always use the count placeholder; otherwise non-English `one` categories produce wrong text - **Copying English's `one`/`other` set into every locale** — each language must include all CLDR plural categories it uses (e.g. Czech needs `one`, `few`, `many`, `other`) -- **Using `` to special-case count=0** — outside Arabic and Welsh, this entry is unreachable: ICU/CLDR maps 0 → `other`, so the runtime never picks the zero item and the user sees `"…0 items"`. Special-case at the call site with a separate `` instead. \ No newline at end of file +- **Using `` to special-case count=0** — outside Arabic, Latvian and Welsh, this entry is unreachable: ICU/CLDR maps 0 → `other`, so the runtime never picks the zero item and the user sees `"…0 items"`. Special-case at the call site with a separate `` instead. +- **Reporting Latvian `quantity="zero"` entries as dead code** — `lv` has a real, integer-bearing `zero` category covering 0, 10, 11–19, 20, 30, … so those entries fire on *most* counts. An earlier version of this skill excluded only `ar`/`cy` from the zero audit and flagged all ~40 correct `values-lv-rLV` entries; acting on that would have deleted working translations. Confirm any locale's category set against CLDR (the babel snippet in Step 4) before calling a `zero` item dead. \ No newline at end of file diff --git a/amethyst/src/main/res/CLAUDE.md b/amethyst/src/main/res/CLAUDE.md index f3da508b02..e7d124f6bb 100644 --- a/amethyst/src/main/res/CLAUDE.md +++ b/amethyst/src/main/res/CLAUDE.md @@ -11,8 +11,11 @@ Always consider Slavic / Baltic / Semitic / Celtic languages when a string conta - English / German / Swedish / Brazilian Portuguese / Hungarian: `one`, `other` - Czech / Polish / Russian / Ukrainian / Croatian: `one`, `few`, `many`, `other` - Arabic: `zero`, `one`, `two`, `few`, `many`, `other` + - Latvian: `zero`, `one`, `other` - Chinese / Japanese: `other` only + Latvian's `zero` does **not** mean "no items" — it covers 0, 10, 11–19, 20, 30, … (`n % 10 = 0` or `n % 100 = 11..19`), so it fires on most counts and must read as a normal plural form. Never strip `` from `values-lv-rLV`; among the locales we ship, only Arabic and Latvian have an integer-bearing `zero`. + ## Anti-patterns to flag When adding or reviewing strings, flag these: From e6f6edd29b12c5abb1d100883843b21835f79d0a Mon Sep 17 00:00:00 2001 From: davotoula Date: Wed, 22 Jul 2026 17:23:43 +0100 Subject: [PATCH 30/34] update cs,sv,de,pt --- amethyst/src/main/res/values-cs/strings.xml | 1 + .../src/commonMain/composeResources/values-cs/strings.xml | 6 ++++++ .../commonMain/composeResources/values-pt-rBR/strings.xml | 5 +++++ .../commonMain/composeResources/values-sv-rSE/strings.xml | 1 + 4 files changed, 13 insertions(+) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index 36a04cbaf9..cb43dfb32b 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -4352,4 +4352,5 @@ Anonymní — jednorázová identita pro každou oblast, nikdy váš npub. Kompatibilní s Bitchatem; zasáhne blízké uživatele na stejných relayích. Veřejné a dočasné — žádná historie, může to číst kdokoli v buňce. + Teleportace diff --git a/commons/src/commonMain/composeResources/values-cs/strings.xml b/commons/src/commonMain/composeResources/values-cs/strings.xml index 2397cb87fd..e03a039b2a 100644 --- a/commons/src/commonMain/composeResources/values-cs/strings.xml +++ b/commons/src/commonMain/composeResources/values-cs/strings.xml @@ -145,4 +145,10 @@ %1$d událostí %1$d událostí + + %1$d relay + %1$d relaye + %1$d relaye + %1$d relayů + diff --git a/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml b/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml index 74e8676117..17d896ccb3 100644 --- a/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml +++ b/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml @@ -137,4 +137,9 @@ %1$d evento %1$d eventos + Site Estático: %1$s + + %1$d relay + %1$d relays + diff --git a/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml b/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml index 50b8bb491d..4b2ee35517 100644 --- a/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml +++ b/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml @@ -140,4 +140,5 @@ %1$d händelse %1$d händelser + Statisk webbplats: %1$s From b86bc5b798519d1ee44e8086f4278400d7bbcf21 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 16:25:23 +0000 Subject: [PATCH 31/34] ci: replace abandoned android-test-report-action with mikepenz/action-junit-report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit asadmansr/android-test-report-action@v1.2.0 is a Docker action whose Dockerfile is `FROM ubuntu:18.04` + `apt-get install python` (Python 2). Ubuntu 18.04 (bionic) is end-of-life, so its apt archives are now unreliable and Python 2 has no installation candidate — the image rebuild runs on every CI invocation, takes ~8 minutes, and has started hard-failing the test-and-build-android job (`E: Package 'python' has no installation candidate`). The action was last released in 2020 and is unmaintained, and it was referenced by a movable tag rather than a pinned commit. Swap it for mikepenz/action-junit-report (actively maintained, Apache-2.0, JS action — no Docker rebuild), pinned to the v6.4.2 commit SHA. Use annotate_only so it needs no `checks: write` permission and keeps working on pull requests from forks; fail_on_failure keeps the job red when a unit test fails, matching the previous step's behavior. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01HMtpSuyr8FBh2ZQ22BV4ZP --- .github/workflows/build.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6a42fec122..cbb3d82f62 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -231,9 +231,23 @@ jobs: name: Android Lint Reports path: amethyst/build/reports/lint-results-*.html + # Publishes the JUnit XML produced by the unit-test tasks above as inline + # annotations plus a job summary. Replaces asadmansr/android-test-report-action, + # which was abandoned (last release 2020) and rebuilt an EOL Ubuntu 18.04 + + # Python 2 Docker image on every run — bionic's apt archives have since gone + # unreliable and broke this job. Pinned to a commit SHA (not the movable + # v6.4.2 tag) to close the supply-chain hole. annotate_only avoids needing + # `checks: write`, so it keeps working on pull requests from forks (where the + # GITHUB_TOKEN is read-only). fail_on_failure preserves the old step's + # behavior of marking the job red when a test fails. - name: Android Test Report - uses: asadmansr/android-test-report-action@v1.2.0 + uses: mikepenz/action-junit-report@d9f48fc87bc235f7e214acf696ca5abc0a986f16 # v6.4.2 if: always() + with: + report_paths: '**/build/test-results/**/TEST-*.xml' + annotate_only: true + detailed_summary: true + fail_on_failure: true - name: Upload Test Results uses: actions/upload-artifact@v7 From adf9b68c7a667eca190aad5831712a048a085bfc Mon Sep 17 00:00:00 2001 From: davotoula <1747287+davotoula@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:32:32 +0000 Subject: [PATCH 32/34] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-cs/strings.xml | 2 +- .../src/main/res/values-pl-rPL/strings.xml | 45 +++++++++++++++++++ .../composeResources/values-cs/strings.xml | 6 --- .../values-pt-rBR/strings.xml | 6 +-- .../values-sv-rSE/strings.xml | 2 +- 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index cb43dfb32b..e8966aebbb 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -2125,6 +2125,7 @@ Obnovit výchozí nastavení Zveřejnit polohu jako Přidá Geohash vaší polohy do příspěvku. Veřejnost bude vědět, že se nacházíte do 5 km od aktuální polohy + Teleportace ✈ Teleportovat sem Vyberte místo na mapě Změnit místo na mapě @@ -4352,5 +4353,4 @@ Anonymní — jednorázová identita pro každou oblast, nikdy váš npub. Kompatibilní s Bitchatem; zasáhne blízké uživatele na stejných relayích. Veřejné a dočasné — žádná historie, může to číst kdokoli v buňce. - Teleportace diff --git a/amethyst/src/main/res/values-pl-rPL/strings.xml b/amethyst/src/main/res/values-pl-rPL/strings.xml index 7d1daa2575..3d71fd0436 100644 --- a/amethyst/src/main/res/values-pl-rPL/strings.xml +++ b/amethyst/src/main/res/values-pl-rPL/strings.xml @@ -32,6 +32,7 @@ Przywołane zdarzenie nie zostało znalezione Nie można odszyfrować wiadomości + Rozszyfrowywanie… Zdjęcie grupy Niedozwolona zawartość Uwagi transmitera @@ -308,6 +309,10 @@ Ten link do zaproszenia jest nieprawidłowy lub nie można go otworzyć przy użyciu tego konta. Nie można otworzyć tego linku do zaproszenia. Być może jest on nieaktualny, został już zastąpiony nowszym lub został utworzony w nowszej wersji aplikacji. Poproś o nowy link do zaproszenia. Ten link do zaproszenia został unieważniony i nie można już z niego korzystać. Poproś o nowy. + Ten link do zaproszenia wygasł i nie może być już używany. Poproś o nowy link. + Nazwa społeczności zostanie ujawniona dopiero po dołączeniu + Dołączenie do transmiterów tego zaproszenia, publikuje ogłoszenie o dołączeniu podpisane przez twoje konto i dodaje społeczność do listy. Nic nie zostanie wysłane, dopóki nie klikniesz Dołącz. + Transmiter tego zaproszenia skontaktuje się: %1$s Kanały Concord Nie dołączyłeś(aś) jeszcze do kanału Concord. Utwórz kanał lub otwórz link z zaproszeniem. Brak kanałów. @@ -323,6 +328,10 @@ Usunąć kanał? Usunąć #%1$s? Tej operacji nie można cofnąć i nie będzie można odtworzyć kanału z tym samym identyfikatorem. Usuń + Opuść społeczność + Opuścić społeczność? + Chcesz opuścić %1$s? Zostaniesz usunięty z listy członków tej społeczności, a synchronizacja z Twoimi urządzeniami zostanie wstrzymana. Społeczność nie zostanie o tym powiadomiona, a Ty nie zostaniesz usunięty z listy jej członków. Wiadomości, których nie będziesz już mógł odszyfrować, mogą okazać się nie do odzyskania, a powrót do społeczności będzie możliwy wyłącznie po otrzymaniu nowego zaproszenia. + To Ty stworzyłeś tę społeczność. Odejście nie powoduje jej usunięcia ani przekazania komukolwiek innemu, ale powoduje usunięcie klucza właściciela przechowywanego na Twojej liście — nie będziesz mógł już nią zarządzać. Miejsce, w którym publikowane i czytane są zaszyfrowane plany tej społeczności. %1$s pisze… %1$s i %2$s piszą… @@ -373,6 +382,13 @@ Usunąć członka? Spowoduje to zmianę klucza szyfrującego społeczności, przez co ten użytkownik nie będzie już mógł odczytać żadnych wiadomości wysłanych po tej zmianie. Klucze pozostałych użytkowników zostaną automatycznie zaktualizowane. Czynności tej nie można cofnąć. Usuń + Role… + Przypisz rolę + Wybierz każdą rolę, jaką powinien pełnić ten członek. Odznaczenie roli usuwa ją. + Zapisz + Nie masz wyższej rangi od tego członka + Brak ról, które można przydzielić + Nie udało się zaktualizować ról tego członka. Właściciel Admin Zbanowany @@ -498,6 +514,15 @@ Udostępnij jako adres Url obrazu Generowanie podglądu… Udostępnione przez Ametyst + Udostępnij jako QR + Link do strony internetowej + Nostr link + Skanuj za pomocą dowolnej kamery telefonu + Zeskanuj za pomocą aplikacji Nostr + Zdjęcie + Kod QR zawierający link do tej notatki + Kod QR zawierający link Nostr do tej notatki + Miniaturka ukryta ze względu na wrażliwą treść ID autora ID wpisu Skopiuj tekst @@ -834,6 +859,7 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Ta aplikacja nApplet chce korzystać ze swojego prywatnego schowka. Ta aplikacja nApplet chce zapłacić fakturę w systemie Lightning. + ⚠ Ta aplikacja nApplet ma na celu opłacenie faktury Lightning, w której NIE podano kwoty — to odbiorca płatności decyduje, jaka kwota zostanie pobrana. Zezwól na to tylko wtedy, gdy ufasz tej aplikacji. Ten nApplet chce pobrać zasób internetowy. Ten nApplet chce przesłać plik na Twój serwer multimedialny. Ten nApplet chce pokazywać Ci powiadomienia. @@ -848,6 +874,18 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest kind-only summary would hide what is actually being signed. These replaceable lists are already cached on the account, so the dialog diffs the proposed list against the current one and reports what actually changes rather than a raw total. --> + + obserwuje %1$d nowe konto + obserwuje %1$d nowych kont + obserwuje %1$d nowych kont + obserwuje %1$d nowe konta + + + Przestaje obserwować %1$d konto + Przestaje obserwować %1$d kont + Przestaje obserwować %1$d kont + Przestaje obserwować %1$d konta + @@ -889,9 +927,13 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest odczytaj prywatne wiadomości + przeczytaj swoje prywatne wiadomości za pomocą %1$s + Zawsze zezwalaj na %1$s + Amethyst nie zdołała odszyfrować tej wiadomości. Być może nie jest ona adresowana do tego konta. + Wiadomości z Podłączone aplikacje Cofnij wszystkie uprawnienia @@ -1559,6 +1601,7 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Ten serwer wymaga płatności do wysłania: %1$s Wymagana płatność %1$s pobiera opłatę w systemie Lightning za zapisanie tego pliku. Aby kontynuować, dokonaj płatności z podłączonego portfela. + %1$s powiedział: „%2$s” Zapłać Zapłać %1$d sat @@ -2392,6 +2435,8 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Nie można wypłacić Nie można wykupić Cashu Mint dostarczył następujący komunikat błędu: %1$s + Niebezpieczny adres Cashu mint + Amethyst nie skontaktował się z emitentem tego tokena. %1$s Cashu odebrano %1$s satsy zostały wysłane do Twojego portfela. (opłata: %2$s satoszy) W systemie nie znaleziono kompatybilnego portfela Cashu diff --git a/commons/src/commonMain/composeResources/values-cs/strings.xml b/commons/src/commonMain/composeResources/values-cs/strings.xml index e03a039b2a..2397cb87fd 100644 --- a/commons/src/commonMain/composeResources/values-cs/strings.xml +++ b/commons/src/commonMain/composeResources/values-cs/strings.xml @@ -145,10 +145,4 @@ %1$d událostí %1$d událostí - - %1$d relay - %1$d relaye - %1$d relaye - %1$d relayů - diff --git a/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml b/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml index 17d896ccb3..6b65c8acfe 100644 --- a/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml +++ b/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml @@ -65,6 +65,7 @@ respondendo para + Site Estático: %1$s O que ele pode acessar Site raiz Origem: @@ -137,9 +138,4 @@ %1$d evento %1$d eventos - Site Estático: %1$s - - %1$d relay - %1$d relays - diff --git a/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml b/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml index 4b2ee35517..a90c7d5954 100644 --- a/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml +++ b/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml @@ -69,6 +69,7 @@ Svarar till + Statisk webbplats: %1$s Vad det har åtkomst till Rotplats Källa: @@ -140,5 +141,4 @@ %1$d händelse %1$d händelser - Statisk webbplats: %1$s From 8247d49afa7b61c3da3f4628098a0867c03c6e2d Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:46:03 +0000 Subject: [PATCH 33/34] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-cs/strings.xml | 2 +- .../src/main/res/values-pl-rPL/strings.xml | 45 +++++++++++++++++++ .../composeResources/values-cs/strings.xml | 6 --- .../values-pt-rBR/strings.xml | 6 +-- .../values-sv-rSE/strings.xml | 2 +- 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index cb43dfb32b..e8966aebbb 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -2125,6 +2125,7 @@ Obnovit výchozí nastavení Zveřejnit polohu jako Přidá Geohash vaší polohy do příspěvku. Veřejnost bude vědět, že se nacházíte do 5 km od aktuální polohy + Teleportace ✈ Teleportovat sem Vyberte místo na mapě Změnit místo na mapě @@ -4352,5 +4353,4 @@ Anonymní — jednorázová identita pro každou oblast, nikdy váš npub. Kompatibilní s Bitchatem; zasáhne blízké uživatele na stejných relayích. Veřejné a dočasné — žádná historie, může to číst kdokoli v buňce. - Teleportace diff --git a/amethyst/src/main/res/values-pl-rPL/strings.xml b/amethyst/src/main/res/values-pl-rPL/strings.xml index 7d1daa2575..3d71fd0436 100644 --- a/amethyst/src/main/res/values-pl-rPL/strings.xml +++ b/amethyst/src/main/res/values-pl-rPL/strings.xml @@ -32,6 +32,7 @@ Przywołane zdarzenie nie zostało znalezione Nie można odszyfrować wiadomości + Rozszyfrowywanie… Zdjęcie grupy Niedozwolona zawartość Uwagi transmitera @@ -308,6 +309,10 @@ Ten link do zaproszenia jest nieprawidłowy lub nie można go otworzyć przy użyciu tego konta. Nie można otworzyć tego linku do zaproszenia. Być może jest on nieaktualny, został już zastąpiony nowszym lub został utworzony w nowszej wersji aplikacji. Poproś o nowy link do zaproszenia. Ten link do zaproszenia został unieważniony i nie można już z niego korzystać. Poproś o nowy. + Ten link do zaproszenia wygasł i nie może być już używany. Poproś o nowy link. + Nazwa społeczności zostanie ujawniona dopiero po dołączeniu + Dołączenie do transmiterów tego zaproszenia, publikuje ogłoszenie o dołączeniu podpisane przez twoje konto i dodaje społeczność do listy. Nic nie zostanie wysłane, dopóki nie klikniesz Dołącz. + Transmiter tego zaproszenia skontaktuje się: %1$s Kanały Concord Nie dołączyłeś(aś) jeszcze do kanału Concord. Utwórz kanał lub otwórz link z zaproszeniem. Brak kanałów. @@ -323,6 +328,10 @@ Usunąć kanał? Usunąć #%1$s? Tej operacji nie można cofnąć i nie będzie można odtworzyć kanału z tym samym identyfikatorem. Usuń + Opuść społeczność + Opuścić społeczność? + Chcesz opuścić %1$s? Zostaniesz usunięty z listy członków tej społeczności, a synchronizacja z Twoimi urządzeniami zostanie wstrzymana. Społeczność nie zostanie o tym powiadomiona, a Ty nie zostaniesz usunięty z listy jej członków. Wiadomości, których nie będziesz już mógł odszyfrować, mogą okazać się nie do odzyskania, a powrót do społeczności będzie możliwy wyłącznie po otrzymaniu nowego zaproszenia. + To Ty stworzyłeś tę społeczność. Odejście nie powoduje jej usunięcia ani przekazania komukolwiek innemu, ale powoduje usunięcie klucza właściciela przechowywanego na Twojej liście — nie będziesz mógł już nią zarządzać. Miejsce, w którym publikowane i czytane są zaszyfrowane plany tej społeczności. %1$s pisze… %1$s i %2$s piszą… @@ -373,6 +382,13 @@ Usunąć członka? Spowoduje to zmianę klucza szyfrującego społeczności, przez co ten użytkownik nie będzie już mógł odczytać żadnych wiadomości wysłanych po tej zmianie. Klucze pozostałych użytkowników zostaną automatycznie zaktualizowane. Czynności tej nie można cofnąć. Usuń + Role… + Przypisz rolę + Wybierz każdą rolę, jaką powinien pełnić ten członek. Odznaczenie roli usuwa ją. + Zapisz + Nie masz wyższej rangi od tego członka + Brak ról, które można przydzielić + Nie udało się zaktualizować ról tego członka. Właściciel Admin Zbanowany @@ -498,6 +514,15 @@ Udostępnij jako adres Url obrazu Generowanie podglądu… Udostępnione przez Ametyst + Udostępnij jako QR + Link do strony internetowej + Nostr link + Skanuj za pomocą dowolnej kamery telefonu + Zeskanuj za pomocą aplikacji Nostr + Zdjęcie + Kod QR zawierający link do tej notatki + Kod QR zawierający link Nostr do tej notatki + Miniaturka ukryta ze względu na wrażliwą treść ID autora ID wpisu Skopiuj tekst @@ -834,6 +859,7 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Ta aplikacja nApplet chce korzystać ze swojego prywatnego schowka. Ta aplikacja nApplet chce zapłacić fakturę w systemie Lightning. + ⚠ Ta aplikacja nApplet ma na celu opłacenie faktury Lightning, w której NIE podano kwoty — to odbiorca płatności decyduje, jaka kwota zostanie pobrana. Zezwól na to tylko wtedy, gdy ufasz tej aplikacji. Ten nApplet chce pobrać zasób internetowy. Ten nApplet chce przesłać plik na Twój serwer multimedialny. Ten nApplet chce pokazywać Ci powiadomienia. @@ -848,6 +874,18 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest kind-only summary would hide what is actually being signed. These replaceable lists are already cached on the account, so the dialog diffs the proposed list against the current one and reports what actually changes rather than a raw total. --> + + obserwuje %1$d nowe konto + obserwuje %1$d nowych kont + obserwuje %1$d nowych kont + obserwuje %1$d nowe konta + + + Przestaje obserwować %1$d konto + Przestaje obserwować %1$d kont + Przestaje obserwować %1$d kont + Przestaje obserwować %1$d konta + @@ -889,9 +927,13 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest odczytaj prywatne wiadomości + przeczytaj swoje prywatne wiadomości za pomocą %1$s + Zawsze zezwalaj na %1$s + Amethyst nie zdołała odszyfrować tej wiadomości. Być może nie jest ona adresowana do tego konta. + Wiadomości z Podłączone aplikacje Cofnij wszystkie uprawnienia @@ -1559,6 +1601,7 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Ten serwer wymaga płatności do wysłania: %1$s Wymagana płatność %1$s pobiera opłatę w systemie Lightning za zapisanie tego pliku. Aby kontynuować, dokonaj płatności z podłączonego portfela. + %1$s powiedział: „%2$s” Zapłać Zapłać %1$d sat @@ -2392,6 +2435,8 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Nie można wypłacić Nie można wykupić Cashu Mint dostarczył następujący komunikat błędu: %1$s + Niebezpieczny adres Cashu mint + Amethyst nie skontaktował się z emitentem tego tokena. %1$s Cashu odebrano %1$s satsy zostały wysłane do Twojego portfela. (opłata: %2$s satoszy) W systemie nie znaleziono kompatybilnego portfela Cashu diff --git a/commons/src/commonMain/composeResources/values-cs/strings.xml b/commons/src/commonMain/composeResources/values-cs/strings.xml index e03a039b2a..2397cb87fd 100644 --- a/commons/src/commonMain/composeResources/values-cs/strings.xml +++ b/commons/src/commonMain/composeResources/values-cs/strings.xml @@ -145,10 +145,4 @@ %1$d událostí %1$d událostí - - %1$d relay - %1$d relaye - %1$d relaye - %1$d relayů - diff --git a/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml b/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml index 17d896ccb3..6b65c8acfe 100644 --- a/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml +++ b/commons/src/commonMain/composeResources/values-pt-rBR/strings.xml @@ -65,6 +65,7 @@ respondendo para + Site Estático: %1$s O que ele pode acessar Site raiz Origem: @@ -137,9 +138,4 @@ %1$d evento %1$d eventos - Site Estático: %1$s - - %1$d relay - %1$d relays - diff --git a/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml b/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml index 4b2ee35517..a90c7d5954 100644 --- a/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml +++ b/commons/src/commonMain/composeResources/values-sv-rSE/strings.xml @@ -69,6 +69,7 @@ Svarar till + Statisk webbplats: %1$s Vad det har åtkomst till Rotplats Källa: @@ -140,5 +141,4 @@ %1$d händelse %1$d händelser - Statisk webbplats: %1$s From 636dd2afe74d4b3746339f0d0158fdc5e5507729 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 22 Jul 2026 13:57:39 -0400 Subject: [PATCH 34/34] fix(signer): never stamp our client tag on someone else's template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NIP-89 client tag says "this app composed this event", so it belongs only on templates Amethyst authored. NostrSignerWithClientTag was applied at the account level, which meant it also fired on every event we sign on behalf of an external client. That is wrong twice over. It misattributes the event, and — because the tag is appended before signing — it rewrites the exact bytes the caller is about to have hashed into an id. NIP-07 callers routinely re-check the returned event against the template they submitted, and block/buzz compares tags outright (web/src/shared/lib/nostr-signer.ts): JSON.stringify(actual.tags) === JSON.stringify(expected.tags) so joining a Buzz community from the in-app browser failed with "The NIP-07 extension returned an invalid signed event". Probed live over the WebView devtools protocol: kind, created_at, content and pubkey all round-tripped intact and only tags differed, by exactly the ["client","Amethyst"] we append. Add NostrSigner.withoutClientTag() and use it at the two boundaries where the template belongs to someone else: - the napplet broker, covering napplets, nSites and web apps over NIP-07 - Nip46SignerState, where we act as another client's bunker — the same defect, and quieter, since that client never learns why its event changed underneath it Amethyst's own events are untouched and still carry the tag. Unwrapping keeps everything layered below (metering, NIP-13 mining) and leaves pubKey alone. There is no NIP-55 provider surface to fix; we are only ever the client there. Verified against Buzz's own four acceptance conditions after the change: pubkey matches, sameUnsignedEvent true, id and sig present — and the invite join then succeeded. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../com/vitorpamplona/amethyst/model/Account.kt | 6 +++++- .../napplet/gateways/AccountNappletGateways.kt | 6 +++++- .../clientTag/NostrSignerWithClientTag.kt | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt index 4207c80def..6841fe10a8 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt @@ -325,6 +325,7 @@ import com.vitorpamplona.quartz.nip7DThreads.ThreadEvent import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent import com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent import com.vitorpamplona.quartz.nip89AppHandlers.clientTag.NostrSignerWithClientTag +import com.vitorpamplona.quartz.nip89AppHandlers.clientTag.withoutClientTag import com.vitorpamplona.quartz.nip90Dvms.contentDiscoveryRequest.NIP90ContentDiscoveryRequestEvent import com.vitorpamplona.quartz.nip92IMeta.IMetaTag import com.vitorpamplona.quartz.nip92IMeta.imetas @@ -467,7 +468,10 @@ class Account( */ val nip46Signer = Nip46SignerState( - signer = signer, + // Acting as someone else's bunker: the templates arriving here were composed by the + // connected client, so they are signed exactly as received — our client tag would both + // misattribute the event and change the id the client expects back. + signer = signer.withoutClientTag(), client = client, ledger = signerPermissionLedger, clientStore = nip46ClientStore, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/AccountNappletGateways.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/AccountNappletGateways.kt index db32cbbf6c..e7fd4f0d67 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/AccountNappletGateways.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/AccountNappletGateways.kt @@ -60,6 +60,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip47WalletConnect.rpc.NwcErrorResponse import com.vitorpamplona.quartz.nip47WalletConnect.rpc.PayInvoiceErrorResponse import com.vitorpamplona.quartz.nip47WalletConnect.rpc.PayInvoiceSuccessResponse +import com.vitorpamplona.quartz.nip89AppHandlers.clientTag.withoutClientTag import com.vitorpamplona.quartz.utils.sha256.sha256 import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.withTimeout @@ -155,7 +156,10 @@ class AccountNappletGateways( ) } - return NappletBroker(account.signer, ledger, consent, signerLedger = signerLedger, nostrConnectPrompt = connectPrompt, signerConsentPrompt = signerConsent, relay = relay, storage = storage, wallet = wallet, resource = resource, upload = upload, identityReads = identityReads, theme = theme, notify = notify) + // Everything the broker signs belongs to the guest — a napplet, an nSite, or a web app + // calling NIP-07 — never to Amethyst, so our client tag has no business on it. It would also + // corrupt the template a NIP-07 caller re-checks the returned event against. + return NappletBroker(account.signer.withoutClientTag(), ledger, consent, signerLedger = signerLedger, nostrConnectPrompt = connectPrompt, signerConsentPrompt = signerConsent, relay = relay, storage = storage, wallet = wallet, resource = resource, upload = upload, identityReads = identityReads, theme = theme, notify = notify) } /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip89AppHandlers/clientTag/NostrSignerWithClientTag.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip89AppHandlers/clientTag/NostrSignerWithClientTag.kt index b39c69843b..7cd234f925 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip89AppHandlers/clientTag/NostrSignerWithClientTag.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip89AppHandlers/clientTag/NostrSignerWithClientTag.kt @@ -117,3 +117,18 @@ class NostrSignerWithClientTag( return tags + arrayOf(clientTag) } } + +/** + * The same signer with the NIP-89 client-tag decorator peeled off, or the receiver unchanged when it + * carries no such decorator. + * + * The client tag says "this app composed this event", so it belongs only on templates this app + * authored. When we sign on someone else's behalf — a napplet, an nSite, a web app over NIP-07, a + * client using us as a NIP-46 bunker — the template is theirs, and appending a tag rewrites the very + * bytes they are about to have hashed into an id. NIP-07 callers routinely re-check the returned + * event against the template they submitted (block/buzz compares `JSON.stringify(tags)` outright) + * and reject the result as an invalid signature when it does not match. + * + * Any decoration under this one (metering, NIP-13 mining) is preserved, as is [NostrSigner.pubKey]. + */ +fun NostrSigner.withoutClientTag(): NostrSigner = if (this is NostrSignerWithClientTag) inner else this