From 0c67f63bad0b117a9e4f29da566adc680922a300 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 18 Jun 2026 21:49:06 +0000 Subject: [PATCH 1/3] feat(quartz): add FTS reindex to SQLite and filesystem event stores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The set of event kinds that implement SearchableEvent — and the text each contributes via indexableContent() — is baked into the quartz build, so it changes across app versions. Events stored under older code keep their old (or missing) NIP-50 full-text-search rows, so search silently misses them after an upgrade. Add IEventStore.reindexFullTextSearch() so the app can wipe and rebuild the FTS index from already-stored events when it has spare cycles. Speed: only kinds that currently map to a SearchableEvent are scanned. Kind alone selects the event class in EventFactory, so a single probe per distinct kind is authoritative, letting us push a `kind IN (...)` filter (SQLite) / skip whole idx/kind dirs (filesystem) so the non-searchable bulk — reactions, zaps, follow lists — is never deserialised. - SQLite: FullTextSearchModule.reindexAll drops+recreates the virtual table (O(1) wipe) then streams only searchable-kind rows in one write transaction, reusing a single INSERT statement. - Filesystem: rebuilds only idx/fts/, driving the walk from idx/kind// via the new FsIndexer.linkFts. - Wrappers (EventStore, ObservableEventStore, InterningEventStore) delegate; the observable layer emits nothing since no event changes. - cli: `amy store reindex-fts`. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BZqPFds2TPPUKkMmBngwys --- .../com/vitorpamplona/amethyst/cli/Main.kt | 1 + .../amethyst/cli/commands/StoreCommands.kt | 22 ++++- .../cache/interning/InterningEventStore.kt | 2 + .../quartz/nip01Core/store/IEventStore.kt | 23 +++++ .../nip01Core/store/ObservableEventStore.kt | 5 + .../nip01Core/store/sqlite/EventStore.kt | 2 + .../store/sqlite/FullTextSearchModule.kt | 93 ++++++++++++++++++- .../store/sqlite/SQLiteEventStore.kt | 13 +++ .../nip01Core/store/sqlite/SearchTest.kt | 38 ++++++++ .../quartz/nip01Core/store/fs/FsEventStore.kt | 44 +++++++++ .../quartz/nip01Core/store/fs/FsIndexer.kt | 15 +++ .../quartz/nip01Core/store/fs/FsSearchTest.kt | 48 ++++++++++ 12 files changed, 304 insertions(+), 2 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 03b6ac97d8..3ebfac44f2 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -442,6 +442,7 @@ private fun printUsage() { | store sweep-expired delete events past their NIP-40 expiration | store scrub rebuild idx/ from canonical events (after edits / crashes) | store compact drop dangling idx entries (canonical gone) + | store reindex-fts rebuild the NIP-50 search index (after a searchable-kinds change) """.trimMargin(), ) } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt index 470c42bbf5..ff9037d6bf 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt @@ -45,19 +45,23 @@ import kotlin.io.path.exists * external edits. * - `compact` drop dangling `idx/` entries whose canonical is * gone. Cheaper than scrub. + * - `reindex-fts` wipe and rebuild only the NIP-50 full-text search + * index from the stored events. Run after a quartz + * upgrade that changes which kinds are searchable. */ object StoreCommands { suspend fun dispatch( dataDir: DataDir, tail: Array, ): Int { - if (tail.isEmpty()) return Output.error("bad_args", "store ") + if (tail.isEmpty()) return Output.error("bad_args", "store ") val rest = tail.drop(1).toTypedArray() return when (tail[0]) { "stat" -> stat(dataDir) "sweep-expired" -> sweepExpired(dataDir) "scrub" -> scrub(dataDir) "compact" -> compact(dataDir) + "reindex-fts" -> reindexFts(dataDir) else -> Output.error("bad_args", "store ${tail[0]}") } } @@ -161,6 +165,22 @@ object StoreCommands { 0 } + private suspend fun reindexFts(dataDir: DataDir): Int = + withStore(dataDir) { store -> + val ftsDir = dataDir.eventsDir.toPath().resolve("idx/fts") + val before = countEntries(ftsDir) + store.reindexFullTextSearch() + val after = countEntries(ftsDir) + Output.emit( + mapOf( + "ok" to true, + "tokens_before" to before, + "tokens_after" to after, + ), + ) + 0 + } + /** * Maintenance verbs only need the store — not identity, not relays, * not the signer. Skip [Context.open] (which throws if no identity diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt index 85fbc1c6fa..48d1e2faca 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt @@ -123,5 +123,7 @@ class InterningEventStore( override suspend fun deleteExpiredEvents() = inner.deleteExpiredEvents() + override suspend fun reindexFullTextSearch() = inner.reindexFullTextSearch() + override fun close() = inner.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index 3300185a43..6c7210ba94 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -133,5 +133,28 @@ interface IEventStore : AutoCloseable { suspend fun deleteExpiredEvents() + /** + * Wipe and rebuild the NIP-50 full-text search index from the + * events already in storage. + * + * Which kinds are searchable — i.e. implement + * `SearchableEvent` — and what text each one contributes is baked + * into the quartz build, so it can change across app versions: a + * kind that was opaque before may start implementing + * `SearchableEvent`, or an existing one may change what its + * `indexableContent()` returns. Events that were inserted under the + * old code keep their old (or missing) FTS rows until they are + * re-indexed, so search silently misses them. Call this once, off + * the hot path, after such an upgrade — the app decides when it has + * the spare cycles to process the whole store. + * + * Implementations rebuild from scratch (so the result is the same + * whether or not an index already existed) and only visit kinds + * that currently map to a searchable event, leaving the bulk of + * non-searchable rows (reactions, zaps, follow lists, …) untouched + * to keep the scan as cheap as possible. + */ + suspend fun reindexFullTextSearch() + override fun close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt index b11d1b27e4..d65002ebc4 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt @@ -215,5 +215,10 @@ class ObservableEventStore( _changes.emit(StoreChange.DeleteExpired(asOf)) } + // Pure index maintenance: it rewrites the FTS index without adding, + // removing, or changing any event, so there is nothing for + // projections to observe and no [StoreChange] is emitted. + override suspend fun reindexFullTextSearch() = inner.reindexFullTextSearch() + override fun close() = inner.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index faad3204d3..ce336b1e13 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -79,5 +79,7 @@ class EventStore( override suspend fun deleteExpiredEvents() = store.deleteExpiredEvents() + override suspend fun reindexFullTextSearch() = store.reindexFullTextSearch() + override fun close() = store.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index f39da4fe85..456e7d28c7 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -23,10 +23,13 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.SQLiteConnection import androidx.sqlite.SQLiteException import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip50Search.SearchableEvent +import com.vitorpamplona.quartz.utils.EventFactory class FullTextSearchModule : IModule { val tableName = "event_fts" + val triggerName = "fts_foreign_key" val eventHeaderRowIdName = "event_header_row_id" val contentName = "content" @@ -42,7 +45,7 @@ class FullTextSearchModule : IModule { // Foreign key cleanup for full text search db.execSQL( """ - CREATE TRIGGER fts_foreign_key + CREATE TRIGGER $triggerName AFTER DELETE ON event_headers FOR EACH ROW BEGIN @@ -57,6 +60,17 @@ class FullTextSearchModule : IModule { db.execSQL("DROP TABLE IF EXISTS $tableName") } + /** + * Drop the cleanup trigger on its own. [drop] only removes the FTS + * table; the trigger lives on `event_headers`, so a rebuild that + * recreates the table without first dropping the trigger would fail + * with "trigger already exists". (A schema upgrade doesn't hit this + * because dropping `event_headers` takes its triggers with it.) + */ + fun dropTrigger(db: SQLiteConnection) { + db.execSQL("DROP TRIGGER IF EXISTS $triggerName") + } + val insertFTS = """ INSERT OR ROLLBACK INTO $tableName ($eventHeaderRowIdName, $contentName) @@ -109,4 +123,81 @@ class FullTextSearchModule : IModule { override fun deleteAll(db: SQLiteConnection) { db.execSQL("DELETE FROM event_fts") } + + /** + * Wipe and rebuild the whole FTS index from `event_headers`. + * + * Wiping is done by dropping and recreating the virtual table, which + * empties it in O(1) — far cheaper than DELETE-ing every row out of a + * populated FTS index. The rebuild then streams only the rows whose + * kind currently parses to a [SearchableEvent] (see + * [searchableKindsPresent]) so the common non-searchable bulk — + * reactions, zaps, follow lists — is never deserialised. A single + * shared INSERT statement is reused across the scan. + * + * Must run inside the caller's write transaction. + */ + fun reindexAll(db: SQLiteConnection) { + dropTrigger(db) + drop(db) + create(db) + + val kinds = searchableKindsPresent(db) + if (kinds.isEmpty()) return + + val selectSql = + "SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " + + "FROM event_headers WHERE kind IN (${kinds.joinToString(",")})" + + db.prepare(insertFTS).use { write -> + db.prepare(selectSql).use { read -> + while (read.step()) { + val event = + EventFactory.create( + read.getText(1), + read.getText(2), + read.getLong(3), + read.getInt(4), + OptimizedJsonMapper.fromJsonToTagArray(read.getText(5)), + read.getText(6), + read.getText(7), + ) + if (event is SearchableEvent) { + write.bindLong(1, read.getLong(0)) + write.bindText(2, event.indexableContent()) + write.step() + write.reset() + } + } + } + } + } + + /** + * The distinct kinds present in `event_headers` that currently parse + * to a [SearchableEvent]. Kind alone selects the event class in + * [EventFactory], so one probe per distinct kind is authoritative; + * the result drives a `kind IN (...)` filter on the rebuild scan so + * non-searchable rows are skipped at the SQL layer. + */ + private fun searchableKindsPresent(db: SQLiteConnection): List { + val out = ArrayList() + db.prepare("SELECT DISTINCT kind FROM event_headers").use { stmt -> + while (stmt.step()) { + val kind = stmt.getInt(0) + if (isSearchableKind(kind)) out.add(kind) + } + } + return out + } + + private fun isSearchableKind(kind: Int): Boolean = EventFactory.create(PROBE_ID, PROBE_ID, 0L, kind, EMPTY_TAGS, "", "") is SearchableEvent + + companion object { + // A non-blank id keeps kinds that lazily hash a missing id (e.g. + // NIP-17 chat messages) from doing that work — the probe only + // inspects the resulting runtime type. + private const val PROBE_ID = "0" + private val EMPTY_TAGS = emptyArray>() + } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index e67f7b4646..47e2ac3cfa 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -340,6 +340,19 @@ class SQLiteEventStore( suspend fun deleteExpiredEvents() = pool.useWriter { expirationModule.deleteExpiredEvents(it) } + /** + * Wipe and rebuild the NIP-50 full-text search index for every + * stored event. See [IEventStore.reindexFullTextSearch] for when to + * call this. The whole rebuild runs in a single write transaction so + * the WAL append + sync cost is paid once. + */ + suspend fun reindexFullTextSearch() = + pool.useWriter { db -> + db.transaction { + fullTextSearchModule.reindexAll(db) + } + } + fun close() = pool.close() } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt index ebcc4081b7..b79d0d2679 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt @@ -192,6 +192,44 @@ class SearchTest : BaseDBTest() { db.assertQuery(p, Filter(kinds = listOf(MetadataEvent.KIND), search = "uniqalice")) } + @Test + fun testReindexFullTextSearchRebuildsTheWholeIndex() = + forEachDB { db -> + // A calendar event (title tag + content) and a plain note. Both + // are searchable kinds, but we'll erase the FTS index to mimic + // events that were stored before their kind became searchable. + val cal = + signer.sign( + CalendarEvent.build( + title = "uniqtitle Meetup", + content = "annual uniqbody gathering", + ), + ) + val note = signer.sign(TextNoteEvent.build("reindex uniqnote please", createdAt = TimeUtils.now())) + + db.store.insertEvent(cal) + db.store.insertEvent(note) + + // Wipe the FTS rows but keep the canonical event rows — this is + // the state a store ends up in after an upgrade adds search + // support for a kind that was inserted under the old code. + db.store.pool.useWriter { db.store.fullTextSearchModule.deleteAll(it) } + db.store.assertQuery(null, Filter(search = "uniqbody")) + db.store.assertQuery(null, Filter(search = "uniqnote")) + + // Rebuilding from storage brings every searchable field back. + db.store.reindexFullTextSearch() + db.store.assertQuery(cal, Filter(search = "uniqtitle")) + db.store.assertQuery(cal, Filter(search = "uniqbody")) + db.store.assertQuery(note, Filter(search = "uniqnote")) + + // Running it again must not duplicate rows (assertQuery expects + // exactly one match), proving the rebuild starts from a clean slate. + db.store.reindexFullTextSearch() + db.store.assertQuery(cal, Filter(search = "uniqbody")) + db.store.assertQuery(note, Filter(search = "uniqnote")) + } + @Test fun testChannelJsonFieldsAreSearchable() = forEachDB { db -> diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index c91cf7445c..e15f810a6a 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -34,7 +34,9 @@ import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy import com.vitorpamplona.quartz.nip01Core.store.sqlite.TagNameValueHasher import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip40Expiration.expiration +import com.vitorpamplona.quartz.nip50Search.SearchableEvent import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent +import com.vitorpamplona.quartz.utils.EventFactory import com.vitorpamplona.quartz.utils.TimeUtils import java.nio.file.FileAlreadyExistsException import java.nio.file.Files @@ -493,6 +495,48 @@ open class FsEventStore( } } + /** + * Wipe and rebuild only the `idx/fts/` tree from the stored events. + * See [IEventStore.reindexFullTextSearch] for when to call this. + * + * Unlike [scrub] (which rebuilds every index tree from a full + * `events/` walk), this touches nothing but FTS and reads only the + * events whose kind is searchable today: it drives the walk from + * `idx/kind//`, which already lists exactly the live canonical + * events per kind, and skips kind directories that don't map to a + * [SearchableEvent]. The bulk of non-searchable events (reactions, + * zaps, follow lists, …) is never opened. + */ + override suspend fun reindexFullTextSearch() = + lockManager.withWriteLock { + deleteRecursively(layout.idxFts) + Files.createDirectories(layout.idxFts) + + if (!Files.isDirectory(layout.idxKind)) return@withWriteLock + Files.list(layout.idxKind).use { kindDirs -> + for (kindDir in kindDirs) { + val kind = kindDir.fileName.toString().toIntOrNull() ?: continue + if (!isSearchableKind(kind)) continue + Files.list(kindDir).use { entries -> + for (entry in entries) { + val id = FsLayout.parseEntry(entry.fileName.toString())?.second ?: continue + val event = readEvent(id) ?: continue + indexer.linkFts(event, layout.canonical(id)) + } + } + } + } + } + + /** + * True when [kind] currently parses to a [SearchableEvent]. Kind + * alone selects the event class in [EventFactory], so a single probe + * per kind is authoritative. The id is non-blank so kinds that lazily + * hash a missing id (NIP-17 chat) skip that work — only the runtime + * type matters here. + */ + private fun isSearchableKind(kind: Int): Boolean = EventFactory.create("0", "0", 0L, kind, emptyArray(), "", "") is SearchableEvent + private fun deleteRecursively(p: java.nio.file.Path) { if (!Files.exists(p)) return Files.walk(p).use { stream -> diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt index 6a99bec6cc..198e4aea25 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt @@ -86,6 +86,21 @@ internal class FsIndexer( } } + /** + * Create only the NIP-50 FTS hardlinks for this event (no-op when it + * isn't a [SearchableEvent]). Used by the FTS-only rebuild so an + * existing kind/author/owner/tag tree is left untouched. Idempotent. + */ + fun linkFts( + event: Event, + canonical: Path, + ) { + if (event !is SearchableEvent) return + for (token in FsSearchTokenizer.tokenize(event.indexableContent())) { + createLink(layout.ftsEntry(token, event.createdAt, event.id), canonical) + } + } + /** Remove every index hardlink for this event. Missing entries ignored. */ fun unlink(event: Event) { for (path in pathsFor(event)) { diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt index df24d3dcb6..766644a93f 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt @@ -131,6 +131,54 @@ class FsSearchTest { assertEquals(0, ftsRoot.listDirectoryEntries().size, "unknown kind is not SearchableEvent") } + @Test + fun `reindexFullTextSearch rebuilds fts entries after a wipe`() = + runBlocking { + val a = note("bitcoin reindex", ts = 100) + val b = note("nostr reindex", ts = 200) + store.insert(a) + store.insert(b) + + // Mimic a store written before this kind was searchable: drop + // the whole idx/fts/ tree, leaving canonical events intact. + val ftsRoot = root.resolve("idx/fts") + Files.walk(ftsRoot).use { stream -> + stream.sorted(Comparator.reverseOrder()).forEach { p -> if (p != ftsRoot) Files.deleteIfExists(p) } + } + assertEquals(0, ftsRoot.listDirectoryEntries().size, "precondition: fts wiped") + assertTrue(store.query(Filter(search = "bitcoin")).isEmpty(), "no index, no match") + + store.reindexFullTextSearch() + + assertEquals(listOf(a.id), store.query(Filter(search = "bitcoin")).map { it.id }) + assertEquals(listOf(b.id), store.query(Filter(search = "nostr")).map { it.id }) + // Reindexing twice must not duplicate entries. + store.reindexFullTextSearch() + assertEquals(1, ftsRoot.resolve("bitcoin").listDirectoryEntries().size) + // "reindex" appears in both notes — one entry per event, no dupes. + assertEquals(2, ftsRoot.resolve("reindex").listDirectoryEntries().size) + } + + @Test + fun `reindexFullTextSearch ignores non-searchable kinds`() = + runBlocking { + val searchable = note("findme bitcoin", ts = 100) + val opaque = + signer.sign( + createdAt = 1, + kind = 9999, + tags = emptyArray(), + content = "findme must not be indexed", + ) + store.insert(searchable) + store.insert(opaque) + + store.reindexFullTextSearch() + + // Only the searchable note contributes the "findme" token. + assertEquals(listOf(searchable.id), store.query(Filter(search = "findme")).map { it.id }) + } + @Test fun `delete removes fts entries`() = runBlocking { From b419db4fe0a84c85389b585b2a086214e5858ce0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 18 Jun 2026 22:16:48 +0000 Subject: [PATCH 2/3] feat(quartz): make FTS reindex pausable/resumable for large stores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full FTS rebuild can run for a long time on a big store, so add a resumable, batched overload alongside the one-shot: reindexFullTextSearch(resumeFrom: String?, batchSize): FtsReindexProgress Each call processes ~batchSize events in its own write transaction and returns an opaque cursor + done flag. The caller loops until done and may stop at any point — the cursor is durable across crash/app-restart, and the writer lock is released between batches, so "pause" is just "don't make the next call". The path is additive/refresh and keeps search usable throughout (no up-front wipe); the one-shot variant remains for a guaranteed-clean rebuild. - SQLite: FullTextSearchModule.reindexBatch walks event_headers ordered by the monotonic row_id (a free, stable cursor), restricted to searchable kinds, delete-then-insert per event so batches are idempotent and never duplicate rows. - Filesystem: FsEventStore walks one idx/kind// dir per step (linear, no re-sort); cursor is the next searchable kind. Idempotent linkFts, so nothing is wiped. Pauses between kinds. - Wrappers delegate; new FtsReindexProgress value type carries cursor + progress + done. - cli: `amy store reindex-fts` now loops the batched path to completion and reports processed/batch counts. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BZqPFds2TPPUKkMmBngwys --- .../amethyst/cli/commands/StoreCommands.kt | 17 +++- .../cache/interning/InterningEventStore.kt | 6 ++ .../nip01Core/store/FtsReindexProgress.kt | 42 ++++++++++ .../quartz/nip01Core/store/IEventStore.kt | 50 ++++++++++++ .../nip01Core/store/ObservableEventStore.kt | 5 ++ .../nip01Core/store/sqlite/EventStore.kt | 6 ++ .../store/sqlite/FullTextSearchModule.kt | 81 +++++++++++++++++++ .../store/sqlite/SQLiteEventStore.kt | 17 ++++ .../nip01Core/store/sqlite/SearchTest.kt | 69 ++++++++++++++++ .../quartz/nip01Core/store/fs/FsEventStore.kt | 58 +++++++++++++ .../quartz/nip01Core/store/fs/FsSearchTest.kt | 40 +++++++++ 11 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/FtsReindexProgress.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt index ff9037d6bf..c1ff1b89eb 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.amethyst.cli.commands import com.vitorpamplona.amethyst.cli.DataDir import com.vitorpamplona.amethyst.cli.Output import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper +import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore import java.io.IOException import java.nio.file.Files @@ -169,11 +170,25 @@ object StoreCommands { withStore(dataDir) { store -> val ftsDir = dataDir.eventsDir.toPath().resolve("idx/fts") val before = countEntries(ftsDir) - store.reindexFullTextSearch() + // Drive the resumable, batched path to completion so a huge + // store is processed without holding the writer lock for the + // whole pass. A real long-running caller would persist the + // cursor between calls; here we just loop until done. + var cursor: String? = null + var processed = 0L + var batches = 0 + do { + val progress = store.reindexFullTextSearch(cursor, IEventStore.DEFAULT_FTS_REINDEX_BATCH) + cursor = progress.cursor + processed += progress.processedThisBatch + batches++ + } while (!progress.done) val after = countEntries(ftsDir) Output.emit( mapOf( "ok" to true, + "processed" to processed, + "batches" to batches, "tokens_before" to before, "tokens_after" to after, ), diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt index 48d1e2faca..2f87070657 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.quartz.nip01Core.cache.interning import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.IdAndTime @@ -125,5 +126,10 @@ class InterningEventStore( override suspend fun reindexFullTextSearch() = inner.reindexFullTextSearch() + override suspend fun reindexFullTextSearch( + resumeFrom: String?, + batchSize: Int, + ): FtsReindexProgress = inner.reindexFullTextSearch(resumeFrom, batchSize) + override fun close() = inner.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/FtsReindexProgress.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/FtsReindexProgress.kt new file mode 100644 index 0000000000..2db5df7cb4 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/FtsReindexProgress.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store + +/** + * Progress token returned by the resumable + * [IEventStore.reindexFullTextSearch] overload. + * + * Treat [cursor] as opaque: persist it (it survives process death) and + * hand it back to the next call to continue where the previous one + * stopped. Each store encodes its own resume position into it. + * + * @property cursor where to resume from on the next call, or `null` once + * [done] is `true` (nothing left to process). + * @property processedThisBatch how many events this call (re)indexed — + * useful to drive a progress indicator. + * @property done `true` when the whole store has been visited; further + * calls are no-ops that keep returning `done = true`. + */ +data class FtsReindexProgress( + val cursor: String?, + val processedThisBatch: Int, + val done: Boolean, +) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index 6c7210ba94..7d68d4fb93 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -25,6 +25,16 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl interface IEventStore : AutoCloseable { + companion object { + /** + * How many events a single resumable + * [reindexFullTextSearch] batch aims to process before yielding. + * Big enough to amortise the per-batch transaction/lock cost, + * small enough that a pause request is honoured promptly. + */ + const val DEFAULT_FTS_REINDEX_BATCH = 1000 + } + /** * Relay URL this store is acting on behalf of, or `null` for an * unscoped store. Used by NIP-62 right-to-vanish handling: only @@ -153,8 +163,48 @@ interface IEventStore : AutoCloseable { * that currently map to a searchable event, leaving the bulk of * non-searchable rows (reactions, zaps, follow lists, …) untouched * to keep the scan as cheap as possible. + * + * This one-shot variant runs to completion under a single lock and + * cannot be paused. For a store large enough that the full pass + * would block for too long, use the resumable overload + * [reindexFullTextSearch] instead. */ suspend fun reindexFullTextSearch() + /** + * Resumable, batched companion to [reindexFullTextSearch] for stores + * large enough that a single pass would take too long to run (or to + * hold a lock) in one go. + * + * Process roughly [batchSize] events starting from [resumeFrom] + * (`null` = from the beginning) and return a [FtsReindexProgress]. + * Drive it in a loop, feeding [FtsReindexProgress.cursor] back in, + * until [FtsReindexProgress.done] is `true`: + * + * ``` + * var cursor: String? = null + * do { + * val p = store.reindexFullTextSearch(cursor) + * cursor = p.cursor + * // optionally persist `cursor` and stop; resume later by passing it back + * } while (!p.done) + * ``` + * + * Each call commits its own batch, so progress is durable across a + * crash or app restart and the writer lock is released between + * batches — the app can pause simply by not making the next call. + * + * Semantics differ slightly from the one-shot variant: this path is + * **additive / refresh** — it makes sure every currently-searchable + * event is indexed (and, on SQLite, refreshes changed content) while + * leaving search usable throughout. It does not purge stale rows left + * behind by a kind that *lost* searchability; run the one-shot + * [reindexFullTextSearch] once for that rarer case. + */ + suspend fun reindexFullTextSearch( + resumeFrom: String?, + batchSize: Int = DEFAULT_FTS_REINDEX_BATCH, + ): FtsReindexProgress + override fun close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt index d65002ebc4..35e0427f25 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt @@ -220,5 +220,10 @@ class ObservableEventStore( // projections to observe and no [StoreChange] is emitted. override suspend fun reindexFullTextSearch() = inner.reindexFullTextSearch() + override suspend fun reindexFullTextSearch( + resumeFrom: String?, + batchSize: Int, + ): FtsReindexProgress = inner.reindexFullTextSearch(resumeFrom, batchSize) + override fun close() = inner.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index ce336b1e13..7e1c912d55 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.IdAndTime @@ -81,5 +82,10 @@ class EventStore( override suspend fun reindexFullTextSearch() = store.reindexFullTextSearch() + override suspend fun reindexFullTextSearch( + resumeFrom: String?, + batchSize: Int, + ): FtsReindexProgress = store.reindexFullTextSearch(resumeFrom, batchSize) + override fun close() = store.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index 456e7d28c7..3de6859e4b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -24,6 +24,7 @@ import androidx.sqlite.SQLiteConnection import androidx.sqlite.SQLiteException import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper +import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress import com.vitorpamplona.quartz.nip50Search.SearchableEvent import com.vitorpamplona.quartz.utils.EventFactory @@ -77,6 +78,11 @@ class FullTextSearchModule : IModule { VALUES (?, ?) """.trimIndent() + val deleteFTSByRowId = + """ + DELETE FROM $tableName WHERE $eventHeaderRowIdName = ? + """.trimIndent() + fun insert( event: Event, headerId: Long, @@ -173,6 +179,81 @@ class FullTextSearchModule : IModule { } } + /** + * Process one batch of a resumable rebuild: re-derive the FTS rows + * for up to [batchSize] events whose `row_id > ` [afterRowId] and + * whose kind is searchable, ordered by `row_id`. + * + * `row_id` is a monotonic AUTOINCREMENT key, so it is a stable + * cursor that needs no extra bookkeeping. Each event is + * delete-then-insert, which keeps the batch idempotent (a replay + * after a crash is harmless) and avoids duplicate FTS rows for + * events that were already indexed by the normal insert path. Rows + * not yet reached keep their previous FTS content, so search stays + * usable while the rebuild is in flight. + * + * Must run inside the caller's per-batch write transaction. + */ + fun reindexBatch( + db: SQLiteConnection, + afterRowId: Long, + batchSize: Int, + ): FtsReindexProgress { + val kinds = searchableKindsPresent(db) + if (kinds.isEmpty()) return FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true) + + val selectSql = + "SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " + + "FROM event_headers WHERE row_id > ? AND kind IN (${kinds.joinToString(",")}) " + + "ORDER BY row_id LIMIT ?" + + var last = afterRowId + var processed = 0 + db.prepare(deleteFTSByRowId).use { del -> + db.prepare(insertFTS).use { write -> + db.prepare(selectSql).use { read -> + read.bindLong(1, afterRowId) + read.bindLong(2, batchSize.toLong()) + while (read.step()) { + val rowId = read.getLong(0) + // Clear any existing row for this event first so a + // replay or an already-indexed event can't duplicate. + del.bindLong(1, rowId) + del.step() + del.reset() + + val event = + EventFactory.create( + read.getText(1), + read.getText(2), + read.getLong(3), + read.getInt(4), + OptimizedJsonMapper.fromJsonToTagArray(read.getText(5)), + read.getText(6), + read.getText(7), + ) + if (event is SearchableEvent) { + write.bindLong(1, rowId) + write.bindText(2, event.indexableContent()) + write.step() + write.reset() + } + last = rowId + processed++ + } + } + } + } + + // Fewer than a full page came back ⇒ we hit the end of the table. + val done = processed < batchSize + return FtsReindexProgress( + cursor = if (done) null else last.toString(), + processedThisBatch = processed, + done = done, + ) + } + /** * The distinct kinds present in `event_headers` that currently parse * to a [SearchableEvent]. Kind alone selects the event class in diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 47e2ac3cfa..ad9088275b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -31,6 +31,7 @@ import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.core.isEphemeral import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.IdAndTime import com.vitorpamplona.quartz.nip40Expiration.isExpired @@ -353,6 +354,22 @@ class SQLiteEventStore( } } + /** + * One batch of a resumable FTS rebuild. See + * [IEventStore.reindexFullTextSearch]. The opaque cursor is the last + * `row_id` processed; `null` (or an unparseable value) starts from + * the beginning. Each batch is its own write transaction. + */ + suspend fun reindexFullTextSearch( + resumeFrom: String?, + batchSize: Int, + ): FtsReindexProgress = + pool.useWriter { db -> + db.transaction { + fullTextSearchModule.reindexBatch(db, resumeFrom?.toLongOrNull() ?: 0L, batchSize) + } + } + fun close() = pool.close() } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt index b79d0d2679..bdf7ae690a 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt @@ -230,6 +230,75 @@ class SearchTest : BaseDBTest() { db.store.assertQuery(note, Filter(search = "uniqnote")) } + @Test + fun testResumableReindexProcessesEveryEventInBatches() = + forEachDB { db -> + // A handful of searchable notes, each with a unique token. + val notes = + (0 until 5).map { i -> + signer.sign(TextNoteEvent.build("uniqresume$i body", createdAt = TimeUtils.now() + i)) + } + notes.forEach { db.store.insertEvent(it) } + + // Mimic the post-upgrade state: canonical rows present, FTS empty. + db.store.pool.useWriter { db.store.fullTextSearchModule.deleteAll(it) } + db.store.assertQuery(null, Filter(search = "uniqresume0")) + + // Drive the resumable path two events at a time, persisting only + // the opaque cursor between calls — exactly what a paused/resumed + // app would do. + var cursor: String? = null + var batches = 0 + var processed = 0 + do { + val progress = db.store.reindexFullTextSearch(cursor, batchSize = 2) + cursor = progress.cursor + processed += progress.processedThisBatch + batches++ + } while (!progress.done) + + // 5 events at 2 per batch ⇒ 3 batches (2 + 2 + 1). + kotlin.test.assertEquals(5, processed) + kotlin.test.assertEquals(3, batches) + + // Every note is searchable again after the full run. + notes.forEachIndexed { i, n -> db.store.assertQuery(n, Filter(search = "uniqresume$i")) } + } + + @Test + fun testResumableReindexKeepsSearchLiveAndIsIdempotent() = + forEachDB { db -> + val a = signer.sign(TextNoteEvent.build("uniqalpha note", createdAt = TimeUtils.now())) + val b = signer.sign(TextNoteEvent.build("uniqbeta note", createdAt = TimeUtils.now() + 1)) + db.store.insertEvent(a) + db.store.insertEvent(b) + + // First batch (size 1) reindexes only the lowest row_id; the + // untouched event keeps the FTS row it already had from insert, + // so search stays live for both throughout. + val first = db.store.reindexFullTextSearch(null, batchSize = 1) + kotlin.test.assertEquals(false, first.done) + db.store.assertQuery(a, Filter(search = "uniqalpha")) + db.store.assertQuery(b, Filter(search = "uniqbeta")) + + // Finish, then run the whole loop again from scratch: delete-then- + // insert per event means no duplicate rows (assertQuery wants 1). + var cursor = first.cursor + do { + val p = db.store.reindexFullTextSearch(cursor, batchSize = 1) + cursor = p.cursor + } while (!p.done) + + cursor = null + do { + val p = db.store.reindexFullTextSearch(cursor, batchSize = 10) + cursor = p.cursor + } while (!p.done) + + db.store.assertQuery(a, Filter(search = "uniqalpha")) + db.store.assertQuery(b, Filter(search = "uniqbeta")) + } + @Test fun testChannelJsonFieldsAreSearchable() = forEachDB { db -> diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index e15f810a6a..0313eebda5 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.quartz.nip01Core.core.isEphemeral import com.vitorpamplona.quartz.nip01Core.core.isReplaceable import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy @@ -528,6 +529,63 @@ open class FsEventStore( } } + /** + * Resumable, additive FTS rebuild. See + * [IEventStore.reindexFullTextSearch] for the loop contract. + * + * The cursor is the next searchable kind to process, so the store is + * walked one `idx/kind//` directory at a time — that keeps the + * scan linear (each directory is listed once, never re-sorted) at the + * cost of pausing only between kinds: a single huge kind is processed + * in one batch. [batchSize] is a soft floor — whole kinds are + * processed until at least that many events have been (re)linked, + * then the call yields. Linking is idempotent ([FsIndexer.linkFts] + * ignores an existing hardlink), so nothing is wiped and search stays + * usable throughout; replaying a kind after a crash is harmless. + */ + override suspend fun reindexFullTextSearch( + resumeFrom: String?, + batchSize: Int, + ): FtsReindexProgress = + lockManager.withWriteLock { + if (!Files.isDirectory(layout.idxKind)) { + return@withWriteLock FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true) + } + Files.createDirectories(layout.idxFts) + + val resumeKind = resumeFrom?.toIntOrNull() + val pending = + Files + .list(layout.idxKind) + .use { dirs -> dirs.map { it.fileName.toString() }.toList() } + .mapNotNull { it.toIntOrNull() } + .filter { isSearchableKind(it) && (resumeKind == null || it >= resumeKind) } + .sorted() + + var processed = 0 + var index = 0 + while (index < pending.size) { + val kind = pending[index] + Files.list(layout.kindDir(kind)).use { entries -> + for (entry in entries) { + val id = FsLayout.parseEntry(entry.fileName.toString())?.second ?: continue + val event = readEvent(id) ?: continue + indexer.linkFts(event, layout.canonical(id)) + processed++ + } + } + index++ + if (processed >= batchSize) break + } + + val done = index >= pending.size + FtsReindexProgress( + cursor = if (done) null else pending[index].toString(), + processedThisBatch = processed, + done = done, + ) + } + /** * True when [kind] currently parses to a [SearchableEvent]. Kind * alone selects the event class in [EventFactory], so a single probe diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt index 766644a93f..9251311df9 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt @@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance import kotlinx.coroutines.runBlocking import java.nio.file.Files @@ -159,6 +160,45 @@ class FsSearchTest { assertEquals(2, ftsRoot.resolve("reindex").listDirectoryEntries().size) } + @Test + fun `resumable reindex covers every kind across batches`() = + runBlocking { + // Two searchable kinds (note = kind 1, long-form = kind 30023) so + // the kind-granular cursor must advance across more than one dir. + val n = note("uniqnote bitcoin", ts = 100) + val long = + signer.sign( + LongTextNoteEvent.build( + "uniqlong body", + title = "title", + dTag = "d1", + createdAt = 200, + ), + ) + store.insert(n) + store.insert(long) + + // Wipe the index, then drive the resumable path one kind at a time. + val ftsRoot = root.resolve("idx/fts") + Files.walk(ftsRoot).use { stream -> + stream.sorted(Comparator.reverseOrder()).forEach { p -> if (p != ftsRoot) Files.deleteIfExists(p) } + } + assertTrue(store.query(Filter(search = "uniqnote")).isEmpty()) + + var cursor: String? = null + var batches = 0 + do { + val progress = store.reindexFullTextSearch(cursor, batchSize = 1) + cursor = progress.cursor + batches++ + } while (!progress.done) + + // Two searchable kind dirs, batchSize 1 ⇒ at least two batches. + assertTrue(batches >= 2, "expected the cursor to span both kinds, got $batches batch(es)") + assertEquals(listOf(n.id), store.query(Filter(search = "uniqnote")).map { it.id }) + assertEquals(listOf(long.id), store.query(Filter(search = "uniqlong")).map { it.id }) + } + @Test fun `reindexFullTextSearch ignores non-searchable kinds`() = runBlocking { From af1e34a34ddf8447895dc7cc0a2096673a0907f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 18 Jun 2026 22:32:51 +0000 Subject: [PATCH 3/3] fix(quartz): clamp non-positive FTS reindex batch size A batchSize <= 0 made the SQLite resumable reindex select no rows yet never report done, so a caller's loop would spin forever. Clamp the page size to at least one in both stores and add a regression test. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BZqPFds2TPPUKkMmBngwys --- .../store/sqlite/FullTextSearchModule.kt | 7 +++++-- .../nip01Core/store/sqlite/SearchTest.kt | 21 +++++++++++++++++++ .../quartz/nip01Core/store/fs/FsEventStore.kt | 3 ++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index 3de6859e4b..71f55b1096 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -199,6 +199,9 @@ class FullTextSearchModule : IModule { afterRowId: Long, batchSize: Int, ): FtsReindexProgress { + // A non-positive page would select no rows yet never report done, + // spinning the caller's loop forever — clamp to at least one. + val limit = batchSize.coerceAtLeast(1) val kinds = searchableKindsPresent(db) if (kinds.isEmpty()) return FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true) @@ -213,7 +216,7 @@ class FullTextSearchModule : IModule { db.prepare(insertFTS).use { write -> db.prepare(selectSql).use { read -> read.bindLong(1, afterRowId) - read.bindLong(2, batchSize.toLong()) + read.bindLong(2, limit.toLong()) while (read.step()) { val rowId = read.getLong(0) // Clear any existing row for this event first so a @@ -246,7 +249,7 @@ class FullTextSearchModule : IModule { } // Fewer than a full page came back ⇒ we hit the end of the table. - val done = processed < batchSize + val done = processed < limit return FtsReindexProgress( cursor = if (done) null else last.toString(), processedThisBatch = processed, diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt index bdf7ae690a..615e6fc47e 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt @@ -299,6 +299,27 @@ class SearchTest : BaseDBTest() { db.store.assertQuery(b, Filter(search = "uniqbeta")) } + @Test + fun testResumableReindexClampsNonPositiveBatchSize() = + forEachDB { db -> + val a = signer.sign(TextNoteEvent.build("uniqclamp note", createdAt = TimeUtils.now())) + db.store.insertEvent(a) + db.store.pool.useWriter { db.store.fullTextSearchModule.deleteAll(it) } + + // batchSize 0 must not spin forever: it is clamped to one event + // per call. Cap the loop so a regression fails fast instead of + // hanging the suite. + var cursor: String? = null + var guard = 0 + do { + val progress = db.store.reindexFullTextSearch(cursor, batchSize = 0) + cursor = progress.cursor + check(guard++ < 100) { "reindex did not terminate with batchSize=0" } + } while (!progress.done) + + db.store.assertQuery(a, Filter(search = "uniqclamp")) + } + @Test fun testChannelJsonFieldsAreSearchable() = forEachDB { db -> diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index 0313eebda5..9062867baf 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -554,6 +554,7 @@ open class FsEventStore( Files.createDirectories(layout.idxFts) val resumeKind = resumeFrom?.toIntOrNull() + val target = batchSize.coerceAtLeast(1) val pending = Files .list(layout.idxKind) @@ -575,7 +576,7 @@ open class FsEventStore( } } index++ - if (processed >= batchSize) break + if (processed >= target) break } val done = index >= pending.size