feat(quartz): add FTS reindex to SQLite and filesystem event stores

The set of event kinds that implement SearchableEvent — and the text
each contributes via indexableContent() — is baked into the quartz
build, so it changes across app versions. Events stored under older code
keep their old (or missing) NIP-50 full-text-search rows, so search
silently misses them after an upgrade.

Add IEventStore.reindexFullTextSearch() so the app can wipe and rebuild
the FTS index from already-stored events when it has spare cycles.

Speed: only kinds that currently map to a SearchableEvent are scanned.
Kind alone selects the event class in EventFactory, so a single probe per
distinct kind is authoritative, letting us push a `kind IN (...)` filter
(SQLite) / skip whole idx/kind dirs (filesystem) so the non-searchable
bulk — reactions, zaps, follow lists — is never deserialised.

- SQLite: FullTextSearchModule.reindexAll drops+recreates the virtual
  table (O(1) wipe) then streams only searchable-kind rows in one write
  transaction, reusing a single INSERT statement.
- Filesystem: rebuilds only idx/fts/, driving the walk from
  idx/kind/<searchable kind>/ via the new FsIndexer.linkFts.
- Wrappers (EventStore, ObservableEventStore, InterningEventStore)
  delegate; the observable layer emits nothing since no event changes.
- cli: `amy store reindex-fts`.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BZqPFds2TPPUKkMmBngwys

cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt

@@ -442,6 +442,7 @@ private fun printUsage() {
         |  store sweep-expired                        delete events past their NIP-40 expiration
         |  store scrub                                rebuild idx/ from canonical events (after edits / crashes)
         |  store compact                              drop dangling idx entries (canonical gone)
+        |  store reindex-fts                          rebuild the NIP-50 search index (after a searchable-kinds change)
         """.trimMargin(),
     )
 }

cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt

@@ -45,19 +45,23 @@ import kotlin.io.path.exists
  *                    external edits.
  * - `compact`        drop dangling `idx/` entries whose canonical is
  *                    gone. Cheaper than scrub.
+ * - `reindex-fts`    wipe and rebuild only the NIP-50 full-text search
+ *                    index from the stored events. Run after a quartz
+ *                    upgrade that changes which kinds are searchable.
  */
 object StoreCommands {
     suspend fun dispatch(
         dataDir: DataDir,
         tail: Array<String>,
     ): Int {
-        if (tail.isEmpty()) return Output.error("bad_args", "store <stat|sweep-expired|scrub|compact>")
+        if (tail.isEmpty()) return Output.error("bad_args", "store <stat|sweep-expired|scrub|compact|reindex-fts>")
         val rest = tail.drop(1).toTypedArray()
         return when (tail[0]) {
             "stat" -> stat(dataDir)
             "sweep-expired" -> sweepExpired(dataDir)
             "scrub" -> scrub(dataDir)
             "compact" -> compact(dataDir)
+            "reindex-fts" -> reindexFts(dataDir)
             else -> Output.error("bad_args", "store ${tail[0]}")
         }
     }
@@ -161,6 +165,22 @@ object StoreCommands {
             0
         }
 
+    private suspend fun reindexFts(dataDir: DataDir): Int =
+        withStore(dataDir) { store ->
+            val ftsDir = dataDir.eventsDir.toPath().resolve("idx/fts")
+            val before = countEntries(ftsDir)
+            store.reindexFullTextSearch()
+            val after = countEntries(ftsDir)
+            Output.emit(
+                mapOf(
+                    "ok" to true,
+                    "tokens_before" to before,
+                    "tokens_after" to after,
+                ),
+            )
+            0
+        }
+
     /**
      * Maintenance verbs only need the store — not identity, not relays,
      * not the signer. Skip [Context.open] (which throws if no identity

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt

@@ -123,5 +123,7 @@ class InterningEventStore(
 
     override suspend fun deleteExpiredEvents() = inner.deleteExpiredEvents()
 
+    override suspend fun reindexFullTextSearch() = inner.reindexFullTextSearch()
+
     override fun close() = inner.close()
 }

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt

@@ -133,5 +133,28 @@ interface IEventStore : AutoCloseable {
 
     suspend fun deleteExpiredEvents()
 
+    /**
+     * Wipe and rebuild the NIP-50 full-text search index from the
+     * events already in storage.
+     *
+     * Which kinds are searchable — i.e. implement
+     * `SearchableEvent` — and what text each one contributes is baked
+     * into the quartz build, so it can change across app versions: a
+     * kind that was opaque before may start implementing
+     * `SearchableEvent`, or an existing one may change what its
+     * `indexableContent()` returns. Events that were inserted under the
+     * old code keep their old (or missing) FTS rows until they are
+     * re-indexed, so search silently misses them. Call this once, off
+     * the hot path, after such an upgrade — the app decides when it has
+     * the spare cycles to process the whole store.
+     *
+     * Implementations rebuild from scratch (so the result is the same
+     * whether or not an index already existed) and only visit kinds
+     * that currently map to a searchable event, leaving the bulk of
+     * non-searchable rows (reactions, zaps, follow lists, …) untouched
+     * to keep the scan as cheap as possible.
+     */
+    suspend fun reindexFullTextSearch()
+
     override fun close()
 }

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt

@@ -215,5 +215,10 @@ class ObservableEventStore(
         _changes.emit(StoreChange.DeleteExpired(asOf))
     }
 
+    // Pure index maintenance: it rewrites the FTS index without adding,
+    // removing, or changing any event, so there is nothing for
+    // projections to observe and no [StoreChange] is emitted.
+    override suspend fun reindexFullTextSearch() = inner.reindexFullTextSearch()
+
     override fun close() = inner.close()
 }

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt

@@ -79,5 +79,7 @@ class EventStore(
 
     override suspend fun deleteExpiredEvents() = store.deleteExpiredEvents()
 
+    override suspend fun reindexFullTextSearch() = store.reindexFullTextSearch()
+
     override fun close() = store.close()
 }

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt

@@ -23,10 +23,13 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite
 import androidx.sqlite.SQLiteConnection
 import androidx.sqlite.SQLiteException
 import com.vitorpamplona.quartz.nip01Core.core.Event
+import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
 import com.vitorpamplona.quartz.nip50Search.SearchableEvent
+import com.vitorpamplona.quartz.utils.EventFactory
 
 class FullTextSearchModule : IModule {
     val tableName = "event_fts"
+    val triggerName = "fts_foreign_key"
     val eventHeaderRowIdName = "event_header_row_id"
     val contentName = "content"
 
@@ -42,7 +45,7 @@ class FullTextSearchModule : IModule {
         // Foreign key cleanup for full text search
         db.execSQL(
             """
-                CREATE TRIGGER fts_foreign_key
+                CREATE TRIGGER $triggerName
                 AFTER DELETE ON event_headers
                 FOR EACH ROW
                 BEGIN
@@ -57,6 +60,17 @@ class FullTextSearchModule : IModule {
         db.execSQL("DROP TABLE IF EXISTS $tableName")
     }
 
+    /**
+     * Drop the cleanup trigger on its own. [drop] only removes the FTS
+     * table; the trigger lives on `event_headers`, so a rebuild that
+     * recreates the table without first dropping the trigger would fail
+     * with "trigger already exists". (A schema upgrade doesn't hit this
+     * because dropping `event_headers` takes its triggers with it.)
+     */
+    fun dropTrigger(db: SQLiteConnection) {
+        db.execSQL("DROP TRIGGER IF EXISTS $triggerName")
+    }
+
     val insertFTS =
         """
         INSERT OR ROLLBACK INTO $tableName ($eventHeaderRowIdName, $contentName)
@@ -109,4 +123,81 @@ class FullTextSearchModule : IModule {
     override fun deleteAll(db: SQLiteConnection) {
         db.execSQL("DELETE FROM event_fts")
     }
+
+    /**
+     * Wipe and rebuild the whole FTS index from `event_headers`.
+     *
+     * Wiping is done by dropping and recreating the virtual table, which
+     * empties it in O(1) — far cheaper than DELETE-ing every row out of a
+     * populated FTS index. The rebuild then streams only the rows whose
+     * kind currently parses to a [SearchableEvent] (see
+     * [searchableKindsPresent]) so the common non-searchable bulk —
+     * reactions, zaps, follow lists — is never deserialised. A single
+     * shared INSERT statement is reused across the scan.
+     *
+     * Must run inside the caller's write transaction.
+     */
+    fun reindexAll(db: SQLiteConnection) {
+        dropTrigger(db)
+        drop(db)
+        create(db)
+
+        val kinds = searchableKindsPresent(db)
+        if (kinds.isEmpty()) return
+
+        val selectSql =
+            "SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
+                "FROM event_headers WHERE kind IN (${kinds.joinToString(",")})"
+
+        db.prepare(insertFTS).use { write ->
+            db.prepare(selectSql).use { read ->
+                while (read.step()) {
+                    val event =
+                        EventFactory.create<Event>(
+                            read.getText(1),
+                            read.getText(2),
+                            read.getLong(3),
+                            read.getInt(4),
+                            OptimizedJsonMapper.fromJsonToTagArray(read.getText(5)),
+                            read.getText(6),
+                            read.getText(7),
+                        )
+                    if (event is SearchableEvent) {
+                        write.bindLong(1, read.getLong(0))
+                        write.bindText(2, event.indexableContent())
+                        write.step()
+                        write.reset()
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * The distinct kinds present in `event_headers` that currently parse
+     * to a [SearchableEvent]. Kind alone selects the event class in
+     * [EventFactory], so one probe per distinct kind is authoritative;
+     * the result drives a `kind IN (...)` filter on the rebuild scan so
+     * non-searchable rows are skipped at the SQL layer.
+     */
+    private fun searchableKindsPresent(db: SQLiteConnection): List<Int> {
+        val out = ArrayList<Int>()
+        db.prepare("SELECT DISTINCT kind FROM event_headers").use { stmt ->
+            while (stmt.step()) {
+                val kind = stmt.getInt(0)
+                if (isSearchableKind(kind)) out.add(kind)
+            }
+        }
+        return out
+    }
+
+    private fun isSearchableKind(kind: Int): Boolean = EventFactory.create<Event>(PROBE_ID, PROBE_ID, 0L, kind, EMPTY_TAGS, "", "") is SearchableEvent
+
+    companion object {
+        // A non-blank id keeps kinds that lazily hash a missing id (e.g.
+        // NIP-17 chat messages) from doing that work — the probe only
+        // inspects the resulting runtime type.
+        private const val PROBE_ID = "0"
+        private val EMPTY_TAGS = emptyArray<Array<String>>()
+    }
 }

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt

@@ -340,6 +340,19 @@ class SQLiteEventStore(
 
     suspend fun deleteExpiredEvents() = pool.useWriter { expirationModule.deleteExpiredEvents(it) }
 
+    /**
+     * Wipe and rebuild the NIP-50 full-text search index for every
+     * stored event. See [IEventStore.reindexFullTextSearch] for when to
+     * call this. The whole rebuild runs in a single write transaction so
+     * the WAL append + sync cost is paid once.
+     */
+    suspend fun reindexFullTextSearch() =
+        pool.useWriter { db ->
+            db.transaction {
+                fullTextSearchModule.reindexAll(db)
+            }
+        }
+
     fun close() = pool.close()
 }
 

quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt

@@ -192,6 +192,44 @@ class SearchTest : BaseDBTest() {
             db.assertQuery(p, Filter(kinds = listOf(MetadataEvent.KIND), search = "uniqalice"))
         }
 
+    @Test
+    fun testReindexFullTextSearchRebuildsTheWholeIndex() =
+        forEachDB { db ->
+            // A calendar event (title tag + content) and a plain note. Both
+            // are searchable kinds, but we'll erase the FTS index to mimic
+            // events that were stored before their kind became searchable.
+            val cal =
+                signer.sign(
+                    CalendarEvent.build(
+                        title = "uniqtitle Meetup",
+                        content = "annual uniqbody gathering",
+                    ),
+                )
+            val note = signer.sign(TextNoteEvent.build("reindex uniqnote please", createdAt = TimeUtils.now()))
+
+            db.store.insertEvent(cal)
+            db.store.insertEvent(note)
+
+            // Wipe the FTS rows but keep the canonical event rows — this is
+            // the state a store ends up in after an upgrade adds search
+            // support for a kind that was inserted under the old code.
+            db.store.pool.useWriter { db.store.fullTextSearchModule.deleteAll(it) }
+            db.store.assertQuery(null, Filter(search = "uniqbody"))
+            db.store.assertQuery(null, Filter(search = "uniqnote"))
+
+            // Rebuilding from storage brings every searchable field back.
+            db.store.reindexFullTextSearch()
+            db.store.assertQuery(cal, Filter(search = "uniqtitle"))
+            db.store.assertQuery(cal, Filter(search = "uniqbody"))
+            db.store.assertQuery(note, Filter(search = "uniqnote"))
+
+            // Running it again must not duplicate rows (assertQuery expects
+            // exactly one match), proving the rebuild starts from a clean slate.
+            db.store.reindexFullTextSearch()
+            db.store.assertQuery(cal, Filter(search = "uniqbody"))
+            db.store.assertQuery(note, Filter(search = "uniqnote"))
+        }
+
     @Test
     fun testChannelJsonFieldsAreSearchable() =
         forEachDB { db ->

quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt

@@ -34,7 +34,9 @@ import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy
 import com.vitorpamplona.quartz.nip01Core.store.sqlite.TagNameValueHasher
 import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
 import com.vitorpamplona.quartz.nip40Expiration.expiration
+import com.vitorpamplona.quartz.nip50Search.SearchableEvent
 import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent
+import com.vitorpamplona.quartz.utils.EventFactory
 import com.vitorpamplona.quartz.utils.TimeUtils
 import java.nio.file.FileAlreadyExistsException
 import java.nio.file.Files
@@ -493,6 +495,48 @@ open class FsEventStore(
             }
         }
 
+    /**
+     * Wipe and rebuild only the `idx/fts/` tree from the stored events.
+     * See [IEventStore.reindexFullTextSearch] for when to call this.
+     *
+     * Unlike [scrub] (which rebuilds every index tree from a full
+     * `events/` walk), this touches nothing but FTS and reads only the
+     * events whose kind is searchable today: it drives the walk from
+     * `idx/kind/<k>/`, which already lists exactly the live canonical
+     * events per kind, and skips kind directories that don't map to a
+     * [SearchableEvent]. The bulk of non-searchable events (reactions,
+     * zaps, follow lists, …) is never opened.
+     */
+    override suspend fun reindexFullTextSearch() =
+        lockManager.withWriteLock {
+            deleteRecursively(layout.idxFts)
+            Files.createDirectories(layout.idxFts)
+
+            if (!Files.isDirectory(layout.idxKind)) return@withWriteLock
+            Files.list(layout.idxKind).use { kindDirs ->
+                for (kindDir in kindDirs) {
+                    val kind = kindDir.fileName.toString().toIntOrNull() ?: continue
+                    if (!isSearchableKind(kind)) continue
+                    Files.list(kindDir).use { entries ->
+                        for (entry in entries) {
+                            val id = FsLayout.parseEntry(entry.fileName.toString())?.second ?: continue
+                            val event = readEvent(id) ?: continue
+                            indexer.linkFts(event, layout.canonical(id))
+                        }
+                    }
+                }
+            }
+        }
+
+    /**
+     * True when [kind] currently parses to a [SearchableEvent]. Kind
+     * alone selects the event class in [EventFactory], so a single probe
+     * per kind is authoritative. The id is non-blank so kinds that lazily
+     * hash a missing id (NIP-17 chat) skip that work — only the runtime
+     * type matters here.
+     */
+    private fun isSearchableKind(kind: Int): Boolean = EventFactory.create<Event>("0", "0", 0L, kind, emptyArray(), "", "") is SearchableEvent
+
     private fun deleteRecursively(p: java.nio.file.Path) {
         if (!Files.exists(p)) return
         Files.walk(p).use { stream ->

quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt

@@ -86,6 +86,21 @@ internal class FsIndexer(
         }
     }
 
+    /**
+     * Create only the NIP-50 FTS hardlinks for this event (no-op when it
+     * isn't a [SearchableEvent]). Used by the FTS-only rebuild so an
+     * existing kind/author/owner/tag tree is left untouched. Idempotent.
+     */
+    fun linkFts(
+        event: Event,
+        canonical: Path,
+    ) {
+        if (event !is SearchableEvent) return
+        for (token in FsSearchTokenizer.tokenize(event.indexableContent())) {
+            createLink(layout.ftsEntry(token, event.createdAt, event.id), canonical)
+        }
+    }
+
     /** Remove every index hardlink for this event. Missing entries ignored. */
     fun unlink(event: Event) {
         for (path in pathsFor(event)) {

quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt

@@ -131,6 +131,54 @@ class FsSearchTest {
             assertEquals(0, ftsRoot.listDirectoryEntries().size, "unknown kind is not SearchableEvent")
         }
 
+    @Test
+    fun `reindexFullTextSearch rebuilds fts entries after a wipe`() =
+        runBlocking {
+            val a = note("bitcoin reindex", ts = 100)
+            val b = note("nostr reindex", ts = 200)
+            store.insert(a)
+            store.insert(b)
+
+            // Mimic a store written before this kind was searchable: drop
+            // the whole idx/fts/ tree, leaving canonical events intact.
+            val ftsRoot = root.resolve("idx/fts")
+            Files.walk(ftsRoot).use { stream ->
+                stream.sorted(Comparator.reverseOrder()).forEach { p -> if (p != ftsRoot) Files.deleteIfExists(p) }
+            }
+            assertEquals(0, ftsRoot.listDirectoryEntries().size, "precondition: fts wiped")
+            assertTrue(store.query<TextNoteEvent>(Filter(search = "bitcoin")).isEmpty(), "no index, no match")
+
+            store.reindexFullTextSearch()
+
+            assertEquals(listOf(a.id), store.query<TextNoteEvent>(Filter(search = "bitcoin")).map { it.id })
+            assertEquals(listOf(b.id), store.query<TextNoteEvent>(Filter(search = "nostr")).map { it.id })
+            // Reindexing twice must not duplicate entries.
+            store.reindexFullTextSearch()
+            assertEquals(1, ftsRoot.resolve("bitcoin").listDirectoryEntries().size)
+            // "reindex" appears in both notes — one entry per event, no dupes.
+            assertEquals(2, ftsRoot.resolve("reindex").listDirectoryEntries().size)
+        }
+
+    @Test
+    fun `reindexFullTextSearch ignores non-searchable kinds`() =
+        runBlocking {
+            val searchable = note("findme bitcoin", ts = 100)
+            val opaque =
+                signer.sign<Event>(
+                    createdAt = 1,
+                    kind = 9999,
+                    tags = emptyArray(),
+                    content = "findme must not be indexed",
+                )
+            store.insert(searchable)
+            store.insert(opaque)
+
+            store.reindexFullTextSearch()
+
+            // Only the searchable note contributes the "findme" token.
+            assertEquals(listOf(searchable.id), store.query<Event>(Filter(search = "findme")).map { it.id })
+        }
+
     @Test
     fun `delete removes fts entries`() =
         runBlocking {