Merge pull request #3276 from vitorpamplona/claude/wizardly-mendel-o3ar8d

Add resumable FTS reindex for NIP-50 search index rebuilds
2026-08-09 16:14:40 +00:00 · 2026-06-18 18:35:38 -04:00
parent c84333c753 af1e34a34d
commit bdb16bfc2c
13 changed files with 718 additions and 2 deletions
@@ -442,6 +442,7 @@ private fun printUsage() {
        |  store sweep-expired                        delete events past their NIP-40 expiration
        |  store scrub                                rebuild idx/ from canonical events (after edits / crashes)
        |  store compact                              drop dangling idx entries (canonical gone)
+        |  store reindex-fts                          rebuild the NIP-50 search index (after a searchable-kinds change)
        """.trimMargin(),
    )
 }
@@ -23,6 +23,7 @@ package com.vitorpamplona.amethyst.cli.commands
 import com.vitorpamplona.amethyst.cli.DataDir
 import com.vitorpamplona.amethyst.cli.Output
 import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper
+import com.vitorpamplona.quartz.nip01Core.store.IEventStore
 import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore
 import java.io.IOException
 import java.nio.file.Files
@@ -45,19 +46,23 @@ import kotlin.io.path.exists
 *                    external edits.
 * - `compact`        drop dangling `idx/` entries whose canonical is
 *                    gone. Cheaper than scrub.
+ * - `reindex-fts`    wipe and rebuild only the NIP-50 full-text search
+ *                    index from the stored events. Run after a quartz
+ *                    upgrade that changes which kinds are searchable.
 */
 object StoreCommands {
    suspend fun dispatch(
        dataDir: DataDir,
        tail: Array<String>,
    ): Int {
-        if (tail.isEmpty()) return Output.error("bad_args", "store <stat|sweep-expired|scrub|compact>")
+        if (tail.isEmpty()) return Output.error("bad_args", "store <stat|sweep-expired|scrub|compact|reindex-fts>")
        val rest = tail.drop(1).toTypedArray()
        return when (tail[0]) {
            "stat" -> stat(dataDir)
            "sweep-expired" -> sweepExpired(dataDir)
            "scrub" -> scrub(dataDir)
            "compact" -> compact(dataDir)
+            "reindex-fts" -> reindexFts(dataDir)
            else -> Output.error("bad_args", "store ${tail[0]}")
        }
    }
@@ -161,6 +166,36 @@ object StoreCommands {
            0
        }

+    private suspend fun reindexFts(dataDir: DataDir): Int =
+        withStore(dataDir) { store ->
+            val ftsDir = dataDir.eventsDir.toPath().resolve("idx/fts")
+            val before = countEntries(ftsDir)
+            // Drive the resumable, batched path to completion so a huge
+            // store is processed without holding the writer lock for the
+            // whole pass. A real long-running caller would persist the
+            // cursor between calls; here we just loop until done.
+            var cursor: String? = null
+            var processed = 0L
+            var batches = 0
+            do {
+                val progress = store.reindexFullTextSearch(cursor, IEventStore.DEFAULT_FTS_REINDEX_BATCH)
+                cursor = progress.cursor
+                processed += progress.processedThisBatch
+                batches++
+            } while (!progress.done)
+            val after = countEntries(ftsDir)
+            Output.emit(
+                mapOf(
+                    "ok" to true,
+                    "processed" to processed,
+                    "batches" to batches,
+                    "tokens_before" to before,
+                    "tokens_after" to after,
+                ),
+            )
+            0
+        }
+
    /**
     * Maintenance verbs only need the store — not identity, not relays,
     * not the signer. Skip [Context.open] (which throws if no identity
@@ -23,6 +23,7 @@ package com.vitorpamplona.quartz.nip01Core.cache.interning
 import com.vitorpamplona.quartz.nip01Core.core.Event
 import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
 import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
+import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress
 import com.vitorpamplona.quartz.nip01Core.store.IEventStore
 import com.vitorpamplona.quartz.nip01Core.store.IdAndTime

@@ -123,5 +124,12 @@ class InterningEventStore(

    override suspend fun deleteExpiredEvents() = inner.deleteExpiredEvents()

+    override suspend fun reindexFullTextSearch() = inner.reindexFullTextSearch()
+
+    override suspend fun reindexFullTextSearch(
+        resumeFrom: String?,
+        batchSize: Int,
+    ): FtsReindexProgress = inner.reindexFullTextSearch(resumeFrom, batchSize)
+
    override fun close() = inner.close()
 }
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.quartz.nip01Core.store
+
+/**
+ * Progress token returned by the resumable
+ * [IEventStore.reindexFullTextSearch] overload.
+ *
+ * Treat [cursor] as opaque: persist it (it survives process death) and
+ * hand it back to the next call to continue where the previous one
+ * stopped. Each store encodes its own resume position into it.
+ *
+ * @property cursor where to resume from on the next call, or `null` once
+ *   [done] is `true` (nothing left to process).
+ * @property processedThisBatch how many events this call (re)indexed —
+ *   useful to drive a progress indicator.
+ * @property done `true` when the whole store has been visited; further
+ *   calls are no-ops that keep returning `done = true`.
+ */
+data class FtsReindexProgress(
+    val cursor: String?,
+    val processedThisBatch: Int,
+    val done: Boolean,
+)
@@ -25,6 +25,16 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
 import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl

 interface IEventStore : AutoCloseable {
+    companion object {
+        /**
+         * How many events a single resumable
+         * [reindexFullTextSearch] batch aims to process before yielding.
+         * Big enough to amortise the per-batch transaction/lock cost,
+         * small enough that a pause request is honoured promptly.
+         */
+        const val DEFAULT_FTS_REINDEX_BATCH = 1000
+    }
+
    /**
     * Relay URL this store is acting on behalf of, or `null` for an
     * unscoped store. Used by NIP-62 right-to-vanish handling: only
@@ -133,5 +143,68 @@ interface IEventStore : AutoCloseable {

    suspend fun deleteExpiredEvents()

+    /**
+     * Wipe and rebuild the NIP-50 full-text search index from the
+     * events already in storage.
+     *
+     * Which kinds are searchable — i.e. implement
+     * `SearchableEvent` — and what text each one contributes is baked
+     * into the quartz build, so it can change across app versions: a
+     * kind that was opaque before may start implementing
+     * `SearchableEvent`, or an existing one may change what its
+     * `indexableContent()` returns. Events that were inserted under the
+     * old code keep their old (or missing) FTS rows until they are
+     * re-indexed, so search silently misses them. Call this once, off
+     * the hot path, after such an upgrade — the app decides when it has
+     * the spare cycles to process the whole store.
+     *
+     * Implementations rebuild from scratch (so the result is the same
+     * whether or not an index already existed) and only visit kinds
+     * that currently map to a searchable event, leaving the bulk of
+     * non-searchable rows (reactions, zaps, follow lists, …) untouched
+     * to keep the scan as cheap as possible.
+     *
+     * This one-shot variant runs to completion under a single lock and
+     * cannot be paused. For a store large enough that the full pass
+     * would block for too long, use the resumable overload
+     * [reindexFullTextSearch] instead.
+     */
+    suspend fun reindexFullTextSearch()
+
+    /**
+     * Resumable, batched companion to [reindexFullTextSearch] for stores
+     * large enough that a single pass would take too long to run (or to
+     * hold a lock) in one go.
+     *
+     * Process roughly [batchSize] events starting from [resumeFrom]
+     * (`null` = from the beginning) and return a [FtsReindexProgress].
+     * Drive it in a loop, feeding [FtsReindexProgress.cursor] back in,
+     * until [FtsReindexProgress.done] is `true`:
+     *
+     * ```
+     * var cursor: String? = null
+     * do {
+     *     val p = store.reindexFullTextSearch(cursor)
+     *     cursor = p.cursor
+     *     // optionally persist `cursor` and stop; resume later by passing it back
+     * } while (!p.done)
+     * ```
+     *
+     * Each call commits its own batch, so progress is durable across a
+     * crash or app restart and the writer lock is released between
+     * batches — the app can pause simply by not making the next call.
+     *
+     * Semantics differ slightly from the one-shot variant: this path is
+     * **additive / refresh** — it makes sure every currently-searchable
+     * event is indexed (and, on SQLite, refreshes changed content) while
+     * leaving search usable throughout. It does not purge stale rows left
+     * behind by a kind that *lost* searchability; run the one-shot
+     * [reindexFullTextSearch] once for that rarer case.
+     */
+    suspend fun reindexFullTextSearch(
+        resumeFrom: String?,
+        batchSize: Int = DEFAULT_FTS_REINDEX_BATCH,
+    ): FtsReindexProgress
+
    override fun close()
 }
@@ -215,5 +215,15 @@ class ObservableEventStore(
        _changes.emit(StoreChange.DeleteExpired(asOf))
    }

+    // Pure index maintenance: it rewrites the FTS index without adding,
+    // removing, or changing any event, so there is nothing for
+    // projections to observe and no [StoreChange] is emitted.
+    override suspend fun reindexFullTextSearch() = inner.reindexFullTextSearch()
+
+    override suspend fun reindexFullTextSearch(
+        resumeFrom: String?,
+        batchSize: Int,
+    ): FtsReindexProgress = inner.reindexFullTextSearch(resumeFrom, batchSize)
+
    override fun close() = inner.close()
 }
@@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event
 import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
 import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
 import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl
+import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress
 import com.vitorpamplona.quartz.nip01Core.store.IEventStore
 import com.vitorpamplona.quartz.nip01Core.store.IdAndTime

@@ -79,5 +80,12 @@ class EventStore(

    override suspend fun deleteExpiredEvents() = store.deleteExpiredEvents()

+    override suspend fun reindexFullTextSearch() = store.reindexFullTextSearch()
+
+    override suspend fun reindexFullTextSearch(
+        resumeFrom: String?,
+        batchSize: Int,
+    ): FtsReindexProgress = store.reindexFullTextSearch(resumeFrom, batchSize)
+
    override fun close() = store.close()
 }
@@ -23,10 +23,14 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite
 import androidx.sqlite.SQLiteConnection
 import androidx.sqlite.SQLiteException
 import com.vitorpamplona.quartz.nip01Core.core.Event
+import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
+import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress
 import com.vitorpamplona.quartz.nip50Search.SearchableEvent
+import com.vitorpamplona.quartz.utils.EventFactory

 class FullTextSearchModule : IModule {
    val tableName = "event_fts"
+    val triggerName = "fts_foreign_key"
    val eventHeaderRowIdName = "event_header_row_id"
    val contentName = "content"

@@ -42,7 +46,7 @@ class FullTextSearchModule : IModule {
        // Foreign key cleanup for full text search
        db.execSQL(
            """
-                CREATE TRIGGER fts_foreign_key
+                CREATE TRIGGER $triggerName
                AFTER DELETE ON event_headers
                FOR EACH ROW
                BEGIN
@@ -57,12 +61,28 @@ class FullTextSearchModule : IModule {
        db.execSQL("DROP TABLE IF EXISTS $tableName")
    }

+    /**
+     * Drop the cleanup trigger on its own. [drop] only removes the FTS
+     * table; the trigger lives on `event_headers`, so a rebuild that
+     * recreates the table without first dropping the trigger would fail
+     * with "trigger already exists". (A schema upgrade doesn't hit this
+     * because dropping `event_headers` takes its triggers with it.)
+     */
+    fun dropTrigger(db: SQLiteConnection) {
+        db.execSQL("DROP TRIGGER IF EXISTS $triggerName")
+    }
+
    val insertFTS =
        """
        INSERT OR ROLLBACK INTO $tableName ($eventHeaderRowIdName, $contentName)
        VALUES (?, ?)
        """.trimIndent()

+    val deleteFTSByRowId =
+        """
+        DELETE FROM $tableName WHERE $eventHeaderRowIdName = ?
+        """.trimIndent()
+
    fun insert(
        event: Event,
        headerId: Long,
@@ -109,4 +129,159 @@ class FullTextSearchModule : IModule {
    override fun deleteAll(db: SQLiteConnection) {
        db.execSQL("DELETE FROM event_fts")
    }
+
+    /**
+     * Wipe and rebuild the whole FTS index from `event_headers`.
+     *
+     * Wiping is done by dropping and recreating the virtual table, which
+     * empties it in O(1) — far cheaper than DELETE-ing every row out of a
+     * populated FTS index. The rebuild then streams only the rows whose
+     * kind currently parses to a [SearchableEvent] (see
+     * [searchableKindsPresent]) so the common non-searchable bulk —
+     * reactions, zaps, follow lists — is never deserialised. A single
+     * shared INSERT statement is reused across the scan.
+     *
+     * Must run inside the caller's write transaction.
+     */
+    fun reindexAll(db: SQLiteConnection) {
+        dropTrigger(db)
+        drop(db)
+        create(db)
+
+        val kinds = searchableKindsPresent(db)
+        if (kinds.isEmpty()) return
+
+        val selectSql =
+            "SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
+                "FROM event_headers WHERE kind IN (${kinds.joinToString(",")})"
+
+        db.prepare(insertFTS).use { write ->
+            db.prepare(selectSql).use { read ->
+                while (read.step()) {
+                    val event =
+                        EventFactory.create<Event>(
+                            read.getText(1),
+                            read.getText(2),
+                            read.getLong(3),
+                            read.getInt(4),
+                            OptimizedJsonMapper.fromJsonToTagArray(read.getText(5)),
+                            read.getText(6),
+                            read.getText(7),
+                        )
+                    if (event is SearchableEvent) {
+                        write.bindLong(1, read.getLong(0))
+                        write.bindText(2, event.indexableContent())
+                        write.step()
+                        write.reset()
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Process one batch of a resumable rebuild: re-derive the FTS rows
+     * for up to [batchSize] events whose `row_id > ` [afterRowId] and
+     * whose kind is searchable, ordered by `row_id`.
+     *
+     * `row_id` is a monotonic AUTOINCREMENT key, so it is a stable
+     * cursor that needs no extra bookkeeping. Each event is
+     * delete-then-insert, which keeps the batch idempotent (a replay
+     * after a crash is harmless) and avoids duplicate FTS rows for
+     * events that were already indexed by the normal insert path. Rows
+     * not yet reached keep their previous FTS content, so search stays
+     * usable while the rebuild is in flight.
+     *
+     * Must run inside the caller's per-batch write transaction.
+     */
+    fun reindexBatch(
+        db: SQLiteConnection,
+        afterRowId: Long,
+        batchSize: Int,
+    ): FtsReindexProgress {
+        // A non-positive page would select no rows yet never report done,
+        // spinning the caller's loop forever — clamp to at least one.
+        val limit = batchSize.coerceAtLeast(1)
+        val kinds = searchableKindsPresent(db)
+        if (kinds.isEmpty()) return FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true)
+
+        val selectSql =
+            "SELECT row_id, id, pubkey, created_at, kind, tags, content, sig " +
+                "FROM event_headers WHERE row_id > ? AND kind IN (${kinds.joinToString(",")}) " +
+                "ORDER BY row_id LIMIT ?"
+
+        var last = afterRowId
+        var processed = 0
+        db.prepare(deleteFTSByRowId).use { del ->
+            db.prepare(insertFTS).use { write ->
+                db.prepare(selectSql).use { read ->
+                    read.bindLong(1, afterRowId)
+                    read.bindLong(2, limit.toLong())
+                    while (read.step()) {
+                        val rowId = read.getLong(0)
+                        // Clear any existing row for this event first so a
+                        // replay or an already-indexed event can't duplicate.
+                        del.bindLong(1, rowId)
+                        del.step()
+                        del.reset()
+
+                        val event =
+                            EventFactory.create<Event>(
+                                read.getText(1),
+                                read.getText(2),
+                                read.getLong(3),
+                                read.getInt(4),
+                                OptimizedJsonMapper.fromJsonToTagArray(read.getText(5)),
+                                read.getText(6),
+                                read.getText(7),
+                            )
+                        if (event is SearchableEvent) {
+                            write.bindLong(1, rowId)
+                            write.bindText(2, event.indexableContent())
+                            write.step()
+                            write.reset()
+                        }
+                        last = rowId
+                        processed++
+                    }
+                }
+            }
+        }
+
+        // Fewer than a full page came back ⇒ we hit the end of the table.
+        val done = processed < limit
+        return FtsReindexProgress(
+            cursor = if (done) null else last.toString(),
+            processedThisBatch = processed,
+            done = done,
+        )
+    }
+
+    /**
+     * The distinct kinds present in `event_headers` that currently parse
+     * to a [SearchableEvent]. Kind alone selects the event class in
+     * [EventFactory], so one probe per distinct kind is authoritative;
+     * the result drives a `kind IN (...)` filter on the rebuild scan so
+     * non-searchable rows are skipped at the SQL layer.
+     */
+    private fun searchableKindsPresent(db: SQLiteConnection): List<Int> {
+        val out = ArrayList<Int>()
+        db.prepare("SELECT DISTINCT kind FROM event_headers").use { stmt ->
+            while (stmt.step()) {
+                val kind = stmt.getInt(0)
+                if (isSearchableKind(kind)) out.add(kind)
+            }
+        }
+        return out
+    }
+
+    private fun isSearchableKind(kind: Int): Boolean = EventFactory.create<Event>(PROBE_ID, PROBE_ID, 0L, kind, EMPTY_TAGS, "", "") is SearchableEvent
+
+    companion object {
+        // A non-blank id keeps kinds that lazily hash a missing id (e.g.
+        // NIP-17 chat messages) from doing that work — the probe only
+        // inspects the resulting runtime type.
+        private const val PROBE_ID = "0"
+        private val EMPTY_TAGS = emptyArray<Array<String>>()
+    }
 }
@@ -31,6 +31,7 @@ import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
 import com.vitorpamplona.quartz.nip01Core.core.isEphemeral
 import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
 import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
+import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress
 import com.vitorpamplona.quartz.nip01Core.store.IEventStore
 import com.vitorpamplona.quartz.nip01Core.store.IdAndTime
 import com.vitorpamplona.quartz.nip40Expiration.isExpired
@@ -340,6 +341,35 @@ class SQLiteEventStore(

    suspend fun deleteExpiredEvents() = pool.useWriter { expirationModule.deleteExpiredEvents(it) }

+    /**
+     * Wipe and rebuild the NIP-50 full-text search index for every
+     * stored event. See [IEventStore.reindexFullTextSearch] for when to
+     * call this. The whole rebuild runs in a single write transaction so
+     * the WAL append + sync cost is paid once.
+     */
+    suspend fun reindexFullTextSearch() =
+        pool.useWriter { db ->
+            db.transaction {
+                fullTextSearchModule.reindexAll(db)
+            }
+        }
+
+    /**
+     * One batch of a resumable FTS rebuild. See
+     * [IEventStore.reindexFullTextSearch]. The opaque cursor is the last
+     * `row_id` processed; `null` (or an unparseable value) starts from
+     * the beginning. Each batch is its own write transaction.
+     */
+    suspend fun reindexFullTextSearch(
+        resumeFrom: String?,
+        batchSize: Int,
+    ): FtsReindexProgress =
+        pool.useWriter { db ->
+            db.transaction {
+                fullTextSearchModule.reindexBatch(db, resumeFrom?.toLongOrNull() ?: 0L, batchSize)
+            }
+        }
+
    fun close() = pool.close()
 }

@@ -192,6 +192,134 @@ class SearchTest : BaseDBTest() {
            db.assertQuery(p, Filter(kinds = listOf(MetadataEvent.KIND), search = "uniqalice"))
        }

+    @Test
+    fun testReindexFullTextSearchRebuildsTheWholeIndex() =
+        forEachDB { db ->
+            // A calendar event (title tag + content) and a plain note. Both
+            // are searchable kinds, but we'll erase the FTS index to mimic
+            // events that were stored before their kind became searchable.
+            val cal =
+                signer.sign(
+                    CalendarEvent.build(
+                        title = "uniqtitle Meetup",
+                        content = "annual uniqbody gathering",
+                    ),
+                )
+            val note = signer.sign(TextNoteEvent.build("reindex uniqnote please", createdAt = TimeUtils.now()))
+
+            db.store.insertEvent(cal)
+            db.store.insertEvent(note)
+
+            // Wipe the FTS rows but keep the canonical event rows — this is
+            // the state a store ends up in after an upgrade adds search
+            // support for a kind that was inserted under the old code.
+            db.store.pool.useWriter { db.store.fullTextSearchModule.deleteAll(it) }
+            db.store.assertQuery(null, Filter(search = "uniqbody"))
+            db.store.assertQuery(null, Filter(search = "uniqnote"))
+
+            // Rebuilding from storage brings every searchable field back.
+            db.store.reindexFullTextSearch()
+            db.store.assertQuery(cal, Filter(search = "uniqtitle"))
+            db.store.assertQuery(cal, Filter(search = "uniqbody"))
+            db.store.assertQuery(note, Filter(search = "uniqnote"))
+
+            // Running it again must not duplicate rows (assertQuery expects
+            // exactly one match), proving the rebuild starts from a clean slate.
+            db.store.reindexFullTextSearch()
+            db.store.assertQuery(cal, Filter(search = "uniqbody"))
+            db.store.assertQuery(note, Filter(search = "uniqnote"))
+        }
+
+    @Test
+    fun testResumableReindexProcessesEveryEventInBatches() =
+        forEachDB { db ->
+            // A handful of searchable notes, each with a unique token.
+            val notes =
+                (0 until 5).map { i ->
+                    signer.sign(TextNoteEvent.build("uniqresume$i body", createdAt = TimeUtils.now() + i))
+                }
+            notes.forEach { db.store.insertEvent(it) }
+
+            // Mimic the post-upgrade state: canonical rows present, FTS empty.
+            db.store.pool.useWriter { db.store.fullTextSearchModule.deleteAll(it) }
+            db.store.assertQuery(null, Filter(search = "uniqresume0"))
+
+            // Drive the resumable path two events at a time, persisting only
+            // the opaque cursor between calls — exactly what a paused/resumed
+            // app would do.
+            var cursor: String? = null
+            var batches = 0
+            var processed = 0
+            do {
+                val progress = db.store.reindexFullTextSearch(cursor, batchSize = 2)
+                cursor = progress.cursor
+                processed += progress.processedThisBatch
+                batches++
+            } while (!progress.done)
+
+            // 5 events at 2 per batch ⇒ 3 batches (2 + 2 + 1).
+            kotlin.test.assertEquals(5, processed)
+            kotlin.test.assertEquals(3, batches)
+
+            // Every note is searchable again after the full run.
+            notes.forEachIndexed { i, n -> db.store.assertQuery(n, Filter(search = "uniqresume$i")) }
+        }
+
+    @Test
+    fun testResumableReindexKeepsSearchLiveAndIsIdempotent() =
+        forEachDB { db ->
+            val a = signer.sign(TextNoteEvent.build("uniqalpha note", createdAt = TimeUtils.now()))
+            val b = signer.sign(TextNoteEvent.build("uniqbeta note", createdAt = TimeUtils.now() + 1))
+            db.store.insertEvent(a)
+            db.store.insertEvent(b)
+
+            // First batch (size 1) reindexes only the lowest row_id; the
+            // untouched event keeps the FTS row it already had from insert,
+            // so search stays live for both throughout.
+            val first = db.store.reindexFullTextSearch(null, batchSize = 1)
+            kotlin.test.assertEquals(false, first.done)
+            db.store.assertQuery(a, Filter(search = "uniqalpha"))
+            db.store.assertQuery(b, Filter(search = "uniqbeta"))
+
+            // Finish, then run the whole loop again from scratch: delete-then-
+            // insert per event means no duplicate rows (assertQuery wants 1).
+            var cursor = first.cursor
+            do {
+                val p = db.store.reindexFullTextSearch(cursor, batchSize = 1)
+                cursor = p.cursor
+            } while (!p.done)
+
+            cursor = null
+            do {
+                val p = db.store.reindexFullTextSearch(cursor, batchSize = 10)
+                cursor = p.cursor
+            } while (!p.done)
+
+            db.store.assertQuery(a, Filter(search = "uniqalpha"))
+            db.store.assertQuery(b, Filter(search = "uniqbeta"))
+        }
+
+    @Test
+    fun testResumableReindexClampsNonPositiveBatchSize() =
+        forEachDB { db ->
+            val a = signer.sign(TextNoteEvent.build("uniqclamp note", createdAt = TimeUtils.now()))
+            db.store.insertEvent(a)
+            db.store.pool.useWriter { db.store.fullTextSearchModule.deleteAll(it) }
+
+            // batchSize 0 must not spin forever: it is clamped to one event
+            // per call. Cap the loop so a regression fails fast instead of
+            // hanging the suite.
+            var cursor: String? = null
+            var guard = 0
+            do {
+                val progress = db.store.reindexFullTextSearch(cursor, batchSize = 0)
+                cursor = progress.cursor
+                check(guard++ < 100) { "reindex did not terminate with batchSize=0" }
+            } while (!progress.done)
+
+            db.store.assertQuery(a, Filter(search = "uniqclamp"))
+        }
+
    @Test
    fun testChannelJsonFieldsAreSearchable() =
        forEachDB { db ->
@@ -28,13 +28,16 @@ import com.vitorpamplona.quartz.nip01Core.core.isEphemeral
 import com.vitorpamplona.quartz.nip01Core.core.isReplaceable
 import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
 import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
+import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress
 import com.vitorpamplona.quartz.nip01Core.store.IEventStore
 import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
 import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy
 import com.vitorpamplona.quartz.nip01Core.store.sqlite.TagNameValueHasher
 import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
 import com.vitorpamplona.quartz.nip40Expiration.expiration
+import com.vitorpamplona.quartz.nip50Search.SearchableEvent
 import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent
+import com.vitorpamplona.quartz.utils.EventFactory
 import com.vitorpamplona.quartz.utils.TimeUtils
 import java.nio.file.FileAlreadyExistsException
 import java.nio.file.Files
@@ -493,6 +496,106 @@ open class FsEventStore(
            }
        }

+    /**
+     * Wipe and rebuild only the `idx/fts/` tree from the stored events.
+     * See [IEventStore.reindexFullTextSearch] for when to call this.
+     *
+     * Unlike [scrub] (which rebuilds every index tree from a full
+     * `events/` walk), this touches nothing but FTS and reads only the
+     * events whose kind is searchable today: it drives the walk from
+     * `idx/kind/<k>/`, which already lists exactly the live canonical
+     * events per kind, and skips kind directories that don't map to a
+     * [SearchableEvent]. The bulk of non-searchable events (reactions,
+     * zaps, follow lists, …) is never opened.
+     */
+    override suspend fun reindexFullTextSearch() =
+        lockManager.withWriteLock {
+            deleteRecursively(layout.idxFts)
+            Files.createDirectories(layout.idxFts)
+
+            if (!Files.isDirectory(layout.idxKind)) return@withWriteLock
+            Files.list(layout.idxKind).use { kindDirs ->
+                for (kindDir in kindDirs) {
+                    val kind = kindDir.fileName.toString().toIntOrNull() ?: continue
+                    if (!isSearchableKind(kind)) continue
+                    Files.list(kindDir).use { entries ->
+                        for (entry in entries) {
+                            val id = FsLayout.parseEntry(entry.fileName.toString())?.second ?: continue
+                            val event = readEvent(id) ?: continue
+                            indexer.linkFts(event, layout.canonical(id))
+                        }
+                    }
+                }
+            }
+        }
+
+    /**
+     * Resumable, additive FTS rebuild. See
+     * [IEventStore.reindexFullTextSearch] for the loop contract.
+     *
+     * The cursor is the next searchable kind to process, so the store is
+     * walked one `idx/kind/<k>/` directory at a time — that keeps the
+     * scan linear (each directory is listed once, never re-sorted) at the
+     * cost of pausing only between kinds: a single huge kind is processed
+     * in one batch. [batchSize] is a soft floor — whole kinds are
+     * processed until at least that many events have been (re)linked,
+     * then the call yields. Linking is idempotent ([FsIndexer.linkFts]
+     * ignores an existing hardlink), so nothing is wiped and search stays
+     * usable throughout; replaying a kind after a crash is harmless.
+     */
+    override suspend fun reindexFullTextSearch(
+        resumeFrom: String?,
+        batchSize: Int,
+    ): FtsReindexProgress =
+        lockManager.withWriteLock {
+            if (!Files.isDirectory(layout.idxKind)) {
+                return@withWriteLock FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true)
+            }
+            Files.createDirectories(layout.idxFts)
+
+            val resumeKind = resumeFrom?.toIntOrNull()
+            val target = batchSize.coerceAtLeast(1)
+            val pending =
+                Files
+                    .list(layout.idxKind)
+                    .use { dirs -> dirs.map { it.fileName.toString() }.toList() }
+                    .mapNotNull { it.toIntOrNull() }
+                    .filter { isSearchableKind(it) && (resumeKind == null || it >= resumeKind) }
+                    .sorted()
+
+            var processed = 0
+            var index = 0
+            while (index < pending.size) {
+                val kind = pending[index]
+                Files.list(layout.kindDir(kind)).use { entries ->
+                    for (entry in entries) {
+                        val id = FsLayout.parseEntry(entry.fileName.toString())?.second ?: continue
+                        val event = readEvent(id) ?: continue
+                        indexer.linkFts(event, layout.canonical(id))
+                        processed++
+                    }
+                }
+                index++
+                if (processed >= target) break
+            }
+
+            val done = index >= pending.size
+            FtsReindexProgress(
+                cursor = if (done) null else pending[index].toString(),
+                processedThisBatch = processed,
+                done = done,
+            )
+        }
+
+    /**
+     * True when [kind] currently parses to a [SearchableEvent]. Kind
+     * alone selects the event class in [EventFactory], so a single probe
+     * per kind is authoritative. The id is non-blank so kinds that lazily
+     * hash a missing id (NIP-17 chat) skip that work — only the runtime
+     * type matters here.
+     */
+    private fun isSearchableKind(kind: Int): Boolean = EventFactory.create<Event>("0", "0", 0L, kind, emptyArray(), "", "") is SearchableEvent
+
    private fun deleteRecursively(p: java.nio.file.Path) {
        if (!Files.exists(p)) return
        Files.walk(p).use { stream ->
@@ -86,6 +86,21 @@ internal class FsIndexer(
        }
    }

+    /**
+     * Create only the NIP-50 FTS hardlinks for this event (no-op when it
+     * isn't a [SearchableEvent]). Used by the FTS-only rebuild so an
+     * existing kind/author/owner/tag tree is left untouched. Idempotent.
+     */
+    fun linkFts(
+        event: Event,
+        canonical: Path,
+    ) {
+        if (event !is SearchableEvent) return
+        for (token in FsSearchTokenizer.tokenize(event.indexableContent())) {
+            createLink(layout.ftsEntry(token, event.createdAt, event.id), canonical)
+        }
+    }
+
    /** Remove every index hardlink for this event. Missing entries ignored. */
    fun unlink(event: Event) {
        for (path in pathsFor(event)) {
@@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event
 import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
 import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
 import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
+import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent
 import com.vitorpamplona.quartz.utils.Secp256k1Instance
 import kotlinx.coroutines.runBlocking
 import java.nio.file.Files
@@ -131,6 +132,93 @@ class FsSearchTest {
            assertEquals(0, ftsRoot.listDirectoryEntries().size, "unknown kind is not SearchableEvent")
        }

+    @Test
+    fun `reindexFullTextSearch rebuilds fts entries after a wipe`() =
+        runBlocking {
+            val a = note("bitcoin reindex", ts = 100)
+            val b = note("nostr reindex", ts = 200)
+            store.insert(a)
+            store.insert(b)
+
+            // Mimic a store written before this kind was searchable: drop
+            // the whole idx/fts/ tree, leaving canonical events intact.
+            val ftsRoot = root.resolve("idx/fts")
+            Files.walk(ftsRoot).use { stream ->
+                stream.sorted(Comparator.reverseOrder()).forEach { p -> if (p != ftsRoot) Files.deleteIfExists(p) }
+            }
+            assertEquals(0, ftsRoot.listDirectoryEntries().size, "precondition: fts wiped")
+            assertTrue(store.query<TextNoteEvent>(Filter(search = "bitcoin")).isEmpty(), "no index, no match")
+
+            store.reindexFullTextSearch()
+
+            assertEquals(listOf(a.id), store.query<TextNoteEvent>(Filter(search = "bitcoin")).map { it.id })
+            assertEquals(listOf(b.id), store.query<TextNoteEvent>(Filter(search = "nostr")).map { it.id })
+            // Reindexing twice must not duplicate entries.
+            store.reindexFullTextSearch()
+            assertEquals(1, ftsRoot.resolve("bitcoin").listDirectoryEntries().size)
+            // "reindex" appears in both notes — one entry per event, no dupes.
+            assertEquals(2, ftsRoot.resolve("reindex").listDirectoryEntries().size)
+        }
+
+    @Test
+    fun `resumable reindex covers every kind across batches`() =
+        runBlocking {
+            // Two searchable kinds (note = kind 1, long-form = kind 30023) so
+            // the kind-granular cursor must advance across more than one dir.
+            val n = note("uniqnote bitcoin", ts = 100)
+            val long =
+                signer.sign(
+                    LongTextNoteEvent.build(
+                        "uniqlong body",
+                        title = "title",
+                        dTag = "d1",
+                        createdAt = 200,
+                    ),
+                )
+            store.insert(n)
+            store.insert(long)
+
+            // Wipe the index, then drive the resumable path one kind at a time.
+            val ftsRoot = root.resolve("idx/fts")
+            Files.walk(ftsRoot).use { stream ->
+                stream.sorted(Comparator.reverseOrder()).forEach { p -> if (p != ftsRoot) Files.deleteIfExists(p) }
+            }
+            assertTrue(store.query<Event>(Filter(search = "uniqnote")).isEmpty())
+
+            var cursor: String? = null
+            var batches = 0
+            do {
+                val progress = store.reindexFullTextSearch(cursor, batchSize = 1)
+                cursor = progress.cursor
+                batches++
+            } while (!progress.done)
+
+            // Two searchable kind dirs, batchSize 1 ⇒ at least two batches.
+            assertTrue(batches >= 2, "expected the cursor to span both kinds, got $batches batch(es)")
+            assertEquals(listOf(n.id), store.query<Event>(Filter(search = "uniqnote")).map { it.id })
+            assertEquals(listOf(long.id), store.query<Event>(Filter(search = "uniqlong")).map { it.id })
+        }
+
+    @Test
+    fun `reindexFullTextSearch ignores non-searchable kinds`() =
+        runBlocking {
+            val searchable = note("findme bitcoin", ts = 100)
+            val opaque =
+                signer.sign<Event>(
+                    createdAt = 1,
+                    kind = 9999,
+                    tags = emptyArray(),
+                    content = "findme must not be indexed",
+                )
+            store.insert(searchable)
+            store.insert(opaque)
+
+            store.reindexFullTextSearch()
+
+            // Only the searchable note contributes the "findme" token.
+            assertEquals(listOf(searchable.id), store.query<Event>(Filter(search = "findme")).map { it.id })
+        }
+
    @Test
    fun `delete removes fts entries`() =
        runBlocking {