revert: keep original event_fts table design, drop FTS migration

Reverts the event_fts rowid-alignment refactor and the background reindex
that it required. Aligning the FTS rowid with event_headers.row_id was a
schema change, which forced a v2->v3 migration to rebuild the index from
~all cached events — and on large caches that reindex was the expensive,
risky part (slow startup, all-or-nothing transaction, resumability and
malformed-row concerns). The cleanup it bought (not tokenizing the numeric
foreign key into the index) isn't worth that cost.

Restores the original design: event_fts keeps its dedicated
event_header_row_id column, queries join on it, DATABASE_VERSION stays 2,
and there is no FTS migration or reindex at all.

Kept: the newly searchable event kinds (they implement SearchableEvent and
work unchanged with the original table) and their test.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RFdWREvyvixRXnmNXNzmmN
This commit is contained in:
Claude
2026-06-18 18:47:37 +00:00
parent 1027324da8
commit a689d5cb19
5 changed files with 26 additions and 274 deletions
@@ -27,14 +27,7 @@ import com.vitorpamplona.quartz.nip50Search.SearchableEvent
class FullTextSearchModule : IModule {
val tableName = "event_fts"
// We don't store the event_headers foreign key as its own indexed column —
// an FTS column is tokenized into the searchable text, so a numeric FK would
// pollute MATCH results and waste index space. Instead we align the FTS
// table's implicit rowid with event_headers.row_id at insert time and join
// on it. rowid joins are also the fastest possible, and `rowid` works across
// fts3/4/5 (fts4/3 expose it as `docid`, but `rowid` is accepted too).
val rowIdName = "rowid"
val eventHeaderRowIdName = "event_header_row_id"
val contentName = "content"
override fun create(db: SQLiteConnection) {
@@ -42,12 +35,11 @@ class FullTextSearchModule : IModule {
db.execSQL(
"""
CREATE VIRTUAL TABLE $tableName
USING fts$ftsVersion($contentName)
USING fts$ftsVersion($eventHeaderRowIdName, $contentName)
""",
)
// Foreign key cleanup for full text search. Because the FTS rowid is the
// event_headers.row_id, we can delete the matching row directly.
// Foreign key cleanup for full text search
db.execSQL(
"""
CREATE TRIGGER fts_foreign_key
@@ -55,23 +47,19 @@ class FullTextSearchModule : IModule {
FOR EACH ROW
BEGIN
DELETE FROM $tableName
WHERE $tableName.$rowIdName = old.row_id;
WHERE old.row_id = $tableName.$eventHeaderRowIdName;
END;
""",
)
}
override fun drop(db: SQLiteConnection) {
// The trigger lives on event_headers, so dropping our table alone won't
// remove it. Drop it explicitly so drop() is self-contained and a later
// create() doesn't fail with "trigger already exists".
db.execSQL("DROP TRIGGER IF EXISTS fts_foreign_key")
db.execSQL("DROP TABLE IF EXISTS $tableName")
}
val insertFTS =
"""
INSERT OR ROLLBACK INTO $tableName ($rowIdName, $contentName)
INSERT OR ROLLBACK INTO $tableName ($eventHeaderRowIdName, $contentName)
VALUES (?, ?)
""".trimIndent()
@@ -89,30 +77,6 @@ class FullTextSearchModule : IModule {
}
}
// Idempotent variant used by the background reindex backfill: a row may
// already have been indexed by the live insert path (event_headers.row_id
// is monotonic, so freshly received events land above the backfill cursor),
// and re-indexing it must be a no-op rather than a constraint failure.
val insertIfAbsentFTS =
"""
INSERT OR IGNORE INTO $tableName ($rowIdName, $contentName)
VALUES (?, ?)
""".trimIndent()
fun insertIfAbsent(
event: Event,
headerId: Long,
db: SQLiteConnection,
) {
if (event is SearchableEvent) {
db.prepare(insertIfAbsentFTS).use { stmt ->
stmt.bindLong(1, headerId)
stmt.bindText(2, event.indexableContent())
stmt.step()
}
}
}
fun versionFinder(db: SQLiteConnection): Int {
// Defensive cleanup in case a previous probe left these behind
// (e.g. a partial create() during an upgrade) — without this,
@@ -360,7 +360,7 @@ class QueryBuilder(
val sql =
buildString {
append("SELECT event_headers.id, event_headers.created_at FROM event_headers")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
if (clause.conditions.isNotEmpty()) {
append("\nWHERE ${clause.conditions}")
}
@@ -678,13 +678,13 @@ class QueryBuilder(
}
if (mustJoinSearch) {
append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.rowIdName} = event_tags.event_header_row_id")
append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id")
}
} else if (mustJoinSearch) {
append("SELECT ${fts.tableName}.${fts.rowIdName} as row_id FROM ${fts.tableName}")
append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName}")
if (hasHeaders) {
append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}")
append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
}
} else {
// no tags and no search.
@@ -864,7 +864,7 @@ class QueryBuilder(
val sql =
buildString {
append("SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
if (clause.conditions.isNotEmpty()) {
append("\nWHERE ${clause.conditions}")
}
@@ -35,12 +35,6 @@ import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip01Core.store.IdAndTime
import com.vitorpamplona.quartz.nip40Expiration.isExpired
import com.vitorpamplona.quartz.utils.EventFactory
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
import kotlinx.coroutines.yield
class SQLiteEventStore(
val driver: SQLiteDriver = BundledSQLiteDriver(),
@@ -50,12 +44,7 @@ class SQLiteEventStore(
val numReaders: Int = 4,
) {
companion object {
const val DATABASE_VERSION = 3
// Rows per background-reindex batch. Each batch is one writer
// transaction; the writer mutex is released between batches so live
// inserts/queries interleave instead of waiting for the whole reindex.
const val REINDEX_BATCH_SIZE = 500
const val DATABASE_VERSION = 2
}
val seedModule = SeedModule()
@@ -142,26 +131,6 @@ class SQLiteEventStore(
)
}
// Background worker for one-off maintenance that must not block app
// startup — currently the post-migration full-text reindex backfill.
private val maintenanceScope = CoroutineScope(Dispatchers.IO + SupervisorJob())
init {
// Kicked off asynchronously: it returns immediately when there is no
// pending reindex (the common case), and otherwise backfills the FTS
// index in the background while the rest of the app runs normally.
maintenanceScope.launch {
try {
reindexFullTextIfPending()
} catch (_: Throwable) {
// Best-effort maintenance. A failure here only leaves search
// degraded until the next launch retries; it must never crash
// the store. (CancellationException from close() lands here too
// and is intentionally swallowed.)
}
}
}
private fun getUserVersion(db: SQLiteConnection): Int =
db.prepare("PRAGMA user_version").use { stmt ->
stmt.step()
@@ -195,159 +164,10 @@ class SQLiteEventStore(
modules.reversed().forEach { it.drop(db) }
modules.forEach { it.create(db) }
}
2 -> {
// Upgrade from version 2 to 3
// The full-text index dropped its dedicated foreign-key
// column and now aligns the FTS rowid with
// event_headers.row_id. Recreate the (now empty) FTS table
// structure cheaply inside the migration and record a
// persistent marker; the actual repopulation from
// event_headers happens later in the background via
// [reindexFullTextIfPending] so the migration — and app
// startup — never blocks on a large reindex.
fullTextSearchModule.drop(db)
fullTextSearchModule.create(db)
createReindexMarker(db)
}
}
}
}
// ------------------------------------------------------------------
// Background full-text reindex
//
// When a migration recreates the FTS table it leaves a persistent
// `fts_reindex` marker holding a progress cursor (the highest
// event_headers.row_id already backfilled). The backfill walks
// event_headers in row_id order in small committed batches, so it
// interleaves with normal relay inserts/queries and survives process
// death: the cursor is persisted, and on the next launch the marker is
// still present so the work resumes where it stopped. Search is merely
// degraded (partial results) until it finishes — never blocked.
// ------------------------------------------------------------------
private val reindexMarkerTable = "fts_reindex"
private fun createReindexMarker(db: SQLiteConnection) {
db.execSQL("CREATE TABLE IF NOT EXISTS $reindexMarkerTable (next_row_id INTEGER NOT NULL)")
db.execSQL("DELETE FROM $reindexMarkerTable")
// row_id is AUTOINCREMENT starting at 1, so 0 means "nothing done yet".
db.execSQL("INSERT INTO $reindexMarkerTable (next_row_id) VALUES (0)")
}
private fun hasReindexMarker(db: SQLiteConnection): Boolean = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '$reindexMarkerTable'").use { it.step() }
private fun getReindexCursor(db: SQLiteConnection): Long =
db.prepare("SELECT next_row_id FROM $reindexMarkerTable LIMIT 1").use {
if (it.step()) it.getLong(0) else 0L
}
private fun setReindexCursor(
db: SQLiteConnection,
value: Long,
) {
db.prepare("UPDATE $reindexMarkerTable SET next_row_id = ?").use {
it.bindLong(1, value)
it.step()
}
}
private fun dropReindexMarker(db: SQLiteConnection) {
db.execSQL("DROP TABLE IF EXISTS $reindexMarkerTable")
}
/**
* Drives the background backfill loop while a reindex marker exists.
* Each iteration processes one batch in its own writer transaction, then
* releases the writer mutex so other writes get a turn. [yield] makes the
* loop cancellation-aware (see [close]).
*/
internal suspend fun reindexFullTextIfPending() {
if (!pool.useReader { hasReindexMarker(it) }) return
while (true) {
val done = pool.useWriter { db -> reindexNextBatch(db, REINDEX_BATCH_SIZE) }
if (done) break
yield()
}
}
/**
* Indexes up to [limit] not-yet-processed events (row_id greater than the
* persisted cursor) into the FTS table and advances the cursor. Returns
* true once there is nothing left, after dropping the marker. A single
* malformed cached row is skipped — the cursor still moves past it, so the
* backfill can never get stuck retrying the same row.
*/
private fun reindexNextBatch(
db: SQLiteConnection,
limit: Int,
): Boolean =
db.transaction {
// The marker may have been dropped by a previous batch (or, in
// theory, another backfiller) — treat its absence as "done" rather
// than reading a cursor from a missing table.
if (!hasReindexMarker(db)) {
true
} else {
val cursor = getReindexCursor(db)
var lastRowId = cursor
var count = 0
db
.prepare(
"SELECT row_id, id, pubkey, created_at, kind, tags, content, sig FROM event_headers " +
"WHERE row_id > ? ORDER BY row_id LIMIT ?",
).use { stmt ->
stmt.bindLong(1, cursor)
stmt.bindLong(2, limit.toLong())
while (stmt.step()) {
val rowId = stmt.getLong(0)
try {
val event =
EventFactory.create<Event>(
stmt.getText(1),
stmt.getText(2),
stmt.getLong(3),
stmt.getInt(4),
OptimizedJsonMapper.fromJsonToTagArray(stmt.getText(5)),
stmt.getText(6),
stmt.getText(7),
)
fullTextSearchModule.insertIfAbsent(event, rowId, db)
} catch (_: Throwable) {
// Skip a row that fails to parse/index; advancing the
// cursor below guarantees forward progress regardless.
}
lastRowId = rowId
count++
}
}
if (count == 0) {
dropReindexMarker(db)
true
} else {
setReindexCursor(db, lastRowId)
false
}
}
}
/**
* Test hook: simulates the post-migration state by clearing the FTS table
* and arming the reindex marker, so a test can then drive
* [reindexFullTextIfPending] deterministically.
*/
internal suspend fun dropFtsAndMarkPendingForTest() =
pool.useWriter { db ->
db.transaction {
fullTextSearchModule.drop(db)
fullTextSearchModule.create(db)
createReindexMarker(db)
}
}
suspend fun clearDB() =
pool.useWriter { db ->
modules.reversed().forEach { it.deleteAll(db) }
@@ -520,10 +340,7 @@ class SQLiteEventStore(
suspend fun deleteExpiredEvents() = pool.useWriter { expirationModule.deleteExpiredEvents(it) }
fun close() {
maintenanceScope.cancel()
pool.close()
}
fun close() = pool.close()
}
class RawEvent(
@@ -238,9 +238,9 @@ class QueryAssemblerTest : BaseDBTest() {
INNER JOIN (
SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10)
UNION
SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100)
SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100)
UNION
SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30)
SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30)
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY $orderBy
@@ -252,13 +252,13 @@ class QueryAssemblerTest : BaseDBTest() {
│ │ └── SCAN (subquery-1)
│ ├── UNION USING TEMP B-TREE
│ │ ├── CO-ROUTINE (subquery-3)
│ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
│ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
│ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ │ │ └── USE TEMP B-TREE FOR ORDER BY
│ │ └── SCAN (subquery-3)
│ └── UNION USING TEMP B-TREE
│ ├── CO-ROUTINE (subquery-5)
│ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
│ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
│ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ │ └── USE TEMP B-TREE FOR ORDER BY
│ └── SCAN (subquery-5)
@@ -275,9 +275,9 @@ class QueryAssemblerTest : BaseDBTest() {
INNER JOIN (
SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10)
UNION
SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100)
SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100)
UNION
SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30)
SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30)
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY $orderBy
@@ -290,13 +290,13 @@ class QueryAssemblerTest : BaseDBTest() {
│ │ └── SCAN (subquery-1)
│ ├── UNION USING TEMP B-TREE
│ │ ├── CO-ROUTINE (subquery-3)
│ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
│ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
│ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ │ │ └── USE TEMP B-TREE FOR ORDER BY
│ │ └── SCAN (subquery-3)
│ └── UNION USING TEMP B-TREE
│ ├── CO-ROUTINE (subquery-5)
│ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
│ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
│ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ │ └── USE TEMP B-TREE FOR ORDER BY
│ └── SCAN (subquery-5)
@@ -712,10 +712,10 @@ class QueryAssemblerTest : BaseDBTest() {
assertEquals(
"""
SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers
INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid
INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id
WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9"))
ORDER BY event_headers.created_at DESC, event_headers.id ASC
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
""".trimIndent(),
@@ -725,10 +725,10 @@ class QueryAssemblerTest : BaseDBTest() {
assertEquals(
"""
SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers
INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid
INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id
WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9"))
ORDER BY event_headers.created_at DESC
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
""".trimIndent(),
@@ -745,10 +745,10 @@ class QueryAssemblerTest : BaseDBTest() {
assertEquals(
"""
SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers
INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid
INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id
WHERE (event_fts MATCH "keywords") AND (event_headers.kind IN ("1", "1111", "10000"))
ORDER BY $orderBy
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
""".trimIndent(),
@@ -165,33 +165,4 @@ class SearchTest : BaseDBTest() {
db.assertQuery(repo, Filter(search = "uniqdesc"))
db.assertQuery(repo, Filter(kinds = listOf(GitRepositoryEvent.KIND), search = "uniqdesc"))
}
@Test
fun testBackgroundReindexBackfillsExistingEvents() =
forEachDB { db ->
val a = signer.sign(CalendarEvent.build(title = "uniqalpha", content = "uniqbody"))
val b = signer.sign(GitRepositoryEvent.build(name = "uniqgamma", description = "uniqdelta"))
db.store.insertEvent(a)
db.store.insertEvent(b)
// Live insert path already indexed them.
db.assertQuery(a, Filter(search = "uniqalpha"))
db.assertQuery(b, Filter(search = "uniqdelta"))
// Simulate the post-migration state: FTS table emptied + marker armed.
db.store.dropFtsAndMarkPendingForTest()
db.assertQuery(null, Filter(search = "uniqalpha"))
db.assertQuery(null, Filter(search = "uniqdelta"))
// Background backfill rebuilds the index from event_headers.
db.store.reindexFullTextIfPending()
db.assertQuery(a, Filter(search = "uniqalpha"))
db.assertQuery(a, Filter(search = "uniqbody"))
db.assertQuery(b, Filter(search = "uniqgamma"))
db.assertQuery(b, Filter(search = "uniqdelta"))
// Marker is cleared, so a second run is a no-op and search still works.
db.store.reindexFullTextIfPending()
db.assertQuery(a, Filter(search = "uniqalpha"))
}
}