From 6a2d8baf43ff96350cb7fb4e7ee3578fe517d4e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 18 Jun 2026 15:51:06 +0000 Subject: [PATCH] refactor: align event_fts rowid with event_headers.row_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FTS table declared event_header_row_id as a regular full-text column, which means the numeric foreign key was tokenized into the searchable index — a bare MATCH could match an event by its internal row id, and the column wasted index space. Drop the dedicated column and instead align the FTS table's implicit rowid with event_headers.row_id at insert time, joining on it (rowid joins are also the fastest possible). This works across fts3/4/5. Also make FullTextSearchModule.drop() remove its trigger explicitly so the module is self-contained, and add a v2->v3 migration that rebuilds the FTS index in place from event_headers, preserving the cached events. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01RFdWREvyvixRXnmNXNzmmN --- .../store/sqlite/FullTextSearchModule.kt | 22 ++++++++--- .../nip01Core/store/sqlite/QueryBuilder.kt | 10 ++--- .../store/sqlite/SQLiteEventStore.kt | 37 ++++++++++++++++++- .../store/sqlite/QueryAssemblerTest.kt | 28 +++++++------- 4 files changed, 72 insertions(+), 25 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index f39da4fe85..94b5ac2413 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -27,7 +27,14 @@ import com.vitorpamplona.quartz.nip50Search.SearchableEvent class FullTextSearchModule : IModule { val tableName = "event_fts" - val eventHeaderRowIdName = "event_header_row_id" + + // We don't store the event_headers foreign key as its own indexed column — + // an FTS column is tokenized into the searchable text, so a numeric FK would + // pollute MATCH results and waste index space. Instead we align the FTS + // table's implicit rowid with event_headers.row_id at insert time and join + // on it. rowid joins are also the fastest possible, and `rowid` works across + // fts3/4/5 (fts4/3 expose it as `docid`, but `rowid` is accepted too). + val rowIdName = "rowid" val contentName = "content" override fun create(db: SQLiteConnection) { @@ -35,11 +42,12 @@ class FullTextSearchModule : IModule { db.execSQL( """ CREATE VIRTUAL TABLE $tableName - USING fts$ftsVersion($eventHeaderRowIdName, $contentName) + USING fts$ftsVersion($contentName) """, ) - // Foreign key cleanup for full text search + // Foreign key cleanup for full text search. Because the FTS rowid is the + // event_headers.row_id, we can delete the matching row directly. db.execSQL( """ CREATE TRIGGER fts_foreign_key @@ -47,19 +55,23 @@ class FullTextSearchModule : IModule { FOR EACH ROW BEGIN DELETE FROM $tableName - WHERE old.row_id = $tableName.$eventHeaderRowIdName; + WHERE $tableName.$rowIdName = old.row_id; END; """, ) } override fun drop(db: SQLiteConnection) { + // The trigger lives on event_headers, so dropping our table alone won't + // remove it. Drop it explicitly so drop() is self-contained and a later + // create() doesn't fail with "trigger already exists". + db.execSQL("DROP TRIGGER IF EXISTS fts_foreign_key") db.execSQL("DROP TABLE IF EXISTS $tableName") } val insertFTS = """ - INSERT OR ROLLBACK INTO $tableName ($eventHeaderRowIdName, $contentName) + INSERT OR ROLLBACK INTO $tableName ($rowIdName, $contentName) VALUES (?, ?) """.trimIndent() diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 46fa1579a4..b4c9660d31 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -360,7 +360,7 @@ class QueryBuilder( val sql = buildString { append("SELECT event_headers.id, event_headers.created_at FROM event_headers") - append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}") if (clause.conditions.isNotEmpty()) { append("\nWHERE ${clause.conditions}") } @@ -678,13 +678,13 @@ class QueryBuilder( } if (mustJoinSearch) { - append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id") + append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.rowIdName} = event_tags.event_header_row_id") } } else if (mustJoinSearch) { - append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName}") + append("SELECT ${fts.tableName}.${fts.rowIdName} as row_id FROM ${fts.tableName}") if (hasHeaders) { - append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}") } } else { // no tags and no search. @@ -864,7 +864,7 @@ class QueryBuilder( val sql = buildString { append("SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers") - append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}") if (clause.conditions.isNotEmpty()) { append("\nWHERE ${clause.conditions}") } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index e67f7b4646..31758b9da1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -44,7 +44,7 @@ class SQLiteEventStore( val numReaders: Int = 4, ) { companion object { - const val DATABASE_VERSION = 2 + const val DATABASE_VERSION = 3 } val seedModule = SeedModule() @@ -164,6 +164,41 @@ class SQLiteEventStore( modules.reversed().forEach { it.drop(db) } modules.forEach { it.create(db) } } + 2 -> { + // Upgrade from version 2 to 3 + // The full-text index dropped its dedicated foreign-key + // column and now aligns the FTS rowid with + // event_headers.row_id. Rebuild only the FTS index in place + // so the cached events themselves survive the upgrade. + fullTextSearchModule.drop(db) + fullTextSearchModule.create(db) + reindexFullText(db) + } + } + } + } + + /** + * Repopulates [FullTextSearchModule] from the events already stored in + * event_headers. Used by migrations that recreate the FTS table without + * touching the source events. Reading event_headers while inserting into + * event_fts is safe because they are different tables. + */ + private fun reindexFullText(db: SQLiteConnection) { + db.prepare("SELECT row_id, id, pubkey, created_at, kind, tags, content, sig FROM event_headers").use { stmt -> + while (stmt.step()) { + val rowId = stmt.getLong(0) + val event = + EventFactory.create( + stmt.getText(1), + stmt.getText(2), + stmt.getLong(3), + stmt.getInt(4), + OptimizedJsonMapper.fromJsonToTagArray(stmt.getText(5)), + stmt.getText(6), + stmt.getText(7), + ) + fullTextSearchModule.insert(event, rowId, db) } } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt index 32621025e3..f3995f7b8f 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -238,9 +238,9 @@ class QueryAssemblerTest : BaseDBTest() { INNER JOIN ( SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY $orderBy @@ -252,13 +252,13 @@ class QueryAssemblerTest : BaseDBTest() { │ │ └── SCAN (subquery-1) │ ├── UNION USING TEMP B-TREE │ │ ├── CO-ROUTINE (subquery-3) - │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ │ └── USE TEMP B-TREE FOR ORDER BY │ │ └── SCAN (subquery-3) │ └── UNION USING TEMP B-TREE │ ├── CO-ROUTINE (subquery-5) - │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ └── USE TEMP B-TREE FOR ORDER BY │ └── SCAN (subquery-5) @@ -275,9 +275,9 @@ class QueryAssemblerTest : BaseDBTest() { INNER JOIN ( SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY $orderBy @@ -290,13 +290,13 @@ class QueryAssemblerTest : BaseDBTest() { │ │ └── SCAN (subquery-1) │ ├── UNION USING TEMP B-TREE │ │ ├── CO-ROUTINE (subquery-3) - │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ │ └── USE TEMP B-TREE FOR ORDER BY │ │ └── SCAN (subquery-3) │ └── UNION USING TEMP B-TREE │ ├── CO-ROUTINE (subquery-5) - │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ └── USE TEMP B-TREE FOR ORDER BY │ └── SCAN (subquery-5) @@ -712,10 +712,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) ORDER BY event_headers.created_at DESC, event_headers.id ASC - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(), @@ -725,10 +725,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) ORDER BY event_headers.created_at DESC - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(), @@ -745,10 +745,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.kind IN ("1", "1111", "10000")) ORDER BY $orderBy - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(),