diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index f39da4fe85..94b5ac2413 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -27,7 +27,14 @@ import com.vitorpamplona.quartz.nip50Search.SearchableEvent class FullTextSearchModule : IModule { val tableName = "event_fts" - val eventHeaderRowIdName = "event_header_row_id" + + // We don't store the event_headers foreign key as its own indexed column — + // an FTS column is tokenized into the searchable text, so a numeric FK would + // pollute MATCH results and waste index space. Instead we align the FTS + // table's implicit rowid with event_headers.row_id at insert time and join + // on it. rowid joins are also the fastest possible, and `rowid` works across + // fts3/4/5 (fts4/3 expose it as `docid`, but `rowid` is accepted too). + val rowIdName = "rowid" val contentName = "content" override fun create(db: SQLiteConnection) { @@ -35,11 +42,12 @@ class FullTextSearchModule : IModule { db.execSQL( """ CREATE VIRTUAL TABLE $tableName - USING fts$ftsVersion($eventHeaderRowIdName, $contentName) + USING fts$ftsVersion($contentName) """, ) - // Foreign key cleanup for full text search + // Foreign key cleanup for full text search. Because the FTS rowid is the + // event_headers.row_id, we can delete the matching row directly. db.execSQL( """ CREATE TRIGGER fts_foreign_key @@ -47,19 +55,23 @@ class FullTextSearchModule : IModule { FOR EACH ROW BEGIN DELETE FROM $tableName - WHERE old.row_id = $tableName.$eventHeaderRowIdName; + WHERE $tableName.$rowIdName = old.row_id; END; """, ) } override fun drop(db: SQLiteConnection) { + // The trigger lives on event_headers, so dropping our table alone won't + // remove it. Drop it explicitly so drop() is self-contained and a later + // create() doesn't fail with "trigger already exists". + db.execSQL("DROP TRIGGER IF EXISTS fts_foreign_key") db.execSQL("DROP TABLE IF EXISTS $tableName") } val insertFTS = """ - INSERT OR ROLLBACK INTO $tableName ($eventHeaderRowIdName, $contentName) + INSERT OR ROLLBACK INTO $tableName ($rowIdName, $contentName) VALUES (?, ?) """.trimIndent() diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 46fa1579a4..b4c9660d31 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -360,7 +360,7 @@ class QueryBuilder( val sql = buildString { append("SELECT event_headers.id, event_headers.created_at FROM event_headers") - append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}") if (clause.conditions.isNotEmpty()) { append("\nWHERE ${clause.conditions}") } @@ -678,13 +678,13 @@ class QueryBuilder( } if (mustJoinSearch) { - append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id") + append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.rowIdName} = event_tags.event_header_row_id") } } else if (mustJoinSearch) { - append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName}") + append("SELECT ${fts.tableName}.${fts.rowIdName} as row_id FROM ${fts.tableName}") if (hasHeaders) { - append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}") } } else { // no tags and no search. @@ -864,7 +864,7 @@ class QueryBuilder( val sql = buildString { append("SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers") - append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}") if (clause.conditions.isNotEmpty()) { append("\nWHERE ${clause.conditions}") } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index e67f7b4646..31758b9da1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -44,7 +44,7 @@ class SQLiteEventStore( val numReaders: Int = 4, ) { companion object { - const val DATABASE_VERSION = 2 + const val DATABASE_VERSION = 3 } val seedModule = SeedModule() @@ -164,6 +164,41 @@ class SQLiteEventStore( modules.reversed().forEach { it.drop(db) } modules.forEach { it.create(db) } } + 2 -> { + // Upgrade from version 2 to 3 + // The full-text index dropped its dedicated foreign-key + // column and now aligns the FTS rowid with + // event_headers.row_id. Rebuild only the FTS index in place + // so the cached events themselves survive the upgrade. + fullTextSearchModule.drop(db) + fullTextSearchModule.create(db) + reindexFullText(db) + } + } + } + } + + /** + * Repopulates [FullTextSearchModule] from the events already stored in + * event_headers. Used by migrations that recreate the FTS table without + * touching the source events. Reading event_headers while inserting into + * event_fts is safe because they are different tables. + */ + private fun reindexFullText(db: SQLiteConnection) { + db.prepare("SELECT row_id, id, pubkey, created_at, kind, tags, content, sig FROM event_headers").use { stmt -> + while (stmt.step()) { + val rowId = stmt.getLong(0) + val event = + EventFactory.create( + stmt.getText(1), + stmt.getText(2), + stmt.getLong(3), + stmt.getInt(4), + OptimizedJsonMapper.fromJsonToTagArray(stmt.getText(5)), + stmt.getText(6), + stmt.getText(7), + ) + fullTextSearchModule.insert(event, rowId, db) } } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt index 32621025e3..f3995f7b8f 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -238,9 +238,9 @@ class QueryAssemblerTest : BaseDBTest() { INNER JOIN ( SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY $orderBy @@ -252,13 +252,13 @@ class QueryAssemblerTest : BaseDBTest() { │ │ └── SCAN (subquery-1) │ ├── UNION USING TEMP B-TREE │ │ ├── CO-ROUTINE (subquery-3) - │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ │ └── USE TEMP B-TREE FOR ORDER BY │ │ └── SCAN (subquery-3) │ └── UNION USING TEMP B-TREE │ ├── CO-ROUTINE (subquery-5) - │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ └── USE TEMP B-TREE FOR ORDER BY │ └── SCAN (subquery-5) @@ -275,9 +275,9 @@ class QueryAssemblerTest : BaseDBTest() { INNER JOIN ( SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) + SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY $orderBy @@ -290,13 +290,13 @@ class QueryAssemblerTest : BaseDBTest() { │ │ └── SCAN (subquery-1) │ ├── UNION USING TEMP B-TREE │ │ ├── CO-ROUTINE (subquery-3) - │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ │ └── USE TEMP B-TREE FOR ORDER BY │ │ └── SCAN (subquery-3) │ └── UNION USING TEMP B-TREE │ ├── CO-ROUTINE (subquery-5) - │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) │ │ └── USE TEMP B-TREE FOR ORDER BY │ └── SCAN (subquery-5) @@ -712,10 +712,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) ORDER BY event_headers.created_at DESC, event_headers.id ASC - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(), @@ -725,10 +725,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) ORDER BY event_headers.created_at DESC - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(), @@ -745,10 +745,10 @@ class QueryAssemblerTest : BaseDBTest() { assertEquals( """ SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid WHERE (event_fts MATCH "keywords") AND (event_headers.kind IN ("1", "1111", "10000")) ORDER BY $orderBy - ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2 + ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1 ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY """.trimIndent(),