laantungir/amethyst
1
0
Fork 0
mirror of https://github.com/vitorpamplona/amethyst.git synced 2026-08-09 08:04:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: align event_fts rowid with event_headers.row_id

Browse Source 
The FTS table declared event_header_row_id as a regular full-text column,
which means the numeric foreign key was tokenized into the searchable
index — a bare MATCH could match an event by its internal row id, and the
column wasted index space.

Drop the dedicated column and instead align the FTS table's implicit
rowid with event_headers.row_id at insert time, joining on it (rowid
joins are also the fastest possible). This works across fts3/4/5.

Also make FullTextSearchModule.drop() remove its trigger explicitly so
the module is self-contained, and add a v2->v3 migration that rebuilds
the FTS index in place from event_headers, preserving the cached events.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RFdWREvyvixRXnmNXNzmmN
This commit is contained in:
Claude
2026-06-18 15:51:06 +00:00
parent 1bef6ab2ed
commit 6a2d8baf43
4 changed files with 72 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt
+17 -5
View File
@@ -27,7 +27,14 @@ import com.vitorpamplona.quartz.nip50Search.SearchableEvent
class FullTextSearchModule : IModule {
val tableName = "event_fts"
val eventHeaderRowIdName = "event_header_row_id"
// We don't store the event_headers foreign key as its own indexed column —
// an FTS column is tokenized into the searchable text, so a numeric FK would
// pollute MATCH results and waste index space. Instead we align the FTS
// table's implicit rowid with event_headers.row_id at insert time and join
// on it. rowid joins are also the fastest possible, and `rowid` works across
// fts3/4/5 (fts4/3 expose it as `docid`, but `rowid` is accepted too).
val rowIdName = "rowid"
val contentName = "content"
override fun create(db: SQLiteConnection) {
@@ -35,11 +42,12 @@ class FullTextSearchModule : IModule {
db.execSQL(
"""
CREATE VIRTUAL TABLE $tableName
USING fts$ftsVersion($eventHeaderRowIdName, $contentName)
USING fts$ftsVersion($contentName)
""",
)
// Foreign key cleanup for full text search
// Foreign key cleanup for full text search. Because the FTS rowid is the
// event_headers.row_id, we can delete the matching row directly.
db.execSQL(
"""
CREATE TRIGGER fts_foreign_key
@@ -47,19 +55,23 @@ class FullTextSearchModule : IModule {
FOR EACH ROW
BEGIN
DELETE FROM $tableName
WHERE old.row_id = $tableName.$eventHeaderRowIdName;
WHERE $tableName.$rowIdName = old.row_id;
END;
""",
)
}
override fun drop(db: SQLiteConnection) {
// The trigger lives on event_headers, so dropping our table alone won't
// remove it. Drop it explicitly so drop() is self-contained and a later
// create() doesn't fail with "trigger already exists".
db.execSQL("DROP TRIGGER IF EXISTS fts_foreign_key")
db.execSQL("DROP TABLE IF EXISTS $tableName")
}
val insertFTS =
"""
INSERT OR ROLLBACK INTO $tableName ($eventHeaderRowIdName, $contentName)
INSERT OR ROLLBACK INTO $tableName ($rowIdName, $contentName)
VALUES (?, ?)
""".trimIndent()
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt
+5 -5
View File
@@ -360,7 +360,7 @@ class QueryBuilder(
val sql =
buildString {
append("SELECT event_headers.id, event_headers.created_at FROM event_headers")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}")
if (clause.conditions.isNotEmpty()) {
append("\nWHERE ${clause.conditions}")
}
@@ -678,13 +678,13 @@ class QueryBuilder(
}
if (mustJoinSearch) {
append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id")
append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.rowIdName} = event_tags.event_header_row_id")
}
} else if (mustJoinSearch) {
append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName}")
append("SELECT ${fts.tableName}.${fts.rowIdName} as row_id FROM ${fts.tableName}")
if (hasHeaders) {
append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}")
}
} else {
// no tags and no search.
@@ -864,7 +864,7 @@ class QueryBuilder(
val sql =
buildString {
append("SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.rowIdName}")
if (clause.conditions.isNotEmpty()) {
append("\nWHERE ${clause.conditions}")
}
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt
+36 -1
View File
@@ -44,7 +44,7 @@ class SQLiteEventStore(
val numReaders: Int = 4,
) {
companion object {
const val DATABASE_VERSION = 2
const val DATABASE_VERSION = 3
}
val seedModule = SeedModule()
@@ -164,6 +164,41 @@ class SQLiteEventStore(
modules.reversed().forEach { it.drop(db) }
modules.forEach { it.create(db) }
}
2 -> {
// Upgrade from version 2 to 3
// The full-text index dropped its dedicated foreign-key
// column and now aligns the FTS rowid with
// event_headers.row_id. Rebuild only the FTS index in place
// so the cached events themselves survive the upgrade.
fullTextSearchModule.drop(db)
fullTextSearchModule.create(db)
reindexFullText(db)
}
}
}
}
/**
* Repopulates [FullTextSearchModule] from the events already stored in
* event_headers. Used by migrations that recreate the FTS table without
* touching the source events. Reading event_headers while inserting into
* event_fts is safe because they are different tables.
*/
private fun reindexFullText(db: SQLiteConnection) {
db.prepare("SELECT row_id, id, pubkey, created_at, kind, tags, content, sig FROM event_headers").use { stmt ->
while (stmt.step()) {
val rowId = stmt.getLong(0)
val event =
EventFactory.create<Event>(
stmt.getText(1),
stmt.getText(2),
stmt.getLong(3),
stmt.getInt(4),
OptimizedJsonMapper.fromJsonToTagArray(stmt.getText(5)),
stmt.getText(6),
stmt.getText(7),
)
fullTextSearchModule.insert(event, rowId, db)
}
}
}
quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt
+14 -14
View File
@@ -238,9 +238,9 @@ class QueryAssemblerTest : BaseDBTest() {
INNER JOIN (
SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10)
UNION
SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100)
SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100)
UNION
SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30)
SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30)
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY $orderBy
@@ -252,13 +252,13 @@ class QueryAssemblerTest : BaseDBTest() {
│ │ └── SCAN (subquery-1)
│ ├── UNION USING TEMP B-TREE
│ │ ├── CO-ROUTINE (subquery-3)
│ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
│ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
│ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ │ │ └── USE TEMP B-TREE FOR ORDER BY
│ │ └── SCAN (subquery-3)
│ └── UNION USING TEMP B-TREE
│ ├── CO-ROUTINE (subquery-5)
│ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
│ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
│ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ │ └── USE TEMP B-TREE FOR ORDER BY
│ └── SCAN (subquery-5)
@@ -275,9 +275,9 @@ class QueryAssemblerTest : BaseDBTest() {
INNER JOIN (
SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10)
UNION
SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100)
SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100)
UNION
SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30)
SELECT row_id FROM (SELECT event_fts.rowid as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.rowid WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30)
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY $orderBy
@@ -290,13 +290,13 @@ class QueryAssemblerTest : BaseDBTest() {
│ │ └── SCAN (subquery-1)
│ ├── UNION USING TEMP B-TREE
│ │ ├── CO-ROUTINE (subquery-3)
│ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
│ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
│ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ │ │ └── USE TEMP B-TREE FOR ORDER BY
│ │ └── SCAN (subquery-3)
│ └── UNION USING TEMP B-TREE
│ ├── CO-ROUTINE (subquery-5)
│ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
│ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
│ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ │ └── USE TEMP B-TREE FOR ORDER BY
│ └── SCAN (subquery-5)
@@ -712,10 +712,10 @@ class QueryAssemblerTest : BaseDBTest() {
assertEquals(
"""
SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers
INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id
INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid
WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9"))
ORDER BY event_headers.created_at DESC, event_headers.id ASC
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
""".trimIndent(),
@@ -725,10 +725,10 @@ class QueryAssemblerTest : BaseDBTest() {
assertEquals(
"""
SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers
INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id
INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid
WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9"))
ORDER BY event_headers.created_at DESC
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
""".trimIndent(),
@@ -745,10 +745,10 @@ class QueryAssemblerTest : BaseDBTest() {
assertEquals(
"""
SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers
INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id
INNER JOIN event_fts ON event_headers.row_id = event_fts.rowid
WHERE (event_fts MATCH "keywords") AND (event_headers.kind IN ("1", "1111", "10000"))
ORDER BY $orderBy
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M2
├── SCAN event_fts VIRTUAL TABLE INDEX 0:M1
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
""".trimIndent(),