From cd4edefce0884ce00e677cf4390541436e072889 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 21:00:06 +0000 Subject: [PATCH] feat(quartz): allow SQLite event store without FTS indexing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an `enableFullTextSearch` flag (default `true`) to `EventStore` and `SQLiteEventStore` so deployments that never serve NIP-50 search from SQLite — e.g. a relay that offloads search to an external engine like Vespa — can skip the full-text-search write cost. When disabled: - `FullTextSearchModule` becomes an inert no-op: the `event_fts` virtual table and its `fts_foreign_key` delete trigger are never created, inserts skip `indexableContent()` + tokenization, and both reindex entry points return immediately. - `QueryBuilder` short-circuits any query/count/delete filter carrying a non-empty `search` term to a "matches nothing" result (an empty-string search still imposes no constraint), so no SQL ever references the absent `event_fts` table. In a multi-filter union the search branch contributes nothing while the other filters resolve normally. Everything else (replaceable/addressable handling, deletions, expirations, right-to-vanish, negentropy) is unchanged, and the default keeps FTS on for existing callers. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BjzUpY8H31c7ux669zytWg --- .../nip01Core/store/sqlite/EventStore.kt | 10 +- .../store/sqlite/FullTextSearchModule.kt | 21 +- .../nip01Core/store/sqlite/QueryBuilder.kt | 36 ++- .../quartz/nip01Core/store/sqlite/README.md | 30 +++ .../store/sqlite/SQLiteEventStore.kt | 10 +- .../store/sqlite/NoFullTextSearchTest.kt | 208 ++++++++++++++++++ 6 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/NoFullTextSearchTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index 7e1c912d55..b75869f500 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -38,8 +38,16 @@ class EventStore( dbName: String? = "events.db", override val relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(), val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), + /** + * When `false`, NIP-50 full-text search indexing is turned off: no + * `event_fts` table or delete trigger is created, inserts skip the + * FTS tokenization cost, and `search` filters return no matches. + * Use for stores that never serve search from SQLite (e.g. a relay + * that offloads NIP-50 to an external engine like Vespa). + */ + val enableFullTextSearch: Boolean = true, ) : IEventStore { - val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy) + val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy, enableFullTextSearch = enableFullTextSearch) override suspend fun insert(event: Event) = store.insertEvent(event) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt index 71f55b1096..1283b78e16 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt @@ -28,13 +28,28 @@ import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress import com.vitorpamplona.quartz.nip50Search.SearchableEvent import com.vitorpamplona.quartz.utils.EventFactory -class FullTextSearchModule : IModule { +/** + * NIP-50 full-text search index over event content. + * + * When [enabled] is `false` the module becomes an inert no-op: no + * `event_fts` virtual table and no `fts_foreign_key` delete trigger are + * created, inserts skip the per-event tokenization cost, and the reindex + * entry points do nothing. This is for deployments that never query the + * SQLite store for search — e.g. a relay that offloads NIP-50 to an + * external engine such as Vespa — and don't want to pay the write-time + * FTS indexing (or the per-delete trigger) overhead. Search filters are + * handled by [QueryBuilder], which returns no matches when search is off. + */ +class FullTextSearchModule( + val enabled: Boolean = true, +) : IModule { val tableName = "event_fts" val triggerName = "fts_foreign_key" val eventHeaderRowIdName = "event_header_row_id" val contentName = "content" override fun create(db: SQLiteConnection) { + if (!enabled) return val ftsVersion = versionFinder(db) db.execSQL( """ @@ -88,6 +103,7 @@ class FullTextSearchModule : IModule { headerId: Long, db: SQLiteConnection, ) { + if (!enabled) return if (event is SearchableEvent) { db.prepare(insertFTS).use { stmt -> stmt.bindLong(1, headerId) @@ -127,6 +143,7 @@ class FullTextSearchModule : IModule { } override fun deleteAll(db: SQLiteConnection) { + if (!enabled) return db.execSQL("DELETE FROM event_fts") } @@ -144,6 +161,7 @@ class FullTextSearchModule : IModule { * Must run inside the caller's write transaction. */ fun reindexAll(db: SQLiteConnection) { + if (!enabled) return dropTrigger(db) drop(db) create(db) @@ -199,6 +217,7 @@ class FullTextSearchModule : IModule { afterRowId: Long, batchSize: Int, ): FtsReindexProgress { + if (!enabled) return FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true) // A non-positive page would select no rows yet never report done, // spinning the caller's loop forever — clamp to at least one. val limit = batchSize.coerceAtLeast(1) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 46fa1579a4..3be4353679 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -114,6 +114,12 @@ class QueryBuilder( ): QuerySpec { val newFilter = filter.toFilterWithDTags() + // With FTS off there is no event_fts table to MATCH against, so a + // search term can never be satisfied — the filter matches nothing. + if (searchDisabledMatchesNothing(newFilter.search)) { + return QuerySpec(makeMatchesNothingQuery()) + } + if (newFilter.isSimpleQuery()) { return makeSimpleQuery( project = true, @@ -215,6 +221,11 @@ class QueryBuilder( ): QuerySpec { val newFilter = filter.toFilterWithDTags() + // With FTS off a search term can never be satisfied — no matches. + if (searchDisabledMatchesNothing(newFilter.search)) { + return QuerySpec("SELECT id, created_at FROM event_headers WHERE 0") + } + // Simple path — no tag joins, no FTS — collapses to a single // SELECT against event_headers. if (newFilter.isSimpleQuery()) { @@ -389,6 +400,18 @@ class QueryBuilder( results } + /** + * True when full-text search is turned off ([FullTextSearchModule.enabled] + * is `false`) and the filter carries a non-empty `search` term. Such a + * filter can never be satisfied — there is no `event_fts` table to + * MATCH — so callers short-circuit to a "matches nothing" query. An + * empty-string search imposes no constraint and is left to the normal + * (non-search) query path, matching the FTS-enabled behaviour. + */ + private fun searchDisabledMatchesNothing(search: String?): Boolean = !fts.enabled && search != null && search.isNotEmpty() + + private fun makeMatchesNothingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers WHERE 0" + private fun makeEverythingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC${if (indexStrategy.useAndIndexIdOnOrderBy) ", id ASC" else ""}" private fun makeQueryIn(rowIdQuery: String) = @@ -480,6 +503,9 @@ class QueryBuilder( ): Int { val newFilter = filter.toFilterWithDTags() + // With FTS off a search term can never be satisfied — count is 0. + if (searchDisabledMatchesNothing(newFilter.search)) return 0 + if (newFilter.isSimpleQuery()) { val sql = makeSimpleQuery( @@ -622,7 +648,15 @@ class QueryBuilder( ): QuerySpec? { if (filter.isEmpty()) return null - val mustJoinSearch = (filter.search != null) + // With FTS off there is no event_fts table to MATCH/JOIN against. + // A non-empty search can never match, so this filter contributes + // no row ids (returns an empty branch to any UNION/COUNT/DELETE); + // an empty search imposes no constraint and is simply dropped below. + if (searchDisabledMatchesNothing(filter.search)) { + return QuerySpec("SELECT event_headers.row_id as row_id FROM event_headers WHERE 0") + } + + val mustJoinSearch = filter.search != null && fts.enabled val nonDTagsIn = filter.tags?.filter { it.key != "d" } ?: emptyMap() diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/README.md b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/README.md index afac612fcf..42781d1b77 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/README.md +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/README.md @@ -189,6 +189,36 @@ val result = eventStore.query(Filter(search = "bitcoin", limit = 20)) This will match any event whose content contains "bitcoin", returning the most recent 20 results. +#### Disabling full-text search + +FTS indexing has a write-time cost: every inserted `SearchableEvent` is +tokenized into the `event_fts` virtual table, and an `AFTER DELETE` +trigger keeps that table in sync on every deletion. If you never serve +NIP-50 search from this store — e.g. a relay that offloads search to an +external engine like Vespa — pass `enableFullTextSearch = false`: + +```kotlin +val eventStore = EventStore("dbname.db", relayUrlIdentifier, enableFullTextSearch = false) +``` + +With FTS off: + +- The `event_fts` table and its `fts_foreign_key` delete trigger are + never created (no schema dependency on FTS3/4/5 being compiled in). +- Inserts skip the per-event `indexableContent()` + tokenization work, + and deletes skip the trigger. +- Any query/count/delete filter carrying a non-empty `search` term + returns **no matches** (an empty-string search imposes no constraint + and behaves like a normal query). Everything else — replaceable and + addressable handling, deletions, expirations, right-to-vanish, + negentropy — is unchanged. + +The flag is chosen at store-construction time and only governs whether +new writes maintain the index; it does not migrate an existing database +(it won't drop an `event_fts` table left over from a store previously +opened with FTS enabled). Pick the mode when the database is first +created. + ### Periodic cleanup of expired events. The store exposes a `deleteExpiredEvents` to be used in a periodic clean up procedure. Users diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index ad9088275b..a9ef4b5c8f 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -43,6 +43,14 @@ class SQLiteEventStore( val relay: NormalizedRelayUrl? = null, val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), val numReaders: Int = 4, + /** + * When `false`, NIP-50 full-text search indexing is turned off: no + * `event_fts` table or delete trigger is created, inserts skip the + * FTS tokenization cost, and `search` filters return no matches. + * Use for stores that never serve search from SQLite (e.g. a relay + * that offloads NIP-50 to an external engine like Vespa). + */ + val enableFullTextSearch: Boolean = true, ) { companion object { const val DATABASE_VERSION = 2 @@ -50,7 +58,7 @@ class SQLiteEventStore( val seedModule = SeedModule() - val fullTextSearchModule = FullTextSearchModule() + val fullTextSearchModule = FullTextSearchModule(enableFullTextSearch) val eventIndexModule = EventIndexesModule( seedModule::hasher, diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/NoFullTextSearchTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/NoFullTextSearchTest.kt new file mode 100644 index 0000000000..bcd0063f21 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/NoFullTextSearchTest.kt @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import androidx.sqlite.SQLiteConnection +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.utils.Secp256k1Instance +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.runBlocking +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * Exercises a [SQLiteEventStore] created with `enableFullTextSearch = false`. + * + * The store must behave exactly like a normal store for every non-search + * operation while (a) never creating the `event_fts` table or its delete + * trigger and (b) treating any non-empty `search` term as matching nothing. + * This is the mode a relay uses when it offloads NIP-50 search to an + * external engine (e.g. Vespa) and doesn't want to pay the FTS index cost. + */ +class NoFullTextSearchTest { + private val signer = NostrSignerSync() + + @BeforeTest + fun setup() { + Secp256k1Instance + } + + private fun store() = SQLiteEventStore(dbName = null, enableFullTextSearch = false) + + private fun objectExists( + db: SQLiteConnection, + name: String, + ): Boolean = + db.prepare("SELECT name FROM sqlite_master WHERE name = ?").use { stmt -> + stmt.bindText(1, name) + stmt.step() + } + + @Test + fun testNoFtsTableOrTriggerIsCreated() = + runBlocking { + val store = store() + try { + // Force schema creation by opening the pool for a read. + store.pool.useReader { db -> + assertFalse(objectExists(db, "event_fts"), "event_fts table must not exist when FTS is disabled") + assertFalse(objectExists(db, "fts_foreign_key"), "fts_foreign_key trigger must not exist when FTS is disabled") + // Sanity: the canonical table is still there. + assertTrue(objectExists(db, "event_headers"), "event_headers must still be created") + } + } finally { + store.close() + } + } + + @Test + fun testInsertAndPlainQueriesStillWork() = + runBlocking { + val store = store() + try { + val note = signer.sign(TextNoteEvent.build("hello nofts world", createdAt = TimeUtils.now())) + store.insertEvent(note) + + // Query by id, kind, and author all resolve normally. + store.assertQuery(note, Filter(ids = listOf(note.id))) + store.assertQuery(note, Filter(kinds = listOf(TextNoteEvent.KIND))) + store.assertQuery(note, Filter(authors = listOf(note.pubKey))) + assertEquals(1, store.count(Filter(kinds = listOf(TextNoteEvent.KIND)))) + } finally { + store.close() + } + } + + @Test + fun testSearchFilterMatchesNothing() = + runBlocking { + val store = store() + try { + // Content clearly contains the term; with FTS off it must not match. + val note = signer.sign(TextNoteEvent.build("searchable bitcoin content", createdAt = TimeUtils.now())) + store.insertEvent(note) + + store.assertQuery(null, Filter(search = "bitcoin")) + store.assertQuery(null, Filter(kinds = listOf(TextNoteEvent.KIND), search = "bitcoin")) + store.assertQuery(null, Filter(authors = listOf(note.pubKey), search = "bitcoin")) + assertEquals(0, store.count(Filter(search = "bitcoin"))) + } finally { + store.close() + } + } + + @Test + fun testMultiFilterUnionDropsSearchBranchButKeepsOthers() = + runBlocking { + val store = store() + try { + val note = signer.sign(TextNoteEvent.build("multi filter bitcoin note", createdAt = TimeUtils.now())) + store.insertEvent(note) + + // The search branch contributes nothing; the kind branch still returns the note. + val results = + store.query( + listOf( + Filter(search = "bitcoin"), + Filter(kinds = listOf(TextNoteEvent.KIND)), + ), + ) + assertEquals(1, results.size) + assertEquals(note.id, results.first().id) + + assertEquals( + 1, + store.count( + listOf( + Filter(search = "bitcoin"), + Filter(kinds = listOf(TextNoteEvent.KIND)), + ), + ), + ) + } finally { + store.close() + } + } + + @Test + fun testDeleteWithSearchFilterRemovesNothing() = + runBlocking { + val store = store() + try { + val note = signer.sign(TextNoteEvent.build("delete me bitcoin", createdAt = TimeUtils.now())) + store.insertEvent(note) + + store.delete(Filter(search = "bitcoin")) + + // The event survives — a search delete cannot resolve to any rows. + store.assertQuery(note, Filter(ids = listOf(note.id))) + } finally { + store.close() + } + } + + @Test + fun testEmptySearchStringImposesNoConstraint() = + runBlocking { + val store = store() + try { + val note = signer.sign(TextNoteEvent.build("empty search body", createdAt = TimeUtils.now())) + store.insertEvent(note) + + // An empty search term is not a search; the filter still returns the note. + store.assertQuery(note, Filter(kinds = listOf(TextNoteEvent.KIND), search = "")) + } finally { + store.close() + } + } + + @Test + fun testReindexIsANoOp() = + runBlocking { + val store = store() + try { + val note = signer.sign(TextNoteEvent.build("reindex noop body", createdAt = TimeUtils.now())) + store.insertEvent(note) + + // Neither reindex entry point should throw or create an index. + store.reindexFullTextSearch() + + val progress = store.reindexFullTextSearch(resumeFrom = null, batchSize = 100) + assertTrue(progress.done, "batched reindex must report done immediately when FTS is disabled") + assertEquals(0, progress.processedThisBatch) + + store.pool.useReader { db -> + assertFalse(objectExists(db, "event_fts"), "reindex must not create event_fts when FTS is disabled") + } + + // Search still matches nothing after the no-op rebuild. + store.assertQuery(null, Filter(search = "reindex")) + } finally { + store.close() + } + } +}