mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-08 23:54:39 +00:00
feat(quartz): allow SQLite event store without FTS indexing
Add an `enableFullTextSearch` flag (default `true`) to `EventStore` and `SQLiteEventStore` so deployments that never serve NIP-50 search from SQLite — e.g. a relay that offloads search to an external engine like Vespa — can skip the full-text-search write cost. When disabled: - `FullTextSearchModule` becomes an inert no-op: the `event_fts` virtual table and its `fts_foreign_key` delete trigger are never created, inserts skip `indexableContent()` + tokenization, and both reindex entry points return immediately. - `QueryBuilder` short-circuits any query/count/delete filter carrying a non-empty `search` term to a "matches nothing" result (an empty-string search still imposes no constraint), so no SQL ever references the absent `event_fts` table. In a multi-filter union the search branch contributes nothing while the other filters resolve normally. Everything else (replaceable/addressable handling, deletions, expirations, right-to-vanish, negentropy) is unchanged, and the default keeps FTS on for existing callers. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BjzUpY8H31c7ux669zytWg
This commit is contained in:
Claude
+9
-1
@@ -38,8 +38,16 @@ class EventStore(
|
||||
dbName: String? = "events.db",
|
||||
override val relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(),
|
||||
val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
|
||||
/**
|
||||
* When `false`, NIP-50 full-text search indexing is turned off: no
|
||||
* `event_fts` table or delete trigger is created, inserts skip the
|
||||
* FTS tokenization cost, and `search` filters return no matches.
|
||||
* Use for stores that never serve search from SQLite (e.g. a relay
|
||||
* that offloads NIP-50 to an external engine like Vespa).
|
||||
*/
|
||||
val enableFullTextSearch: Boolean = true,
|
||||
) : IEventStore {
|
||||
val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy)
|
||||
val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy, enableFullTextSearch = enableFullTextSearch)
|
||||
|
||||
override suspend fun insert(event: Event) = store.insertEvent(event)
|
||||
|
||||
|
||||
+20
-1
@@ -28,13 +28,28 @@ import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress
|
||||
import com.vitorpamplona.quartz.nip50Search.SearchableEvent
|
||||
import com.vitorpamplona.quartz.utils.EventFactory
|
||||
|
||||
class FullTextSearchModule : IModule {
|
||||
/**
|
||||
* NIP-50 full-text search index over event content.
|
||||
*
|
||||
* When [enabled] is `false` the module becomes an inert no-op: no
|
||||
* `event_fts` virtual table and no `fts_foreign_key` delete trigger are
|
||||
* created, inserts skip the per-event tokenization cost, and the reindex
|
||||
* entry points do nothing. This is for deployments that never query the
|
||||
* SQLite store for search — e.g. a relay that offloads NIP-50 to an
|
||||
* external engine such as Vespa — and don't want to pay the write-time
|
||||
* FTS indexing (or the per-delete trigger) overhead. Search filters are
|
||||
* handled by [QueryBuilder], which returns no matches when search is off.
|
||||
*/
|
||||
class FullTextSearchModule(
|
||||
val enabled: Boolean = true,
|
||||
) : IModule {
|
||||
val tableName = "event_fts"
|
||||
val triggerName = "fts_foreign_key"
|
||||
val eventHeaderRowIdName = "event_header_row_id"
|
||||
val contentName = "content"
|
||||
|
||||
override fun create(db: SQLiteConnection) {
|
||||
if (!enabled) return
|
||||
val ftsVersion = versionFinder(db)
|
||||
db.execSQL(
|
||||
"""
|
||||
@@ -88,6 +103,7 @@ class FullTextSearchModule : IModule {
|
||||
headerId: Long,
|
||||
db: SQLiteConnection,
|
||||
) {
|
||||
if (!enabled) return
|
||||
if (event is SearchableEvent) {
|
||||
db.prepare(insertFTS).use { stmt ->
|
||||
stmt.bindLong(1, headerId)
|
||||
@@ -127,6 +143,7 @@ class FullTextSearchModule : IModule {
|
||||
}
|
||||
|
||||
override fun deleteAll(db: SQLiteConnection) {
|
||||
if (!enabled) return
|
||||
db.execSQL("DELETE FROM event_fts")
|
||||
}
|
||||
|
||||
@@ -144,6 +161,7 @@ class FullTextSearchModule : IModule {
|
||||
* Must run inside the caller's write transaction.
|
||||
*/
|
||||
fun reindexAll(db: SQLiteConnection) {
|
||||
if (!enabled) return
|
||||
dropTrigger(db)
|
||||
drop(db)
|
||||
create(db)
|
||||
@@ -199,6 +217,7 @@ class FullTextSearchModule : IModule {
|
||||
afterRowId: Long,
|
||||
batchSize: Int,
|
||||
): FtsReindexProgress {
|
||||
if (!enabled) return FtsReindexProgress(cursor = null, processedThisBatch = 0, done = true)
|
||||
// A non-positive page would select no rows yet never report done,
|
||||
// spinning the caller's loop forever — clamp to at least one.
|
||||
val limit = batchSize.coerceAtLeast(1)
|
||||
|
||||
+35
-1
@@ -114,6 +114,12 @@ class QueryBuilder(
|
||||
): QuerySpec {
|
||||
val newFilter = filter.toFilterWithDTags()
|
||||
|
||||
// With FTS off there is no event_fts table to MATCH against, so a
|
||||
// search term can never be satisfied — the filter matches nothing.
|
||||
if (searchDisabledMatchesNothing(newFilter.search)) {
|
||||
return QuerySpec(makeMatchesNothingQuery())
|
||||
}
|
||||
|
||||
if (newFilter.isSimpleQuery()) {
|
||||
return makeSimpleQuery(
|
||||
project = true,
|
||||
@@ -215,6 +221,11 @@ class QueryBuilder(
|
||||
): QuerySpec {
|
||||
val newFilter = filter.toFilterWithDTags()
|
||||
|
||||
// With FTS off a search term can never be satisfied — no matches.
|
||||
if (searchDisabledMatchesNothing(newFilter.search)) {
|
||||
return QuerySpec("SELECT id, created_at FROM event_headers WHERE 0")
|
||||
}
|
||||
|
||||
// Simple path — no tag joins, no FTS — collapses to a single
|
||||
// SELECT against event_headers.
|
||||
if (newFilter.isSimpleQuery()) {
|
||||
@@ -389,6 +400,18 @@ class QueryBuilder(
|
||||
results
|
||||
}
|
||||
|
||||
/**
|
||||
* True when full-text search is turned off ([FullTextSearchModule.enabled]
|
||||
* is `false`) and the filter carries a non-empty `search` term. Such a
|
||||
* filter can never be satisfied — there is no `event_fts` table to
|
||||
* MATCH — so callers short-circuit to a "matches nothing" query. An
|
||||
* empty-string search imposes no constraint and is left to the normal
|
||||
* (non-search) query path, matching the FTS-enabled behaviour.
|
||||
*/
|
||||
private fun searchDisabledMatchesNothing(search: String?): Boolean = !fts.enabled && search != null && search.isNotEmpty()
|
||||
|
||||
private fun makeMatchesNothingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers WHERE 0"
|
||||
|
||||
private fun makeEverythingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC${if (indexStrategy.useAndIndexIdOnOrderBy) ", id ASC" else ""}"
|
||||
|
||||
private fun makeQueryIn(rowIdQuery: String) =
|
||||
@@ -480,6 +503,9 @@ class QueryBuilder(
|
||||
): Int {
|
||||
val newFilter = filter.toFilterWithDTags()
|
||||
|
||||
// With FTS off a search term can never be satisfied — count is 0.
|
||||
if (searchDisabledMatchesNothing(newFilter.search)) return 0
|
||||
|
||||
if (newFilter.isSimpleQuery()) {
|
||||
val sql =
|
||||
makeSimpleQuery(
|
||||
@@ -622,7 +648,15 @@ class QueryBuilder(
|
||||
): QuerySpec? {
|
||||
if (filter.isEmpty()) return null
|
||||
|
||||
val mustJoinSearch = (filter.search != null)
|
||||
// With FTS off there is no event_fts table to MATCH/JOIN against.
|
||||
// A non-empty search can never match, so this filter contributes
|
||||
// no row ids (returns an empty branch to any UNION/COUNT/DELETE);
|
||||
// an empty search imposes no constraint and is simply dropped below.
|
||||
if (searchDisabledMatchesNothing(filter.search)) {
|
||||
return QuerySpec("SELECT event_headers.row_id as row_id FROM event_headers WHERE 0")
|
||||
}
|
||||
|
||||
val mustJoinSearch = filter.search != null && fts.enabled
|
||||
|
||||
val nonDTagsIn = filter.tags?.filter { it.key != "d" } ?: emptyMap()
|
||||
|
||||
|
||||
+30
@@ -189,6 +189,36 @@ val result = eventStore.query(Filter(search = "bitcoin", limit = 20))
|
||||
|
||||
This will match any event whose content contains "bitcoin", returning the most recent 20 results.
|
||||
|
||||
#### Disabling full-text search
|
||||
|
||||
FTS indexing has a write-time cost: every inserted `SearchableEvent` is
|
||||
tokenized into the `event_fts` virtual table, and an `AFTER DELETE`
|
||||
trigger keeps that table in sync on every deletion. If you never serve
|
||||
NIP-50 search from this store — e.g. a relay that offloads search to an
|
||||
external engine like Vespa — pass `enableFullTextSearch = false`:
|
||||
|
||||
```kotlin
|
||||
val eventStore = EventStore("dbname.db", relayUrlIdentifier, enableFullTextSearch = false)
|
||||
```
|
||||
|
||||
With FTS off:
|
||||
|
||||
- The `event_fts` table and its `fts_foreign_key` delete trigger are
|
||||
never created (no schema dependency on FTS3/4/5 being compiled in).
|
||||
- Inserts skip the per-event `indexableContent()` + tokenization work,
|
||||
and deletes skip the trigger.
|
||||
- Any query/count/delete filter carrying a non-empty `search` term
|
||||
returns **no matches** (an empty-string search imposes no constraint
|
||||
and behaves like a normal query). Everything else — replaceable and
|
||||
addressable handling, deletions, expirations, right-to-vanish,
|
||||
negentropy — is unchanged.
|
||||
|
||||
The flag is chosen at store-construction time and only governs whether
|
||||
new writes maintain the index; it does not migrate an existing database
|
||||
(it won't drop an `event_fts` table left over from a store previously
|
||||
opened with FTS enabled). Pick the mode when the database is first
|
||||
created.
|
||||
|
||||
### Periodic cleanup of expired events.
|
||||
|
||||
The store exposes a `deleteExpiredEvents` to be used in a periodic clean up procedure. Users
|
||||
|
||||
+9
-1
@@ -43,6 +43,14 @@ class SQLiteEventStore(
|
||||
val relay: NormalizedRelayUrl? = null,
|
||||
val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
|
||||
val numReaders: Int = 4,
|
||||
/**
|
||||
* When `false`, NIP-50 full-text search indexing is turned off: no
|
||||
* `event_fts` table or delete trigger is created, inserts skip the
|
||||
* FTS tokenization cost, and `search` filters return no matches.
|
||||
* Use for stores that never serve search from SQLite (e.g. a relay
|
||||
* that offloads NIP-50 to an external engine like Vespa).
|
||||
*/
|
||||
val enableFullTextSearch: Boolean = true,
|
||||
) {
|
||||
companion object {
|
||||
const val DATABASE_VERSION = 2
|
||||
@@ -50,7 +58,7 @@ class SQLiteEventStore(
|
||||
|
||||
val seedModule = SeedModule()
|
||||
|
||||
val fullTextSearchModule = FullTextSearchModule()
|
||||
val fullTextSearchModule = FullTextSearchModule(enableFullTextSearch)
|
||||
val eventIndexModule =
|
||||
EventIndexesModule(
|
||||
seedModule::hasher,
|
||||
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.quartz.nip01Core.store.sqlite
|
||||
|
||||
import androidx.sqlite.SQLiteConnection
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
|
||||
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
|
||||
import com.vitorpamplona.quartz.utils.Secp256k1Instance
|
||||
import com.vitorpamplona.quartz.utils.TimeUtils
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlin.test.BeforeTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
/**
|
||||
* Exercises a [SQLiteEventStore] created with `enableFullTextSearch = false`.
|
||||
*
|
||||
* The store must behave exactly like a normal store for every non-search
|
||||
* operation while (a) never creating the `event_fts` table or its delete
|
||||
* trigger and (b) treating any non-empty `search` term as matching nothing.
|
||||
* This is the mode a relay uses when it offloads NIP-50 search to an
|
||||
* external engine (e.g. Vespa) and doesn't want to pay the FTS index cost.
|
||||
*/
|
||||
class NoFullTextSearchTest {
|
||||
private val signer = NostrSignerSync()
|
||||
|
||||
@BeforeTest
|
||||
fun setup() {
|
||||
Secp256k1Instance
|
||||
}
|
||||
|
||||
private fun store() = SQLiteEventStore(dbName = null, enableFullTextSearch = false)
|
||||
|
||||
private fun objectExists(
|
||||
db: SQLiteConnection,
|
||||
name: String,
|
||||
): Boolean =
|
||||
db.prepare("SELECT name FROM sqlite_master WHERE name = ?").use { stmt ->
|
||||
stmt.bindText(1, name)
|
||||
stmt.step()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testNoFtsTableOrTriggerIsCreated() =
|
||||
runBlocking {
|
||||
val store = store()
|
||||
try {
|
||||
// Force schema creation by opening the pool for a read.
|
||||
store.pool.useReader { db ->
|
||||
assertFalse(objectExists(db, "event_fts"), "event_fts table must not exist when FTS is disabled")
|
||||
assertFalse(objectExists(db, "fts_foreign_key"), "fts_foreign_key trigger must not exist when FTS is disabled")
|
||||
// Sanity: the canonical table is still there.
|
||||
assertTrue(objectExists(db, "event_headers"), "event_headers must still be created")
|
||||
}
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testInsertAndPlainQueriesStillWork() =
|
||||
runBlocking {
|
||||
val store = store()
|
||||
try {
|
||||
val note = signer.sign(TextNoteEvent.build("hello nofts world", createdAt = TimeUtils.now()))
|
||||
store.insertEvent(note)
|
||||
|
||||
// Query by id, kind, and author all resolve normally.
|
||||
store.assertQuery(note, Filter(ids = listOf(note.id)))
|
||||
store.assertQuery(note, Filter(kinds = listOf(TextNoteEvent.KIND)))
|
||||
store.assertQuery(note, Filter(authors = listOf(note.pubKey)))
|
||||
assertEquals(1, store.count(Filter(kinds = listOf(TextNoteEvent.KIND))))
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testSearchFilterMatchesNothing() =
|
||||
runBlocking {
|
||||
val store = store()
|
||||
try {
|
||||
// Content clearly contains the term; with FTS off it must not match.
|
||||
val note = signer.sign(TextNoteEvent.build("searchable bitcoin content", createdAt = TimeUtils.now()))
|
||||
store.insertEvent(note)
|
||||
|
||||
store.assertQuery(null, Filter(search = "bitcoin"))
|
||||
store.assertQuery(null, Filter(kinds = listOf(TextNoteEvent.KIND), search = "bitcoin"))
|
||||
store.assertQuery(null, Filter(authors = listOf(note.pubKey), search = "bitcoin"))
|
||||
assertEquals(0, store.count(Filter(search = "bitcoin")))
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testMultiFilterUnionDropsSearchBranchButKeepsOthers() =
|
||||
runBlocking {
|
||||
val store = store()
|
||||
try {
|
||||
val note = signer.sign(TextNoteEvent.build("multi filter bitcoin note", createdAt = TimeUtils.now()))
|
||||
store.insertEvent(note)
|
||||
|
||||
// The search branch contributes nothing; the kind branch still returns the note.
|
||||
val results =
|
||||
store.query<Event>(
|
||||
listOf(
|
||||
Filter(search = "bitcoin"),
|
||||
Filter(kinds = listOf(TextNoteEvent.KIND)),
|
||||
),
|
||||
)
|
||||
assertEquals(1, results.size)
|
||||
assertEquals(note.id, results.first().id)
|
||||
|
||||
assertEquals(
|
||||
1,
|
||||
store.count(
|
||||
listOf(
|
||||
Filter(search = "bitcoin"),
|
||||
Filter(kinds = listOf(TextNoteEvent.KIND)),
|
||||
),
|
||||
),
|
||||
)
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testDeleteWithSearchFilterRemovesNothing() =
|
||||
runBlocking {
|
||||
val store = store()
|
||||
try {
|
||||
val note = signer.sign(TextNoteEvent.build("delete me bitcoin", createdAt = TimeUtils.now()))
|
||||
store.insertEvent(note)
|
||||
|
||||
store.delete(Filter(search = "bitcoin"))
|
||||
|
||||
// The event survives — a search delete cannot resolve to any rows.
|
||||
store.assertQuery(note, Filter(ids = listOf(note.id)))
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testEmptySearchStringImposesNoConstraint() =
|
||||
runBlocking {
|
||||
val store = store()
|
||||
try {
|
||||
val note = signer.sign(TextNoteEvent.build("empty search body", createdAt = TimeUtils.now()))
|
||||
store.insertEvent(note)
|
||||
|
||||
// An empty search term is not a search; the filter still returns the note.
|
||||
store.assertQuery(note, Filter(kinds = listOf(TextNoteEvent.KIND), search = ""))
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testReindexIsANoOp() =
|
||||
runBlocking {
|
||||
val store = store()
|
||||
try {
|
||||
val note = signer.sign(TextNoteEvent.build("reindex noop body", createdAt = TimeUtils.now()))
|
||||
store.insertEvent(note)
|
||||
|
||||
// Neither reindex entry point should throw or create an index.
|
||||
store.reindexFullTextSearch()
|
||||
|
||||
val progress = store.reindexFullTextSearch(resumeFrom = null, batchSize = 100)
|
||||
assertTrue(progress.done, "batched reindex must report done immediately when FTS is disabled")
|
||||
assertEquals(0, progress.processedThisBatch)
|
||||
|
||||
store.pool.useReader { db ->
|
||||
assertFalse(objectExists(db, "event_fts"), "reindex must not create event_fts when FTS is disabled")
|
||||
}
|
||||
|
||||
// Search still matches nothing after the no-op rebuild.
|
||||
store.assertQuery(null, Filter(search = "reindex"))
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user