diff --git a/quartz/RELAY.md b/quartz/RELAY.md index 8ff7ee7dcb..807b65caee 100644 --- a/quartz/RELAY.md +++ b/quartz/RELAY.md @@ -257,6 +257,16 @@ q.domain // "example.com" q.nsfwIncluded // false (NIP-50 default is true when the token is absent) ``` +The SQLite store expects `Filter.search` to be plain FTS text: `:` is FTS5 +column-filter syntax, so a raw `include:spam` reaching MATCH raises "no such +column: include" instead of matching. Strip the tokens before querying with +`SearchQuery.stripExtensions(raw)` or `filter.strippingSearchExtensions()` — +an extensions-only query collapses to an empty search, which imposes no +constraint (NIP-50: unsupported extensions are ignored, not match-nothing). +The storage-backed server path (`NostrServer` / `LiveEventStore`) already does +this, so relays like geode comply out of the box; `EventSource` backends get +the raw string because a real search backend wants the extensions. + A search/redirector relay is just a custom policy (or, for computed results, a custom `IEventStore` whose `query` answers the REQ) that reads the parsed query: diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt index 9fc4ece2e3..83f3947800 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.filters.FilterIndex import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.IdAndTime +import com.vitorpamplona.quartz.nip50Search.strippingSearchExtensions import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.awaitCancellation import kotlin.concurrent.atomics.AtomicBoolean @@ -44,6 +45,15 @@ import kotlin.concurrent.atomics.ExperimentalAtomicApi * match. This avoids the quadratic O(N_subscribers × N_filters_per_sub) per-event walk * that a SharedFlow-based broadcast would do. * + * NIP-50 `search` strings are handed to the store with their `key:value` + * extension tokens stripped ([strippingSearchExtensions]): the SQLite FTS + * backend treats `:` as column-filter syntax, so a raw `include:spam` + * would raise "no such column" instead of matching. Per NIP-50, an + * unsupported extension is ignored — an extensions-only search therefore + * becomes unconstrained, not match-nothing. Relays that *do* implement + * extensions serve search through an [EventSource] backend, which + * receives the raw string. + * * @property store The underlying persistent storage for events. * @property ingest The group-commit writer pipeline. Accepted events fan out via the * index; rejected events do not. @@ -177,7 +187,7 @@ class LiveEventStore( index.register(filters, sub) try { - store.query(filters) { event -> + store.query(filters.strippingSearchExtensions()) { event -> seenLocked { seenIds?.add(event.id) } onEach(event) } @@ -198,14 +208,14 @@ class LiveEventStore( override suspend fun count( ctx: RequestContext, filters: List, - ): Int = store.count(filters) + ): Int = store.count(filters.strippingSearchExtensions()) /** * One-shot snapshot query. Used by NIP-77 negentropy: the server * needs the full set of event ids matching the filter at the * moment the NEG-OPEN arrives, not a streamed/live result. */ - suspend fun snapshotQuery(filter: Filter): List = store.query(filter) + suspend fun snapshotQuery(filter: Filter): List = store.query(filter.strippingSearchExtensions()) /** * Multi-filter snapshot. Unions the per-filter results and @@ -218,7 +228,7 @@ class LiveEventStore( val seen = HashSet() val merged = ArrayList() for (f in filters) { - for (e in store.query(f)) { + for (e in store.query(f.strippingSearchExtensions())) { if (seen.add(e.id)) merged += e } } @@ -238,5 +248,5 @@ class LiveEventStore( override suspend fun snapshotIdsForNegentropy( filters: List, maxEntries: Int?, - ): List = store.snapshotIdsForNegentropy(filters, maxEntries) + ): List = store.snapshotIdsForNegentropy(filters.strippingSearchExtensions(), maxEntries) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt index 0945a50a2f..12b3af995d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt @@ -20,6 +20,8 @@ */ package com.vitorpamplona.quartz.nip50Search +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter + /** * Parsed representation of a NIP-50 `search` filter string. * @@ -163,9 +165,55 @@ class SearchQuery( } private fun isExtensionKey(key: String): Boolean = key.isNotEmpty() && key.all { it in 'a'..'z' } + + /** + * Returns [search] with every `key:value` extension token removed, + * leaving only the free-text terms (tokenized as in [parse]). + * + * Backends that hand the search string to an engine with its own + * query syntax — e.g. SQLite FTS, where `:` is column-filter + * syntax and `include:spam` raises "no such column" — must call + * this (or apply the extensions themselves) before querying. + * + * A string with no extension tokens is returned unchanged. An + * extensions-only query collapses to `""`, which event stores + * treat as "no search constraint" — the NIP-50 behaviour for + * relays that don't support an extension is to ignore it, not to + * return nothing. + */ + fun stripExtensions(search: String?): String? { + if (search.isNullOrBlank()) return search + val parsed = parse(search) + return if (parsed.extensions.isEmpty()) search else parsed.terms + } } } +/** + * Returns a copy of this filter whose `search` string has all NIP-50 + * extension tokens removed (see [SearchQuery.stripExtensions]), or this + * same instance when there is nothing to strip. + */ +fun Filter.strippingSearchExtensions(): Filter { + val stripped = SearchQuery.stripExtensions(search) + return if (stripped == search) this else copy(search = stripped) +} + +/** + * Applies [strippingSearchExtensions] to every filter, returning this + * same list when no filter carried extension tokens. + */ +fun List.strippingSearchExtensions(): List { + var changed = false + val out = + map { + val stripped = it.strippingSearchExtensions() + if (stripped !== it) changed = true + stripped + } + return if (changed) out else this +} + /** NIP-50 `sentiment:` extension values. */ enum class Sentiment( val code: String, diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NostrServerTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NostrServerTest.kt index d0b1e9df46..7171a9e4a0 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NostrServerTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NostrServerTest.kt @@ -30,6 +30,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.server.policies.EmptyPolicy import com.vitorpamplona.quartz.nip01Core.relay.server.policies.IRelayPolicy import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.utils.EventFactory import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.UnconfinedTestDispatcher import kotlinx.coroutines.test.runTest @@ -199,6 +200,44 @@ class NostrServerTest { server.close() } + @Test + fun reqWithSearchExtensionsIsAnsweredNotErrored() = + runTest { + val dispatcher = UnconfinedTestDispatcher(testScheduler) + val store = EventStore(null) + val server = createServer(dispatcher, store) + + // Build through EventFactory so kind 1 materializes as a + // TextNoteEvent — only SearchableEvent instances are FTS-indexed. + store.insert(EventFactory.create(hexId(1), pubkey, 100L, 1, emptyArray(), "bitcoin is money", sig)) + store.insert(EventFactory.create(hexId(2), pubkey, 200L, 1, emptyArray(), "coffee time", sig)) + + val collector = MessageCollector() + val c1 = server.connect(collector.sendCallback) + + // The raw string would be FTS5 syntax ("no such column: include") + // if it reached MATCH unstripped — the REQ must answer, not CLOSE. + c1.receive("""["REQ","sub1",{"search":"bitcoin include:spam"}]""") + + var events = collector.parsedEventMessages().filterIsInstance() + assertEquals(1, events.size) + assertEquals("bitcoin is money", events[0].event.content) + assertEquals(1, collector.parsedEventMessages().filterIsInstance().size) + assertTrue(collector.rawMessagesContaining("CLOSED").isEmpty()) + + // Extensions-only search: NIP-50 ignores unsupported extensions, + // so the search constraint drops and the filter is unconstrained. + c1.receive("""["CLOSE","sub1"]""") + collector.messages.clear() + c1.receive("""["REQ","sub2",{"search":"language:en"}]""") + + events = collector.parsedEventMessages().filterIsInstance() + assertEquals(2, events.size) + assertTrue(collector.rawMessagesContaining("CLOSED").isEmpty()) + + server.close() + } + // -- Live subscription ----------------------------------------------------- @Test diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt index 23bdcee0ce..4864e04162 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt @@ -20,10 +20,12 @@ */ package com.vitorpamplona.quartz.nip50Search +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertNull +import kotlin.test.assertSame import kotlin.test.assertTrue class SearchQueryTest { @@ -144,4 +146,57 @@ class SearchQueryTest { val q = SearchQuery.parse("nsfw:false") assertEquals("nsfw:false", q.toSearchString()) } + + @Test + fun stripExtensionsRemovesTokens() { + assertEquals("bitcoin", SearchQuery.stripExtensions("bitcoin include:spam")) + assertEquals("best nostr apps", SearchQuery.stripExtensions("best domain:example.com nostr language:en apps")) + // Unknown extension keys are stripped too — the store can't apply them either. + assertEquals("hello", SearchQuery.stripExtensions("hello foo:bar")) + } + + @Test + fun stripExtensionsOnExtensionsOnlyYieldsEmpty() { + // Empty, not null: an empty search imposes no constraint (NIP-50 + // says unsupported extensions are ignored, not match-nothing). + assertEquals("", SearchQuery.stripExtensions("include:spam nsfw:false")) + } + + @Test + fun stripExtensionsPassesPlainQueriesThrough() { + assertNull(SearchQuery.stripExtensions(null)) + assertEquals("", SearchQuery.stripExtensions("")) + assertEquals(" ", SearchQuery.stripExtensions(" ")) + assertEquals("best nostr apps", SearchQuery.stripExtensions("best nostr apps")) + // URLs and uppercase-key tokens are terms, not extensions. + assertEquals("check https://example.com out", SearchQuery.stripExtensions("check https://example.com out")) + assertEquals("NASA:cool stuff", SearchQuery.stripExtensions("NASA:cool stuff")) + } + + @Test + fun filterStrippingSearchExtensions() { + val filter = Filter(kinds = listOf(1), search = "bitcoin include:spam") + val stripped = filter.strippingSearchExtensions() + assertEquals("bitcoin", stripped.search) + assertEquals(listOf(1), stripped.kinds) + + // Nothing to strip -> same instance, no copy. + val plain = Filter(kinds = listOf(1), search = "bitcoin") + assertSame(plain, plain.strippingSearchExtensions()) + val noSearch = Filter(kinds = listOf(1)) + assertSame(noSearch, noSearch.strippingSearchExtensions()) + } + + @Test + fun filterListStrippingSearchExtensions() { + val plain = Filter(search = "bitcoin") + val extended = Filter(search = "bitcoin language:en") + + val untouched = listOf(plain, Filter(kinds = listOf(1))) + assertSame(untouched, untouched.strippingSearchExtensions()) + + val mixed = listOf(plain, extended).strippingSearchExtensions() + assertSame(plain, mixed[0]) + assertEquals("bitcoin", mixed[1].search) + } }