fix(quartz): strip NIP-50 extension tokens before FTS MATCH in the relay store path

Raw key:value tokens like include:spam reached SQLite FTS MATCH, where
the colon is column-filter syntax — any REQ carrying an extension token
died with CLOSED "no such column: include" instead of matching.

Adds SearchQuery.stripExtensions() plus Filter/List<Filter>
.strippingSearchExtensions() so EventStore users can drop the tokens
before querying, and applies them in LiveEventStore (query, count,
negentropy snapshots). Per NIP-50, unsupported extensions are ignored:
an extensions-only search becomes unconstrained, not match-nothing.
EventSource-backed search relays still receive the raw string since a
real search backend wants the extensions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Tujoyfc2kNLZNVgiLZAR7F

quartz/RELAY.md

@@ -257,6 +257,16 @@ q.domain       // "example.com"
 q.nsfwIncluded // false  (NIP-50 default is true when the token is absent)
 ```
 
+The SQLite store expects `Filter.search` to be plain FTS text: `:` is FTS5
+column-filter syntax, so a raw `include:spam` reaching MATCH raises "no such
+column: include" instead of matching. Strip the tokens before querying with
+`SearchQuery.stripExtensions(raw)` or `filter.strippingSearchExtensions()` —
+an extensions-only query collapses to an empty search, which imposes no
+constraint (NIP-50: unsupported extensions are ignored, not match-nothing).
+The storage-backed server path (`NostrServer` / `LiveEventStore`) already does
+this, so relays like geode comply out of the box; `EventSource` backends get
+the raw string because a real search backend wants the extensions.
+
 A search/redirector relay is just a custom policy (or, for computed results, a
 custom `IEventStore` whose `query` answers the REQ) that reads the parsed query:
 

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt

@@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
 import com.vitorpamplona.quartz.nip01Core.relay.filters.FilterIndex
 import com.vitorpamplona.quartz.nip01Core.store.IEventStore
 import com.vitorpamplona.quartz.nip01Core.store.IdAndTime
+import com.vitorpamplona.quartz.nip50Search.strippingSearchExtensions
 import kotlinx.coroutines.CompletableDeferred
 import kotlinx.coroutines.awaitCancellation
 import kotlin.concurrent.atomics.AtomicBoolean
@@ -44,6 +45,15 @@ import kotlin.concurrent.atomics.ExperimentalAtomicApi
  * match. This avoids the quadratic O(N_subscribers × N_filters_per_sub) per-event walk
  * that a SharedFlow-based broadcast would do.
  *
+ * NIP-50 `search` strings are handed to the store with their `key:value`
+ * extension tokens stripped ([strippingSearchExtensions]): the SQLite FTS
+ * backend treats `:` as column-filter syntax, so a raw `include:spam`
+ * would raise "no such column" instead of matching. Per NIP-50, an
+ * unsupported extension is ignored — an extensions-only search therefore
+ * becomes unconstrained, not match-nothing. Relays that *do* implement
+ * extensions serve search through an [EventSource] backend, which
+ * receives the raw string.
+ *
  * @property store The underlying persistent storage for events.
  * @property ingest The group-commit writer pipeline. Accepted events fan out via the
  *  index; rejected events do not.
@@ -177,7 +187,7 @@ class LiveEventStore(
 
         index.register(filters, sub)
         try {
-            store.query<Event>(filters) { event ->
+            store.query<Event>(filters.strippingSearchExtensions()) { event ->
                 seenLocked { seenIds?.add(event.id) }
                 onEach(event)
             }
@@ -198,14 +208,14 @@ class LiveEventStore(
     override suspend fun count(
         ctx: RequestContext,
         filters: List<Filter>,
-    ): Int = store.count(filters)
+    ): Int = store.count(filters.strippingSearchExtensions())
 
     /**
      * One-shot snapshot query. Used by NIP-77 negentropy: the server
      * needs the full set of event ids matching the filter at the
      * moment the NEG-OPEN arrives, not a streamed/live result.
      */
-    suspend fun snapshotQuery(filter: Filter): List<Event> = store.query(filter)
+    suspend fun snapshotQuery(filter: Filter): List<Event> = store.query(filter.strippingSearchExtensions())
 
     /**
      * Multi-filter snapshot. Unions the per-filter results and
@@ -218,7 +228,7 @@ class LiveEventStore(
         val seen = HashSet<String>()
         val merged = ArrayList<Event>()
         for (f in filters) {
-            for (e in store.query<Event>(f)) {
+            for (e in store.query<Event>(f.strippingSearchExtensions())) {
                 if (seen.add(e.id)) merged += e
             }
         }
@@ -238,5 +248,5 @@ class LiveEventStore(
     override suspend fun snapshotIdsForNegentropy(
         filters: List<Filter>,
         maxEntries: Int?,
-    ): List<IdAndTime> = store.snapshotIdsForNegentropy(filters, maxEntries)
+    ): List<IdAndTime> = store.snapshotIdsForNegentropy(filters.strippingSearchExtensions(), maxEntries)
 }

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt

@@ -20,6 +20,8 @@
  */
 package com.vitorpamplona.quartz.nip50Search
 
+import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
+
 /**
  * Parsed representation of a NIP-50 `search` filter string.
  *
@@ -163,9 +165,55 @@ class SearchQuery(
         }
 
         private fun isExtensionKey(key: String): Boolean = key.isNotEmpty() && key.all { it in 'a'..'z' }
+
+        /**
+         * Returns [search] with every `key:value` extension token removed,
+         * leaving only the free-text terms (tokenized as in [parse]).
+         *
+         * Backends that hand the search string to an engine with its own
+         * query syntax — e.g. SQLite FTS, where `:` is column-filter
+         * syntax and `include:spam` raises "no such column" — must call
+         * this (or apply the extensions themselves) before querying.
+         *
+         * A string with no extension tokens is returned unchanged. An
+         * extensions-only query collapses to `""`, which event stores
+         * treat as "no search constraint" — the NIP-50 behaviour for
+         * relays that don't support an extension is to ignore it, not to
+         * return nothing.
+         */
+        fun stripExtensions(search: String?): String? {
+            if (search.isNullOrBlank()) return search
+            val parsed = parse(search)
+            return if (parsed.extensions.isEmpty()) search else parsed.terms
+        }
     }
 }
 
+/**
+ * Returns a copy of this filter whose `search` string has all NIP-50
+ * extension tokens removed (see [SearchQuery.stripExtensions]), or this
+ * same instance when there is nothing to strip.
+ */
+fun Filter.strippingSearchExtensions(): Filter {
+    val stripped = SearchQuery.stripExtensions(search)
+    return if (stripped == search) this else copy(search = stripped)
+}
+
+/**
+ * Applies [strippingSearchExtensions] to every filter, returning this
+ * same list when no filter carried extension tokens.
+ */
+fun List<Filter>.strippingSearchExtensions(): List<Filter> {
+    var changed = false
+    val out =
+        map {
+            val stripped = it.strippingSearchExtensions()
+            if (stripped !== it) changed = true
+            stripped
+        }
+    return if (changed) out else this
+}
+
 /** NIP-50 `sentiment:` extension values. */
 enum class Sentiment(
     val code: String,

quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NostrServerTest.kt

@@ -30,6 +30,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.server.policies.EmptyPolicy
 import com.vitorpamplona.quartz.nip01Core.relay.server.policies.IRelayPolicy
 import com.vitorpamplona.quartz.nip01Core.store.IEventStore
 import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
+import com.vitorpamplona.quartz.utils.EventFactory
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.test.UnconfinedTestDispatcher
 import kotlinx.coroutines.test.runTest
@@ -199,6 +200,44 @@ class NostrServerTest {
             server.close()
         }
 
+    @Test
+    fun reqWithSearchExtensionsIsAnsweredNotErrored() =
+        runTest {
+            val dispatcher = UnconfinedTestDispatcher(testScheduler)
+            val store = EventStore(null)
+            val server = createServer(dispatcher, store)
+
+            // Build through EventFactory so kind 1 materializes as a
+            // TextNoteEvent — only SearchableEvent instances are FTS-indexed.
+            store.insert(EventFactory.create(hexId(1), pubkey, 100L, 1, emptyArray(), "bitcoin is money", sig))
+            store.insert(EventFactory.create(hexId(2), pubkey, 200L, 1, emptyArray(), "coffee time", sig))
+
+            val collector = MessageCollector()
+            val c1 = server.connect(collector.sendCallback)
+
+            // The raw string would be FTS5 syntax ("no such column: include")
+            // if it reached MATCH unstripped — the REQ must answer, not CLOSE.
+            c1.receive("""["REQ","sub1",{"search":"bitcoin include:spam"}]""")
+
+            var events = collector.parsedEventMessages().filterIsInstance<EventMessage>()
+            assertEquals(1, events.size)
+            assertEquals("bitcoin is money", events[0].event.content)
+            assertEquals(1, collector.parsedEventMessages().filterIsInstance<EoseMessage>().size)
+            assertTrue(collector.rawMessagesContaining("CLOSED").isEmpty())
+
+            // Extensions-only search: NIP-50 ignores unsupported extensions,
+            // so the search constraint drops and the filter is unconstrained.
+            c1.receive("""["CLOSE","sub1"]""")
+            collector.messages.clear()
+            c1.receive("""["REQ","sub2",{"search":"language:en"}]""")
+
+            events = collector.parsedEventMessages().filterIsInstance<EventMessage>()
+            assertEquals(2, events.size)
+            assertTrue(collector.rawMessagesContaining("CLOSED").isEmpty())
+
+            server.close()
+        }
+
     // -- Live subscription -----------------------------------------------------
 
     @Test

quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt

@@ -20,10 +20,12 @@
  */
 package com.vitorpamplona.quartz.nip50Search
 
+import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertFalse
 import kotlin.test.assertNull
+import kotlin.test.assertSame
 import kotlin.test.assertTrue
 
 class SearchQueryTest {
@@ -144,4 +146,57 @@ class SearchQueryTest {
         val q = SearchQuery.parse("nsfw:false")
         assertEquals("nsfw:false", q.toSearchString())
     }
+
+    @Test
+    fun stripExtensionsRemovesTokens() {
+        assertEquals("bitcoin", SearchQuery.stripExtensions("bitcoin include:spam"))
+        assertEquals("best nostr apps", SearchQuery.stripExtensions("best domain:example.com nostr language:en apps"))
+        // Unknown extension keys are stripped too — the store can't apply them either.
+        assertEquals("hello", SearchQuery.stripExtensions("hello foo:bar"))
+    }
+
+    @Test
+    fun stripExtensionsOnExtensionsOnlyYieldsEmpty() {
+        // Empty, not null: an empty search imposes no constraint (NIP-50
+        // says unsupported extensions are ignored, not match-nothing).
+        assertEquals("", SearchQuery.stripExtensions("include:spam nsfw:false"))
+    }
+
+    @Test
+    fun stripExtensionsPassesPlainQueriesThrough() {
+        assertNull(SearchQuery.stripExtensions(null))
+        assertEquals("", SearchQuery.stripExtensions(""))
+        assertEquals("  ", SearchQuery.stripExtensions("  "))
+        assertEquals("best nostr apps", SearchQuery.stripExtensions("best nostr apps"))
+        // URLs and uppercase-key tokens are terms, not extensions.
+        assertEquals("check https://example.com out", SearchQuery.stripExtensions("check https://example.com out"))
+        assertEquals("NASA:cool stuff", SearchQuery.stripExtensions("NASA:cool stuff"))
+    }
+
+    @Test
+    fun filterStrippingSearchExtensions() {
+        val filter = Filter(kinds = listOf(1), search = "bitcoin include:spam")
+        val stripped = filter.strippingSearchExtensions()
+        assertEquals("bitcoin", stripped.search)
+        assertEquals(listOf(1), stripped.kinds)
+
+        // Nothing to strip -> same instance, no copy.
+        val plain = Filter(kinds = listOf(1), search = "bitcoin")
+        assertSame(plain, plain.strippingSearchExtensions())
+        val noSearch = Filter(kinds = listOf(1))
+        assertSame(noSearch, noSearch.strippingSearchExtensions())
+    }
+
+    @Test
+    fun filterListStrippingSearchExtensions() {
+        val plain = Filter(search = "bitcoin")
+        val extended = Filter(search = "bitcoin language:en")
+
+        val untouched = listOf(plain, Filter(kinds = listOf(1)))
+        assertSame(untouched, untouched.strippingSearchExtensions())
+
+        val mixed = listOf(plain, extended).strippingSearchExtensions()
+        assertSame(plain, mixed[0])
+        assertEquals("bitcoin", mixed[1].search)
+    }
 }