diff --git a/.claude/skills/searchable-events/references/searchable-kinds.md b/.claude/skills/searchable-events/references/searchable-kinds.md index 6c5ef8797d..dfd087a07e 100644 --- a/.claude/skills/searchable-events/references/searchable-kinds.md +++ b/.claude/skills/searchable-events/references/searchable-kinds.md @@ -2,7 +2,7 @@ Every concrete `SearchableEvent` implementor in Quartz, with the exact `indexableContent()` expression. **Update this file in the same PR as any change to the searchable set or to an -`indexableContent()` body** (see SKILL.md). Verified against the code 2026-09-17. +`indexableContent()` body** (see SKILL.md). Verified against the code 2026-10-04. Counts: 139 concrete classes covering 140 kind values (`GitStatusEvent` spans 4 kinds; kind 30063 is shared by two NIPs and kind 38000 by three classes — see the footnotes). File paths are under @@ -65,7 +65,7 @@ Separator legend: **NL** = `joinToString("\n")`, **SP** = `joinToString(" ")`. | 9736 | Bolt12ZapEvent | nipB1Bolt12Zaps/zap | `content` | | 9737 | Bolt12ZapIntentEvent | nipB1Bolt12Zaps/intent | `content` | | 9802 | HighlightEvent | nip84Highlights | `listOfNotNull(comment(), context(), content)` NL | -| 9998 | ListHeaderEvent | experimental/decentralizedLists/header | `tags.searchableListContent()` NL — `names` (singular, plural), `titles` (singular, plural), `name`, `title`, `description`, `comments`, then every `t` value; ids/pubkeys/coordinates are left to tag filters | +| 9998 | ListHeaderEvent | experimental/decentralizedLists/header | `tags.searchableListContent()` NL — `names` (singular, plural), `titles` (singular, plural), `name`, `title`, `description`, `comments`, then every `t` value that is not `isMachineValue`, then every value of every other tag (except `alt`, `client`) that passes `isNaturalLanguageValue` (has whitespace or non-ASCII, or is one capitalized letters-only word; never JSON, numbers, URIs of any scheme, addresses, hex ids, UUIDs, bech32) | | 9999 | ListItemEvent | experimental/decentralizedLists/item | same as 9998 | | 10003 | BookmarkListEvent | nip51Lists/bookmarkList | `listOfNotNull(title())` NL | | 10100 | AgentProfileEvent | buzz/agentProfiles | `profileOrNull()?.let { listOfNotNull(it.name, it.displayName).joinToString("\n") } ?: ""` | diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/README.md b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/README.md index 060f22cfb4..aad465db68 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/README.md +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/README.md @@ -85,7 +85,18 @@ Filter( All four kinds are `SearchableEvent`s sharing one walk (`forEachSearchableListField`): `names` and `titles` (singular then plural), -`name`, `title`, `description`, `comments`, then every `t` item value. +`name`, `title`, `description`, `comments`, then every `t` item value, then +every value of every other tag that reads as natural language. + +The tag set is open — deployments add `author`, `subject`, `artist`, +`relationshipType` and tags nobody has named yet — so that last step decides +by the value, not the tag (`nip50Search/NaturalLanguageValue.kt`): a value is +indexed when it has whitespace or a non-ASCII character, or is a single +capitalized word ("Fiction", "Aristotle"), and is never a machine value (JSON, +a number, a URI of any scheme, an address, a hex id, a UUID, bech32). `t` +values skip the natural-language test but not the machine one. `alt` (NIP-31 +fallback text that restates other tags) and `client` are not indexed. + `content` is not part of the spec, and ids, pubkeys and coordinates are served by `#p`/`#e`/`#a`/`#z` filters, so none of them are indexed. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt index 27557c2bbd..5aaefaf5f4 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt @@ -35,12 +35,17 @@ import com.vitorpamplona.quartz.experimental.decentralizedLists.tags.Description import com.vitorpamplona.quartz.nip01Core.core.TagArray import com.vitorpamplona.quartz.nip01Core.core.fastForEach import com.vitorpamplona.quartz.nip01Core.tags.hashtags.HashtagTag +import com.vitorpamplona.quartz.nip31Alts.AltTag import com.vitorpamplona.quartz.nip50Search.IndexableFieldVisitor +import com.vitorpamplona.quartz.nip50Search.isMachineValue +import com.vitorpamplona.quartz.nip50Search.isNaturalLanguageValue +import com.vitorpamplona.quartz.nip89AppHandlers.clientTag.ClientTag /** * The human-authored text of any kind in the family, in a fixed order: the header's `names` * and `titles` (singular, then plural), the item's `name` and `title`, `description`, - * `comments`, then each `t` item value. Headers and items share one walk because the spec's + * `comments`, then each `t` value that is not a machine value, then + * [forEachSearchableListExtraField]. Headers and items share one walk because the spec's * nonstandard method lets an item carry header tags; each kind simply has fewer of them set. * * `content` is not part of the spec and ids/pubkeys/coordinates are served by tag filters, @@ -84,11 +89,50 @@ fun TagArray.forEachSearchableListField(visitor: IndexableFieldVisitor): Boolean description?.let { if (!visitor.visit(it)) return false } comments?.let { if (!visitor.visit(it)) return false } fastForEach { tag -> - HashtagTag.parse(tag)?.let { if (!visitor.visit(it)) return false } + HashtagTag.parse(tag)?.let { if (!isMachineValue(it) && !visitor.visit(it)) return false } + } + return forEachSearchableListExtraField(visitor) +} + +/** + * Every value of every other tag that reads as natural language ([isNaturalLanguageValue]), in + * tag order. The family's tag set is open — deployments add `author`, `subject`, `artist`, + * `relationshipType` and tags nobody has named yet — so this walk decides by what a value looks + * like rather than by the tag it sits in. + * + * Skips the tags [forEachSearchableListField] already visits, plus two NIP-defined metadata tags + * that are not the event's own text: `alt` (NIP-31 fallback text, which here restates `title` + * and `artist` behind a fixed "Song: … by …" prefix) and `client` (NIP-89 app name). + * + * @return false when the visitor stopped the walk. + */ +fun TagArray.forEachSearchableListExtraField(visitor: IndexableFieldVisitor): Boolean { + fastForEach { tag -> + if (tag.size < 2 || !isExtraFieldTagName(tag[0])) return@fastForEach + for (i in 1 until tag.size) { + val value = tag[i] + if (isNaturalLanguageValue(value) && !visitor.visit(value)) return false + } } return true } +private fun isExtraFieldTagName(name: String) = + when (name) { + NamesTag.TAG_NAME, + TitlesTag.TAG_NAME, + NameTag.TAG_NAME, + TitleTag.TAG_NAME, + DescriptionTag.TAG_NAME, + CommentsTag.TAG_NAME, + HashtagTag.TAG_NAME, + AltTag.TAG_NAME, + ClientTag.TAG_NAME, + -> false + + else -> true + } + /** * The TITLE role of [forEachSearchableListField], for search engines that weight fields: what the * list or item is called — the header's `names` and `titles` (singular, then plural) and the @@ -103,6 +147,22 @@ fun TagArray.searchableListTitles(): List { /** The DESCRIPTION role of [forEachSearchableListField]: `description`, then `comments`. */ fun TagArray.searchableListDescriptions(): List = listOf(description(), comments()) +/** + * The TEXT role of [forEachSearchableListField]: the [forEachSearchableListExtraField] values, one + * per line, or null when the event has none. The `t` values are not here; the search funnel + * carries them as hashtags. + */ +fun TagArray.searchableListExtraText(): String? = + buildString { + forEachSearchableListExtraField { field -> + if (field != null) { + if (isNotEmpty()) append('\n') + append(field) + } + true + } + }.ifEmpty { null } + /** The write-path join of [forEachSearchableListField]: one field per line. */ fun TagArray.searchableListContent() = buildString { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/NaturalLanguageValue.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/NaturalLanguageValue.kt new file mode 100644 index 0000000000..328f09df10 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/NaturalLanguageValue.kt @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip50Search + +/** + * Whether a free-form tag value reads as natural language, for kinds whose tag set is open + * (anyone can invent a tag name) and so cannot be indexed from a list of known tag names. + * + * A value qualifies when it is not [isMachineValue] and it either contains whitespace, contains a + * non-ASCII character, or is a single capitalized ASCII word made of letters only. That keeps + * "Social history", "刘慈欣", "Fiction" and "Aristotle", and drops the single lowercase or mixed + * tokens that label data rather than describe it: `music`, `eng`, `openlibrary`, `IS_A_PROPERTY_OF`, + * `concept-header`, `OL19722168W`, `2026-06-01`. + * + * Allocation-free: the read path calls it once per value per event per keystroke. + */ +fun isNaturalLanguageValue(value: String): Boolean { + var start = 0 + var end = value.length + while (start < end && value[start].isWhitespace()) start++ + while (end > start && value[end - 1].isWhitespace()) end-- + if (start == end || isMachineValue(value, start, end)) return false + + for (i in start until end) { + val c = value[i] + if (c.isWhitespace() || c.code > 127) return true + } + return isCapitalizedWord(value, start, end) +} + +/** + * Whether a tag value is an identifier or a structure rather than text: a JSON object or array, + * a number (including a signed one and an ISBN-10 ending in `X`), a URI of any scheme + * (`https://…`, `wss://…`, `tag:…`, `urn:…`, `isbn:…`), an event address (`kind:pubkey:d`), a + * long hex id, a UUID, or a NIP-19 bech32 entity. Empty and blank values count as machine + * values too: there is nothing in them to index. + */ +fun isMachineValue(value: String): Boolean { + var start = 0 + var end = value.length + while (start < end && value[start].isWhitespace()) start++ + while (end > start && value[end - 1].isWhitespace()) end-- + return start == end || isMachineValue(value, start, end) +} + +private fun isMachineValue( + v: String, + start: Int, + end: Int, +): Boolean { + val first = v[start] + val last = v[end - 1] + if ((first == '{' && last == '}') || (first == '[' && last == ']')) return true + + // Every other shape is a single token. + for (i in start until end) if (v[i].isWhitespace()) return false + + return isNumber(v, start, end) || + isUri(v, start, end) || + isAddress(v, start, end) || + isHexId(v, start, end) || + isUuid(v, start, end) || + isBech32(v, start) +} + +private fun Char.isAsciiDigit() = this in '0'..'9' + +private fun Char.isAsciiLetter() = this in 'a'..'z' || this in 'A'..'Z' + +private fun Char.isHex() = this in '0'..'9' || this in 'a'..'f' || this in 'A'..'F' + +/** `-1`, `473`, `0.5`, an ISBN-13, and an ISBN-10 with its `X` check digit. */ +private fun isNumber( + v: String, + start: Int, + end: Int, +): Boolean { + var i = start + if (v[i] == '-' || v[i] == '+') i++ + val digitsStart = i + while (i < end && v[i].isAsciiDigit()) i++ + if (i == digitsStart) return false + if (i < end && v[i] == '.') { + i++ + val fractionStart = i + while (i < end && v[i].isAsciiDigit()) i++ + if (i == fractionStart) return false + } + if (i < end && (v[i] == 'X' || v[i] == 'x')) i++ + return i == end +} + +/** An RFC 3986 scheme followed by `:` and something more, as a single token. */ +private fun isUri( + v: String, + start: Int, + end: Int, +): Boolean { + if (!v[start].isAsciiLetter()) return false + var i = start + 1 + while (i < end) { + val c = v[i] + if (c == ':') return i + 1 < end + if (!(c.isAsciiLetter() || c.isAsciiDigit() || c == '+' || c == '.' || c == '-')) return false + i++ + } + return false +} + +/** `:<64-hex pubkey>` optionally followed by `:`. */ +private fun isAddress( + v: String, + start: Int, + end: Int, +): Boolean { + var i = start + while (i < end && v[i].isAsciiDigit()) i++ + if (i == start || i >= end || v[i] != ':') return false + i++ + val pubkeyEnd = i + 64 + if (pubkeyEnd > end) return false + while (i < pubkeyEnd) { + if (!v[i].isHex()) return false + i++ + } + return i == end || v[i] == ':' +} + +/** Event ids, pubkeys, and the shorter hashes (32+ hex characters) feeds use as ids. */ +private fun isHexId( + v: String, + start: Int, + end: Int, +): Boolean { + if (end - start < 32) return false + for (i in start until end) if (!v[i].isHex()) return false + return true +} + +private fun isUuid( + v: String, + start: Int, + end: Int, +): Boolean { + if (end - start != 36) return false + for (i in 0 until 36) { + val c = v[start + i] + val hyphen = i == 8 || i == 13 || i == 18 || i == 23 + if (if (hyphen) c != '-' else !c.isHex()) return false + } + return true +} + +private val BECH32_PREFIXES = arrayOf("npub1", "nsec1", "note1", "nevent1", "naddr1", "nprofile1", "nrelay1") + +private fun isBech32( + v: String, + start: Int, +): Boolean { + for (prefix in BECH32_PREFIXES) if (v.startsWith(prefix, start)) return true + return false +} + +/** `Fiction`, `Aristotle`, `RTL`, `AT&T`: an ASCII capital followed by letters only. */ +private fun isCapitalizedWord( + v: String, + start: Int, + end: Int, +): Boolean { + if (v[start] !in 'A'..'Z') return false + for (i in start + 1 until end) { + val c = v[i] + if (!(c.isAsciiLetter() || c == '\'' || c == '&' || c == '.')) return false + } + return true +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt index dc43fd3943..b91786e25c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt @@ -31,6 +31,7 @@ import com.vitorpamplona.quartz.experimental.birdstar.BirdDetectionEvent import com.vitorpamplona.quartz.experimental.birdstar.BirdexEvent import com.vitorpamplona.quartz.experimental.decentralizedLists.DecentralizedListEvent import com.vitorpamplona.quartz.experimental.decentralizedLists.searchableListDescriptions +import com.vitorpamplona.quartz.experimental.decentralizedLists.searchableListExtraText import com.vitorpamplona.quartz.experimental.decentralizedLists.searchableListTitles import com.vitorpamplona.quartz.experimental.edits.TextNoteModificationEvent import com.vitorpamplona.quartz.experimental.fitness.workout.ExerciseTemplateEvent @@ -629,9 +630,10 @@ object SearchFieldExtractor { // already, so -- as for 1111/1311/30382 -- they are not passed again, and // content is not part of the spec. One branch for all four: an item may // carry header tags (the spec's nonstandard method), and the tag helpers - // read whichever are present. + // read whichever are present. Every other natural-language tag value + // (author, subject, artist, ...) is the text tier, below both. is DecentralizedListEvent -> { - tiers(event, event.tags.searchableListTitles(), event.tags.searchableListDescriptions(), null) + tiers(event, event.tags.searchableListTitles(), event.tags.searchableListDescriptions(), event.tags.searchableListExtraText()) } // kind 1 LAST among the explicit branches, defensively: a future diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/DecentralizedListsTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/DecentralizedListsTest.kt index 4a0e5511f0..0522964b34 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/DecentralizedListsTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/DecentralizedListsTest.kt @@ -299,7 +299,7 @@ class DecentralizedListsTest { ) assertEquals("Fido\nVery good\nFido", item.indexableContent()) - listOf(header, item).forEach { event -> + listOf(header, item, book).forEach { event -> val fields = mutableListOf() event.forEachIndexableField { field -> field?.let { fields.add(it) } @@ -310,4 +310,54 @@ class DecentralizedListsTest { assertEquals("", assertIs(event(9999, emptyArray())).indexableContent()) } + + // A book item as a Decentralized Lists deployment publishes it: deployment-specific tags + // around the spec's own, most of them machine data. + private val book = + assertIs( + event( + 39999, + arrayOf( + arrayOf("d", "ol-ol1141875w"), + arrayOf("z", "39998:$author:books"), + arrayOf("title", "Herfsttij der Middeleeuwen"), + arrayOf("t", "history"), + arrayOf("t", "tag:soundcloud,2010:tracks/711391888"), + arrayOf("t", "https://example.com/ep/1"), + arrayOf("author", "Johan Huizinga"), + arrayOf("subject", "Middle Ages"), + arrayOf("subject", "Fiction"), + arrayOf("subject", "Civilisation médiévale"), + arrayOf("open-library-id", "OL1141875W"), + arrayOf("isbn", "9780226360935"), + arrayOf("isbn10", "057507681X"), + arrayOf("year", "1919"), + arrayOf("lang", "eng"), + arrayOf("source", "openlibrary"), + arrayOf("cover", "https://covers.openlibrary.org/b/id/7245546-L.jpg"), + arrayOf("json", "{\"word\": {\"name\": \"Herfsttij der Middeleeuwen\"}}"), + arrayOf("p", author, "wss://relay.example.com", "Johan Fan"), + arrayOf("e", headerId, "wss://relay.example.com", "mention"), + arrayOf("alt", "Book: Herfsttij der Middeleeuwen by Johan Huizinga"), + arrayOf("client", "Brainstorm"), + ), + ), + ) + + @Test + fun searchIndexesEveryNaturalLanguageTagValue() { + assertEquals( + "Herfsttij der Middeleeuwen\nhistory\nJohan Huizinga\nMiddle Ages\nFiction\nCivilisation médiévale\nJohan Fan", + book.indexableContent(), + ) + } + + @Test + fun searchSkipsMachineHashtagsButKeepsSingleWordOnes() { + val item = + assertIs( + event(9999, arrayOf(arrayOf("t", "romance"), arrayOf("t", "8ad7c296-67e9-5f57-ba52-2ff732e62e87"), arrayOf("t", "473"))), + ) + assertEquals("romance", item.indexableContent()) + } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt index 23024b8be5..0d646b93fc 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SearchTest.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.quartz.nip01Core.store.sqlite +import com.vitorpamplona.quartz.experimental.decentralizedLists.item.AddressableListItemEvent import com.vitorpamplona.quartz.experimental.trustedLists.metric import com.vitorpamplona.quartz.experimental.trustedLists.title import com.vitorpamplona.quartz.experimental.trustedLists.users.UserTrustedListEvent @@ -221,6 +222,35 @@ class SearchTest : BaseDBTest() { db.assertQuery(null, Filter(search = memberKey)) } + @Test + fun testDecentralizedListItemsAreSearchableByEveryNaturalLanguageTag() = + forEachDB { db -> + val item = + signer.sign( + TimeUtils.now(), + AddressableListItemEvent.KIND, + arrayOf( + arrayOf("d", "ol-uniqbookslug"), + arrayOf("title", "Uniqtitle Middeleeuwen"), + arrayOf("author", "Johan Uniqauthor"), + arrayOf("subject", "Uniqsubject"), + arrayOf("lang", "uniqlang"), + arrayOf("alt", "Book: Uniqalt"), + ), + "", + ) + db.store.insertEvent(item) + + db.assertQuery(item, Filter(search = "uniqtitle")) + db.assertQuery(item, Filter(search = "uniqauthor")) + db.assertQuery(item, Filter(search = "uniqsubject")) + + // the slug, a lowercase data label and the NIP-31 fallback text are not indexed + db.assertQuery(null, Filter(search = "uniqbookslug")) + db.assertQuery(null, Filter(search = "uniqlang")) + db.assertQuery(null, Filter(search = "uniqalt")) + } + @Test fun testProfileJsonFieldsAreSearchable() = forEachDB { db -> diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/NaturalLanguageValueTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/NaturalLanguageValueTest.kt new file mode 100644 index 0000000000..1dd4c9aa55 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/NaturalLanguageValueTest.kt @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip50Search + +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** Values taken from the kind 9999/39999 corpus on a Decentralized Lists relay. */ +class NaturalLanguageValueTest { + @Test + fun phrasesAndNonAsciiTextAreNaturalLanguage() { + listOf( + "Social history", + "Johan Huizinga", + "Guevara, ernesto, 1928-1967", + "Climbing The Mountain (1979-1990)", + "[Live] Closer To Somewhere", + "Song: Gold by Torcon 7", + "刘慈欣", + "老子", + "Moyen Âge", + " padded phrase ", + ).forEach { assertTrue(isNaturalLanguageValue(it), it) } + } + + @Test + fun capitalizedWordsAreNaturalLanguage() { + listOf("Fiction", "History", "Aristotle", "Plutarch", "RTL", "PromoDJ", "AT&T").forEach { + assertTrue(isNaturalLanguageValue(it), it) + } + } + + @Test + fun singleLowercaseOrMixedTokensAreNot() { + listOf( + "music", + "eng", + "openlibrary", + "reference", + "string", + "forward", + "IS_THE_CONCEPT_FOR", + "concept-header", + "tagassert--ol-ol49463w--science-fiction--6aca05b8", + "OL19722168W", + "2026-06-01", + "sorita67", + "Musician:", + ).forEach { assertFalse(isNaturalLanguageValue(it), it) } + } + + @Test + fun machineValuesAreNotNaturalLanguageEvenWithSpaces() { + listOf( + "", + " ", + "{\"word\": {\"slug\": \"ol-ol98624w\", \"name\": \"Waking with Enemies\"}}", + "[\"title\", \"artist\"]", + "Re:Zero", + ).forEach { assertFalse(isNaturalLanguageValue(it), it) } + } + + @Test + fun machineValues() { + listOf( + "473", + "-1", + "0.5", + "9781984820983", + "057507681X", + "https://covers.openlibrary.org/b/id/7245546-L.jpg", + "wss://relay.damus.io", + "tag:soundcloud,2010:tracks/711391888", + "urn:isbn:9780679428329", + "39999:6aca05b812da97601151776d13de04ae71afc9d86da1408f0e72cffef72ece4b:ol-ol2838774w", + "39998:6aca05b812da97601151776d13de04ae71afc9d86da1408f0e72cffef72ece4b", + "6aca05b812da97601151776d13de04ae71afc9d86da1408f0e72cffef72ece4b", + "c65d75f4d058f4746e12e44441cacea0", + "8ad7c296-67e9-5f57-ba52-2ff732e62e87", + "npub1f5pre6wl6ad87vr4hr5wppqq30sh58m4p33mthnjreh03qadcajs7gwt3z", + "", + ).forEach { assertTrue(isMachineValue(it), it) } + } + + @Test + fun wordsAndPhrasesAreNotMachineValues() { + listOf("history", "literary-fiction", "Song: Gold", "Fiction", "Paris, 1920", "1984 Orwell").forEach { + assertFalse(isMachineValue(it), it) + } + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt index 617a4fe607..81e8821edb 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt @@ -476,6 +476,30 @@ class SearchFieldExtractorTest { ) } + @Test + fun decentralizedListItemExtraTagsAreTheTextTier() { + // Deployment tags rank below what the item is called and what it is about; machine + // values and the NIP-31 alt text are not indexed at all. + val tags = + arrayOf( + arrayOf("d", "ol-ol98624w"), + arrayOf("title", "Waking with Enemies"), + arrayOf("author", "Christopher Pike"), + arrayOf("subject", "Horror tales"), + arrayOf("lang", "eng"), + arrayOf("isbn", "9781534445145"), + arrayOf("alt", "Book: Waking with Enemies by Christopher Pike"), + ) + val fields = SearchFieldExtractor.extract(AddressableListItemEvent("3c".repeat(32), alice, 1L, tags, "", "")) + assertEquals( + IndexableFields.Tiered( + primary = listOf("Waking with Enemies"), + text = "Christopher Pike\nHorror tales", + ), + fields, + ) + } + @Test fun textTracksIndexWhatIsSaidNotTheTimings() { val vtt = "WEBVTT\n\n1\n00:00:00.000 --> 00:00:02.000 align:start\nHello nostr\n" diff --git a/quartz/src/jvmTest/resources/indexable-content.golden b/quartz/src/jvmTest/resources/indexable-content.golden index ba9a7f11a5..3fddf127b2 100644 --- a/quartz/src/jvmTest/resources/indexable-content.golden +++ b/quartz/src/jvmTest/resources/indexable-content.golden @@ -58,8 +58,8 @@ 9736 The content body. 9737 The content body. 9802 The Comment\nThe Context\nThe content body. -9998 The Name\nThe Title\nThe Description\nhashtag1\nhashtag2 -9999 The Name\nThe Title\nThe Description\nhashtag1\nhashtag2 +9998 The Name\nThe Title\nThe Description\nhashtag1\nhashtag2\nThe Subject\nThe Summary\nThe About\nThe Comment\nThe Context\nThe Rules\nThe Text\nThe Instructions +9999 The Name\nThe Title\nThe Description\nhashtag1\nhashtag2\nThe Subject\nThe Summary\nThe About\nThe Comment\nThe Context\nThe Rules\nThe Text\nThe Instructions 10003 The Title 10100 10154 The Title\nThe Description @@ -141,8 +141,8 @@ 39092 The Title\nThe Description 39307 The content body. 39701 The Title\nThe content body. -39998 The Name\nThe Title\nThe Description\nhashtag1\nhashtag2 -39999 The Name\nThe Title\nThe Description\nhashtag1\nhashtag2 +39998 The Name\nThe Title\nThe Description\nhashtag1\nhashtag2\nThe Subject\nThe Summary\nThe About\nThe Comment\nThe Context\nThe Rules\nThe Text\nThe Instructions +39999 The Name\nThe Title\nThe Description\nhashtag1\nhashtag2\nThe Subject\nThe Summary\nThe About\nThe Comment\nThe Context\nThe Rules\nThe Text\nThe Instructions 40002 The content body. 40100 The content body. 45001 The content body.