From ca7ffd2ad21ff0eab5bf8225770b90b050734b6b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 03:07:39 +0000 Subject: [PATCH] fix(quartz): give search the release notes, list roles and caption words it was missing Three gaps between what a kind's indexableContent() says and what a field-weighting search engine got from SearchFieldExtractor. Kind 30063: the NIP-82 release notes reached indexableContent() but not the extractor, whose branch passed title and description only. So an engine reading the extractor could not find a release by what changed in it, while SQLiteEventStore could. searchableReleaseNotes() becomes public and the branch passes it as the body. The NIP-51 encrypted-content guard lives in that one function, so both paths apply the same check. Kinds 9998/39998/9999/39999 (Decentralized Lists): no branch, so the catch-all put the whole blob in the body, titles included. The `t` values it appends were ALSO carried by the funnel's hashtag role, so every list hashtag was indexed twice, the doubling already removed for 1111/1311/30382. One `is DecentralizedListEvent` branch now puts names and titles in the title tier and description and comments in the secondary tier, and leaves hashtags to the funnel. The role helpers sit beside forEachSearchableListField in SearchExt.kt so the two stay in sync. Kind 39307 (text tracks): the whole WebVTT document was indexed, so the header, every `-->` timing line, cue settings, ids, NOTE/STYLE blocks and inline tags sat in the index beside the dialogue. A search for "00" or "align" matched every caption. WebVttText reads the cue payload lines in one pass, drops tags, and decodes character references. SRT fits the same shape. Content with no timing line is not a caption file and is indexed as written. indexableContent() and forEachIndexableField() both go through it, so the SQLite FTS index is fixed too. Stores should run reindexFullTextSearch() for 39307; the searchable-kinds table is updated. Tests: SearchFieldExtractorTest covers both 30063 shapes, a list header, a list item's single-counted hashtags and a text track. WebVttTextTest covers cue selection, markup, references, CRLF/SRT, the visitor rejoin contract and early stop. The golden file is unchanged: its fixture has no timing line. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AdSGU88PYrRVyVHjiEc5sw --- .../references/searchable-kinds.md | 2 +- .../decentralizedLists/SearchExt.kt | 19 ++ .../nip50Search/SearchFieldExtractor.kt | 19 +- .../ReleaseArtifactSetEvent.kt | 12 +- .../nip71Video/textTrack/TextTrackEvent.kt | 10 +- .../quartz/nip71Video/textTrack/WebVttText.kt | 219 ++++++++++++++++++ .../nip50Search/SearchFieldExtractorTest.kt | 71 ++++++ .../nip71Video/textTrack/WebVttTextTest.kt | 111 +++++++++ 8 files changed, 454 insertions(+), 9 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttText.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttTextTest.kt diff --git a/.claude/skills/searchable-events/references/searchable-kinds.md b/.claude/skills/searchable-events/references/searchable-kinds.md index d9942a6530..532ecc3d68 100644 --- a/.claude/skills/searchable-events/references/searchable-kinds.md +++ b/.claude/skills/searchable-events/references/searchable-kinds.md @@ -140,7 +140,7 @@ Separator legend: **NL** = `joinToString("\n")`, **SP** = `joinToString(" ")`. | 39000 | GroupMetadataEvent | nip29RelayGroups/metadata | `listOfNotNull(name(), about())` NL | | 39089 | StarterPackEvent | nip51Lists/starterPack | `listOfNotNull(title(), description())` NL | | 39092 | MediaStarterPackEvent | nip51Lists/mediaStarterPack | `listOfNotNull(title(), description())` NL | -| 39307 | TextTrackEvent | nip71Video/textTrack | `content` (the WebVTT cue text) | +| 39307 | TextTrackEvent | nip71Video/textTrack | `WebVttText.cueText(content)` NL — each cue's payload lines with tags removed and character references decoded; no header, timings, cue settings, ids, `NOTE`/`STYLE`/`REGION` blocks. Content with no `-->` timing line is not a caption file and is indexed as written | | 39701 | WebBookmarkEvent | nipB0WebBookmarks | `listOfNotNull(title(), description())` NL | | 39998 | AddressableListHeaderEvent | experimental/decentralizedLists/header | same as 9998 | | 39999 | AddressableListItemEvent | experimental/decentralizedLists/item | same as 9998 | diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt index e866f645b9..27557c2bbd 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt @@ -20,12 +20,17 @@ */ package com.vitorpamplona.quartz.experimental.decentralizedLists +import com.vitorpamplona.quartz.experimental.decentralizedLists.header.names import com.vitorpamplona.quartz.experimental.decentralizedLists.header.tags.NamesTag import com.vitorpamplona.quartz.experimental.decentralizedLists.header.tags.SingularPlural import com.vitorpamplona.quartz.experimental.decentralizedLists.header.tags.TitlesTag +import com.vitorpamplona.quartz.experimental.decentralizedLists.header.titles +import com.vitorpamplona.quartz.experimental.decentralizedLists.item.comments +import com.vitorpamplona.quartz.experimental.decentralizedLists.item.name import com.vitorpamplona.quartz.experimental.decentralizedLists.item.tags.CommentsTag import com.vitorpamplona.quartz.experimental.decentralizedLists.item.tags.NameTag import com.vitorpamplona.quartz.experimental.decentralizedLists.item.tags.TitleTag +import com.vitorpamplona.quartz.experimental.decentralizedLists.item.title import com.vitorpamplona.quartz.experimental.decentralizedLists.tags.DescriptionTag import com.vitorpamplona.quartz.nip01Core.core.TagArray import com.vitorpamplona.quartz.nip01Core.core.fastForEach @@ -84,6 +89,20 @@ fun TagArray.forEachSearchableListField(visitor: IndexableFieldVisitor): Boolean return true } +/** + * The TITLE role of [forEachSearchableListField], for search engines that weight fields: what the + * list or item is called — the header's `names` and `titles` (singular, then plural) and the + * item's `name` and `title`. Same first-well-formed-tag rule as the walk. + */ +fun TagArray.searchableListTitles(): List { + val names = names() + val titles = titles() + return listOf(names?.singular, names?.plural, titles?.singular, titles?.plural, name(), title()) +} + +/** The DESCRIPTION role of [forEachSearchableListField]: `description`, then `comments`. */ +fun TagArray.searchableListDescriptions(): List = listOf(description(), comments()) + /** The write-path join of [forEachSearchableListField]: one field per line. */ fun TagArray.searchableListContent() = buildString { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt index 139530d089..b9a679d8b6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt @@ -29,6 +29,9 @@ import com.vitorpamplona.quartz.experimental.agora.FundraiserEvent import com.vitorpamplona.quartz.experimental.audio.track.AudioTrackEvent import com.vitorpamplona.quartz.experimental.birdstar.BirdDetectionEvent import com.vitorpamplona.quartz.experimental.birdstar.BirdexEvent +import com.vitorpamplona.quartz.experimental.decentralizedLists.DecentralizedListEvent +import com.vitorpamplona.quartz.experimental.decentralizedLists.searchableListDescriptions +import com.vitorpamplona.quartz.experimental.decentralizedLists.searchableListTitles import com.vitorpamplona.quartz.experimental.edits.TextNoteModificationEvent import com.vitorpamplona.quartz.experimental.fitness.workout.ExerciseTemplateEvent import com.vitorpamplona.quartz.experimental.fitness.workout.WorkoutRecordEvent @@ -380,8 +383,11 @@ object SearchFieldExtractor { tiers(event, event.title(), event.description(), null) } + // kind 30063 -- the body is a NIP-82 release's notes, and nothing for a + // NIP-51 set (its content may be encrypted private items); the event + // owns that guard, so both search paths apply the same one. is ReleaseArtifactSetEvent -> { - tiers(event, event.title(), event.description(), null) + tiers(event, event.title(), event.description(), event.searchableReleaseNotes()) } is AppCurationSetEvent -> { @@ -617,6 +623,17 @@ object SearchFieldExtractor { tiers(event, null, null, event.content) } + // kinds 9998/39998/9999/39999 -- a list's or item's names and titles are + // what it is called, its description and comments what it is about. The + // `t` values indexableContent() appends are the funnel's hashtag role + // already, so -- as for 1111/1311/30382 -- they are not passed again, and + // content is not part of the spec. One branch for all four: an item may + // carry header tags (the spec's nonstandard method), and the tag helpers + // read whichever are present. + is DecentralizedListEvent -> { + tiers(event, event.tags.searchableListTitles(), event.tags.searchableListDescriptions(), null) + } + // kind 1 LAST among the explicit branches, defensively: a future // kind extending the text-note base must hit its own branch first. is TextNoteEvent -> { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip51Lists/releaseArtifactSet/ReleaseArtifactSetEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip51Lists/releaseArtifactSet/ReleaseArtifactSetEvent.kt index 8926ed25cb..bd151437b0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip51Lists/releaseArtifactSet/ReleaseArtifactSetEvent.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip51Lists/releaseArtifactSet/ReleaseArtifactSetEvent.kt @@ -81,10 +81,14 @@ class ReleaseArtifactSetEvent( visitor.visit(searchableReleaseNotes()) } - // Only NIP-82 releases carry release notes in `content`. A NIP-51 set may keep - // encrypted private items there, which must never reach the index. The blank - // check runs first so empty sets skip the two tag scans on every search. - private fun searchableReleaseNotes() = if (content.isNotBlank() && isNip82SoftwareRelease()) content else null + /** + * The release notes when they are safe to index, else null. Only NIP-82 releases carry + * release notes in `content`; a NIP-51 set may keep encrypted private items there, which + * must never reach the index. The blank check runs first so empty sets skip the two tag + * scans on every search. `SearchFieldExtractor` reads this too, so both search paths + * apply the same guard. + */ + fun searchableReleaseNotes() = if (content.isNotBlank() && isNip82SoftwareRelease()) content else null override fun eventHints() = tags.mapNotNull(EventBookmark::parseAsHint) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/TextTrackEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/TextTrackEvent.kt index 91e0acab39..e9e31c9e13 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/TextTrackEvent.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/TextTrackEvent.kt @@ -59,11 +59,15 @@ class TextTrackEvent( AddressHintProvider, SearchableEvent { // The cue text is the only searchable thing here: the `d` tag is a slug and the rest is - // plumbing. Indexing it makes a video findable by what is said in it. - override fun indexableContent() = content + // plumbing. Indexing it makes a video findable by what is said in it — the words only, + // not the timings, cue settings and markup around them (see WebVttText). Content with no + // cue at all is not a caption file and is indexed as written. + override fun indexableContent() = if (WebVttText.hasCues(content)) WebVttText.cueText(content) else content + // The read path: the same lines cueText() joins, one visit per line so a hit early in the + // track stops the walk before the rest is cleaned. override fun forEachIndexableField(visitor: IndexableFieldVisitor) { - visitor.visit(content) + if (WebVttText.hasCues(content)) WebVttText.forEachCueLine(content, visitor) else visitor.visit(content) } override fun addressHints() = tags.mapNotNull(ATag::parseAsHint) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttText.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttText.kt new file mode 100644 index 0000000000..9fdf875374 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttText.kt @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip71Video.textTrack + +import com.vitorpamplona.quartz.nip50Search.IndexableFieldVisitor + +/** + * The words a WebVTT document says: every cue's payload lines with their markup removed, and + * nothing else. + * + * Search is the reason this exists. A text track's `content` is the whole document, so indexing + * it raw puts `WEBVTT`, every `00:01:02.500 --> 00:01:04.000` timing line, cue settings + * (`align:start`), cue ids, `NOTE` and `STYLE` blocks, and inline tags (``, ``, + * `<00:00:01.000>`) into the index beside the dialogue. A query for "00" or "align" then + * matches every caption ever published, and the terms that matter are diluted by markup. + * + * ## What counts as a cue line + * + * One pass over the lines, no model built. A line containing `-->` opens a cue (it is the + * timing line; the optional identifier line before it is skipped because nothing is open yet), + * and every non-blank line after it until the next blank line is payload. Everything outside a + * cue — the `WEBVTT` header, `NOTE`, `STYLE`, `REGION` — is skipped by the same rule, since none + * of it may contain `-->`. SRT happens to fit the same shape (a numeric id, a timing line, the + * text), so a publisher that wraps SRT instead of WebVTT is read correctly too. + * + * A document with no timing line at all is not a caption file, so it is indexed as written + * rather than discarded: [hasCues] is the switch, and callers pass such content through + * unchanged. + * + * ## Markup + * + * Inline tags are removed, not their contents: `really` is `really`. Voice spans lose the + * speaker name with the tag (`Hello` is `Hello`): the name is an annotation, not + * something said. The HTML character references WebVTT allows are decoded — the named ones the + * spec lists, plus numeric ones — so `Tom & Jerry` is indexed as `Tom & Jerry`. An unknown + * reference is kept as written. + */ +object WebVttText { + private const val TIMING_ARROW = "-->" + + /** Whether [document] has at least one cue. Without one it is not a caption file. */ + fun hasCues(document: String) = document.contains(TIMING_ARROW) + + /** + * The cue text of [document], one payload line per line. Callers should check [hasCues] + * first; a document without cues yields an empty string here. + */ + fun cueText(document: String): String { + val out = StringBuilder(document.length) + forEachCueLine(document) { line -> + if (out.isNotEmpty()) out.append('\n') + out.append(line) + true + } + return out.toString() + } + + /** + * Hands each cleaned, non-empty payload line of [document] to [visitor] in order. Joining + * them with `"\n"` gives exactly [cueText]. Returns false when the visitor stopped the walk. + * + * Lines without markup are passed as substrings; only a line carrying a tag or a character + * reference is rebuilt. + */ + fun forEachCueLine( + document: String, + visitor: IndexableFieldVisitor, + ): Boolean { + var inCue = false + var start = 0 + val length = document.length + while (start <= length) { + var end = start + while (end < length && document[end] != '\n' && document[end] != '\r') end++ + + if (isBlank(document, start, end)) { + inCue = false + } else if (!inCue) { + // Header, NOTE / STYLE / REGION blocks and cue identifiers all end up here and + // are skipped; only a timing line opens a cue. + if (hasArrow(document, start, end)) inCue = true + } else { + val line = clean(document, start, end) + if (line != null && !visitor.visit(line)) return false + } + + if (end >= length) break + // A CRLF pair is one line break. + start = if (document[end] == '\r' && end + 1 < length && document[end + 1] == '\n') end + 2 else end + 1 + } + return true + } + + /** Bounded to the line: an unbounded indexOf would rescan up to the next cue from every skipped line. */ + private fun hasArrow( + s: String, + start: Int, + end: Int, + ): Boolean { + for (i in start until end - 2) { + if (s[i] == '-' && s[i + 1] == '-' && s[i + 2] == '>') return true + } + return false + } + + private fun isBlank( + s: String, + start: Int, + end: Int, + ): Boolean { + for (i in start until end) if (!s[i].isWhitespace()) return false + return true + } + + /** The payload line in `[start, end)` without tags, references decoded and trimmed; null when nothing is left. */ + private fun clean( + s: String, + start: Int, + end: Int, + ): String? { + var markup = false + for (i in start until end) { + val c = s[i] + if (c == '<' || c == '&') { + markup = true + break + } + } + if (!markup) return s.substring(start, end).trim().ifEmpty { null } + + val out = StringBuilder(end - start) + var i = start + while (i < end) { + val c = s[i] + when (c) { + '<' -> { + val close = s.indexOf('>', i + 1) + // An unterminated tag runs to the end of the line, as the WebVTT parser reads it. + i = if (close < 0 || close >= end) end else close + 1 + } + + '&' -> { + val semi = s.indexOf(';', i + 1) + val decoded = if (semi in (i + 1) until end) decodeReference(s, i + 1, semi) else null + if (decoded == null) { + out.append(c) + i++ + } else { + appendCodePoint(out, decoded) + i = semi + 1 + } + } + + else -> { + out.append(c) + i++ + } + } + } + return out.toString().trim().ifEmpty { null } + } + + /** The code point named by the reference body in `[start, end)` (between `&` and `;`), or null. */ + private fun decodeReference( + s: String, + start: Int, + end: Int, + ): Int? { + if (end - start > 10) return null + if (s[start] == '#') { + val hex = start + 1 < end && (s[start + 1] == 'x' || s[start + 1] == 'X') + val digits = s.substring(if (hex) start + 2 else start + 1, end) + val code = digits.toIntOrNull(if (hex) 16 else 10) ?: return null + return if (code in 1..0x10FFFF && code !in 0xD800..0xDFFF) code else null + } + return when (s.substring(start, end)) { + "amp" -> '&'.code + "lt" -> '<'.code + "gt" -> '>'.code + "quot" -> '"'.code + "apos" -> '\''.code + "nbsp" -> 0x00A0 + "lrm" -> 0x200E + "rlm" -> 0x200F + else -> null + } + } + + private fun appendCodePoint( + out: StringBuilder, + code: Int, + ) { + if (code <= 0xFFFF) { + out.append(code.toChar()) + } else { + val v = code - 0x10000 + out.append((0xD800 + (v shr 10)).toChar()) + out.append((0xDC00 + (v and 0x3FF)).toChar()) + } + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt index 51b77ca13f..617a4fe607 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt @@ -23,6 +23,8 @@ package com.vitorpamplona.quartz.nip50Search import com.vitorpamplona.quartz.buzz.agentProfiles.AgentProfileEvent import com.vitorpamplona.quartz.experimental.birdstar.BirdDetectionEvent import com.vitorpamplona.quartz.experimental.birdstar.BirdexEvent +import com.vitorpamplona.quartz.experimental.decentralizedLists.header.ListHeaderEvent +import com.vitorpamplona.quartz.experimental.decentralizedLists.item.AddressableListItemEvent import com.vitorpamplona.quartz.experimental.nip95.header.FileStorageHeaderEvent import com.vitorpamplona.quartz.experimental.ps1saves.Ps1SaveEvent import com.vitorpamplona.quartz.experimental.trustedLists.users.UserTrustedListEvent @@ -38,9 +40,11 @@ import com.vitorpamplona.quartz.nip29RelayGroups.moderation.GroupEditMetadataEve import com.vitorpamplona.quartz.nip32Labeling.LabelEvent import com.vitorpamplona.quartz.nip34Git.repository.GitRepositoryEvent import com.vitorpamplona.quartz.nip35Torrents.TorrentEvent +import com.vitorpamplona.quartz.nip51Lists.releaseArtifactSet.ReleaseArtifactSetEvent import com.vitorpamplona.quartz.nip53LiveActivities.meetingSpaces.MeetingSpaceEvent import com.vitorpamplona.quartz.nip5aStaticWebsites.NamedSiteEvent import com.vitorpamplona.quartz.nip69P2pOrderEvents.P2POrderEvent +import com.vitorpamplona.quartz.nip71Video.textTrack.TextTrackEvent import com.vitorpamplona.quartz.nip85TrustedAssertions.users.UserAssertionEvent import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent import com.vitorpamplona.quartz.nip89AppHandlers.definition.AppDefinitionEvent @@ -412,6 +416,73 @@ class SearchFieldExtractorTest { assertEquals(IndexableFields.Tiered(secondary = listOf("spam"), text = "obvious bot"), fields) } + @Test + fun nip82ReleaseNotesAreTheBodyOfARelease() { + // indexableContent() already carried the notes; the branch used to drop them, so an + // engine reading this extractor could not find a release by what changed in it. + val tags = arrayOf(arrayOf("d", "com.example.app@1.2.0"), arrayOf("i", "com.example.app"), arrayOf("version", "1.2.0"), arrayOf("title", "Example 1.2")) + val fields = SearchFieldExtractor.extract(ReleaseArtifactSetEvent("2e".repeat(32), alice, 1L, tags, "fixes the offline sync crash", "")) + assertEquals(IndexableFields.Tiered(primary = listOf("Example 1.2"), text = "fixes the offline sync crash"), fields) + } + + @Test + fun nip51ReleaseSetContentIsNeverTheBody() { + // A NIP-51 set (no `i` + `version`) may keep encrypted private items in content. + val tags = arrayOf(arrayOf("d", "release"), arrayOf("title", "Example"), arrayOf("description", "all builds")) + val fields = SearchFieldExtractor.extract(ReleaseArtifactSetEvent("2f".repeat(32), alice, 1L, tags, "AgJ0b3BzZWNyZXQ=", "")) + assertEquals(IndexableFields.Tiered(primary = listOf("Example"), secondary = listOf("all builds")), fields) + } + + @Test + fun decentralizedListHeadersSplitWhatTheyAreCalledFromWhatTheyAreAbout() { + val tags = + arrayOf( + arrayOf("names", "podcast", "podcasts"), + arrayOf("titles", "Podcast", "Podcasts"), + arrayOf("description", "shows worth a listen"), + arrayOf("t", "audio"), + ) + val fields = SearchFieldExtractor.extract(ListHeaderEvent("3a".repeat(32), alice, 1L, tags, "", "")) + assertEquals( + IndexableFields.Tiered( + primary = listOf("podcast", "podcasts", "Podcast", "Podcasts"), + secondary = listOf("shows worth a listen"), + hashtags = listOf("audio"), + ), + fields, + ) + } + + @Test + fun decentralizedListItemHashtagsAreNotIndexedTwice() { + // indexableContent() appends the `t` values; the funnel already carries them as + // hashtags, so the branch leaves them out of every tier (the 1111 reasoning). + val tags = + arrayOf( + arrayOf("z", "podcasts"), + arrayOf("title", "Bitcoin Audible"), + arrayOf("description", "long reads, read aloud"), + arrayOf("comments", "start with the classics"), + arrayOf("t", "bitcoin"), + ) + val fields = SearchFieldExtractor.extract(AddressableListItemEvent("3b".repeat(32), alice, 1L, tags + arrayOf(arrayOf("d", "ba")), "", "")) + assertEquals( + IndexableFields.Tiered( + primary = listOf("Bitcoin Audible"), + secondary = listOf("long reads, read aloud", "start with the classics"), + hashtags = listOf("bitcoin"), + ), + fields, + ) + } + + @Test + fun textTracksIndexWhatIsSaidNotTheTimings() { + val vtt = "WEBVTT\n\n1\n00:00:00.000 --> 00:00:02.000 align:start\nHello nostr\n" + val fields = SearchFieldExtractor.extract(TextTrackEvent("3c".repeat(32), alice, 1L, arrayOf(arrayOf("d", "subtitles:v1")), vtt, "")) + assertEquals(IndexableFields.Tiered(text = "Hello nostr"), fields) + } + @Test fun commentHashtagsAreNotIndexedTwice() { // indexableContent() concatenates the `t` tags INTO the body for kind diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttTextTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttTextTest.kt new file mode 100644 index 0000000000..e435eef505 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttTextTest.kt @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip71Video.textTrack + +import com.vitorpamplona.quartz.nip50Search.IndexableFieldVisitor +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class WebVttTextTest { + private val document = + """ + WEBVTT - the whole header line is skipped + Kind: captions + + STYLE + ::cue { color: yellow } + + NOTE this block is a comment + and so is this line + + intro + 00:00:00.000 --> 00:00:02.000 align:start position:10% + We are in New York City + + 00:00:02.500 --> 00:00:04.000 + Really? Tom & Jerry! + on <00:00:03.000>two lines + + 00:00:05.000 --> 00:00:06.000 + + """.trimIndent() + + @Test + fun keepsOnlyTheWordsOfEachCue() { + assertEquals("We are in New York City\nReally? Tom & Jerry!\non two lines", WebVttText.cueText(document)) + } + + @Test + fun theVisitedLinesRejoinToTheCueText() { + val lines = mutableListOf() + assertTrue(WebVttText.forEachCueLine(document, IndexableFieldVisitor { lines.add(it!!) })) + assertEquals(WebVttText.cueText(document), lines.joinToString("\n")) + } + + @Test + fun aVisitorCanStopTheWalk() { + val lines = mutableListOf() + assertFalse( + WebVttText.forEachCueLine(document, IndexableFieldVisitor { lines.add(it!!) && false }), + ) + assertEquals(listOf("We are in New York City"), lines) + } + + @Test + fun crlfAndSrtAreReadTheSameWay() { + val srt = "1\r\n00:00:01,000 --> 00:00:02,000\r\nHello\r\n\r\n2\r\n00:00:03,000 --> 00:00:04,000\r\nworld\r\n" + assertEquals("Hello\nworld", WebVttText.cueText(srt)) + } + + @Test + fun unknownReferencesAndStrayAmpersandsStayAsWritten() { + val vtt = "WEBVTT\n\n00:00.000 --> 00:01.000\nfish & chips &unknown; &#xZZ;" + assertEquals("fish & chips &unknown; &#xZZ;", WebVttText.cueText(vtt)) + } + + @Test + fun anUnterminatedTagRunsToTheEndOfTheLine() { + val vtt = "WEBVTT\n\n00:00.000 --> 00:01.000\nkept () + track.forEachIndexableField { visited.add(it) } + assertEquals(track.indexableContent(), visited.joinToString(track.indexableSeparator())) + + val plain = TextTrackEvent("2".repeat(64), "a1".repeat(32), 1L, tags, "no cues here", "") + assertEquals("no cues here", plain.indexableContent()) + } +}