Merge pull request #4215 from vitorpamplona/claude/search-roles-lists-releases-webvtt

fix(quartz): give search the release notes, list roles and caption words it was missing

.claude/skills/searchable-events/references/searchable-kinds.md

@@ -140,7 +140,7 @@ Separator legend: **NL** = `joinToString("\n")`, **SP** = `joinToString(" ")`.
 | 39000 | GroupMetadataEvent | nip29RelayGroups/metadata | `listOfNotNull(name(), about())` NL |
 | 39089 | StarterPackEvent | nip51Lists/starterPack | `listOfNotNull(title(), description())` NL |
 | 39092 | MediaStarterPackEvent | nip51Lists/mediaStarterPack | `listOfNotNull(title(), description())` NL |
-| 39307 | TextTrackEvent | nip71Video/textTrack | `content` (the WebVTT cue text) |
+| 39307 | TextTrackEvent | nip71Video/textTrack | `WebVttText.cueText(content)` NL — each cue's payload lines with tags removed and character references decoded; no header, timings, cue settings, ids, `NOTE`/`STYLE`/`REGION` blocks. Content with no `-->` timing line is not a caption file and is indexed as written |
 | 39701 | WebBookmarkEvent | nipB0WebBookmarks | `listOfNotNull(title(), description())` NL |
 | 39998 | AddressableListHeaderEvent | experimental/decentralizedLists/header | same as 9998 |
 | 39999 | AddressableListItemEvent | experimental/decentralizedLists/item | same as 9998 |

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/SearchExt.kt

@@ -20,12 +20,17 @@
  */
 package com.vitorpamplona.quartz.experimental.decentralizedLists
 
+import com.vitorpamplona.quartz.experimental.decentralizedLists.header.names
 import com.vitorpamplona.quartz.experimental.decentralizedLists.header.tags.NamesTag
 import com.vitorpamplona.quartz.experimental.decentralizedLists.header.tags.SingularPlural
 import com.vitorpamplona.quartz.experimental.decentralizedLists.header.tags.TitlesTag
+import com.vitorpamplona.quartz.experimental.decentralizedLists.header.titles
+import com.vitorpamplona.quartz.experimental.decentralizedLists.item.comments
+import com.vitorpamplona.quartz.experimental.decentralizedLists.item.name
 import com.vitorpamplona.quartz.experimental.decentralizedLists.item.tags.CommentsTag
 import com.vitorpamplona.quartz.experimental.decentralizedLists.item.tags.NameTag
 import com.vitorpamplona.quartz.experimental.decentralizedLists.item.tags.TitleTag
+import com.vitorpamplona.quartz.experimental.decentralizedLists.item.title
 import com.vitorpamplona.quartz.experimental.decentralizedLists.tags.DescriptionTag
 import com.vitorpamplona.quartz.nip01Core.core.TagArray
 import com.vitorpamplona.quartz.nip01Core.core.fastForEach
@@ -84,6 +89,20 @@ fun TagArray.forEachSearchableListField(visitor: IndexableFieldVisitor): Boolean
     return true
 }
 
+/**
+ * The TITLE role of [forEachSearchableListField], for search engines that weight fields: what the
+ * list or item is called — the header's `names` and `titles` (singular, then plural) and the
+ * item's `name` and `title`. Same first-well-formed-tag rule as the walk.
+ */
+fun TagArray.searchableListTitles(): List<String?> {
+    val names = names()
+    val titles = titles()
+    return listOf(names?.singular, names?.plural, titles?.singular, titles?.plural, name(), title())
+}
+
+/** The DESCRIPTION role of [forEachSearchableListField]: `description`, then `comments`. */
+fun TagArray.searchableListDescriptions(): List<String?> = listOf(description(), comments())
+
 /** The write-path join of [forEachSearchableListField]: one field per line. */
 fun TagArray.searchableListContent() =
     buildString {

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt

@@ -29,6 +29,9 @@ import com.vitorpamplona.quartz.experimental.agora.FundraiserEvent
 import com.vitorpamplona.quartz.experimental.audio.track.AudioTrackEvent
 import com.vitorpamplona.quartz.experimental.birdstar.BirdDetectionEvent
 import com.vitorpamplona.quartz.experimental.birdstar.BirdexEvent
+import com.vitorpamplona.quartz.experimental.decentralizedLists.DecentralizedListEvent
+import com.vitorpamplona.quartz.experimental.decentralizedLists.searchableListDescriptions
+import com.vitorpamplona.quartz.experimental.decentralizedLists.searchableListTitles
 import com.vitorpamplona.quartz.experimental.edits.TextNoteModificationEvent
 import com.vitorpamplona.quartz.experimental.fitness.workout.ExerciseTemplateEvent
 import com.vitorpamplona.quartz.experimental.fitness.workout.WorkoutRecordEvent
@@ -380,8 +383,11 @@ object SearchFieldExtractor {
                 tiers(event, event.title(), event.description(), null)
             }
 
+            // kind 30063 -- the body is a NIP-82 release's notes, and nothing for a
+            // NIP-51 set (its content may be encrypted private items); the event
+            // owns that guard, so both search paths apply the same one.
             is ReleaseArtifactSetEvent -> {
-                tiers(event, event.title(), event.description(), null)
+                tiers(event, event.title(), event.description(), event.searchableReleaseNotes())
             }
 
             is AppCurationSetEvent -> {
@@ -617,6 +623,17 @@ object SearchFieldExtractor {
                 tiers(event, null, null, event.content)
             }
 
+            // kinds 9998/39998/9999/39999 -- a list's or item's names and titles are
+            // what it is called, its description and comments what it is about. The
+            // `t` values indexableContent() appends are the funnel's hashtag role
+            // already, so -- as for 1111/1311/30382 -- they are not passed again, and
+            // content is not part of the spec. One branch for all four: an item may
+            // carry header tags (the spec's nonstandard method), and the tag helpers
+            // read whichever are present.
+            is DecentralizedListEvent -> {
+                tiers(event, event.tags.searchableListTitles(), event.tags.searchableListDescriptions(), null)
+            }
+
             // kind 1 LAST among the explicit branches, defensively: a future
             // kind extending the text-note base must hit its own branch first.
             is TextNoteEvent -> {

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip51Lists/releaseArtifactSet/ReleaseArtifactSetEvent.kt

@@ -81,10 +81,14 @@ class ReleaseArtifactSetEvent(
         visitor.visit(searchableReleaseNotes())
     }
 
-    // Only NIP-82 releases carry release notes in `content`. A NIP-51 set may keep
-    // encrypted private items there, which must never reach the index. The blank
-    // check runs first so empty sets skip the two tag scans on every search.
-    private fun searchableReleaseNotes() = if (content.isNotBlank() && isNip82SoftwareRelease()) content else null
+    /**
+     * The release notes when they are safe to index, else null. Only NIP-82 releases carry
+     * release notes in `content`; a NIP-51 set may keep encrypted private items there, which
+     * must never reach the index. The blank check runs first so empty sets skip the two tag
+     * scans on every search. `SearchFieldExtractor` reads this too, so both search paths
+     * apply the same guard.
+     */
+    fun searchableReleaseNotes() = if (content.isNotBlank() && isNip82SoftwareRelease()) content else null
 
     override fun eventHints() = tags.mapNotNull(EventBookmark::parseAsHint)
 

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/TextTrackEvent.kt

@@ -59,11 +59,15 @@ class TextTrackEvent(
     AddressHintProvider,
     SearchableEvent {
     // The cue text is the only searchable thing here: the `d` tag is a slug and the rest is
-    // plumbing. Indexing it makes a video findable by what is said in it.
-    override fun indexableContent() = content
+    // plumbing. Indexing it makes a video findable by what is said in it — the words only,
+    // not the timings, cue settings and markup around them (see WebVttText). Content with no
+    // cue at all is not a caption file and is indexed as written.
+    override fun indexableContent() = if (WebVttText.hasCues(content)) WebVttText.cueText(content) else content
 
+    // The read path: the same lines cueText() joins, one visit per line so a hit early in the
+    // track stops the walk before the rest is cleaned.
     override fun forEachIndexableField(visitor: IndexableFieldVisitor) {
-        visitor.visit(content)
+        if (WebVttText.hasCues(content)) WebVttText.forEachCueLine(content, visitor) else visitor.visit(content)
     }
 
     override fun addressHints() = tags.mapNotNull(ATag::parseAsHint)

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttText.kt

@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.quartz.nip71Video.textTrack
+
+import com.vitorpamplona.quartz.nip50Search.IndexableFieldVisitor
+
+/**
+ * The words a WebVTT document says: every cue's payload lines with their markup removed, and
+ * nothing else.
+ *
+ * Search is the reason this exists. A text track's `content` is the whole document, so indexing
+ * it raw puts `WEBVTT`, every `00:01:02.500 --> 00:01:04.000` timing line, cue settings
+ * (`align:start`), cue ids, `NOTE` and `STYLE` blocks, and inline tags (`<v Roger>`, `<i>`,
+ * `<00:00:01.000>`) into the index beside the dialogue. A query for "00" or "align" then
+ * matches every caption ever published, and the terms that matter are diluted by markup.
+ *
+ * ## What counts as a cue line
+ *
+ * One pass over the lines, no model built. A line containing `-->` opens a cue (it is the
+ * timing line; the optional identifier line before it is skipped because nothing is open yet),
+ * and every non-blank line after it until the next blank line is payload. Everything outside a
+ * cue — the `WEBVTT` header, `NOTE`, `STYLE`, `REGION` — is skipped by the same rule, since none
+ * of it may contain `-->`. SRT happens to fit the same shape (a numeric id, a timing line, the
+ * text), so a publisher that wraps SRT instead of WebVTT is read correctly too.
+ *
+ * A document with no timing line at all is not a caption file, so it is indexed as written
+ * rather than discarded: [hasCues] is the switch, and callers pass such content through
+ * unchanged.
+ *
+ * ## Markup
+ *
+ * Inline tags are removed, not their contents: `<i>really</i>` is `really`. Voice spans lose the
+ * speaker name with the tag (`<v Roger>Hello` is `Hello`): the name is an annotation, not
+ * something said. The HTML character references WebVTT allows are decoded — the named ones the
+ * spec lists, plus numeric ones — so `Tom &amp; Jerry` is indexed as `Tom & Jerry`. An unknown
+ * reference is kept as written.
+ */
+object WebVttText {
+    private const val TIMING_ARROW = "-->"
+
+    /** Whether [document] has at least one cue. Without one it is not a caption file. */
+    fun hasCues(document: String) = document.contains(TIMING_ARROW)
+
+    /**
+     * The cue text of [document], one payload line per line. Callers should check [hasCues]
+     * first; a document without cues yields an empty string here.
+     */
+    fun cueText(document: String): String {
+        val out = StringBuilder(document.length)
+        forEachCueLine(document) { line ->
+            if (out.isNotEmpty()) out.append('\n')
+            out.append(line)
+            true
+        }
+        return out.toString()
+    }
+
+    /**
+     * Hands each cleaned, non-empty payload line of [document] to [visitor] in order. Joining
+     * them with `"\n"` gives exactly [cueText]. Returns false when the visitor stopped the walk.
+     *
+     * Lines without markup are passed as substrings; only a line carrying a tag or a character
+     * reference is rebuilt.
+     */
+    fun forEachCueLine(
+        document: String,
+        visitor: IndexableFieldVisitor,
+    ): Boolean {
+        var inCue = false
+        var start = 0
+        val length = document.length
+        while (start <= length) {
+            var end = start
+            while (end < length && document[end] != '\n' && document[end] != '\r') end++
+
+            if (isBlank(document, start, end)) {
+                inCue = false
+            } else if (!inCue) {
+                // Header, NOTE / STYLE / REGION blocks and cue identifiers all end up here and
+                // are skipped; only a timing line opens a cue.
+                if (hasArrow(document, start, end)) inCue = true
+            } else {
+                val line = clean(document, start, end)
+                if (line != null && !visitor.visit(line)) return false
+            }
+
+            if (end >= length) break
+            // A CRLF pair is one line break.
+            start = if (document[end] == '\r' && end + 1 < length && document[end + 1] == '\n') end + 2 else end + 1
+        }
+        return true
+    }
+
+    /** Bounded to the line: an unbounded indexOf would rescan up to the next cue from every skipped line. */
+    private fun hasArrow(
+        s: String,
+        start: Int,
+        end: Int,
+    ): Boolean {
+        for (i in start until end - 2) {
+            if (s[i] == '-' && s[i + 1] == '-' && s[i + 2] == '>') return true
+        }
+        return false
+    }
+
+    private fun isBlank(
+        s: String,
+        start: Int,
+        end: Int,
+    ): Boolean {
+        for (i in start until end) if (!s[i].isWhitespace()) return false
+        return true
+    }
+
+    /** The payload line in `[start, end)` without tags, references decoded and trimmed; null when nothing is left. */
+    private fun clean(
+        s: String,
+        start: Int,
+        end: Int,
+    ): String? {
+        var markup = false
+        for (i in start until end) {
+            val c = s[i]
+            if (c == '<' || c == '&') {
+                markup = true
+                break
+            }
+        }
+        if (!markup) return s.substring(start, end).trim().ifEmpty { null }
+
+        val out = StringBuilder(end - start)
+        var i = start
+        while (i < end) {
+            val c = s[i]
+            when (c) {
+                '<' -> {
+                    val close = s.indexOf('>', i + 1)
+                    // An unterminated tag runs to the end of the line, as the WebVTT parser reads it.
+                    i = if (close < 0 || close >= end) end else close + 1
+                }
+
+                '&' -> {
+                    val semi = s.indexOf(';', i + 1)
+                    val decoded = if (semi in (i + 1) until end) decodeReference(s, i + 1, semi) else null
+                    if (decoded == null) {
+                        out.append(c)
+                        i++
+                    } else {
+                        appendCodePoint(out, decoded)
+                        i = semi + 1
+                    }
+                }
+
+                else -> {
+                    out.append(c)
+                    i++
+                }
+            }
+        }
+        return out.toString().trim().ifEmpty { null }
+    }
+
+    /** The code point named by the reference body in `[start, end)` (between `&` and `;`), or null. */
+    private fun decodeReference(
+        s: String,
+        start: Int,
+        end: Int,
+    ): Int? {
+        if (end - start > 10) return null
+        if (s[start] == '#') {
+            val hex = start + 1 < end && (s[start + 1] == 'x' || s[start + 1] == 'X')
+            val digits = s.substring(if (hex) start + 2 else start + 1, end)
+            val code = digits.toIntOrNull(if (hex) 16 else 10) ?: return null
+            return if (code in 1..0x10FFFF && code !in 0xD800..0xDFFF) code else null
+        }
+        return when (s.substring(start, end)) {
+            "amp" -> '&'.code
+            "lt" -> '<'.code
+            "gt" -> '>'.code
+            "quot" -> '"'.code
+            "apos" -> '\''.code
+            "nbsp" -> 0x00A0
+            "lrm" -> 0x200E
+            "rlm" -> 0x200F
+            else -> null
+        }
+    }
+
+    private fun appendCodePoint(
+        out: StringBuilder,
+        code: Int,
+    ) {
+        if (code <= 0xFFFF) {
+            out.append(code.toChar())
+        } else {
+            val v = code - 0x10000
+            out.append((0xD800 + (v shr 10)).toChar())
+            out.append((0xDC00 + (v and 0x3FF)).toChar())
+        }
+    }
+}

quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt

@@ -23,6 +23,8 @@ package com.vitorpamplona.quartz.nip50Search
 import com.vitorpamplona.quartz.buzz.agentProfiles.AgentProfileEvent
 import com.vitorpamplona.quartz.experimental.birdstar.BirdDetectionEvent
 import com.vitorpamplona.quartz.experimental.birdstar.BirdexEvent
+import com.vitorpamplona.quartz.experimental.decentralizedLists.header.ListHeaderEvent
+import com.vitorpamplona.quartz.experimental.decentralizedLists.item.AddressableListItemEvent
 import com.vitorpamplona.quartz.experimental.nip95.header.FileStorageHeaderEvent
 import com.vitorpamplona.quartz.experimental.ps1saves.Ps1SaveEvent
 import com.vitorpamplona.quartz.experimental.trustedLists.users.UserTrustedListEvent
@@ -38,9 +40,11 @@ import com.vitorpamplona.quartz.nip29RelayGroups.moderation.GroupEditMetadataEve
 import com.vitorpamplona.quartz.nip32Labeling.LabelEvent
 import com.vitorpamplona.quartz.nip34Git.repository.GitRepositoryEvent
 import com.vitorpamplona.quartz.nip35Torrents.TorrentEvent
+import com.vitorpamplona.quartz.nip51Lists.releaseArtifactSet.ReleaseArtifactSetEvent
 import com.vitorpamplona.quartz.nip53LiveActivities.meetingSpaces.MeetingSpaceEvent
 import com.vitorpamplona.quartz.nip5aStaticWebsites.NamedSiteEvent
 import com.vitorpamplona.quartz.nip69P2pOrderEvents.P2POrderEvent
+import com.vitorpamplona.quartz.nip71Video.textTrack.TextTrackEvent
 import com.vitorpamplona.quartz.nip85TrustedAssertions.users.UserAssertionEvent
 import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent
 import com.vitorpamplona.quartz.nip89AppHandlers.definition.AppDefinitionEvent
@@ -412,6 +416,73 @@ class SearchFieldExtractorTest {
         assertEquals(IndexableFields.Tiered(secondary = listOf("spam"), text = "obvious bot"), fields)
     }
 
+    @Test
+    fun nip82ReleaseNotesAreTheBodyOfARelease() {
+        // indexableContent() already carried the notes; the branch used to drop them, so an
+        // engine reading this extractor could not find a release by what changed in it.
+        val tags = arrayOf(arrayOf("d", "com.example.app@1.2.0"), arrayOf("i", "com.example.app"), arrayOf("version", "1.2.0"), arrayOf("title", "Example 1.2"))
+        val fields = SearchFieldExtractor.extract(ReleaseArtifactSetEvent("2e".repeat(32), alice, 1L, tags, "fixes the offline sync crash", ""))
+        assertEquals(IndexableFields.Tiered(primary = listOf("Example 1.2"), text = "fixes the offline sync crash"), fields)
+    }
+
+    @Test
+    fun nip51ReleaseSetContentIsNeverTheBody() {
+        // A NIP-51 set (no `i` + `version`) may keep encrypted private items in content.
+        val tags = arrayOf(arrayOf("d", "release"), arrayOf("title", "Example"), arrayOf("description", "all builds"))
+        val fields = SearchFieldExtractor.extract(ReleaseArtifactSetEvent("2f".repeat(32), alice, 1L, tags, "AgJ0b3BzZWNyZXQ=", ""))
+        assertEquals(IndexableFields.Tiered(primary = listOf("Example"), secondary = listOf("all builds")), fields)
+    }
+
+    @Test
+    fun decentralizedListHeadersSplitWhatTheyAreCalledFromWhatTheyAreAbout() {
+        val tags =
+            arrayOf(
+                arrayOf("names", "podcast", "podcasts"),
+                arrayOf("titles", "Podcast", "Podcasts"),
+                arrayOf("description", "shows worth a listen"),
+                arrayOf("t", "audio"),
+            )
+        val fields = SearchFieldExtractor.extract(ListHeaderEvent("3a".repeat(32), alice, 1L, tags, "", ""))
+        assertEquals(
+            IndexableFields.Tiered(
+                primary = listOf("podcast", "podcasts", "Podcast", "Podcasts"),
+                secondary = listOf("shows worth a listen"),
+                hashtags = listOf("audio"),
+            ),
+            fields,
+        )
+    }
+
+    @Test
+    fun decentralizedListItemHashtagsAreNotIndexedTwice() {
+        // indexableContent() appends the `t` values; the funnel already carries them as
+        // hashtags, so the branch leaves them out of every tier (the 1111 reasoning).
+        val tags =
+            arrayOf(
+                arrayOf("z", "podcasts"),
+                arrayOf("title", "Bitcoin Audible"),
+                arrayOf("description", "long reads, read aloud"),
+                arrayOf("comments", "start with the classics"),
+                arrayOf("t", "bitcoin"),
+            )
+        val fields = SearchFieldExtractor.extract(AddressableListItemEvent("3b".repeat(32), alice, 1L, tags + arrayOf(arrayOf("d", "ba")), "", ""))
+        assertEquals(
+            IndexableFields.Tiered(
+                primary = listOf("Bitcoin Audible"),
+                secondary = listOf("long reads, read aloud", "start with the classics"),
+                hashtags = listOf("bitcoin"),
+            ),
+            fields,
+        )
+    }
+
+    @Test
+    fun textTracksIndexWhatIsSaidNotTheTimings() {
+        val vtt = "WEBVTT\n\n1\n00:00:00.000 --> 00:00:02.000 align:start\n<v Roger>Hello nostr\n"
+        val fields = SearchFieldExtractor.extract(TextTrackEvent("3c".repeat(32), alice, 1L, arrayOf(arrayOf("d", "subtitles:v1")), vtt, ""))
+        assertEquals(IndexableFields.Tiered(text = "Hello nostr"), fields)
+    }
+
     @Test
     fun commentHashtagsAreNotIndexedTwice() {
         // indexableContent() concatenates the `t` tags INTO the body for kind

quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip71Video/textTrack/WebVttTextTest.kt

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.quartz.nip71Video.textTrack
+
+import com.vitorpamplona.quartz.nip50Search.IndexableFieldVisitor
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+class WebVttTextTest {
+    private val document =
+        """
+        WEBVTT - the whole header line is skipped
+        Kind: captions
+
+        STYLE
+        ::cue { color: yellow }
+
+        NOTE this block is a comment
+        and so is this line
+
+        intro
+        00:00:00.000 --> 00:00:02.000 align:start position:10%
+        <v Roger Bingham>We are in New York City
+
+        00:00:02.500 --> 00:00:04.000
+        <i>Really</i>? Tom &amp; Jerry&#x21;
+        <c.loud>on <00:00:03.000>two lines</c>
+
+        00:00:05.000 --> 00:00:06.000
+        <b></b>
+        """.trimIndent()
+
+    @Test
+    fun keepsOnlyTheWordsOfEachCue() {
+        assertEquals("We are in New York City\nReally? Tom & Jerry!\non two lines", WebVttText.cueText(document))
+    }
+
+    @Test
+    fun theVisitedLinesRejoinToTheCueText() {
+        val lines = mutableListOf<String>()
+        assertTrue(WebVttText.forEachCueLine(document, IndexableFieldVisitor { lines.add(it!!) }))
+        assertEquals(WebVttText.cueText(document), lines.joinToString("\n"))
+    }
+
+    @Test
+    fun aVisitorCanStopTheWalk() {
+        val lines = mutableListOf<String>()
+        assertFalse(
+            WebVttText.forEachCueLine(document, IndexableFieldVisitor { lines.add(it!!) && false }),
+        )
+        assertEquals(listOf("We are in New York City"), lines)
+    }
+
+    @Test
+    fun crlfAndSrtAreReadTheSameWay() {
+        val srt = "1\r\n00:00:01,000 --> 00:00:02,000\r\nHello\r\n\r\n2\r\n00:00:03,000 --> 00:00:04,000\r\nworld\r\n"
+        assertEquals("Hello\nworld", WebVttText.cueText(srt))
+    }
+
+    @Test
+    fun unknownReferencesAndStrayAmpersandsStayAsWritten() {
+        val vtt = "WEBVTT\n\n00:00.000 --> 00:01.000\nfish & chips &unknown; &#xZZ;"
+        assertEquals("fish & chips &unknown; &#xZZ;", WebVttText.cueText(vtt))
+    }
+
+    @Test
+    fun anUnterminatedTagRunsToTheEndOfTheLine() {
+        val vtt = "WEBVTT\n\n00:00.000 --> 00:01.000\nkept <i unterminated\nnext line"
+        assertEquals("kept\nnext line", WebVttText.cueText(vtt))
+    }
+
+    @Test
+    fun plainTextIsNotACaptionFile() {
+        assertFalse(WebVttText.hasCues("just a description of the track"))
+        assertTrue(WebVttText.hasCues(document))
+    }
+
+    @Test
+    fun theEventIndexesCueTextAndPassesPlainContentThrough() {
+        val tags = arrayOf(arrayOf("d", "subtitles:v1"))
+        val track = TextTrackEvent("1".repeat(64), "a1".repeat(32), 1L, tags, document, "")
+        assertEquals(WebVttText.cueText(document), track.indexableContent())
+
+        val visited = mutableListOf<String?>()
+        track.forEachIndexableField { visited.add(it) }
+        assertEquals(track.indexableContent(), visited.joinToString(track.indexableSeparator()))
+
+        val plain = TextTrackEvent("2".repeat(64), "a1".repeat(32), 1L, tags, "no cues here", "")
+        assertEquals("no cues here", plain.indexableContent())
+    }
+}