Merge pull request #3963 from vitorpamplona/claude/opengraph-preview-rendering-na1cul

Optimize MetaTagsParser: fix quote tracking in comments/scripts
This commit is contained in:
Vitor Pamplona
2026-08-21 17:20:24 -04:00
committed by GitHub
7 changed files with 1098 additions and 112 deletions
@@ -21,7 +21,6 @@
package com.vitorpamplona.amethyst.commons.preview
import com.vitorpamplona.amethyst.commons.util.codePointToChars
import kotlinx.collections.immutable.toImmutableMap
data class MetaTag(
private val attrs: Map<String, String>,
@@ -32,9 +31,44 @@ data class MetaTag(
fun attr(name: String): String = attrs[name.lowercase()] ?: ""
}
/**
* Extracts `<meta>` tags out of a (possibly partial) HTML document.
*
* This runs on every link preview, over bytes straight off the network, so the scan touches each
* character once and allocates nothing until an actual `<meta>` shows up: `<` is found with
* [String.indexOf], tag names are compared in place against the four names that matter, and only a
* meta tag's attribute span is ever handed to [parseAttrs].
*/
object MetaTagsParser {
private val NON_ATTR_NAME_CHARS = setOf(Char(0x0), '"', '\'', '>', '/')
private val NON_UNQUOTED_ATTR_VALUE_CHARS = setOf('"', '\'', '=', '>', '<', '`')
private const val NO_QUOTE = ' '
private const val META = "meta"
private const val HEAD = "head"
// Elements whose content is text rather than markup: script and style hold raw text, title and
// textarea hold character data. A `<` inside any of them is not a tag.
private const val SCRIPT = "script"
private const val STYLE = "style"
private const val TITLE = "title"
private const val TEXTAREA = "textarea"
private const val SCRIPT_END = "</script"
private const val STYLE_END = "</style"
private const val TITLE_END = "</title"
private const val TEXTAREA_END = "</textarea"
private const val COMMENT_START = "!--"
private const val COMMENT_END = "-->"
private enum class TagKind {
/** A `<meta …>` start tag: its attribute span is [TagScanner.attrsStart]..<[TagScanner.attrsEnd]. */
META,
/** The `</head>` that ends the interesting part of the document. */
HEAD_END,
/** Anything else: other elements, comments, declarations, unparseable markup. */
OTHER,
}
/**
* Lazily parse a partial HTML document and extract meta tags.
@@ -43,100 +77,176 @@ object MetaTagsParser {
sequence {
val s = TagScanner(input)
while (!s.exhausted()) {
val t = s.nextTag() ?: continue
if (t.name == "head" && t.isEnd) {
break
}
if (t.name == "meta") {
val attrs = parseAttrs(t.attrPart) ?: continue
val kind = s.nextTag()
if (kind == TagKind.HEAD_END) break
if (kind == TagKind.META) {
val attrs = parseAttrs(input, s.attrsStart, s.attrsEnd) ?: continue
yield(MetaTag(attrs))
}
}
}
private data class RawTag(
val isEnd: Boolean,
val name: String,
val attrPart: String,
)
private class TagScanner(
private val input: String,
) {
private val length = input.length
private var p = 0
fun exhausted(): Boolean = p >= input.length
/** Attribute span of the tag [nextTag] last reported as [TagKind.META]. */
var attrsStart = 0
private set
var attrsEnd = 0
private set
private fun peek(): Char = input[p]
fun exhausted(): Boolean = p >= length
private fun consume(): Char = input[p++]
/**
* True when `input[from..<to]` equals [lower], ASCII-case-insensitively. [lower] must hold
* only lowercase ASCII letters: `code or 0x20` lands on such a letter only when the input
* char is that same letter in either case, so the fold cannot produce a false positive.
*/
private fun nameIs(
from: Int,
to: Int,
lower: String,
): Boolean {
if (to - from != lower.length) return false
for (i in lower.indices) {
val c = input[from + i]
if (c != lower[i] && (c.code or 0x20).toChar() != lower[i]) return false
}
return true
}
private fun skipWhile(pred: (Char) -> Boolean) {
while (!this.exhausted() && pred(this.peek())) {
this.consume()
private fun skipComment() {
val end = input.indexOf(COMMENT_END, p)
p = if (end < 0) length else end + COMMENT_END.length
}
private fun skipToTagEnd() {
val end = input.indexOf('>', p)
p = if (end < 0) length else end + 1
}
/** Leaves [p] on the `</name` that closes a raw-text element, or at the end of the input. */
private fun skipRawText(endTag: String) {
var i = p
while (true) {
i = input.indexOf('<', i)
if (i < 0) {
p = length
return
}
if (input.regionMatches(i, endTag, 0, endTag.length, ignoreCase = true)) {
p = i
return
}
i++
}
}
private fun skipSpaces() {
this.skipWhile { it.isWhitespace() }
}
fun nextTag(): RawTag? {
skipWhile { it != '<' }
if (this.exhausted()) return null
consume()
// read tag name
val isEnd = peek() == '/'
if (isEnd) {
consume()
fun nextTag(): TagKind {
val lt = input.indexOf('<', p)
if (lt < 0) {
p = length
return TagKind.OTHER
}
p = lt + 1
if (p >= length) return TagKind.OTHER
// `<!-- ... -->`, `<!DOCTYPE ...>` and `<?...?>` are not element markup, so the
// attribute-quote tracking below must not run over them. A comment holding an odd
// number of quote characters -- an apostrophe in "we don't", a lone `"` -- would
// otherwise leave the scanner inside a phantom quoted attribute value and make it
// swallow every tag that follows, until the next matching quote character. That is
// enough to hide a page's whole `<meta property="og:*">` block from the preview.
val first = input[p]
if (first == '!' || first == '?') {
if (input.startsWith(COMMENT_START, p)) {
skipComment()
} else {
skipToTagEnd()
}
return TagKind.OTHER
}
// read the tag name
val isEnd = first == '/'
if (isEnd) p++
val nameStart = p
skipWhile { !it.isWhitespace() && it != '>' }
while (p < length && !input[p].isWhitespace() && input[p] != '>') p++
val nameEnd = p
// seek to start of attrs part
skipSpaces()
val attrsStart = p
// seek to the start of the attrs part
while (p < length && input[p].isWhitespace()) p++
attrsStart = p
// skip until end of tag
var quote: Char? = null
while (!exhausted()) {
val c = consume()
when {
// `/>` out of quote -> end of tag
quote == null && c == '/' && peek() == '>' -> {
consume()
// skip to the end of the tag, tracking quoted values so a `>` inside one doesn't end it
var i = p
var quote = NO_QUOTE
while (i < length) {
val c = input[i]
if (quote == NO_QUOTE) {
// `>` or `/>` out of quote -> end of tag
if (c == '>') {
i++
break
}
// `>` out of quote -> end of tag
quote == null && c == '>' -> {
if (c == '/' && i + 1 < length && input[i + 1] == '>') {
i += 2
break
}
// entering quote
quote == null && (c == '\'' || c == '"') -> {
quote = c
}
// leaving quote
quote != null && c == quote -> {
quote = null
}
if (c == '"' || c == '\'') quote = c
} else if (c == quote) {
quote = NO_QUOTE
}
i++
}
val attrsEnd = p - 1
p = i
attrsEnd = i - 1
val name = input.slice(nameStart..<nameEnd)
if (!name.matches(Regex("""[0-9a-zA-Z]+"""))) {
return null
if (isEnd) {
return if (nameIs(nameStart, nameEnd, HEAD)) TagKind.HEAD_END else TagKind.OTHER
}
val attrsPart = input.slice(attrsStart..<attrsEnd)
return RawTag(isEnd, name.lowercase(), attrsPart)
// Text-only elements are skipped whole: `for (i = 0; i < n; i++)` in a script, a quote
// in a JS string, or the `<` and the apostrophe in `<title>5 < 6, that's math</title>`
// are content, not markup, and scanning them as markup hides the tags that follow --
// the same way an unbalanced quote inside a comment does. Switching on the name length
// first keeps the common tag (a `<div>`, a `<link>`) down to one comparison.
when (nameEnd - nameStart) {
META.length -> if (nameIs(nameStart, nameEnd, META)) return TagKind.META
STYLE.length ->
if (nameIs(nameStart, nameEnd, STYLE)) {
skipRawText(STYLE_END)
} else if (nameIs(nameStart, nameEnd, TITLE)) {
skipRawText(TITLE_END)
}
SCRIPT.length -> if (nameIs(nameStart, nameEnd, SCRIPT)) skipRawText(SCRIPT_END)
TEXTAREA.length -> if (nameIs(nameStart, nameEnd, TEXTAREA)) skipRawText(TEXTAREA_END)
}
return TagKind.OTHER
}
}
// These two are `when` branches rather than a `Set<Char>` because `Set<Char>.contains` boxes
// the char, once per attribute character of every meta tag.
private fun isNonAttrNameChar(c: Char): Boolean =
when (c) {
'\u0000', '"', '\'', '>', '/' -> true
else -> false
}
private fun isNonUnquotedAttrValueChar(c: Char): Boolean =
when (c) {
'"', '\'', '=', '>', '<', '`' -> true
else -> false
}
// map of HTML element attribute name to its value, with additional logics:
// - attribute names are matched in a case-insensitive manner
// - attribute names never duplicate
@@ -216,16 +326,20 @@ object MetaTagsParser {
private val attrs = mutableMapOf<String, String>()
fun add(attr: Pair<String, String>) {
val name = attr.first.lowercase()
if (attrs.containsKey(name)) {
throw IllegalArgumentException("duplicated attribute name: $name")
}
val value = attr.second.replace(RE_CHAR_REF, Companion::replaceCharRefs)
attrs += Pair(name, value)
/** Adds an attribute, returning false if that name was already set (the first value wins). */
fun add(
name: String,
value: String,
): Boolean {
val key = name.lowercase()
if (attrs.containsKey(key)) return false
// Resolving character references is the expensive half of an attribute and almost no
// value has an `&` in it, so the scan for one pays for itself.
attrs[key] = if (value.indexOf('&') < 0) value else value.replace(RE_CHAR_REF, Companion::replaceCharRefs)
return true
}
fun freeze(): Map<String, String> = attrs.toImmutableMap()
fun freeze(): Map<String, String> = attrs
}
private enum class State {
@@ -236,16 +350,22 @@ object MetaTagsParser {
SPACE,
}
private fun parseAttrs(input: String): Map<String, String>? {
/** Parses the attributes of a single tag, held in `input[from..<to]`. */
private fun parseAttrs(
input: String,
from: Int,
to: Int,
): Map<String, String>? {
val attrs = Attrs()
var state = State.NAME
var nameBegin = 0
var nameEnd = 0
var valueBegin = 0
var valueQuote: Char? = null
var nameBegin = from
var nameEnd = from
var valueBegin = from
var valueQuote = NO_QUOTE
input.forEachIndexed { i, c ->
for (i in from..<to) {
val c = input[i]
when (state) {
State.NAME -> {
when {
@@ -259,7 +379,7 @@ object MetaTagsParser {
state = State.BEFORE_EQ
}
NON_ATTR_NAME_CHARS.contains(c) || c.isISOControl() || !c.isDefined() -> {
isNonAttrNameChar(c) || c.isISOControl() || !c.isDefined() -> {
return null
}
}
@@ -275,7 +395,7 @@ object MetaTagsParser {
else -> {
// if it is expecting = but gets another name, starts another property
runCatching { attrs.add(Pair(input.slice(nameBegin..<nameEnd), "")) }
attrs.add(input.substring(nameBegin, nameEnd), "")
nameBegin = i
state = State.NAME
@@ -295,47 +415,33 @@ object MetaTagsParser {
else -> {
valueBegin = i
valueQuote = null
valueQuote = NO_QUOTE
state = State.VALUE
}
}
}
State.VALUE -> {
var attr: Pair<String, String>? = null
if (valueQuote != null) {
if (c == valueQuote) {
attr =
Pair(
input.slice(nameBegin..<nameEnd),
input.slice(valueBegin..<i),
)
}
// -1 while the value is still running; otherwise the index one past its last char
var valueEnd = -1
if (valueQuote != NO_QUOTE) {
if (c == valueQuote) valueEnd = i
} else {
when {
c.isWhitespace() -> {
attr =
Pair(
input.slice(nameBegin..<nameEnd),
input.slice(valueBegin..<i),
)
}
c.isWhitespace() -> valueEnd = i
i == input.length - 1 -> {
attr =
Pair(
input.slice(nameBegin..<nameEnd),
input.slice(valueBegin..i),
)
}
i == to - 1 -> valueEnd = i + 1
NON_UNQUOTED_ATTR_VALUE_CHARS.contains(c) -> {
return null
}
isNonUnquotedAttrValueChar(c) -> return null
}
}
if (attr != null) {
runCatching { attrs.add(attr) }.getOrNull() ?: return null
if (valueEnd >= 0) {
val added =
attrs.add(
input.substring(nameBegin, nameEnd),
input.substring(valueBegin, valueEnd),
)
if (!added) return null
state = State.SPACE
}
}
@@ -0,0 +1,120 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
import kotlinx.coroutines.test.runTest
import kotlin.test.Test
import kotlin.test.assertEquals
/**
* The charset half of the meta scan: `<meta charset>` and `<meta http-equiv="content-type">` are
* the tags that decide how the rest of the document -- including every og: value -- is decoded.
* Get this wrong and a preview renders mojibake rather than nothing, so it fails quietly.
*/
class HtmlParserCharsetTest {
private suspend fun firstContentOf(
bytes: ByteArray,
charsetName: String?,
): String =
HtmlParser()
.parseHtml(bytes, charsetName)
.last()
.attr("content")
// -- HtmlCharsetParser --------------------------------------------------------------------
@Test
fun sniffsTheCharsetAttribute() {
assertEquals("iso-8859-1", HtmlCharsetParser.detectCharset("""<head><meta charset="iso-8859-1">""".encodeToByteArray()))
}
@Test
fun sniffsTheHttpEquivContentType() {
val html = """<head><meta http-equiv="Content-Type" content="text/html; charset=shift_jis">"""
assertEquals("shift_jis", HtmlCharsetParser.detectCharset(html.encodeToByteArray()))
}
@Test
fun defaultsToUtf8WhenNothingIsDeclared() {
assertEquals("UTF-8", HtmlCharsetParser.detectCharset("""<head><title>x</title>""".encodeToByteArray()))
}
@Test
fun onlySniffsTheFirstKilobyte() {
// The window is deliberate -- the declaration is required to be early -- but it means a
// charset pushed past 1 KB by a banner comment is not found, and UTF-8 is assumed.
val pushedOut = "<head>" + "<!-- " + "x".repeat(1100) + " -->" + """<meta charset="iso-8859-1">"""
assertEquals("UTF-8", HtmlCharsetParser.detectCharset(pushedOut.encodeToByteArray()))
}
@Test
fun aCommentedOutCharsetIsNotSniffed() {
val html = """<head><!-- <meta charset="iso-8859-1"> --><meta charset="utf-8">"""
assertEquals("utf-8", HtmlCharsetParser.detectCharset(html.encodeToByteArray()))
}
// -- HtmlParser: which charset wins --------------------------------------------------------
@Test
fun anExplicitCharsetWinsOverTheDocumentDeclaration() =
runTest {
// Content-Type said windows-1252; the document claims utf-8 and must not be believed.
val bytes =
byteArrayOf(0x3C) + // '<'
"""head><meta charset="utf-8"><meta property="og:title" content="caf""".encodeToByteArray() +
byteArrayOf(0xE9.toByte()) + // 'é' in windows-1252
""""></head>""".encodeToByteArray()
assertEquals("café", firstContentOf(bytes, "windows-1252"))
}
@Test
fun aByteOrderMarkWinsOverTheDocumentDeclaration() =
runTest {
val utf16 = """<head><meta charset="iso-8859-1"><meta property="og:title" content="café"></head>"""
val bytes = byteArrayOf(0xFE.toByte(), 0xFF.toByte()) + utf16.encodeToUtf16Be()
assertEquals("café", firstContentOf(bytes, null))
}
@Test
fun fallsBackToTheDocumentDeclarationWhenTheResponseHasNoCharset() =
runTest {
val bytes =
"""<head><meta charset="windows-1252"><meta property="og:title" content="caf""".encodeToByteArray() +
byteArrayOf(0xE9.toByte()) +
""""></head>""".encodeToByteArray()
assertEquals("café", firstContentOf(bytes, null))
}
private fun String.encodeToUtf16Be(): ByteArray {
val out = ByteArray(length * 2)
forEachIndexed { i, c ->
out[i * 2] = (c.code shr 8).toByte()
out[i * 2 + 1] = (c.code and 0xFF).toByte()
}
return out
}
}
@@ -0,0 +1,238 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
import kotlin.test.Test
import kotlin.test.assertEquals
/**
* Comments, declarations and the text-only elements (script, style, title, textarea) are not
* element markup. Scanning them for
* attribute quotes lets an odd apostrophe -- "we don't" is enough -- leave the scanner
* inside a phantom quoted value, swallowing every tag up to the next quote character.
* That is what hid the entire og: block of https://brainstorm.world from link previews.
*/
class MetaTagsParserCommentTest {
@Test
fun commentWithUnbalancedApostropheDoesNotSwallowFollowingMetaTags() {
val input =
"""
|<html><head>
| <!-- Fresh visitors stay LIGHT -- we don't auto-dark a dark-OS visitor.
| Honors 'dark' and 'system'. -->
| <meta property="og:title" content="Brainstorm">
| <meta property="og:image" content="https://example.com/og-image.png">
|</head></html>
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(2, metaTags.size)
assertEquals("Brainstorm", metaTags[0].attr("content"))
assertEquals("https://example.com/og-image.png", metaTags[1].attr("content"))
}
@Test
fun metaTagsInsideCommentsAreNotParsed() {
val input =
"""
|<html><head>
| <!-- <meta property="og:title" content="Commented Out"> -->
| <meta property="og:title" content="Real Title">
|</head></html>
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(1, metaTags.size)
assertEquals("Real Title", metaTags[0].attr("content"))
}
@Test
fun scriptBodyDoesNotSwallowFollowingMetaTags() {
val input =
"""
|<html><head>
| <script>
| var t = localStorage.getItem('theme');
| for (var i = 0; i < 3; i++) { console.log("<meta property=\"og:title\" content=\"Fake\">"); }
| </script>
| <meta property="og:title" content="Real Title">
|</head></html>
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(1, metaTags.size)
assertEquals("Real Title", metaTags[0].attr("content"))
}
@Test
fun styleBodyDoesNotSwallowFollowingMetaTags() {
val input =
"""
|<html><head>
| <style>body { font-family: 'Figtree', sans-serif; }</style>
| <meta property="og:title" content="Real Title">
|</head></html>
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(1, metaTags.size)
assertEquals("Real Title", metaTags[0].attr("content"))
}
@Test
fun titleTextWithALessThanDoesNotSwallowFollowingMetaTags() {
// `<title>5 < 6, that's math</title>` is ordinary HTML: an unescaped `<` in title text,
// then an apostrophe. Scanned as markup, the `<` opens a phantom tag and the apostrophe
// opens a phantom attribute value that runs to the end of the document.
val input =
"""
|<html><head>
| <title>5 < 6, that's math</title>
| <meta property="og:title" content="Real Title">
|</head></html>
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(1, metaTags.size)
assertEquals("Real Title", metaTags[0].attr("content"))
}
@Test
fun metaTagsInsideTitleTextAreNotParsed() {
// Title content is character data, so this is a title that reads literally
// `a <meta property="og:title" content="Fake"> b`, not a second og:title.
val input =
"""
|<html><head>
| <title>a <meta property="og:title" content="Fake"> b</title>
| <meta property="og:title" content="Real Title">
|</head></html>
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(1, metaTags.size)
assertEquals("Real Title", metaTags[0].attr("content"))
}
@Test
fun textareaTextDoesNotSwallowFollowingMetaTags() {
val input =
"""
|<html><head>
| <textarea>x < y's z</textarea>
| <meta property="og:title" content="Real Title">
|</head></html>
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(1, metaTags.size)
assertEquals("Real Title", metaTags[0].attr("content"))
}
@Test
fun aSelfClosedScriptStillOpensRawText() {
// Deliberate, and what a browser does: `/` on a script start tag is ignored, so everything
// up to `</script>` is script data. A page written this way shows nothing after it either.
// Pinned so that "fixing" it never turns a JS string into an og: tag.
val input =
"""
|<html><head>
| <script src="a.js"/>
| <meta property="og:title" content="Unreachable">
|</head></html>
""".trimMargin()
assertEquals(0, MetaTagsParser.parse(input).count())
}
@Test
fun doctypeAndProcessingInstructionsAreSkipped() {
val input =
"""
|<?xml version="1.0" encoding="utf-8"?>
|<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
|<html><head>
| <meta property="og:title" content="Real Title">
|</head></html>
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(1, metaTags.size)
assertEquals("Real Title", metaTags[0].attr("content"))
}
@Test
fun unterminatedCommentEndsTheDocument() {
val input =
"""
|<html><head>
| <meta property="og:title" content="Real Title">
| <!-- truncated download cuts the comment here
""".trimMargin()
val metaTags = MetaTagsParser.parse(input).toList()
assertEquals(1, metaTags.size)
assertEquals("Real Title", metaTags[0].attr("content"))
}
@Test
fun extractsOpenGraphFromAHeadWithCommentsAndScripts() {
// Shape of https://brainstorm.world/ (any /p/<npub> route serves the same index.html).
val input =
"""
|<!DOCTYPE html>
|<html lang="en">
| <head>
| <meta charset="UTF-8" />
| <!-- No-flash theme: set class="dark" synchronously before first paint.
| Fresh visitors (no stored choice) stay LIGHT -- we don't auto-dark a
| dark-OS visitor. When ready, change to: 't === dark || (!t || t === system)'. -->
| <script>
| (function () {
| var t = localStorage.getItem("brainstorm_theme");
| if (t === "dark") document.documentElement.classList.add("dark");
| })();
| </script>
| <title>Brainstorm - Web of Trust for Nostr</title>
| <meta property="og:title" content="Brainstorm - Your Network. Your Rules." />
| <meta property="og:description" content="The decentralized Web of Trust layer for Nostr." />
| <meta property="og:image" content="https://brainstorm.nosfabrica.com/og-image.png" />
| </head>
| <body><div id="root"></div></body>
|</html>
""".trimMargin()
val info = OpenGraphParser().extractUrlInfo(MetaTagsParser.parse(input))
assertEquals("Brainstorm - Your Network. Your Rules.", info.title)
assertEquals("The decentralized Web of Trust layer for Nostr.", info.description)
assertEquals("https://brainstorm.nosfabrica.com/og-image.png", info.image)
}
}
@@ -0,0 +1,208 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Tag and attribute shapes a preview fetch can meet in the wild, and the ones a truncated or
* hostile response can produce. The server picks this input, so "it throws" and "it silently eats
* the rest of the head" both have to be ruled out for every shape here.
*/
class MetaTagsParserEdgeCaseTest {
private fun contents(html: String) = MetaTagsParser.parse(html).map { it.attr("content") }.toList()
// -- the end of the scan ------------------------------------------------------------------
@Test
fun stopsAtAnUppercaseHeadEndTag() {
val metas = contents("""<head><meta name="a" content="1"></HEAD><meta name="b" content="2">""")
assertEquals(listOf("1"), metas)
}
@Test
fun stopsAtAHeadEndTagWithTrailingSpace() {
val metas = contents("""<head><meta name="a" content="1"></head ><meta name="b" content="2">""")
assertEquals(listOf("1"), metas)
}
@Test
fun aHeadEndTagInsideAnAttributeValueDoesNotEndTheScan() {
val metas = contents("""<head><meta name="a" content="</head>"><meta name="b" content="2"></head>""")
assertEquals(listOf("</head>", "2"), metas)
}
@Test
fun scansTheWholeDocumentWhenThereIsNoHeadEndTag() {
val metas = contents("""<head><meta name="a" content="1"><body><meta name="b" content="2">""")
assertEquals(listOf("1", "2"), metas)
}
// -- truncated responses ------------------------------------------------------------------
@Test
fun aBodyTruncatedRightAfterASlashDoesNotThrow() {
// The `/` of a `/>` as the very last byte: the self-closing check must not read past it.
val metas = contents("""<head><meta name="a" content="1" /""")
assertEquals(listOf("1"), metas)
}
@Test
fun aBodyTruncatedInsideAnAttributeValueYieldsNoValue() {
val metas = MetaTagsParser.parse("""<head><meta property="og:title" content="Trunc""").toList()
assertEquals(1, metas.size)
assertEquals("og:title", metas[0].attr("property"))
assertEquals("", metas[0].attr("content"))
}
@Test
fun aBodyTruncatedRightAfterALessThanDoesNotThrow() {
assertEquals(listOf("1"), contents("""<head><meta name="a" content="1"><"""))
}
// -- tag shapes ---------------------------------------------------------------------------
@Test
fun readsAnUppercaseMetaTagAndUppercaseAttributeNames() {
val metas = contents("""<head><META PROPERTY="og:title" CONTENT="Real"></head>""")
assertEquals(listOf("Real"), metas)
}
@Test
fun anEmptySelfClosedMetaIsSkippedWithoutDerailingTheScan() {
// `<meta/>` has no separator before the `/`, so the name reads as `meta/` and the tag is
// dropped. It carries nothing anyway; what matters is that the next tag still parses.
val metas = contents("""<head><meta/><meta property="og:title" content="Real"></head>""")
assertEquals(listOf("Real"), metas)
}
@Test
fun aSelfClosingSequenceInsideAQuotedValueDoesNotEndTheTag() {
val metas =
contents(
"""<head><meta property="og:title" content="a /> b"><meta name="after" content="ok"></head>""",
)
assertEquals(listOf("a /> b", "ok"), metas)
}
@Test
fun readsMetaTagsInsideNoscript() {
// noscript content is markup for a parser that isn't running scripts, and a redirect meta
// hidden in there is exactly the kind a preview wants to see.
val metas =
contents(
"""<head><noscript><meta http-equiv="refresh" content="0"></noscript><meta property="og:title" content="Real"></head>""",
)
assertEquals(listOf("0", "Real"), metas)
}
@Test
fun skipsCdataSections() {
val metas = contents("""<head><![CDATA[ x > y ]]><meta property="og:title" content="Real"></head>""")
assertEquals(listOf("Real"), metas)
}
// -- attribute shapes ---------------------------------------------------------------------
@Test
fun keepsAttributesWhenTheTagEndsWithAValuelessOne() {
val metas = MetaTagsParser.parse("""<head><meta property="og:title" content="T" data-foo></head>""").toList()
assertEquals(1, metas.size)
assertEquals("og:title", metas[0].attr("property"))
assertEquals("T", metas[0].attr("content"))
}
@Test
fun keepsAValueThatSpansLines() {
val metas = contents("<head><meta property=\"og:title\" content=\"line1\nline2\"></head>")
assertEquals(listOf("line1\nline2"), metas)
}
@Test
fun anUnknownAttributeIsIgnoredNotFatal() {
val metas = contents("""<head><meta data-rh="true" property="og:title" content="Real"></head>""")
assertEquals(listOf("Real"), metas)
}
// -- character references in values --------------------------------------------------------
@Test
fun keepsQueryStringAmpersandsIntact() {
// og:image URLs are full of `&`; only a real character reference may be resolved.
val metas =
contents(
"""<head><meta property="og:image" content="https://x.com/i.png?w=1200&h=630&fit=crop&amp;q=80"></head>""",
)
assertEquals(listOf("https://x.com/i.png?w=1200&h=630&fit=crop&q=80"), metas)
}
@Test
fun decodesCharacterReferencesOutsideTheBasicMultilingualPlane() {
val metas = contents("""<head><meta property="og:title" content="&#128512; &#x1F600; hi"></head>""")
assertEquals(listOf("😀 😀 hi"), metas)
}
@Test
fun leavesAnUnknownCharacterReferenceAlone() {
val metas = contents("""<head><meta property="og:title" content="AT&T &notareference; &#xZZ;"></head>""")
assertEquals(listOf("AT&T &notareference; &#xZZ;"), metas)
}
// -- laziness -------------------------------------------------------------------------------
@Test
fun stopsReadingOnceTheConsumerStops() {
// OpenGraphParser bails as soon as it has title+description+image; the sequence must not
// have scanned the rest of the document by then. A tag after an unterminated comment is
// unreachable, so seeing the first one proves the scan was still lazy.
val html =
"""
|<head>
| <meta property="og:title" content="T">
| <!-- an unterminated comment swallows everything after it
| <meta property="og:description" content="D">
""".trimMargin()
val first = MetaTagsParser.parse(html).first()
assertEquals("T", first.attr("content"))
assertTrue(MetaTagsParser.parse(html).count() == 1)
}
}
@@ -20,9 +20,13 @@
*/
package com.vitorpamplona.amethyst.commons.preview
import org.junit.Assert.assertEquals
import org.junit.Test
import kotlin.test.Test
import kotlin.test.assertEquals
/**
* Lived in `androidDeviceTest` until it was moved here: nothing in it is Android-specific, and on
* a device-only source set it never ran in CI -- every attribute shape below was unguarded.
*/
class MetaTagsParserTest {
@Test
fun testParse() {
@@ -0,0 +1,145 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
import kotlin.test.Test
import kotlin.test.assertEquals
/**
* The three attributes a preview can be declared under -- `property` (Open Graph), `name`
* (Twitter cards and plain HTML) and `itemprop` (schema.org) -- and what happens when a page
* declares the same thing under more than one.
*/
class OpenGraphParserTest {
private fun extract(html: String) = OpenGraphParser().extractUrlInfo(MetaTagsParser.parse(html))
@Test
fun readsOpenGraphProperties() {
val info =
extract(
"""
|<head>
| <meta property="og:title" content="T">
| <meta property="og:description" content="D">
| <meta property="og:image" content="https://example.com/i.png">
|</head>
""".trimMargin(),
)
assertEquals("T", info.title)
assertEquals("D", info.description)
assertEquals("https://example.com/i.png", info.image)
}
@Test
fun fallsBackToTwitterCardNames() {
val info =
extract(
"""
|<head>
| <meta name="twitter:title" content="T">
| <meta name="twitter:description" content="D">
| <meta name="twitter:image" content="https://example.com/i.png">
|</head>
""".trimMargin(),
)
assertEquals("T", info.title)
assertEquals("D", info.description)
assertEquals("https://example.com/i.png", info.image)
}
@Test
fun fallsBackToPlainNames() {
val info =
extract(
"""
|<head>
| <meta name="title" content="T">
| <meta name="description" content="D">
| <meta name="image" content="https://example.com/i.png">
|</head>
""".trimMargin(),
)
assertEquals("T", info.title)
assertEquals("D", info.description)
assertEquals("https://example.com/i.png", info.image)
}
@Test
fun readsSchemaOrgItemprops() {
val info =
extract(
"""
|<head>
| <meta itemprop="name" content="ignored, not a title key">
| <meta itemprop="title" content="T">
| <meta itemprop="description" content="D">
| <meta itemprop="image" content="https://example.com/i.png">
|</head>
""".trimMargin(),
)
assertEquals("T", info.title)
assertEquals("D", info.description)
assertEquals("https://example.com/i.png", info.image)
}
@Test
fun takesEachFieldFromWhicheverTagCarriesItFirst() {
// NOTE: this is document order, not source priority. A page that puts a plain
// <meta name="description"> above its <meta property="og:description"> -- a very common
// CMS layout -- has the plain one win, even though og: is the more specific declaration.
// Pinned as the current behavior; changing it means preferring og: over name: explicitly.
val info =
extract(
"""
|<head>
| <meta name="description" content="plain, comes first">
| <meta property="og:description" content="og, comes second">
| <meta property="og:title" content="T">
| <meta property="og:image" content="https://example.com/i.png">
|</head>
""".trimMargin(),
)
assertEquals("plain, comes first", info.description)
}
@Test
fun missingFieldsComeBackEmptyRatherThanNull() {
val info = extract("""<head><meta property="og:title" content="T"></head>""")
assertEquals("T", info.title)
assertEquals("", info.description)
assertEquals("", info.image)
}
@Test
fun ignoresAMetaTagWithNoRecognizedKey() {
val info = extract("""<head><meta name="viewport" content="width=device-width"></head>""")
assertEquals("", info.title)
assertEquals("", info.description)
assertEquals("", info.image)
}
}
@@ -0,0 +1,165 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.prodbench
import com.vitorpamplona.amethyst.commons.preview.MetaTagsParser
import com.vitorpamplona.amethyst.commons.preview.OpenGraphParser
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Measures the `<meta>` scan behind every link preview.
*
* Every URL in a rendered note can reach [MetaTagsParser] with a whole HTML document in hand, so
* the scan runs on user-visible paths with attacker-shaped input (any site can serve a 1 MB head).
* The numbers below are the guard against that: the parser must stay linear and allocation-light,
* scanning at hundreds of MB/s rather than degrading with page size.
*
* Deterministic and offline. Prints ns/op and MB/s; the assertions only check that the scan still
* finds the right tags, never wall time (CI machines vary).
*/
class MetaTagsParserBenchmark {
companion object {
/** The shape of a Vite/React SPA head: theme comment, inline script, then the og: block. */
fun spaHead(): String =
"""
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<!-- No-flash theme: set class="dark" synchronously before first paint.
Fresh visitors (no stored choice) stay LIGHT -- we don't auto-dark a
dark-OS visitor until dark mode is fully reviewed. -->
<script>
(function () {
var t = localStorage.getItem("app_theme");
for (var i = 0; i < 2; i++) {
if (t === "dark") document.documentElement.classList.add("dark");
}
})();
</script>
<title>Example - A Site</title>
<meta name="description" content="A description that is long enough to look real." />
<meta property="og:title" content="Example &mdash; Your Network. Your Rules." />
<meta property="og:description" content="The decentralized layer for everything." />
<meta property="og:image" content="https://example.com/og-image.png" />
<meta property="og:image:width" content="1200" />
<meta property="og:image:height" content="630" />
<meta name="twitter:card" content="summary_large_image" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<link rel="manifest" href="/site.webmanifest" />
<link rel="stylesheet" crossorigin href="/assets/index-Cxc5egnN.css" />
</head>
<body><div id="root"></div></body>
</html>
""".trimIndent()
/**
* A news/CMS head: [tags] worth of meta+link noise, analytics scripts, JSON-LD and
* boilerplate comments, with the og: block near the end -- the worst realistic ordering.
*/
fun heavyHead(tags: Int): String {
val sb = StringBuilder(64 * 1024)
sb.append("<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n")
sb.append("<meta charset=\"utf-8\">\n")
repeat(tags) { i ->
sb.append("<!-- section $i: don't reorder, the CMS won't regenerate it -->\n")
sb.append("<meta name=\"cms:field-$i\" content=\"value $i for a field nobody reads\">\n")
sb.append("<link rel=\"preload\" as=\"font\" href=\"/fonts/f$i.woff2\" crossorigin>\n")
sb.append("<script>window.__cfg$i = { id: $i, path: \"/a/b/c\", t: 0 < 1 };</script>\n")
sb.append("<style>.cls-$i { font-family: 'Figtree', sans-serif; }</style>\n")
}
sb.append("<script type=\"application/ld+json\">{\"@type\":\"Article\",\"headline\":\"x < y\"}</script>\n")
sb.append("<meta property=\"og:title\" content=\"The Headline\">\n")
sb.append("<meta property=\"og:description\" content=\"The standfirst, in full.\">\n")
sb.append("<meta property=\"og:image\" content=\"https://example.com/lead.jpg\">\n")
sb.append("</head>\n<body>\n")
// Body the parser must never reach: it stops at </head>.
repeat(tags * 40) { i -> sb.append("<p class=\"para\">Paragraph $i with <em>markup</em> and \"quotes\".</p>\n") }
sb.append("</body>\n</html>\n")
return sb.toString()
}
/** Same content, but with no `</head>` to stop at -- the scan runs over the whole document. */
fun unterminatedHead(tags: Int): String = heavyHead(tags).replace("</head>", "")
fun bench(
label: String,
input: String,
reps: Int,
op: (String) -> Int,
) {
repeat(maxOf(reps / 4, 2)) { op(input) } // warmup
val t0 = System.nanoTime()
var sink = 0
repeat(reps) { sink += op(input) }
val ns = (System.nanoTime() - t0) / reps
val mbps = input.length.toDouble() / ns * 1000.0 // bytes/ns -> MB/s
println(
String.format(
"%-34s %9d B %9d ns/op %8.1f MB/s (hits=%d)",
label,
input.length,
ns,
mbps,
sink / reps,
),
)
}
}
@Test
fun metaScans() {
val spa = spaHead()
val heavy = heavyHead(60)
val open = unterminatedHead(60)
// correctness first: a benchmark that finds nothing measures nothing
val spaInfo = OpenGraphParser().extractUrlInfo(MetaTagsParser.parse(spa))
assertEquals("Example — Your Network. Your Rules.", spaInfo.title)
assertEquals("https://example.com/og-image.png", spaInfo.image)
val heavyInfo = OpenGraphParser().extractUrlInfo(MetaTagsParser.parse(heavy))
assertEquals("The Headline", heavyInfo.title)
assertEquals("https://example.com/lead.jpg", heavyInfo.image)
// the body after </head> is never scanned
assertEquals(1 + 60 + 3, MetaTagsParser.parse(heavy).count())
// Note: the two "heavy head" rows report MB/s over the whole document, of which only the
// ~46 KB head is actually scanned -- the scan stops at </head>. The last row is the same
// page with no </head>, i.e. what a hostile server can force us to read end to end.
println("MetaTagsParser")
bench("spa head, all tags", spa, 50_000) { MetaTagsParser.parse(it).count() }
bench("spa head, og: extraction", spa, 50_000) {
OpenGraphParser().extractUrlInfo(MetaTagsParser.parse(it)).title.length
}
bench("heavy head, to </head>", heavy, 2_000) { MetaTagsParser.parse(it).count() }
bench("heavy head, og: extraction", heavy, 2_000) {
OpenGraphParser().extractUrlInfo(MetaTagsParser.parse(it)).title.length
}
bench("no </head>, whole doc", open, 2_000) { MetaTagsParser.parse(it).count() }
assertTrue(MetaTagsParser.parse(open).count() >= 64)
}
}