From 53ee02ea69a74aafe3702e333227853e27b6b124 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 28 May 2026 21:14:55 +0000 Subject: [PATCH] refactor(richtext): CommonMark intraword rule for _ / * MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The markdown detector now applies CommonMark §6.2 flanking rules: - `_` skips intraword positions (preceded by a letter, digit, or another `_`). This is the same "snake_case" carve-out that lets identifiers like `foo_bar_baz` and `snake_case_identifier` render literally — and it incidentally protects base64url payloads from misclassifying, since every `_` inside a cashuB token sits between word chars. - `*` keeps intraword behavior (CommonMark allows `foo*bar*baz`) but now requires non-whitespace on both sides of the run, so `5 * 3 = 15` and `5 * 3 * 7 = 105` no longer false-fire as italic. The upfront `contains("cashuA"/"cashuB")` shortcut is kept as a safety net for the mixed-content case (a chat that has BOTH a cashu token AND real markdown like "**enjoy** cashuB..."), where routing through the markdown renderer would lose the cashu card because RenderContentAsMarkdown has no CashuSegment support. For cashu-only messages, the intraword rule alone is sufficient — a new test (`arbitraryBase64UrlBlobIsNotMarkdown`) confirms that by feeding the detector a base64url blob without the cashuB prefix. New regression tests cover: - snake_case_identifier - foo__init__bar (intraword `__`) - 0v______ trailing-underscore run - foo*bar*baz (intraword `*`, expected markdown per spec) - 5 * 3 * 7 = 105 (whitespace-flanked `*`) - __init__ surrounded by whitespace (markdown per spec; Python dunders collide here — users escape with backticks) - user_name@example.com - arbitrary base64url blob --- .../amethyst/service/CachedRichTextParser.kt | 86 +++++++++++++++---- .../CachedRichTextParserMarkdownTest.kt | 46 ++++++++++ 2 files changed, 116 insertions(+), 16 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/CachedRichTextParser.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/CachedRichTextParser.kt index 6fa8598a0e..b7a38ca544 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/CachedRichTextParser.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/CachedRichTextParser.kt @@ -82,15 +82,13 @@ object CachedRichTextParser { } private fun computeIsMarkdown(content: String): Boolean { - // Cashu v2/cashuB tokens are base64url payloads that routinely - // contain '__' and lone '_' pairs (the base64url alphabet uses - // '_'; a token whose CBOR ends with a fixed-bytes break easily - // produces a tail like `______`). The detector below treats - // those as markdown bold/italic and routes the chat through - // RenderContentAsMarkdown, which has no CashuSegment support, - // so the user sees raw base64 instead of the redeem card. - // cashuA shares the same alphabet and the same risk. Force the - // rich-text path whenever a cashu token is present. + // Safety net for the mixed-content case: a chat that has BOTH + // a cashu token AND real markdown (e.g. "**enjoy** cashuB...") + // would otherwise be routed to RenderContentAsMarkdown, which + // has no CashuSegment support, and the cashu card disappears. + // The intraword-underscore rule below already handles + // cashu-only messages correctly; this short-circuit only + // matters when markdown chars also appear. if (content.contains("cashuA", true) || content.contains("cashuB", true)) return false val len = content.length @@ -126,13 +124,61 @@ object CachedRichTextParser { if ((c == '-' || c == '*' || c == '+') && i + 1 < len && content[i + 1] == ' ') return true } - if (c == '*' || c == '_') { - if (i + 1 < len && content[i + 1] == c) return true - var j = i + 1 - while (j < len) { - if (content[j] == c) return true - if (content[j] == '\n') break - j++ + if (c == '*') { + // CommonMark allows intraword `*` emphasis + // (`foo*bar*baz` → `foobarbaz`), so no + // word-boundary carve-out here. We still need a + // flanking check though: a `*` followed by + // whitespace can't open emphasis, and a closing + // `*` can't be preceded by whitespace. Without + // these, `5 * 3 = 15` and `5 * 3 * 7` would + // false-fire. + if (i + 1 < len && content[i + 1] == '*') return true + val nextChar = if (i + 1 < len) content[i + 1] else ' ' + if (!nextChar.isMdSpaceOrNewline()) { + var j = i + 1 + while (j < len && content[j] != '\n') { + if (content[j] == '*' && !content[j - 1].isMdSpaceOrNewline()) return true + j++ + } + } + } + if (c == '_') { + // CommonMark §6.2 forbids `_` from opening or + // closing emphasis intraword — the rule that + // makes `snake_case` and `foo_bar_baz` render + // literally. Same rule keeps cashuB/cashuA + // base64url payloads from false-firing, since + // every `_` inside such a token is surrounded + // by word chars. + // + // Practical heuristic: skip when the char before + // this `_` is a word char (letter, digit, or + // another `_` — the latter folds runs of `_` so + // we only evaluate the run's first position). + if (i == 0 || !content[i - 1].isMdWordChar()) { + if (i + 1 < len && content[i + 1] == '_') { + // `__` (or longer) at a non-word + // boundary. Walk to the end of the run + // and confirm it's followed by + // non-whitespace (left-flanking). + var runEnd = i + 1 + while (runEnd < len && content[runEnd] == '_') runEnd++ + if (runEnd < len && !content[runEnd].isMdSpaceOrNewline()) return true + } else { + // Single `_` at a non-word boundary. + // Find a matching `_` on the same line + // that itself is a valid closer (not + // followed by a word char). + var j = i + 1 + while (j < len && content[j] != '\n') { + if (content[j] == '_') { + val nextIsWord = j + 1 < len && content[j + 1].isMdWordChar() + if (!nextIsWord) return true + } + j++ + } + } } } if (c == '[') { @@ -208,6 +254,14 @@ object CachedRichTextParser { return false } + // CommonMark "word" character for the intraword-emphasis rule: + // ASCII letters, ASCII digits, and `_` itself. Used to fold runs of + // `_` (so we only evaluate the start of a run) and to detect + // `snake_case`-style intraword underscores. + private fun Char.isMdWordChar(): Boolean = this.isLetterOrDigit() || this == '_' + + private fun Char.isMdSpaceOrNewline(): Boolean = this == ' ' || this == '\t' || this == '\n' || this == '\r' + // Allocated once at object init; every isMarkdown call does an // O(1) lookup against this table instead of branching through ten // `contains(...)` calls. diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/CachedRichTextParserMarkdownTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/CachedRichTextParserMarkdownTest.kt index b0e37d27ca..7f5c78ba4d 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/CachedRichTextParserMarkdownTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/CachedRichTextParserMarkdownTest.kt @@ -118,6 +118,52 @@ class CachedRichTextParserMarkdownTest { @Test fun underscoreAcrossNewlineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("first line _\nsecond line _")) + // CommonMark §6.2 forbids `_` from opening/closing emphasis intraword. + // These cases pin down the snake_case carve-out that also protects + // base64url cashu tokens from false-firing the markdown route. + @Test fun snakeCaseIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("call snake_case_identifier here")) + + @Test fun pythonDunderSurroundedByWhitespaceIsMarkdown() { + // CommonMark §6.2: `__text__` flanked by whitespace is strong + // emphasis. Python identifiers like `__init__` collide with + // this — there is no language-specific carve-out. Users who + // want literal dunders should wrap them in backticks. + assertTrue(CachedRichTextParser.isMarkdown("typedef __init__ method")) + } + + @Test fun intrawordDoubleUnderscoreIsNotMarkdown() { + // But `foo__init__bar` (intraword) is not strong emphasis + // and must stay rich-text — same rule that protects cashuB. + assertFalse(CachedRichTextParser.isMarkdown("foo__init__bar literal")) + } + + @Test fun intrawordTripleUnderscoreIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("var foo___bar end")) + + @Test fun underscoreRunAtEndOfWordIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("hash 0v______ end")) + + @Test fun intrawordAsteriskIsMarkdown() { + // CommonMark allows intraword `*` emphasis — keep behaving + // the same as a reference renderer so `foo*bar*baz` lights up. + assertTrue(CachedRichTextParser.isMarkdown("call foo*bar*baz here")) + } + + @Test fun mathSpacedTripleStarIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("5 * 3 * 7 = 105")) + + @Test fun emailWithUnderscoreIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("ping me at user_name@example.com")) + + @Test fun arbitraryBase64UrlBlobIsNotMarkdown() { + // Same shape as the cashuB payload but without the `cashuB` + // prefix, so the upfront shortcut can't help — proves the + // intraword rule alone keeps base64url blobs out of the + // markdown route. + val blob = + "v2FteCJodHRwczovL21pbnQubWluaWJpdHMuY2FzaC9CaXRjb2luYXVjc2F0YWRkVEVTVGF0n79haUgAEHk32wzI" + + "ZWFwn79hYQJhc3hAZWM1YWI3Yjc1NjViYjBjZTZhNzg2NzBkMDA0OGExMjVlZGQzMjJhYmVjMTEzYWMwZTBmZGVkZmE3NTQ4Mzg3OWFj" + + "WCED0-ops8Ta6NjKChNJPe_jgIbXLlyxg2KSy2WaSTADo5D_v2FhCGFzeEBkNDZlODU5MDExNjU0NmNjZjAwNTE3ZTQ1NmU0MTY0N2Fm" + + "ZWUxOWNlMzY2N2IzYTcxODZkMzEwZDY1MjM3OTM4YWNYIQMTDGTY943O4ojhKopoYdemsUE2rSLfzwNBODL8WgOX0v______" + assertFalse(CachedRichTextParser.isMarkdown(blob)) + } + // ---- Code ----------------------------------------------------------- @Test fun backtickCodeSpanIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("run `ls -la` here"))