refactor(richtext): CommonMark intraword rule for _ / *

The markdown detector now applies CommonMark §6.2 flanking rules:

- `_` skips intraword positions (preceded by a letter, digit, or
  another `_`). This is the same "snake_case" carve-out that lets
  identifiers like `foo_bar_baz` and `snake_case_identifier`
  render literally — and it incidentally protects base64url
  payloads from misclassifying, since every `_` inside a cashuB
  token sits between word chars.
- `*` keeps intraword behavior (CommonMark allows
  `foo*bar*baz`) but now requires non-whitespace on both sides of
  the run, so `5 * 3 = 15` and `5 * 3 * 7 = 105` no longer
  false-fire as italic.

The upfront `contains("cashuA"/"cashuB")` shortcut is kept as a
safety net for the mixed-content case (a chat that has BOTH a
cashu token AND real markdown like "**enjoy** cashuB..."), where
routing through the markdown renderer would lose the cashu card
because RenderContentAsMarkdown has no CashuSegment support. For
cashu-only messages, the intraword rule alone is sufficient — a
new test (`arbitraryBase64UrlBlobIsNotMarkdown`) confirms that
by feeding the detector a base64url blob without the cashuB
prefix.

New regression tests cover:
- snake_case_identifier
- foo__init__bar (intraword `__`)
- 0v______ trailing-underscore run
- foo*bar*baz (intraword `*`, expected markdown per spec)
- 5 * 3 * 7 = 105 (whitespace-flanked `*`)
- __init__ surrounded by whitespace (markdown per spec; Python
  dunders collide here — users escape with backticks)
- user_name@example.com
- arbitrary base64url blob
This commit is contained in:
Claude
2026-05-28 21:14:55 +00:00
parent a5cd3e0dc1
commit 53ee02ea69
2 changed files with 116 additions and 16 deletions
@@ -82,15 +82,13 @@ object CachedRichTextParser {
}
private fun computeIsMarkdown(content: String): Boolean {
// Cashu v2/cashuB tokens are base64url payloads that routinely
// contain '__' and lone '_' pairs (the base64url alphabet uses
// '_'; a token whose CBOR ends with a fixed-bytes break easily
// produces a tail like `______`). The detector below treats
// those as markdown bold/italic and routes the chat through
// RenderContentAsMarkdown, which has no CashuSegment support,
// so the user sees raw base64 instead of the redeem card.
// cashuA shares the same alphabet and the same risk. Force the
// rich-text path whenever a cashu token is present.
// Safety net for the mixed-content case: a chat that has BOTH
// a cashu token AND real markdown (e.g. "**enjoy** cashuB...")
// would otherwise be routed to RenderContentAsMarkdown, which
// has no CashuSegment support, and the cashu card disappears.
// The intraword-underscore rule below already handles
// cashu-only messages correctly; this short-circuit only
// matters when markdown chars also appear.
if (content.contains("cashuA", true) || content.contains("cashuB", true)) return false
val len = content.length
@@ -126,13 +124,61 @@ object CachedRichTextParser {
if ((c == '-' || c == '*' || c == '+') && i + 1 < len && content[i + 1] == ' ') return true
}
if (c == '*' || c == '_') {
if (i + 1 < len && content[i + 1] == c) return true
var j = i + 1
while (j < len) {
if (content[j] == c) return true
if (content[j] == '\n') break
j++
if (c == '*') {
// CommonMark allows intraword `*` emphasis
// (`foo*bar*baz` → `foo<em>bar</em>baz`), so no
// word-boundary carve-out here. We still need a
// flanking check though: a `*` followed by
// whitespace can't open emphasis, and a closing
// `*` can't be preceded by whitespace. Without
// these, `5 * 3 = 15` and `5 * 3 * 7` would
// false-fire.
if (i + 1 < len && content[i + 1] == '*') return true
val nextChar = if (i + 1 < len) content[i + 1] else ' '
if (!nextChar.isMdSpaceOrNewline()) {
var j = i + 1
while (j < len && content[j] != '\n') {
if (content[j] == '*' && !content[j - 1].isMdSpaceOrNewline()) return true
j++
}
}
}
if (c == '_') {
// CommonMark §6.2 forbids `_` from opening or
// closing emphasis intraword — the rule that
// makes `snake_case` and `foo_bar_baz` render
// literally. Same rule keeps cashuB/cashuA
// base64url payloads from false-firing, since
// every `_` inside such a token is surrounded
// by word chars.
//
// Practical heuristic: skip when the char before
// this `_` is a word char (letter, digit, or
// another `_` — the latter folds runs of `_` so
// we only evaluate the run's first position).
if (i == 0 || !content[i - 1].isMdWordChar()) {
if (i + 1 < len && content[i + 1] == '_') {
// `__` (or longer) at a non-word
// boundary. Walk to the end of the run
// and confirm it's followed by
// non-whitespace (left-flanking).
var runEnd = i + 1
while (runEnd < len && content[runEnd] == '_') runEnd++
if (runEnd < len && !content[runEnd].isMdSpaceOrNewline()) return true
} else {
// Single `_` at a non-word boundary.
// Find a matching `_` on the same line
// that itself is a valid closer (not
// followed by a word char).
var j = i + 1
while (j < len && content[j] != '\n') {
if (content[j] == '_') {
val nextIsWord = j + 1 < len && content[j + 1].isMdWordChar()
if (!nextIsWord) return true
}
j++
}
}
}
}
if (c == '[') {
@@ -208,6 +254,14 @@ object CachedRichTextParser {
return false
}
// CommonMark "word" character for the intraword-emphasis rule:
// ASCII letters, ASCII digits, and `_` itself. Used to fold runs of
// `_` (so we only evaluate the start of a run) and to detect
// `snake_case`-style intraword underscores.
private fun Char.isMdWordChar(): Boolean = this.isLetterOrDigit() || this == '_'
private fun Char.isMdSpaceOrNewline(): Boolean = this == ' ' || this == '\t' || this == '\n' || this == '\r'
// Allocated once at object init; every isMarkdown call does an
// O(1) lookup against this table instead of branching through ten
// `contains(...)` calls.
@@ -118,6 +118,52 @@ class CachedRichTextParserMarkdownTest {
@Test fun underscoreAcrossNewlineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("first line _\nsecond line _"))
// CommonMark §6.2 forbids `_` from opening/closing emphasis intraword.
// These cases pin down the snake_case carve-out that also protects
// base64url cashu tokens from false-firing the markdown route.
@Test fun snakeCaseIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("call snake_case_identifier here"))
@Test fun pythonDunderSurroundedByWhitespaceIsMarkdown() {
// CommonMark §6.2: `__text__` flanked by whitespace is strong
// emphasis. Python identifiers like `__init__` collide with
// this — there is no language-specific carve-out. Users who
// want literal dunders should wrap them in backticks.
assertTrue(CachedRichTextParser.isMarkdown("typedef __init__ method"))
}
@Test fun intrawordDoubleUnderscoreIsNotMarkdown() {
// But `foo__init__bar` (intraword) is not strong emphasis
// and must stay rich-text — same rule that protects cashuB.
assertFalse(CachedRichTextParser.isMarkdown("foo__init__bar literal"))
}
@Test fun intrawordTripleUnderscoreIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("var foo___bar end"))
@Test fun underscoreRunAtEndOfWordIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("hash 0v______ end"))
@Test fun intrawordAsteriskIsMarkdown() {
// CommonMark allows intraword `*` emphasis — keep behaving
// the same as a reference renderer so `foo*bar*baz` lights up.
assertTrue(CachedRichTextParser.isMarkdown("call foo*bar*baz here"))
}
@Test fun mathSpacedTripleStarIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("5 * 3 * 7 = 105"))
@Test fun emailWithUnderscoreIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("ping me at user_name@example.com"))
@Test fun arbitraryBase64UrlBlobIsNotMarkdown() {
// Same shape as the cashuB payload but without the `cashuB`
// prefix, so the upfront shortcut can't help — proves the
// intraword rule alone keeps base64url blobs out of the
// markdown route.
val blob =
"v2FteCJodHRwczovL21pbnQubWluaWJpdHMuY2FzaC9CaXRjb2luYXVjc2F0YWRkVEVTVGF0n79haUgAEHk32wzI" +
"ZWFwn79hYQJhc3hAZWM1YWI3Yjc1NjViYjBjZTZhNzg2NzBkMDA0OGExMjVlZGQzMjJhYmVjMTEzYWMwZTBmZGVkZmE3NTQ4Mzg3OWFj" +
"WCED0-ops8Ta6NjKChNJPe_jgIbXLlyxg2KSy2WaSTADo5D_v2FhCGFzeEBkNDZlODU5MDExNjU0NmNjZjAwNTE3ZTQ1NmU0MTY0N2Fm" +
"ZWUxOWNlMzY2N2IzYTcxODZkMzEwZDY1MjM3OTM4YWNYIQMTDGTY943O4ojhKopoYdemsUE2rSLfzwNBODL8WgOX0v______"
assertFalse(CachedRichTextParser.isMarkdown(blob))
}
// ---- Code -----------------------------------------------------------
@Test fun backtickCodeSpanIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("run `ls -la` here"))