mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-11 08:47:33 +00:00
refactor(richtext): CommonMark intraword rule for _ / *
The markdown detector now applies CommonMark §6.2 flanking rules:
- `_` skips intraword positions (preceded by a letter, digit, or
another `_`). This is the same "snake_case" carve-out that lets
identifiers like `foo_bar_baz` and `snake_case_identifier`
render literally — and it incidentally protects base64url
payloads from misclassifying, since every `_` inside a cashuB
token sits between word chars.
- `*` keeps intraword behavior (CommonMark allows
`foo*bar*baz`) but now requires non-whitespace on both sides of
the run, so `5 * 3 = 15` and `5 * 3 * 7 = 105` no longer
false-fire as italic.
The upfront `contains("cashuA"/"cashuB")` shortcut is kept as a
safety net for the mixed-content case (a chat that has BOTH a
cashu token AND real markdown like "**enjoy** cashuB..."), where
routing through the markdown renderer would lose the cashu card
because RenderContentAsMarkdown has no CashuSegment support. For
cashu-only messages, the intraword rule alone is sufficient — a
new test (`arbitraryBase64UrlBlobIsNotMarkdown`) confirms that
by feeding the detector a base64url blob without the cashuB
prefix.
New regression tests cover:
- snake_case_identifier
- foo__init__bar (intraword `__`)
- 0v______ trailing-underscore run
- foo*bar*baz (intraword `*`, expected markdown per spec)
- 5 * 3 * 7 = 105 (whitespace-flanked `*`)
- __init__ surrounded by whitespace (markdown per spec; Python
dunders collide here — users escape with backticks)
- user_name@example.com
- arbitrary base64url blob
This commit is contained in:
@@ -82,15 +82,13 @@ object CachedRichTextParser {
|
||||
}
|
||||
|
||||
private fun computeIsMarkdown(content: String): Boolean {
|
||||
// Cashu v2/cashuB tokens are base64url payloads that routinely
|
||||
// contain '__' and lone '_' pairs (the base64url alphabet uses
|
||||
// '_'; a token whose CBOR ends with a fixed-bytes break easily
|
||||
// produces a tail like `______`). The detector below treats
|
||||
// those as markdown bold/italic and routes the chat through
|
||||
// RenderContentAsMarkdown, which has no CashuSegment support,
|
||||
// so the user sees raw base64 instead of the redeem card.
|
||||
// cashuA shares the same alphabet and the same risk. Force the
|
||||
// rich-text path whenever a cashu token is present.
|
||||
// Safety net for the mixed-content case: a chat that has BOTH
|
||||
// a cashu token AND real markdown (e.g. "**enjoy** cashuB...")
|
||||
// would otherwise be routed to RenderContentAsMarkdown, which
|
||||
// has no CashuSegment support, and the cashu card disappears.
|
||||
// The intraword-underscore rule below already handles
|
||||
// cashu-only messages correctly; this short-circuit only
|
||||
// matters when markdown chars also appear.
|
||||
if (content.contains("cashuA", true) || content.contains("cashuB", true)) return false
|
||||
|
||||
val len = content.length
|
||||
@@ -126,13 +124,61 @@ object CachedRichTextParser {
|
||||
if ((c == '-' || c == '*' || c == '+') && i + 1 < len && content[i + 1] == ' ') return true
|
||||
}
|
||||
|
||||
if (c == '*' || c == '_') {
|
||||
if (i + 1 < len && content[i + 1] == c) return true
|
||||
var j = i + 1
|
||||
while (j < len) {
|
||||
if (content[j] == c) return true
|
||||
if (content[j] == '\n') break
|
||||
j++
|
||||
if (c == '*') {
|
||||
// CommonMark allows intraword `*` emphasis
|
||||
// (`foo*bar*baz` → `foo<em>bar</em>baz`), so no
|
||||
// word-boundary carve-out here. We still need a
|
||||
// flanking check though: a `*` followed by
|
||||
// whitespace can't open emphasis, and a closing
|
||||
// `*` can't be preceded by whitespace. Without
|
||||
// these, `5 * 3 = 15` and `5 * 3 * 7` would
|
||||
// false-fire.
|
||||
if (i + 1 < len && content[i + 1] == '*') return true
|
||||
val nextChar = if (i + 1 < len) content[i + 1] else ' '
|
||||
if (!nextChar.isMdSpaceOrNewline()) {
|
||||
var j = i + 1
|
||||
while (j < len && content[j] != '\n') {
|
||||
if (content[j] == '*' && !content[j - 1].isMdSpaceOrNewline()) return true
|
||||
j++
|
||||
}
|
||||
}
|
||||
}
|
||||
if (c == '_') {
|
||||
// CommonMark §6.2 forbids `_` from opening or
|
||||
// closing emphasis intraword — the rule that
|
||||
// makes `snake_case` and `foo_bar_baz` render
|
||||
// literally. Same rule keeps cashuB/cashuA
|
||||
// base64url payloads from false-firing, since
|
||||
// every `_` inside such a token is surrounded
|
||||
// by word chars.
|
||||
//
|
||||
// Practical heuristic: skip when the char before
|
||||
// this `_` is a word char (letter, digit, or
|
||||
// another `_` — the latter folds runs of `_` so
|
||||
// we only evaluate the run's first position).
|
||||
if (i == 0 || !content[i - 1].isMdWordChar()) {
|
||||
if (i + 1 < len && content[i + 1] == '_') {
|
||||
// `__` (or longer) at a non-word
|
||||
// boundary. Walk to the end of the run
|
||||
// and confirm it's followed by
|
||||
// non-whitespace (left-flanking).
|
||||
var runEnd = i + 1
|
||||
while (runEnd < len && content[runEnd] == '_') runEnd++
|
||||
if (runEnd < len && !content[runEnd].isMdSpaceOrNewline()) return true
|
||||
} else {
|
||||
// Single `_` at a non-word boundary.
|
||||
// Find a matching `_` on the same line
|
||||
// that itself is a valid closer (not
|
||||
// followed by a word char).
|
||||
var j = i + 1
|
||||
while (j < len && content[j] != '\n') {
|
||||
if (content[j] == '_') {
|
||||
val nextIsWord = j + 1 < len && content[j + 1].isMdWordChar()
|
||||
if (!nextIsWord) return true
|
||||
}
|
||||
j++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (c == '[') {
|
||||
@@ -208,6 +254,14 @@ object CachedRichTextParser {
|
||||
return false
|
||||
}
|
||||
|
||||
// CommonMark "word" character for the intraword-emphasis rule:
|
||||
// ASCII letters, ASCII digits, and `_` itself. Used to fold runs of
|
||||
// `_` (so we only evaluate the start of a run) and to detect
|
||||
// `snake_case`-style intraword underscores.
|
||||
private fun Char.isMdWordChar(): Boolean = this.isLetterOrDigit() || this == '_'
|
||||
|
||||
private fun Char.isMdSpaceOrNewline(): Boolean = this == ' ' || this == '\t' || this == '\n' || this == '\r'
|
||||
|
||||
// Allocated once at object init; every isMarkdown call does an
|
||||
// O(1) lookup against this table instead of branching through ten
|
||||
// `contains(...)` calls.
|
||||
|
||||
@@ -118,6 +118,52 @@ class CachedRichTextParserMarkdownTest {
|
||||
|
||||
@Test fun underscoreAcrossNewlineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("first line _\nsecond line _"))
|
||||
|
||||
// CommonMark §6.2 forbids `_` from opening/closing emphasis intraword.
|
||||
// These cases pin down the snake_case carve-out that also protects
|
||||
// base64url cashu tokens from false-firing the markdown route.
|
||||
@Test fun snakeCaseIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("call snake_case_identifier here"))
|
||||
|
||||
@Test fun pythonDunderSurroundedByWhitespaceIsMarkdown() {
|
||||
// CommonMark §6.2: `__text__` flanked by whitespace is strong
|
||||
// emphasis. Python identifiers like `__init__` collide with
|
||||
// this — there is no language-specific carve-out. Users who
|
||||
// want literal dunders should wrap them in backticks.
|
||||
assertTrue(CachedRichTextParser.isMarkdown("typedef __init__ method"))
|
||||
}
|
||||
|
||||
@Test fun intrawordDoubleUnderscoreIsNotMarkdown() {
|
||||
// But `foo__init__bar` (intraword) is not strong emphasis
|
||||
// and must stay rich-text — same rule that protects cashuB.
|
||||
assertFalse(CachedRichTextParser.isMarkdown("foo__init__bar literal"))
|
||||
}
|
||||
|
||||
@Test fun intrawordTripleUnderscoreIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("var foo___bar end"))
|
||||
|
||||
@Test fun underscoreRunAtEndOfWordIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("hash 0v______ end"))
|
||||
|
||||
@Test fun intrawordAsteriskIsMarkdown() {
|
||||
// CommonMark allows intraword `*` emphasis — keep behaving
|
||||
// the same as a reference renderer so `foo*bar*baz` lights up.
|
||||
assertTrue(CachedRichTextParser.isMarkdown("call foo*bar*baz here"))
|
||||
}
|
||||
|
||||
@Test fun mathSpacedTripleStarIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("5 * 3 * 7 = 105"))
|
||||
|
||||
@Test fun emailWithUnderscoreIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("ping me at user_name@example.com"))
|
||||
|
||||
@Test fun arbitraryBase64UrlBlobIsNotMarkdown() {
|
||||
// Same shape as the cashuB payload but without the `cashuB`
|
||||
// prefix, so the upfront shortcut can't help — proves the
|
||||
// intraword rule alone keeps base64url blobs out of the
|
||||
// markdown route.
|
||||
val blob =
|
||||
"v2FteCJodHRwczovL21pbnQubWluaWJpdHMuY2FzaC9CaXRjb2luYXVjc2F0YWRkVEVTVGF0n79haUgAEHk32wzI" +
|
||||
"ZWFwn79hYQJhc3hAZWM1YWI3Yjc1NjViYjBjZTZhNzg2NzBkMDA0OGExMjVlZGQzMjJhYmVjMTEzYWMwZTBmZGVkZmE3NTQ4Mzg3OWFj" +
|
||||
"WCED0-ops8Ta6NjKChNJPe_jgIbXLlyxg2KSy2WaSTADo5D_v2FhCGFzeEBkNDZlODU5MDExNjU0NmNjZjAwNTE3ZTQ1NmU0MTY0N2Fm" +
|
||||
"ZWUxOWNlMzY2N2IzYTcxODZkMzEwZDY1MjM3OTM4YWNYIQMTDGTY943O4ojhKopoYdemsUE2rSLfzwNBODL8WgOX0v______"
|
||||
assertFalse(CachedRichTextParser.isMarkdown(blob))
|
||||
}
|
||||
|
||||
// ---- Code -----------------------------------------------------------
|
||||
|
||||
@Test fun backtickCodeSpanIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("run `ls -la` here"))
|
||||
|
||||
Reference in New Issue
Block a user