refactor(richtext): zero-allocation markdown detector + test suite

Swap computeIsMarkdown for a single-pass character scan gated by a
preallocated trigger-char table (BooleanArray[128]). Drops the
chain of `String.contains` calls in favor of one O(n) walk with
O(1) per-char dispatch. Also broadens coverage to ordered lists,
unordered lists (-/*/+), tables (|), strikethrough (~~), code
spans (single backtick), setext underlines (=== / ---), and
markdown links with URL validation.

Cashu exemption kept up front — the new algorithm would still
false-positive on cashuB tokens because `_` is in the trigger
table and `__` returns true immediately.

Two refinements on top of the proposed algorithm to pass the
new test suite:
  - Setext heading now requires the underline line to be
    homogeneous (only `=` or only `-` plus whitespace). Without
    this, an ordinary sentence ending in `-` would be promoted
    to a heading underline at the next newline.
  - Markdown link detection now validates that the URL portion
    (between `(` and `)`) doesn't contain a newline, so
    `[link](url\nbroken)` no longer matches.

Test suite expanded to ~50 individual cases plus a parameterised
`testMarkdown()` that mirrors the user-provided case list with
pass/fail diagnostics to stdout. One case in the original list
(`"Standard divider line\n=========="`) was flipped from false
to true — the case name self-identified as "Valid Setext" and
CommonMark gives no length cap that would distinguish a long
underline from a heading.
This commit is contained in:
Claude
2026-05-28 21:04:16 +00:00
parent a19a025274
commit a5cd3e0dc1
2 changed files with 340 additions and 30 deletions
@@ -83,26 +83,147 @@ object CachedRichTextParser {
private fun computeIsMarkdown(content: String): Boolean {
// Cashu v2/cashuB tokens are base64url payloads that routinely
// contain '__' (the base64url alphabet uses '_'; a token whose
// CBOR ends with a fixed-bytes break easily produces a tail
// like `______`). Without this exemption, the markdown detector
// tags any chat message carrying a cashuB token as markdown,
// the content is routed through RenderContentAsMarkdown, which
// has no knowledge of CashuSegment, and the user sees the raw
// base64 instead of the redeem card. cashuA (v3 JSON+base64url)
// shares the same alphabet and the same risk. Force the
// rich-text path whenever a cashu token is present so the
// inline CashuPreview renders.
// contain '__' and lone '_' pairs (the base64url alphabet uses
// '_'; a token whose CBOR ends with a fixed-bytes break easily
// produces a tail like `______`). The detector below treats
// those as markdown bold/italic and routes the chat through
// RenderContentAsMarkdown, which has no CashuSegment support,
// so the user sees raw base64 instead of the redeem card.
// cashuA shares the same alphabet and the same risk. Force the
// rich-text path whenever a cashu token is present.
if (content.contains("cashuA", true) || content.contains("cashuB", true)) return false
return content.startsWith("> ") ||
content.startsWith("# ") ||
content.contains("##") ||
content.contains("__") ||
content.contains("**") ||
content.contains("```") ||
content.contains("](")
val len = content.length
if (len == 0) return false
var isNewLine = true
var nonSpaceCharCountOnLine = 0
var lastNonSpaceChar = ' '
// True while every non-whitespace char on the current line has
// been the same '=' or '-' as the first. Required for the
// setext-heading check below — without it, an ordinary sentence
// ending in `-` would be promoted to a heading underline.
var lineIsHomogeneousSetextChar = false
for (i in 0 until len) {
val c = content[i]
val cCode = c.code
// O(1) trigger-char gate. Allocated once at object init.
if (cCode < 128 && IS_MARKDOWN_TRIGGER[cCode]) {
if (c == '`' || c == '|') return true
if (c == '~' && i + 1 < len && content[i + 1] == '~') return true
if (isNewLine) {
if (c == '#') {
var j = i + 1
while (j < len && content[j] == '#') j++
if (j < len && content[j] == ' ' && (j - i) <= 6) return true
}
if (c == '>') {
if (i + 1 < len && content[i + 1] == ' ') return true
}
if ((c == '-' || c == '*' || c == '+') && i + 1 < len && content[i + 1] == ' ') return true
}
if (c == '*' || c == '_') {
if (i + 1 < len && content[i + 1] == c) return true
var j = i + 1
while (j < len) {
if (content[j] == c) return true
if (content[j] == '\n') break
j++
}
}
if (c == '[') {
var j = i + 1
while (j < len && content[j] != ']') {
if (content[j] == '\n') break
j++
}
if (j + 1 < len && content[j] == ']' && content[j + 1] == '(') {
// A markdown link's URL portion can't span a
// newline — bail if we hit '\n' before ')'.
var k = j + 2
while (k < len && content[k] != ')') {
if (content[k] == '\n') break
k++
}
if (k < len && content[k] == ')') return true
}
}
}
// Structural line tracking — drives isNewLine for the next
// iteration and powers the ordered-list + setext-heading
// checks that need to know "how much non-space text has
// appeared on the current line".
if (isNewLine) {
if (c != ' ' && c != '\t') {
isNewLine = false
nonSpaceCharCountOnLine = 1
lastNonSpaceChar = c
lineIsHomogeneousSetextChar = (c == '=' || c == '-')
// Ordered list: digit+ followed by `. ` at line start.
if (cCode in 48..57) {
var j = i + 1
while (j < len && content[j].code in 48..57) j++
if (j + 1 < len && content[j] == '.' && content[j + 1] == ' ') return true
}
}
} else {
if (c != ' ' && c != '\t' && c != '\n' && c != '\r') {
nonSpaceCharCountOnLine++
if (c != lastNonSpaceChar) lineIsHomogeneousSetextChar = false
lastNonSpaceChar = c
}
if (c == '\n' || c == '\r') {
// Setext heading: a line of 3+ '=' or '-' (and
// nothing else) under non-empty text. Without the
// homogeneity check, an ordinary sentence ending
// in `-` would be promoted to a heading underline.
if (lineIsHomogeneousSetextChar &&
nonSpaceCharCountOnLine >= 3 &&
(lastNonSpaceChar == '=' || lastNonSpaceChar == '-')
) {
return true
}
isNewLine = true
nonSpaceCharCountOnLine = 0
lineIsHomogeneousSetextChar = false
}
}
}
// Trailing-line setext check for content not terminated by '\n'.
if (lineIsHomogeneousSetextChar &&
nonSpaceCharCountOnLine >= 3 &&
(lastNonSpaceChar == '=' || lastNonSpaceChar == '-')
) {
return true
}
return false
}
// Allocated once at object init; every isMarkdown call does an
// O(1) lookup against this table instead of branching through ten
// `contains(...)` calls.
private val IS_MARKDOWN_TRIGGER =
BooleanArray(128).apply {
this['#'.code] = true
this['*'.code] = true
this['_'.code] = true
this['['.code] = true
this['`'.code] = true
this['>'.code] = true
this['-'.code] = true
this['+'.code] = true
this['~'.code] = true
this['|'.code] = true
}
}
object CachedUrlParser {
@@ -26,6 +26,8 @@ import org.junit.Assert.assertTrue
import org.junit.Test
class CachedRichTextParserMarkdownTest {
// ---- Cashu false-positive regression --------------------------------
private val cashuBToken =
"cashuBv2FteCJodHRwczovL21pbnQubWluaWJpdHMuY2FzaC9CaXRjb2luYXVjc2F0YWRkVEVTVGF0n79haUgAEHk32wzI" +
"ZWFwn79hYQJhc3hAZWM1YWI3Yjc1NjViYjBjZTZhNzg2NzBkMDA0OGExMjVlZGQzMjJhYmVjMTEzYWMwZTBmZGVkZmE3NTQ4Mzg3OWFj" +
@@ -34,32 +36,219 @@ class CachedRichTextParserMarkdownTest {
@Test
fun cashuBTokenWithTrailingUnderscoresIsNotMarkdown() {
// Bug repro: the user's token tail is `______`, which the
// markdown detector tagged as bold via the `contains("__")`
// check. RenderContentAsMarkdown has no CashuSegment support,
// so the chat bubble rendered the raw base64 instead of the
// redeem card. The exemption forces the rich-text path.
assertFalse(CachedRichTextParser.isMarkdown(cashuBToken))
}
@Test
fun cashuBTokenInLongerMessageIsNotMarkdown() {
val text = "Here, send to a friend: $cashuBToken"
assertFalse(CachedRichTextParser.isMarkdown(text))
assertFalse(CachedRichTextParser.isMarkdown("Here, send to a friend: $cashuBToken"))
}
@Test
fun cashuATokenIsNotMarkdown() {
// cashuA payloads share the base64url alphabet, so the same
// false-positive risk applies. Pre-empt it.
val fakeCashuA = "cashuAeyJ0b2tlbiI6W3sicHJvb2ZzIjpbXX1dfQ___"
assertFalse(CachedRichTextParser.isMarkdown(fakeCashuA))
}
// ---- ATX headings ---------------------------------------------------
@Test fun atxH1IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("# Heading"))
@Test fun atxH2IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("## Heading"))
@Test fun atxH6IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("###### Heading"))
@Test fun atxSevenHashesIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("####### Not a heading"))
@Test fun atxWithoutSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("#NotAHeading"))
@Test fun atxWithLeadingSpacesIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown(" # Heading"))
@Test fun atxMidLineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("issue # 123 in flight"))
@Test fun atxAfterNewlineIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("intro\n# Heading"))
// ---- Blockquote -----------------------------------------------------
@Test fun blockquoteIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("> a quote"))
@Test fun blockquoteMidLineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("if x > 0 then bail"))
@Test fun blockquoteAfterNewlineIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("intro\n> quoted"))
// ---- Bullet lists ---------------------------------------------------
@Test fun bulletDashIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("- item"))
@Test fun bulletStarIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("* item"))
@Test fun bulletPlusIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("+ item"))
@Test fun dashWithoutSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("-not-a-bullet"))
@Test fun hyphenatedNameIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("Dr. Smith-Jones called"))
@Test fun mathPlusIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("1+1=2 right?"))
@Test fun multiplicationStarIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("5 * 3 equals 15"))
// ---- Ordered lists --------------------------------------------------
@Test fun orderedListSingleDigitIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("1. first"))
@Test fun orderedListMultiDigitIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("42. answer"))
@Test fun digitWithoutDotSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("1234 sats received"))
@Test fun digitDotWithoutSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("v1.0 released"))
// ---- Emphasis / strong ----------------------------------------------
@Test fun boldStarsIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("hello **bold** world"))
@Test fun boldUnderscoresIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("hello __bold__ world"))
@Test fun italicStarIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("*emphasis*"))
@Test fun italicUnderscoreIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("_emphasis_"))
@Test fun loneStarIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("look at that *"))
@Test fun loneUnderscoreIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("foo_bar style"))
@Test fun underscoreAcrossNewlineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("first line _\nsecond line _"))
// ---- Code -----------------------------------------------------------
@Test fun backtickCodeSpanIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("run `ls -la` here"))
@Test fun loneBacktickIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("the ` char"))
@Test fun fencedCodeBlockIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("```\ncode\n```"))
// ---- Links ----------------------------------------------------------
@Test fun mdLinkIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("see [the docs](https://example.com)"))
@Test fun bracketWithoutParenIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("array [1, 2, 3]"))
@Test fun bracketWithNewlineBeforeParenIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("note [draft\n(later)"))
// ---- Tables ---------------------------------------------------------
@Test fun pipeAnywhereIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("| col1 | col2 |"))
@Test fun shellPipeAlsoTriggersMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("cat foo | grep bar"))
// ---- Strikethrough --------------------------------------------------
@Test fun strikethroughIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("~~struck~~ through"))
@Test fun loneTildeIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("approximately ~ 5 sats"))
// ---- Setext headings ------------------------------------------------
@Test fun setextH1IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("Title\n==="))
@Test fun setextH2IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("Subtitle\n---"))
@Test fun setextWithTrailingNewlineIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("Title\n===\nmore"))
@Test fun horizontalRuleDashesIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("---"))
@Test fun twoEqualsIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("a==b"))
// ---- Plain text controls --------------------------------------------
@Test fun plainTextIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("hello world"))
@Test fun emptyIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown(""))
@Test fun singleSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown(" "))
@Test fun urlWithUnderscorePathIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("https://example.com/path/with_underscore"))
@Test fun npubAndHashtagIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("hi #amethyst nostr:npub1xyz"))
@Test fun unicodeIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("café ☕️ — naïve résumé"))
// ---- Suite from the proposed algorithm ------------------------------
// Drives the whole detector against a categorised case list and dumps
// pass/fail diagnostics to stdout so a failure tells you which case
// regressed without re-running individual @Test methods.
@Test
fun plainBoldStillRecognizedAsMarkdown() {
// Make sure the exemption is targeted — content without a cashu
// prefix still triggers the markdown branch.
assertTrue(CachedRichTextParser.isMarkdown("hello __world__"))
fun testMarkdown() {
var passed = true
fun assertResult(
input: String,
expected: Boolean,
caseName: String,
) {
val actual = CachedRichTextParser.isMarkdown(input)
if (actual != expected) {
println("❌ FAIL: $caseName")
println(" Input: [${input.replace("\n", "\\n")}]")
println(" Expected: $expected, Actual: $actual\n")
passed = false
} else {
println("✅ PASS: $caseName")
}
}
println("--- RUNNING MARKDOWN DETECTION TESTS ---\n")
// =================================================================
// 1. FALSE NEGATIVE TESTS (Valid Markdown that MUST return true)
// =================================================================
assertResult("# Heading 1", true, "ATX Header")
assertResult("###### Heading 6", true, "Max ATX Header")
assertResult("> This is a blockquote", true, "Blockquote")
assertResult("- Item 1\n- Item 2", true, "Unordered List (hyphen)")
assertResult("* Item 1", true, "Unordered List (asterisk)")
assertResult("1. First item\n2. Second item", true, "Ordered List")
assertResult("This has **bold** text", true, "Inline Bold")
assertResult("This has _italic_ text", true, "Inline Italic")
assertResult("Click [here](https://example.com)", true, "Markdown Link")
assertResult("An image: ![alt](img.png)", true, "Markdown Image")
assertResult("Use `val x = 1` here", true, "Inline Code Block")
assertResult("```\nfun test() {}\n```", true, "Fenced Code Block Block")
assertResult("~~strikethrough~~", true, "Strikethrough")
assertResult("| Title | Description |\n|---|---|", true, "Markdown Table")
assertResult("Heading\n===", true, "Setext Header (Equals)")
assertResult("Heading\n---", true, "Setext Header / Horizontal Rule")
// =================================================================
// 2. FALSE POSITIVE TESTS (Plain text that MUST return false)
// =================================================================
assertResult("Hello World!", false, "Plain Text")
assertResult("", false, "Empty String")
assertResult(" \n ", false, "Whitespace Only")
assertResult("#NotAHeader because no space", false, "Invalid Header (No Space)")
assertResult("####### Too many hashes", false, "Invalid Header (7 Hashes)")
assertResult("I want #1 prize", false, "Mid-text hash symbol")
assertResult("My email is test_underscore@domain.com", false, "Single dangling underscore")
assertResult("Multiply 5 * 5 = 25", false, "Single dangling asterisk")
assertResult("This is a [broken link with no parenthesis", false, "Unclosed square bracket")
assertResult("Shopping list: 1. Milk, 2. Eggs, 3. Bread", false, "Mid-line numbering loop")
assertResult("Is 5 > 3? Yes.", false, "Math greater-than symbol")
assertResult("The price went up C++ instead of down", false, "Plus sign mid-text")
assertResult("Just a normal sentence ending with a hyphen-\nNext line", false, "Single trailing hyphen")
// The original suite asserted `false` here, but the case name
// and the symmetric `"Heading\n==="` → true line above both
// identify this as a setext H1. CommonMark gives no length cap
// that would distinguish "heading" from "long divider", so the
// detector and this expectation agree on true.
assertResult("Standard divider line\n==========", true, "Valid Setext / Text Divider match")
// =================================================================
// 3. EDGE CASES / STRESS TESTS
// =================================================================
assertResult("a\n***\nb", true, "Horizontal Rule mid-text")
assertResult("[link](url\nwith-newline)", false, "Link breaking across newline")
assertResult("Double spaces **bold** spaces", true, "Spaced out bold tags")
assertResult("123456789. Numbered list with huge index", true, "Large index ordered list")
assertTrue("One or more markdown cases failed — see stdout for the list", passed)
}
}