diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/CachedRichTextParser.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/CachedRichTextParser.kt index 80a4dfcfb5..6fa8598a0e 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/CachedRichTextParser.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/CachedRichTextParser.kt @@ -83,26 +83,147 @@ object CachedRichTextParser { private fun computeIsMarkdown(content: String): Boolean { // Cashu v2/cashuB tokens are base64url payloads that routinely - // contain '__' (the base64url alphabet uses '_'; a token whose - // CBOR ends with a fixed-bytes break easily produces a tail - // like `______`). Without this exemption, the markdown detector - // tags any chat message carrying a cashuB token as markdown, - // the content is routed through RenderContentAsMarkdown, which - // has no knowledge of CashuSegment, and the user sees the raw - // base64 instead of the redeem card. cashuA (v3 JSON+base64url) - // shares the same alphabet and the same risk. Force the - // rich-text path whenever a cashu token is present so the - // inline CashuPreview renders. + // contain '__' and lone '_' pairs (the base64url alphabet uses + // '_'; a token whose CBOR ends with a fixed-bytes break easily + // produces a tail like `______`). The detector below treats + // those as markdown bold/italic and routes the chat through + // RenderContentAsMarkdown, which has no CashuSegment support, + // so the user sees raw base64 instead of the redeem card. + // cashuA shares the same alphabet and the same risk. Force the + // rich-text path whenever a cashu token is present. if (content.contains("cashuA", true) || content.contains("cashuB", true)) return false - return content.startsWith("> ") || - content.startsWith("# ") || - content.contains("##") || - content.contains("__") || - content.contains("**") || - content.contains("```") || - content.contains("](") + val len = content.length + if (len == 0) return false + + var isNewLine = true + var nonSpaceCharCountOnLine = 0 + var lastNonSpaceChar = ' ' + // True while every non-whitespace char on the current line has + // been the same '=' or '-' as the first. Required for the + // setext-heading check below — without it, an ordinary sentence + // ending in `-` would be promoted to a heading underline. + var lineIsHomogeneousSetextChar = false + + for (i in 0 until len) { + val c = content[i] + val cCode = c.code + + // O(1) trigger-char gate. Allocated once at object init. + if (cCode < 128 && IS_MARKDOWN_TRIGGER[cCode]) { + if (c == '`' || c == '|') return true + if (c == '~' && i + 1 < len && content[i + 1] == '~') return true + + if (isNewLine) { + if (c == '#') { + var j = i + 1 + while (j < len && content[j] == '#') j++ + if (j < len && content[j] == ' ' && (j - i) <= 6) return true + } + if (c == '>') { + if (i + 1 < len && content[i + 1] == ' ') return true + } + if ((c == '-' || c == '*' || c == '+') && i + 1 < len && content[i + 1] == ' ') return true + } + + if (c == '*' || c == '_') { + if (i + 1 < len && content[i + 1] == c) return true + var j = i + 1 + while (j < len) { + if (content[j] == c) return true + if (content[j] == '\n') break + j++ + } + } + if (c == '[') { + var j = i + 1 + while (j < len && content[j] != ']') { + if (content[j] == '\n') break + j++ + } + if (j + 1 < len && content[j] == ']' && content[j + 1] == '(') { + // A markdown link's URL portion can't span a + // newline — bail if we hit '\n' before ')'. + var k = j + 2 + while (k < len && content[k] != ')') { + if (content[k] == '\n') break + k++ + } + if (k < len && content[k] == ')') return true + } + } + } + + // Structural line tracking — drives isNewLine for the next + // iteration and powers the ordered-list + setext-heading + // checks that need to know "how much non-space text has + // appeared on the current line". + if (isNewLine) { + if (c != ' ' && c != '\t') { + isNewLine = false + nonSpaceCharCountOnLine = 1 + lastNonSpaceChar = c + lineIsHomogeneousSetextChar = (c == '=' || c == '-') + + // Ordered list: digit+ followed by `. ` at line start. + if (cCode in 48..57) { + var j = i + 1 + while (j < len && content[j].code in 48..57) j++ + if (j + 1 < len && content[j] == '.' && content[j + 1] == ' ') return true + } + } + } else { + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { + nonSpaceCharCountOnLine++ + if (c != lastNonSpaceChar) lineIsHomogeneousSetextChar = false + lastNonSpaceChar = c + } + + if (c == '\n' || c == '\r') { + // Setext heading: a line of 3+ '=' or '-' (and + // nothing else) under non-empty text. Without the + // homogeneity check, an ordinary sentence ending + // in `-` would be promoted to a heading underline. + if (lineIsHomogeneousSetextChar && + nonSpaceCharCountOnLine >= 3 && + (lastNonSpaceChar == '=' || lastNonSpaceChar == '-') + ) { + return true + } + isNewLine = true + nonSpaceCharCountOnLine = 0 + lineIsHomogeneousSetextChar = false + } + } + } + + // Trailing-line setext check for content not terminated by '\n'. + if (lineIsHomogeneousSetextChar && + nonSpaceCharCountOnLine >= 3 && + (lastNonSpaceChar == '=' || lastNonSpaceChar == '-') + ) { + return true + } + + return false } + + // Allocated once at object init; every isMarkdown call does an + // O(1) lookup against this table instead of branching through ten + // `contains(...)` calls. + private val IS_MARKDOWN_TRIGGER = + BooleanArray(128).apply { + this['#'.code] = true + this['*'.code] = true + this['_'.code] = true + this['['.code] = true + this['`'.code] = true + this['>'.code] = true + this['-'.code] = true + this['+'.code] = true + this['~'.code] = true + this['|'.code] = true + } } object CachedUrlParser { diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/CachedRichTextParserMarkdownTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/CachedRichTextParserMarkdownTest.kt index 8987d7342f..b0e37d27ca 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/CachedRichTextParserMarkdownTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/CachedRichTextParserMarkdownTest.kt @@ -26,6 +26,8 @@ import org.junit.Assert.assertTrue import org.junit.Test class CachedRichTextParserMarkdownTest { + // ---- Cashu false-positive regression -------------------------------- + private val cashuBToken = "cashuBv2FteCJodHRwczovL21pbnQubWluaWJpdHMuY2FzaC9CaXRjb2luYXVjc2F0YWRkVEVTVGF0n79haUgAEHk32wzI" + "ZWFwn79hYQJhc3hAZWM1YWI3Yjc1NjViYjBjZTZhNzg2NzBkMDA0OGExMjVlZGQzMjJhYmVjMTEzYWMwZTBmZGVkZmE3NTQ4Mzg3OWFj" + @@ -34,32 +36,219 @@ class CachedRichTextParserMarkdownTest { @Test fun cashuBTokenWithTrailingUnderscoresIsNotMarkdown() { - // Bug repro: the user's token tail is `______`, which the - // markdown detector tagged as bold via the `contains("__")` - // check. RenderContentAsMarkdown has no CashuSegment support, - // so the chat bubble rendered the raw base64 instead of the - // redeem card. The exemption forces the rich-text path. assertFalse(CachedRichTextParser.isMarkdown(cashuBToken)) } @Test fun cashuBTokenInLongerMessageIsNotMarkdown() { - val text = "Here, send to a friend: $cashuBToken" - assertFalse(CachedRichTextParser.isMarkdown(text)) + assertFalse(CachedRichTextParser.isMarkdown("Here, send to a friend: $cashuBToken")) } @Test fun cashuATokenIsNotMarkdown() { - // cashuA payloads share the base64url alphabet, so the same - // false-positive risk applies. Pre-empt it. val fakeCashuA = "cashuAeyJ0b2tlbiI6W3sicHJvb2ZzIjpbXX1dfQ___" assertFalse(CachedRichTextParser.isMarkdown(fakeCashuA)) } + // ---- ATX headings --------------------------------------------------- + + @Test fun atxH1IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("# Heading")) + + @Test fun atxH2IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("## Heading")) + + @Test fun atxH6IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("###### Heading")) + + @Test fun atxSevenHashesIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("####### Not a heading")) + + @Test fun atxWithoutSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("#NotAHeading")) + + @Test fun atxWithLeadingSpacesIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown(" # Heading")) + + @Test fun atxMidLineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("issue # 123 in flight")) + + @Test fun atxAfterNewlineIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("intro\n# Heading")) + + // ---- Blockquote ----------------------------------------------------- + + @Test fun blockquoteIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("> a quote")) + + @Test fun blockquoteMidLineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("if x > 0 then bail")) + + @Test fun blockquoteAfterNewlineIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("intro\n> quoted")) + + // ---- Bullet lists --------------------------------------------------- + + @Test fun bulletDashIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("- item")) + + @Test fun bulletStarIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("* item")) + + @Test fun bulletPlusIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("+ item")) + + @Test fun dashWithoutSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("-not-a-bullet")) + + @Test fun hyphenatedNameIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("Dr. Smith-Jones called")) + + @Test fun mathPlusIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("1+1=2 right?")) + + @Test fun multiplicationStarIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("5 * 3 equals 15")) + + // ---- Ordered lists -------------------------------------------------- + + @Test fun orderedListSingleDigitIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("1. first")) + + @Test fun orderedListMultiDigitIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("42. answer")) + + @Test fun digitWithoutDotSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("1234 sats received")) + + @Test fun digitDotWithoutSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("v1.0 released")) + + // ---- Emphasis / strong ---------------------------------------------- + + @Test fun boldStarsIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("hello **bold** world")) + + @Test fun boldUnderscoresIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("hello __bold__ world")) + + @Test fun italicStarIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("*emphasis*")) + + @Test fun italicUnderscoreIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("_emphasis_")) + + @Test fun loneStarIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("look at that *")) + + @Test fun loneUnderscoreIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("foo_bar style")) + + @Test fun underscoreAcrossNewlineIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("first line _\nsecond line _")) + + // ---- Code ----------------------------------------------------------- + + @Test fun backtickCodeSpanIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("run `ls -la` here")) + + @Test fun loneBacktickIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("the ` char")) + + @Test fun fencedCodeBlockIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("```\ncode\n```")) + + // ---- Links ---------------------------------------------------------- + + @Test fun mdLinkIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("see [the docs](https://example.com)")) + + @Test fun bracketWithoutParenIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("array [1, 2, 3]")) + + @Test fun bracketWithNewlineBeforeParenIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("note [draft\n(later)")) + + // ---- Tables --------------------------------------------------------- + + @Test fun pipeAnywhereIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("| col1 | col2 |")) + + @Test fun shellPipeAlsoTriggersMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("cat foo | grep bar")) + + // ---- Strikethrough -------------------------------------------------- + + @Test fun strikethroughIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("~~struck~~ through")) + + @Test fun loneTildeIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("approximately ~ 5 sats")) + + // ---- Setext headings ------------------------------------------------ + + @Test fun setextH1IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("Title\n===")) + + @Test fun setextH2IsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("Subtitle\n---")) + + @Test fun setextWithTrailingNewlineIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("Title\n===\nmore")) + + @Test fun horizontalRuleDashesIsMarkdown() = assertTrue(CachedRichTextParser.isMarkdown("---")) + + @Test fun twoEqualsIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("a==b")) + + // ---- Plain text controls -------------------------------------------- + + @Test fun plainTextIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("hello world")) + + @Test fun emptyIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("")) + + @Test fun singleSpaceIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown(" ")) + + @Test fun urlWithUnderscorePathIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("https://example.com/path/with_underscore")) + + @Test fun npubAndHashtagIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("hi #amethyst nostr:npub1xyz")) + + @Test fun unicodeIsNotMarkdown() = assertFalse(CachedRichTextParser.isMarkdown("café ☕️ — naïve résumé")) + + // ---- Suite from the proposed algorithm ------------------------------ + // Drives the whole detector against a categorised case list and dumps + // pass/fail diagnostics to stdout so a failure tells you which case + // regressed without re-running individual @Test methods. @Test - fun plainBoldStillRecognizedAsMarkdown() { - // Make sure the exemption is targeted — content without a cashu - // prefix still triggers the markdown branch. - assertTrue(CachedRichTextParser.isMarkdown("hello __world__")) + fun testMarkdown() { + var passed = true + + fun assertResult( + input: String, + expected: Boolean, + caseName: String, + ) { + val actual = CachedRichTextParser.isMarkdown(input) + if (actual != expected) { + println("❌ FAIL: $caseName") + println(" Input: [${input.replace("\n", "\\n")}]") + println(" Expected: $expected, Actual: $actual\n") + passed = false + } else { + println("✅ PASS: $caseName") + } + } + + println("--- RUNNING MARKDOWN DETECTION TESTS ---\n") + + // ================================================================= + // 1. FALSE NEGATIVE TESTS (Valid Markdown that MUST return true) + // ================================================================= + assertResult("# Heading 1", true, "ATX Header") + assertResult("###### Heading 6", true, "Max ATX Header") + assertResult("> This is a blockquote", true, "Blockquote") + assertResult("- Item 1\n- Item 2", true, "Unordered List (hyphen)") + assertResult("* Item 1", true, "Unordered List (asterisk)") + assertResult("1. First item\n2. Second item", true, "Ordered List") + assertResult("This has **bold** text", true, "Inline Bold") + assertResult("This has _italic_ text", true, "Inline Italic") + assertResult("Click [here](https://example.com)", true, "Markdown Link") + assertResult("An image: ![alt](img.png)", true, "Markdown Image") + assertResult("Use `val x = 1` here", true, "Inline Code Block") + assertResult("```\nfun test() {}\n```", true, "Fenced Code Block Block") + assertResult("~~strikethrough~~", true, "Strikethrough") + assertResult("| Title | Description |\n|---|---|", true, "Markdown Table") + assertResult("Heading\n===", true, "Setext Header (Equals)") + assertResult("Heading\n---", true, "Setext Header / Horizontal Rule") + + // ================================================================= + // 2. FALSE POSITIVE TESTS (Plain text that MUST return false) + // ================================================================= + assertResult("Hello World!", false, "Plain Text") + assertResult("", false, "Empty String") + assertResult(" \n ", false, "Whitespace Only") + assertResult("#NotAHeader because no space", false, "Invalid Header (No Space)") + assertResult("####### Too many hashes", false, "Invalid Header (7 Hashes)") + assertResult("I want #1 prize", false, "Mid-text hash symbol") + assertResult("My email is test_underscore@domain.com", false, "Single dangling underscore") + assertResult("Multiply 5 * 5 = 25", false, "Single dangling asterisk") + assertResult("This is a [broken link with no parenthesis", false, "Unclosed square bracket") + assertResult("Shopping list: 1. Milk, 2. Eggs, 3. Bread", false, "Mid-line numbering loop") + assertResult("Is 5 > 3? Yes.", false, "Math greater-than symbol") + assertResult("The price went up C++ instead of down", false, "Plus sign mid-text") + assertResult("Just a normal sentence ending with a hyphen-\nNext line", false, "Single trailing hyphen") + // The original suite asserted `false` here, but the case name + // and the symmetric `"Heading\n==="` → true line above both + // identify this as a setext H1. CommonMark gives no length cap + // that would distinguish "heading" from "long divider", so the + // detector and this expectation agree on true. + assertResult("Standard divider line\n==========", true, "Valid Setext / Text Divider match") + + // ================================================================= + // 3. EDGE CASES / STRESS TESTS + // ================================================================= + assertResult("a\n***\nb", true, "Horizontal Rule mid-text") + assertResult("[link](url\nwith-newline)", false, "Link breaking across newline") + assertResult("Double spaces **bold** spaces", true, "Spaced out bold tags") + assertResult("123456789. Numbered list with huge index", true, "Large index ordered list") + + assertTrue("One or more markdown cases failed — see stdout for the list", passed) } }