diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
new file mode 100644
index 0000000000..82b0140470
--- /dev/null
+++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.commons.preview
+
+/**
+ * Decodes [bytes] into a String using the charset named [charsetName].
+ *
+ * [charsetName] is an IANA charset name (e.g. "UTF-8", "ISO-8859-1",
+ * "windows-1252"). When it is null or cannot be resolved on the current
+ * platform, the implementation falls back to UTF-8.
+ *
+ * The decode is the only platform-specific step of link-preview HTML parsing:
+ * the JVM actual delegates to `java.nio.charset`, which supports every charset
+ * the JRE ships; the iOS actual maps the common web charsets to
+ * `NSStringEncoding` and falls back to UTF-8 for anything exotic.
+ */
+expect fun decodeBytes(
+ bytes: ByteArray,
+ charsetName: String?,
+): String
diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlCharsetParser.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlCharsetParser.kt
similarity index 80%
rename from commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlCharsetParser.kt
rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlCharsetParser.kt
index e497d7da40..670808ea62 100644
--- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlCharsetParser.kt
+++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlCharsetParser.kt
@@ -20,37 +20,37 @@
*/
package com.vitorpamplona.amethyst.commons.preview
-import java.nio.charset.Charset
-
object HtmlCharsetParser {
val ATTRIBUTE_VALUE_CHARSET = "charset"
val ATTRIBUTE_VALUE_HTTP_EQUIV = "http-equiv"
val CONTENT = "content"
+ private const val DEFAULT_CHARSET = "UTF-8"
+
private val RE_CONTENT_TYPE_CHARSET = Regex("""charset=([^;]+)""")
- fun detectCharset(bodyBytes: ByteArray): Charset {
+ /**
+ * Sniffs the charset declared in the document's `` tags, returning its
+ * IANA name. Returns [DEFAULT_CHARSET] when no usable declaration is found.
+ */
+ fun detectCharset(bodyBytes: ByteArray): String {
// try to detect charset from meta tags parsed from first 1024 bytes of body
- val firstPart = String(bodyBytes, 0, 1024, Charset.forName("utf-8"))
+ val firstPart = bodyBytes.decodeToString(0, minOf(1024, bodyBytes.size))
val metaTags = MetaTagsParser.parse(firstPart)
metaTags.forEach { meta ->
val charsetAttr = meta.attr(ATTRIBUTE_VALUE_CHARSET)
if (charsetAttr.isNotEmpty()) {
- runCatching { Charset.forName(charsetAttr) }.getOrNull()?.let {
- return it
- }
+ return charsetAttr
}
if (meta.attr(ATTRIBUTE_VALUE_HTTP_EQUIV).lowercase() == "content-type") {
RE_CONTENT_TYPE_CHARSET
.find(meta.attr(CONTENT))
?.let {
- runCatching { Charset.forName(it.groupValues[1]) }.getOrNull()
- }?.let {
- return it
+ return it.groupValues[1]
}
}
}
// defaults to UTF-8
- return Charset.forName("utf-8")
+ return DEFAULT_CHARSET
}
}
diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlParser.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlParser.kt
new file mode 100644
index 0000000000..823862bf31
--- /dev/null
+++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlParser.kt
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.commons.preview
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+
+class HtmlParser {
+ companion object {
+ // Byte-order marks mapped to their IANA charset names, longest first so
+ // a 4-byte BOM is matched before a 2-byte one. (Patterns taken from okhttp.)
+ private val UNICODE_BOMS =
+ listOf(
+ byteArrayOf(0x00, 0x00, 0xFF.toByte(), 0xFF.toByte()) to "UTF-32BE",
+ byteArrayOf(0xFF.toByte(), 0xFF.toByte(), 0x00, 0x00) to "UTF-32LE",
+ byteArrayOf(0xEF.toByte(), 0xBB.toByte(), 0xBF.toByte()) to "UTF-8",
+ byteArrayOf(0xFE.toByte(), 0xFF.toByte()) to "UTF-16BE",
+ byteArrayOf(0xFF.toByte(), 0xFE.toByte()) to "UTF-16LE",
+ )
+ }
+
+ suspend fun parseHtml(
+ bodyBytes: ByteArray,
+ charsetName: String?,
+ ): Sequence =
+ withContext(Dispatchers.Default) {
+ // Precedence: explicit charset (e.g. from Content-Type) > BOM >
+ // charset sniffed from tags (defaults to UTF-8).
+ val name =
+ charsetName
+ ?: bodyBytes.bomCharsetName()
+ ?: HtmlCharsetParser.detectCharset(bodyBytes)
+ val content = decodeBytes(bodyBytes, name)
+ MetaTagsParser.parse(content)
+ }
+
+ private fun ByteArray.bomCharsetName(): String? {
+ for ((bom, name) in UNICODE_BOMS) {
+ if (startsWith(bom)) return name
+ }
+ return null
+ }
+
+ private fun ByteArray.startsWith(prefix: ByteArray): Boolean {
+ if (size < prefix.size) return false
+ for (i in prefix.indices) {
+ if (this[i] != prefix[i]) return false
+ }
+ return true
+ }
+}
diff --git a/commons/src/iosMain/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt b/commons/src/iosMain/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
new file mode 100644
index 0000000000..cf70fa5533
--- /dev/null
+++ b/commons/src/iosMain/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.commons.preview
+
+import kotlinx.cinterop.ExperimentalForeignApi
+import kotlinx.cinterop.addressOf
+import kotlinx.cinterop.usePinned
+import platform.Foundation.NSASCIIStringEncoding
+import platform.Foundation.NSData
+import platform.Foundation.NSISOLatin1StringEncoding
+import platform.Foundation.NSString
+import platform.Foundation.NSStringEncoding
+import platform.Foundation.NSUTF16BigEndianStringEncoding
+import platform.Foundation.NSUTF16LittleEndianStringEncoding
+import platform.Foundation.NSUTF32BigEndianStringEncoding
+import platform.Foundation.NSUTF32LittleEndianStringEncoding
+import platform.Foundation.NSUTF8StringEncoding
+import platform.Foundation.NSWindowsCP1252StringEncoding
+import platform.Foundation.create
+
+/**
+ * iOS decode of HTML bytes by charset name. The common web charsets are mapped
+ * to their `NSStringEncoding`; anything else falls back to UTF-8 (matching the
+ * "defaults to UTF-8" behaviour of the charset sniffer).
+ */
+@OptIn(ExperimentalForeignApi::class)
+actual fun decodeBytes(
+ bytes: ByteArray,
+ charsetName: String?,
+): String {
+ if (bytes.isEmpty()) return ""
+
+ val encoding = encodingFor(charsetName)
+
+ val data =
+ bytes.usePinned { pinned ->
+ NSData.create(bytes = pinned.addressOf(0), length = bytes.size.toULong())
+ }
+
+ return (NSString.create(data, encoding) as String?)
+ ?: bytes.decodeToString()
+}
+
+private fun encodingFor(charsetName: String?): NSStringEncoding =
+ when (charsetName?.trim()?.uppercase()) {
+ "UTF-16", "UTF-16BE", "UTF16" -> NSUTF16BigEndianStringEncoding
+ "UTF-16LE" -> NSUTF16LittleEndianStringEncoding
+ "UTF-32", "UTF-32BE", "UTF32" -> NSUTF32BigEndianStringEncoding
+ "UTF-32LE" -> NSUTF32LittleEndianStringEncoding
+ "ISO-8859-1", "ISO8859-1", "ISO_8859-1", "LATIN1", "L1", "CP819" -> NSISOLatin1StringEncoding
+ "WINDOWS-1252", "CP1252" -> NSWindowsCP1252StringEncoding
+ "US-ASCII", "ASCII", "ANSI_X3.4-1968" -> NSASCIIStringEncoding
+ else -> NSUTF8StringEncoding
+ }
diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
new file mode 100644
index 0000000000..7af77373ae
--- /dev/null
+++ b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.commons.preview
+
+import java.nio.charset.Charset
+
+actual fun decodeBytes(
+ bytes: ByteArray,
+ charsetName: String?,
+): String {
+ val charset =
+ charsetName
+ ?.let { runCatching { Charset.forName(it) }.getOrNull() }
+ ?: Charsets.UTF_8
+ return bytes.toString(charset)
+}
diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlParser.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlParser.kt
deleted file mode 100644
index c6d1d95f61..0000000000
--- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlParser.kt
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright (c) 2025 Vitor Pamplona
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal in
- * the Software without restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
- * Software, and to permit persons to whom the Software is furnished to do so,
- * subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
- * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
- * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-package com.vitorpamplona.amethyst.commons.preview
-
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.withContext
-import okio.BufferedSource
-import okio.ByteString.Companion.decodeHex
-import okio.Options
-import java.nio.charset.Charset
-
-class HtmlParser {
- companion object {
- // taken from okhttp
- private val UNICODE_BOMS =
- Options.of(
- // UTF-8
- "efbbbf".decodeHex(),
- // UTF-16BE
- "feff".decodeHex(),
- // UTF-16LE
- "fffe".decodeHex(),
- // UTF-32BE
- "0000ffff".decodeHex(),
- // UTF-32LE
- "ffff0000".decodeHex(),
- )
- }
-
- suspend fun parseHtml(
- source: BufferedSource,
- type: Charset?,
- ): Sequence =
- parseHtml(
- source.readByteArray(),
- type ?: source.readBomAsCharset(),
- )
-
- suspend fun parseHtml(
- bodyBytes: ByteArray,
- type: Charset?,
- ): Sequence =
- withContext(Dispatchers.IO) {
- // sniff charset from Content-Type header or BOM
- if (type != null) {
- val content = bodyBytes.toString(type)
- return@withContext MetaTagsParser.parse(content)
- }
-
- // if sniffing was failed, detect charset from content
- val charset = HtmlCharsetParser.detectCharset(bodyBytes)
- val content = bodyBytes.toString(charset)
- return@withContext MetaTagsParser.parse(content)
- }
-
- private fun BufferedSource.readBomAsCharset(): Charset? =
- when (select(UNICODE_BOMS)) {
- 0 -> Charsets.UTF_8
- 1 -> Charsets.UTF_16BE
- 2 -> Charsets.UTF_16LE
- 3 -> Charsets.UTF_32BE
- 4 -> Charsets.UTF_32LE
- -1 -> null
- else -> throw AssertionError()
- }
-}
diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/UrlPreview.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/UrlPreview.kt
index 18f5086ba4..63f2d00d83 100644
--- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/UrlPreview.kt
+++ b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/UrlPreview.kt
@@ -63,7 +63,7 @@ class UrlPreview {
?: throw IllegalArgumentException("Website returned unknown mimetype: ${response.headers["Content-Type"]}")
when {
mimeType.type == "text" && mimeType.subtype == "html" -> {
- val metaTags = HtmlParser().parseHtml(response.body.source(), mimeType.charset())
+ val metaTags = HtmlParser().parseHtml(response.body.bytes(), mimeType.charset()?.name())
val data = OpenGraphParser().extractUrlInfo(metaTags)
UrlInfoItem(url, data.title, data.description, data.image, mimeType.toString())
}