laantungir/amethyst
1
0
Fork 0
mirror of https://github.com/vitorpamplona/amethyst.git synced 2026-08-10 16:33:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(commons): make HtmlParser KMP — drop java Charset dependency

Browse Source 
Follow-up to the link-preview move: HtmlParser + HtmlCharsetParser were
stuck in jvmAndroid only because they spoke java.nio.charset.Charset.
There is no common Charset type in the Kotlin stdlib, so this reshapes
the API to speak IANA charset *names* (String) and pushes the single
genuinely-platform operation — byte->String decode — behind expect/actual.

- Move HtmlParser + HtmlCharsetParser to commonMain. Charset detection
  (meta-tag sniff + BOM sniff) is pure string/byte work; BOM detection no
  longer needs okio (manual leading-byte compare).
- Add `expect fun decodeBytes(bytes, charsetName)`:
    * jvmAndroid actual -> java.nio.charset (all JRE charsets, UTF-8 fallback)
    * iosMain actual -> NSStringEncoding for the common web charsets
      (UTF-8/16/32, Latin-1, CP1252, ASCII), UTF-8 fallback for the rest.
- UrlPreview (stays jvmAndroid; needs OkHttp) now reads response.body.bytes()
  and passes mimeType.charset()?.name().

Verified: commons compiles for JVM AND iosSimulatorArm64, verifyKmpPurity
passes, commons jvmTest passes, amethyst play + fdroid compile.
This commit is contained in:
Claude
2026-05-30 23:34:05 +00:00
parent 58ad87c900
commit f26c00add0
7 changed files with 225 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
+38
View File
@@ -0,0 +1,38 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
/**
* Decodes [bytes] into a String using the charset named [charsetName].
*
* [charsetName] is an IANA charset name (e.g. "UTF-8", "ISO-8859-1",
* "windows-1252"). When it is null or cannot be resolved on the current
* platform, the implementation falls back to UTF-8.
*
* The decode is the only platform-specific step of link-preview HTML parsing:
* the JVM actual delegates to `java.nio.charset`, which supports every charset
* the JRE ships; the iOS actual maps the common web charsets to
* `NSStringEncoding` and falls back to UTF-8 for anything exotic.
*/
expect fun decodeBytes(
bytes: ByteArray,
charsetName: String?,
): String
commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlCharsetParser.kt → commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlCharsetParser.kt
+11 -11
View File
@@ -20,37 +20,37 @@
*/
package com.vitorpamplona.amethyst.commons.preview
import java.nio.charset.Charset
object HtmlCharsetParser {
val ATTRIBUTE_VALUE_CHARSET = "charset"
val ATTRIBUTE_VALUE_HTTP_EQUIV = "http-equiv"
val CONTENT = "content"
private const val DEFAULT_CHARSET = "UTF-8"
private val RE_CONTENT_TYPE_CHARSET = Regex("""charset=([^;]+)""")
fun detectCharset(bodyBytes: ByteArray): Charset {
/**
* Sniffs the charset declared in the document's `<meta>` tags, returning its
* IANA name. Returns [DEFAULT_CHARSET] when no usable declaration is found.
*/
fun detectCharset(bodyBytes: ByteArray): String {
// try to detect charset from meta tags parsed from first 1024 bytes of body
val firstPart = String(bodyBytes, 0, 1024, Charset.forName("utf-8"))
val firstPart = bodyBytes.decodeToString(0, minOf(1024, bodyBytes.size))
val metaTags = MetaTagsParser.parse(firstPart)
metaTags.forEach { meta ->
val charsetAttr = meta.attr(ATTRIBUTE_VALUE_CHARSET)
if (charsetAttr.isNotEmpty()) {
runCatching { Charset.forName(charsetAttr) }.getOrNull()?.let {
return it
}
return charsetAttr
}
if (meta.attr(ATTRIBUTE_VALUE_HTTP_EQUIV).lowercase() == "content-type") {
RE_CONTENT_TYPE_CHARSET
.find(meta.attr(CONTENT))
?.let {
runCatching { Charset.forName(it.groupValues[1]) }.getOrNull()
}?.let {
return it
return it.groupValues[1]
}
}
}
// defaults to UTF-8
return Charset.forName("utf-8")
return DEFAULT_CHARSET
}
}
commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlParser.kt
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
class HtmlParser {
companion object {
// Byte-order marks mapped to their IANA charset names, longest first so
// a 4-byte BOM is matched before a 2-byte one. (Patterns taken from okhttp.)
private val UNICODE_BOMS =
listOf(
byteArrayOf(0x00, 0x00, 0xFF.toByte(), 0xFF.toByte()) to "UTF-32BE",
byteArrayOf(0xFF.toByte(), 0xFF.toByte(), 0x00, 0x00) to "UTF-32LE",
byteArrayOf(0xEF.toByte(), 0xBB.toByte(), 0xBF.toByte()) to "UTF-8",
byteArrayOf(0xFE.toByte(), 0xFF.toByte()) to "UTF-16BE",
byteArrayOf(0xFF.toByte(), 0xFE.toByte()) to "UTF-16LE",
)
}
suspend fun parseHtml(
bodyBytes: ByteArray,
charsetName: String?,
): Sequence<MetaTag> =
withContext(Dispatchers.Default) {
// Precedence: explicit charset (e.g. from Content-Type) > BOM >
// charset sniffed from <meta> tags (defaults to UTF-8).
val name =
charsetName
?: bodyBytes.bomCharsetName()
?: HtmlCharsetParser.detectCharset(bodyBytes)
val content = decodeBytes(bodyBytes, name)
MetaTagsParser.parse(content)
}
private fun ByteArray.bomCharsetName(): String? {
for ((bom, name) in UNICODE_BOMS) {
if (startsWith(bom)) return name
}
return null
}
private fun ByteArray.startsWith(prefix: ByteArray): Boolean {
if (size < prefix.size) return false
for (i in prefix.indices) {
if (this[i] != prefix[i]) return false
}
return true
}
}
commons/src/iosMain/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
+72
View File
@@ -0,0 +1,72 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
import kotlinx.cinterop.ExperimentalForeignApi
import kotlinx.cinterop.addressOf
import kotlinx.cinterop.usePinned
import platform.Foundation.NSASCIIStringEncoding
import platform.Foundation.NSData
import platform.Foundation.NSISOLatin1StringEncoding
import platform.Foundation.NSString
import platform.Foundation.NSStringEncoding
import platform.Foundation.NSUTF16BigEndianStringEncoding
import platform.Foundation.NSUTF16LittleEndianStringEncoding
import platform.Foundation.NSUTF32BigEndianStringEncoding
import platform.Foundation.NSUTF32LittleEndianStringEncoding
import platform.Foundation.NSUTF8StringEncoding
import platform.Foundation.NSWindowsCP1252StringEncoding
import platform.Foundation.create
/**
* iOS decode of HTML bytes by charset name. The common web charsets are mapped
* to their `NSStringEncoding`; anything else falls back to UTF-8 (matching the
* "defaults to UTF-8" behaviour of the charset sniffer).
*/
@OptIn(ExperimentalForeignApi::class)
actual fun decodeBytes(
bytes: ByteArray,
charsetName: String?,
): String {
if (bytes.isEmpty()) return ""
val encoding = encodingFor(charsetName)
val data =
bytes.usePinned { pinned ->
NSData.create(bytes = pinned.addressOf(0), length = bytes.size.toULong())
}
return (NSString.create(data, encoding) as String?)
?: bytes.decodeToString()
}
private fun encodingFor(charsetName: String?): NSStringEncoding =
when (charsetName?.trim()?.uppercase()) {
"UTF-16", "UTF-16BE", "UTF16" -> NSUTF16BigEndianStringEncoding
"UTF-16LE" -> NSUTF16LittleEndianStringEncoding
"UTF-32", "UTF-32BE", "UTF32" -> NSUTF32BigEndianStringEncoding
"UTF-32LE" -> NSUTF32LittleEndianStringEncoding
"ISO-8859-1", "ISO8859-1", "ISO_8859-1", "LATIN1", "L1", "CP819" -> NSISOLatin1StringEncoding
"WINDOWS-1252", "CP1252" -> NSWindowsCP1252StringEncoding
"US-ASCII", "ASCII", "ANSI_X3.4-1968" -> NSASCIIStringEncoding
else -> NSUTF8StringEncoding
}
commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/CharsetDecoder.kt
+34
View File
@@ -0,0 +1,34 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
import java.nio.charset.Charset
actual fun decodeBytes(
bytes: ByteArray,
charsetName: String?,
): String {
val charset =
charsetName
?.let { runCatching { Charset.forName(it) }.getOrNull() }
?: Charsets.UTF_8
return bytes.toString(charset)
}
commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/HtmlParser.kt
-84
View File
@@ -1,84 +0,0 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.preview
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import okio.BufferedSource
import okio.ByteString.Companion.decodeHex
import okio.Options
import java.nio.charset.Charset
class HtmlParser {
companion object {
// taken from okhttp
private val UNICODE_BOMS =
Options.of(
// UTF-8
"efbbbf".decodeHex(),
// UTF-16BE
"feff".decodeHex(),
// UTF-16LE
"fffe".decodeHex(),
// UTF-32BE
"0000ffff".decodeHex(),
// UTF-32LE
"ffff0000".decodeHex(),
)
}
suspend fun parseHtml(
source: BufferedSource,
type: Charset?,
): Sequence<MetaTag> =
parseHtml(
source.readByteArray(),
type ?: source.readBomAsCharset(),
)
suspend fun parseHtml(
bodyBytes: ByteArray,
type: Charset?,
): Sequence<MetaTag> =
withContext(Dispatchers.IO) {
// sniff charset from Content-Type header or BOM
if (type != null) {
val content = bodyBytes.toString(type)
return@withContext MetaTagsParser.parse(content)
}
// if sniffing was failed, detect charset from content
val charset = HtmlCharsetParser.detectCharset(bodyBytes)
val content = bodyBytes.toString(charset)
return@withContext MetaTagsParser.parse(content)
}
private fun BufferedSource.readBomAsCharset(): Charset? =
when (select(UNICODE_BOMS)) {
0 -> Charsets.UTF_8
1 -> Charsets.UTF_16BE
2 -> Charsets.UTF_16LE
3 -> Charsets.UTF_32BE
4 -> Charsets.UTF_32LE
-1 -> null
else -> throw AssertionError()
}
}
commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/preview/UrlPreview.kt
+1 -1
View File
@@ -63,7 +63,7 @@ class UrlPreview {
?: throw IllegalArgumentException("Website returned unknown mimetype: ${response.headers["Content-Type"]}")
when {
mimeType.type == "text" && mimeType.subtype == "html" -> {
val metaTags = HtmlParser().parseHtml(response.body.source(), mimeType.charset())
val metaTags = HtmlParser().parseHtml(response.body.bytes(), mimeType.charset()?.name())
val data = OpenGraphParser().extractUrlInfo(metaTags)
UrlInfoItem(url, data.title, data.description, data.image, mimeType.toString())
}