From fc99bed55a41ef9a0f4dbbfe6f99600ba46ef185 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Apr 2026 02:53:38 +0000 Subject: [PATCH] fix(cli): default amy launcher to C.UTF-8 so emoji argv survives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `marmot message react "$gid" "$id" "🍕"` was producing a kind:7 whose inner content was four `U+FFFD` replacement characters instead of the emoji. Whitenoise's `message_aggregator::emoji_utils` rejected it with `Invalid reaction content: ����`, and the openmls receiver retried the event ten times before giving up — by which point the secret tree had already consumed that generation, so the retry surfaced as a confusing `Ciphertext generation out of bounds 1 / SecretReuseError`. That last error was a downstream symptom; the root cause is argv decoding. Why it happens: JEP 400 (Java 18+) pins `file.encoding` to UTF-8 but deliberately leaves `sun.jnu.encoding` — the encoding the JVM uses to decode argv, filenames, and native interop strings — tied to the OS locale, and `sun.jnu.encoding` is read **before** the JVM parses any `-D` flag, so it can't be overridden via `applicationDefaultJvmArgs`. Under POSIX/C locales (no `LANG`/`LC_ALL` set, common on CI runners and stripped-down containers) `sun.jnu.encoding` ends up at `ANSI_X3.4-1968` — i.e. ASCII — and every byte > 0x7F in argv lands as `U+FFFD` before our code ever sees it. The four UTF-8 bytes of 🍕 (F0 9F 8D 95) become four replacement chars, get signed into the kind:7 content, and the test fails on receipt. The only place we can set the encoding from is the launching shell. Patch the gradle-generated start scripts (POSIX `amy`, Windows `amy.bat`) right after `:cli:startScripts`: * POSIX: if neither `LANG` nor `LC_ALL` is set, `export LANG=C.UTF-8` before invoking Java. Existing locale settings are respected. * Windows: `chcp 65001` to switch the console to UTF-8 before Java runs. Tested manually: with `LANG=` (sandbox default) `amy` previously sent `argv[0]=????` for an emoji argument; with the patch it sends `🍕` and `sun.jnu.encoding=UTF-8`, and whitenoise accepts the kind:7 reaction. The interop test 09 polling also needed a fix unrelated to the encoding bug: wn aggregates kind:7 reactions onto the anchor message under `.reactions.by_emoji.` instead of surfacing them as standalone messages whose `.content` is the emoji, so the previous `wait_for_message B "$mls_gid" "🍕"` would have failed even with a correctly-decoded reaction. The new poll inspects each message's `.reactions.by_emoji` keys for the emoji literal. Marmot interop score: 15/16 → **16/16** 🎉 https://claude.ai/code/session_013VYkpz8P1mPh9Ejxy9anhJ --- cli/build.gradle.kts | 78 ++++++++++++++++++++++++++++++++ cli/tests/marmot/tests-extras.sh | 22 ++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index 55871b14b3..93295b50de 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -35,6 +35,84 @@ application { applicationName = "amy" } +// Inject `LANG=C.UTF-8` (and the matching Windows code page) into the +// launcher scripts when `LANG`/`LC_ALL` aren't already set. Without this, +// running amy under a POSIX/C locale leaves `sun.jnu.encoding` as +// `ANSI_X3.4-1968` (i.e. ASCII) — JEP 400 pins `file.encoding` to UTF-8 +// but deliberately leaves `sun.jnu.encoding` tied to the OS locale, and +// it's read by the JVM *before* any `-D` flag has a chance to apply. So +// `-Dsun.jnu.encoding=UTF-8` does nothing; the encoding has to be set in +// the environment that launches Java. +// +// The symptom this fixes: `marmot message react "$gid" "$id" "🍕"` — +// the shell hands the four UTF-8 bytes (F0 9F 8D 95) to the JVM, the +// JVM decodes each one as ASCII (every byte > 0x7F → U+FFFD), and amy +// then signs a kind:7 whose `content` is four replacement characters. +// Whitenoise rejects it with "Invalid reaction content". +val patchAmyLauncherCharset by tasks.registering { + val appName = application.applicationName + val startScriptsTask = tasks.named("startScripts") + dependsOn(startScriptsTask) + // Patch the scripts at their source location so installDist (and any + // downstream packaging like jpackage) copies the patched copies. + doLast { + val unixScript = layout.buildDirectory.file("scripts/$appName").get().asFile + if (unixScript.exists()) { + val text = unixScript.readText() + val marker = "# Default LANG to UTF-8 so the JVM picks UTF-8 for sun.jnu.encoding" + if (!text.contains(marker)) { + // Build the snippet with explicit literal `$` characters to avoid + // regex-replacement-string escapes when we splice it in. + val injected = buildString { + append("\n") + append(marker).append("\n") + append("# (POSIX/C locales force ANSI_X3.4-1968, which mangles non-ASCII argv).\n") + append("if [ -z \"") + append("$").append("{LANG-}").append("$").append("{LC_ALL-}") + append("\" ]; then\n") + append(" export LANG=C.UTF-8\n") + append("fi\n") + } + // Insert right after the shebang. Use indexOf+substring instead + // of regex replaceFirst so the `$` chars in `injected` aren't + // mistaken for backreferences. + val nl = text.indexOf('\n') + val patched = + if (nl >= 0 && text.startsWith("#!")) { + text.substring(0, nl + 1) + injected + text.substring(nl + 1) + } else { + injected.trimStart('\n') + text + } + unixScript.writeText(patched) + } + } + val batScript = layout.buildDirectory.file("scripts/$appName.bat").get().asFile + if (batScript.exists()) { + val text = batScript.readText() + val marker = "rem Pin code page to UTF-8 so sun.jnu.encoding picks UTF-8" + if (!text.contains(marker)) { + val injected = + "$marker\r\n" + + "chcp 65001 > NUL 2>&1\r\n" + val anchor = "@if \"%DEBUG%\"==\"\" @echo off" + val idx = text.indexOf(anchor) + if (idx >= 0) { + val afterAnchor = text.indexOf('\n', idx) + if (afterAnchor >= 0) { + val patched = + text.substring(0, afterAnchor + 1) + injected + text.substring(afterAnchor + 1) + batScript.writeText(patched) + } + } + } + } + } +} + +tasks.named("installDist") { + dependsOn(patchAmyLauncherCharset) +} + // --------------------------------------------------------------------------- // Native distribution (jlink + jpackage) // diff --git a/cli/tests/marmot/tests-extras.sh b/cli/tests/marmot/tests-extras.sh index dd4e03e3db..aa0959670e 100644 --- a/cli/tests/marmot/tests-extras.sh +++ b/cli/tests/marmot/tests-extras.sh @@ -55,8 +55,26 @@ test_09_reply_react_unreact() { record_result "$id" fail "amy marmot message react failed"; return fi - # Round-trip: B should surface amy's kind:7 reaction in its messages stream. - if wait_for_message B "$mls_gid" "🍕" 90; then + # Round-trip: B should surface amy's kind:7 reaction. wn aggregates + # reactions onto the anchor message (`.reactions.by_emoji[]`), + # not as a standalone entry whose `.content` is the emoji — so polling + # `messages list` for an entry whose content equals "🍕" would never + # match, even when the reaction was successfully decrypted. Look for + # the emoji under any message's `reactions.by_emoji` keys instead. + local deadline=$(( $(date +%s) + 90 )) saw=0 + while [[ $(date +%s) -lt $deadline ]]; do + local payload + payload=$(wn_b_json messages list "$mls_gid" --limit 50 2>/dev/null || true) + if [[ -n "$payload" ]] && \ + printf '%s' "$payload" \ + | jq -e '(.result // .) | .[]? | (.reactions.by_emoji // {}) | keys[]?' \ + 2>/dev/null \ + | grep -qF '"🍕"'; then + saw=1; break + fi + sleep 3 + done + if [[ "$saw" -eq 1 ]]; then record_result "$id" pass else record_result "$id" fail "B didn't receive amy's reaction"