From 78986bd38c79f2b1aca2e882d255d7ada94650cb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:23:27 +0000 Subject: [PATCH] fix(quartz): never cut an astral character in half in the tagging d MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An audit pass over this branch found the `d16` truncation unpublishable for a target whose `d` has an astral character straddling the 16th UTF-16 code unit. Reproduced end to end before fixing: - the id is hashed from the in-memory string, lone high surrogate and all, giving 7f0bec88...; - UTF-8 has no encoding for half a character, so the bytes that reach the socket carry 0x3F ("?") in its place — not U+FFFD; - a relay parsing those bytes re-hashes a4bc430d... and rejects the event as id-mismatched. The first attempt to reproduce this compared string to string and came back clean; it only shows up once the comparison is at the byte level, which is where the relay reads. `d16` now drops a dangling high surrogate, giving 15 units instead of 16. The draft calls `d16` decoration that carries no uniqueness, and dropping half a character stays deterministic, so replaceability is unaffected and `hash8` still separates the targets. Pinned two ways: the `d` holds no unpaired surrogate, and it survives a UTF-8 round trip byte for byte. Also softens the KDoc on `prefix`, which claimed two targets can never collide. `hash8` is 32 bits by the draft's own widths, so what this actually removes is the old rule's *certain* collision between any two of an author's addressable events; a birthday-bound residue remains and is upstream's to widen, not ours. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AXvKXakvup4inNFfAhhr4L --- .../taggings/EventTagging.kt | 28 ++++++++++++++++--- .../decentralizedLists/AuditRegressionTest.kt | 24 ++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/taggings/EventTagging.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/taggings/EventTagging.kt index 8a45a5ef82..69d42d9956 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/taggings/EventTagging.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/taggings/EventTagging.kt @@ -43,9 +43,13 @@ import com.vitorpamplona.quartz.utils.sha256.sha256 sealed interface TaggingTarget { /** * What stands for this target in the assertion `d`: `` for a plain event, - * `--` for an addressable one. Two distinct targets must never produce - * the same one — kind 39999 is addressable, so taggings sharing a `d` do not sit side by - * side: the later one replaces the earlier. + * `--` for an addressable one. Kind 39999 is addressable, so two + * taggings that share a `d` do not sit side by side — the later one replaces the earlier. + * + * The draft fixes these widths, so the separation is 32 bits of hash rather than a guarantee: + * it removes the old rule's *certain* collision between any two of an author's events, and + * leaves a birthday-bound residue that the draft accepts. Widening it is an upstream change, + * not ours. */ val prefix: String @@ -62,7 +66,23 @@ sealed interface TaggingTarget { * An empty `dTag` yields an empty `d16`, i.e. `--`, which the draft spells * out. A plain event uses its own id (below): that already covers the whole event. */ - override val prefix get() = "${address.pubKeyHex.take(AUTHOR_CHARS)}-${address.dTag.take(D_TAG_CHARS)}-${hash8()}" + override val prefix get() = "${address.pubKeyHex.take(AUTHOR_CHARS)}-${d16()}-${hash8()}" + + /** + * The draft counts `d16` in UTF-16 code units, which can cut an astral character in half + * and leave a dangling high surrogate. That is not survivable: the id is hashed from the + * intact char, but UTF-8 encoding the event for the wire has no encoding for half a + * character and substitutes `?`, so the relay re-hashes different bytes, gets a different + * id, and rejects the event. Verified end to end — see the test. + * + * So the half character is dropped, giving 15 units instead of 16. `d16` is decoration + * the draft says carries no uniqueness, and dropping it stays deterministic, so nothing + * that matters is lost; `hash8` still separates the targets. + */ + private fun d16(): String { + val cut = address.dTag.take(D_TAG_CHARS) + return if (cut.lastOrNull()?.isHighSurrogate() == true) cut.dropLast(1) else cut + } // Lowercase hex of the UTF-8 coordinate exactly as the `a` tag carries it: no trimming, // no normalization, no reordering. Hex.encode already emits lower case. diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/AuditRegressionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/AuditRegressionTest.kt index fbf7384959..08b44cca86 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/AuditRegressionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/experimental/decentralizedLists/AuditRegressionTest.kt @@ -183,6 +183,30 @@ class AuditRegressionTest { ) } + // Truncating at 16 UTF-16 code units can cut an astral character in half. The id is hashed + // from the intact char while the wire bytes get `?` in its place, so the relay re-hashes + // different bytes and rejects the event. The half character is dropped instead. + @Test + fun aDTagCutMidAstralCharacterDoesNotLeaveHalfACharacterInTheD() { + // 15 ASCII then an emoji: the 16th code unit is the high surrogate of the pair. + val dTag = "a".repeat(15) + "\uD83D\uDE00" + "tail" + val d = EventTagging.dTag("awesome-tag", TaggingTarget.ByAddress(Address(39999, bob, dTag)), alice) + + assertFalse(d.any { it.isHighSurrogate() || it.isLowSurrogate() }, "the `d` still carries half a character: $d") + // 15 units of decoration, not 16, and hash8 is over the untruncated coordinate. + assertEquals("event-tag-awesome-tag-bbbbbbbb-${"a".repeat(15)}-15c09d87-aaaaaaaa", d) + } + + // The property the above protects: what is hashed for the id must survive UTF-8 encoding, + // or the relay recomputes a different id. Pins the bytes, not just the string. + @Test + fun theDSurvivesAUtf8RoundTripByteForByte() { + val dTag = "a".repeat(15) + "\uD83D\uDE00" + "tail" + val d = EventTagging.dTag("awesome-tag", TaggingTarget.ByAddress(Address(39999, bob, dTag)), alice) + + assertEquals(d, d.encodeToByteArray().decodeToString(), "the `d` does not survive UTF-8 encoding") + } + // A plain event is still named by its own id: it already covers the whole event, and hashing // it would only cost a round of SHA-256 per assertion built. The `e` branch is unchanged. @Test