From bf99eb740cede62ea863b25cc3a4d151d049d6e9 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sat, 26 Sep 2026 12:56:24 -0400 Subject: [PATCH] feat(cordn): carry the messages in a backup, so a restore keeps the conversation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restoring a backup handed back the groups with an empty history. The archive carried the MLS state, the cursor and the key packages but not one message, and the cursor it *did* carry told the sync loop the stream had been read to the end — so the coordinator was never asked to resend them either. Verified on the tablet with a true no-op round trip (export, restore that same file minutes later): both groups came back with names, icons, members and health, and every message gone, still gone after a cold start, and gone from the Messages inbox entirely for want of anything to sort on. That is not what the screen offers. Its first line is that losing the device without a backup loses "everything already said in it", which only reads one way. Archive version 2 adds the delivered messages per group, written as the same JSON the on-disk message log already stores so the two cannot drift. Version 1 files still open, and still restore empty — there is nothing in them to do better with. The import writes the messages back before the cursor is trusted again, because nothing will re-deliver them afterwards. The alternative was to reset the cursor on restore and re-pull from the coordinator, which does hold the complete ordered history. It is cheaper, but it only recovers the epochs the restored state can still open; carrying the plaintext is whole regardless of how often the group has rotated. Vitor picked this one. The "What is in the file" copy now says messages are in there, since that materially changes what the file is worth to whoever finds it. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/model/cordn/CordnRuntime.kt | 5 ++ amethyst/src/main/res/values/strings.xml | 2 +- .../amethyst/commons/cordn/CordnBackup.kt | 64 +++++++++++++++++-- .../amethyst/commons/cordn/CordnBackupTest.kt | 35 ++++++++++ 4 files changed, 99 insertions(+), 7 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/cordn/CordnRuntime.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/cordn/CordnRuntime.kt index d938bd0955..934a3f383a 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/cordn/CordnRuntime.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/cordn/CordnRuntime.kt @@ -794,6 +794,7 @@ class CordnRuntime( state = state, cursor = groupStore.loadCursor(gid), joinedViaRequest = groupStore.loadJoinedViaRequest(gid), + messages = groupStore.loadMessages(gid), ) } @@ -847,6 +848,10 @@ class CordnRuntime( store.saveGroup(group.gid, group.state) group.cursor?.let { store.saveCursor(group.gid, it) } if (group.joinedViaRequest) store.saveJoinedViaRequest(group.gid) + // Before the cursor is trusted again. The cursor says the + // stream has been read to here, so nothing will re-deliver + // these -- if they are not written back now they are gone. + group.messages.forEach { store.appendMessage(group.gid, it) } } archive.keyPackages.forEach { keyPackage -> diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 3ed179a4c1..2da4657d8d 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -350,7 +350,7 @@ Your nsec cannot bring cordn groups back. A cordn group lives as encryption state on this device and an ordered stream on a coordinator that cannot read it — lose the device without a backup and the group is gone, including everything already said in it. Passphrase What is in the file - Your group encryption keys, your reading positions, and your key packages. Anyone who opens it can read those groups. The passphrase is the only thing protecting it once it leaves Amethyst. + Your group encryption keys, your key packages, and every message in those groups. Anyone who opens it can read the conversations. The passphrase is the only thing protecting it once it leaves Amethyst. Export Backup saved. Restore diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/cordn/CordnBackup.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/cordn/CordnBackup.kt index ac21404400..cd3790fc0e 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/cordn/CordnBackup.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/cordn/CordnBackup.kt @@ -20,6 +20,8 @@ */ package com.vitorpamplona.amethyst.commons.cordn +import com.vitorpamplona.quartz.cordn.spec02Envelopes.CordnDeliveredMessage +import com.vitorpamplona.quartz.cordn.spec02Envelopes.CordnDeliveredMessageCodec import com.vitorpamplona.quartz.cordn.sync.GroupCursor import com.vitorpamplona.quartz.mls.codec.TlsReader import com.vitorpamplona.quartz.mls.codec.TlsWriter @@ -83,7 +85,25 @@ import com.vitorpamplona.quartz.utils.RandomInstance * that is expected rather than a gap. */ object CordnBackup { - const val VERSION = 1 + /** + * Version 2 adds the delivered messages to each group. + * + * Version 1 carried the MLS state, the cursor and the key packages but not + * a single message, and restoring it handed back the groups with an empty + * history: the messages were not in the file, and the cursor that *was* + * in the file told the sync loop it had already read to the end, so the + * coordinator was never asked to resend them. A backup that loses the + * conversation is not the thing the screen promises. + * + * The history could instead be re-pulled by resetting the cursor, since + * the coordinator holds it, but only for epochs the restored state can + * still open. Carrying the plaintext is what makes the restore whole + * regardless of how many times the group has rotated since. + */ + const val VERSION = 2 + + /** The last version that carried no messages; still readable. */ + private const val VERSION_WITHOUT_MESSAGES = 1 /** * scrypt cost, as `log2(N)`. @@ -115,6 +135,14 @@ object CordnBackup { val state: ByteArray, val cursor: GroupCursor?, val joinedViaRequest: Boolean, + /** + * The group's delivered messages, in cursor order. + * + * Empty when the archive was written by version 1, which carried + * none — a v1 restore still yields an empty conversation, and + * there is nothing in the file to do better with. + */ + val messages: List = emptyList(), ) { override fun equals(other: Any?): Boolean = this === other || @@ -125,7 +153,8 @@ object CordnBackup { state.contentEquals(other.state) && cursor?.fetchCursor == other.cursor?.fetchCursor && cursor?.lastCursor == other.cursor?.lastCursor && - joinedViaRequest == other.joinedViaRequest + joinedViaRequest == other.joinedViaRequest && + messages == other.messages ) override fun hashCode(): Int { @@ -134,6 +163,7 @@ object CordnBackup { result = 31 * result + state.contentHashCode() result = 31 * result + (cursor?.fetchCursor?.hashCode() ?: 0) result = 31 * result + joinedViaRequest.hashCode() + result = 31 * result + messages.hashCode() return result } } @@ -188,7 +218,7 @@ object CordnBackup { val reader = TlsReader(sealed.copyOfRange(MAGIC.size, HEADER_LENGTH)) val version = reader.readUint16() - require(version == VERSION) { "unknown cordn backup version $version" } + require(version == VERSION || version == VERSION_WITHOUT_MESSAGES) { "unknown cordn backup version $version" } val logN = reader.readUint8() val salt = reader.readBytes(SALT_LENGTH) @@ -198,7 +228,7 @@ object CordnBackup { val header = sealed.copyOfRange(0, HEADER_LENGTH) val plaintext = ChaCha20Poly1305.decrypt(sealed.copyOfRange(HEADER_LENGTH, sealed.size), header, nonce, key) - return decode(plaintext) + return decode(plaintext, version) } private fun header( @@ -242,6 +272,13 @@ object CordnBackup { writer.putUint64(cursor.lastCursor) } writer.putUint8(if (it.joinedViaRequest) 1 else 0) + // uint32: a long-running group's log is not bounded by 65535. + // Each entry is the same JSON the on-disk message log stores, so + // the archive and the store cannot drift out of step. + writer.putUint32(it.messages.size.toLong()) + it.messages.forEach { message -> + writer.putOpaque4(CordnDeliveredMessageCodec.encode(message).encodeToByteArray()) + } } writer.putUint16(archive.keyPackages.size) @@ -253,7 +290,10 @@ object CordnBackup { return writer.toByteArray() } - private fun decode(bytes: ByteArray): Archive { + private fun decode( + bytes: ByteArray, + version: Int, + ): Archive { val reader = TlsReader(bytes) val accountPubKey = reader.readOpaque2().decodeToString() val coordinators = CoordinatorListCodec.decode(reader.readOpaque4()) @@ -264,7 +304,19 @@ object CordnBackup { val gid = reader.readOpaque2().decodeToString() val state = reader.readOpaque4() val cursor = if (reader.readUint8() == 1) GroupCursor(reader.readUint64(), reader.readUint64()) else null - Archive.Group(coordinator, gid, state, cursor, reader.readUint8() == 1) + val joinedViaRequest = reader.readUint8() == 1 + val messages = + if (version == VERSION_WITHOUT_MESSAGES) { + emptyList() + } else { + // A message that will not parse is dropped rather than + // failing the whole restore: losing one row beats + // refusing to bring the account back at all. + (0 until reader.readUint32()).mapNotNull { + CordnDeliveredMessageCodec.decodeOrNull(reader.readOpaque4().decodeToString()) + } + } + Archive.Group(coordinator, gid, state, cursor, joinedViaRequest, messages) } val keyPackages = diff --git a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/cordn/CordnBackupTest.kt b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/cordn/CordnBackupTest.kt index c15a135b62..bcaafed52a 100644 --- a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/cordn/CordnBackupTest.kt +++ b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/cordn/CordnBackupTest.kt @@ -20,6 +20,8 @@ */ package com.vitorpamplona.amethyst.commons.cordn +import com.vitorpamplona.quartz.cordn.spec02Envelopes.CordnDeliveredMessage +import com.vitorpamplona.quartz.cordn.spec02Envelopes.CordnEnvelope import com.vitorpamplona.quartz.cordn.sync.GroupCursor import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import kotlin.test.Test @@ -55,6 +57,7 @@ class CordnBackupTest { state = byteArrayOf(1, 2, 3, 4), cursor = GroupCursor(fetchCursor = 7, lastCursor = 9), joinedViaRequest = true, + messages = listOf(delivered("hello", 7), delivered("again", 8)), ), CordnBackup.Archive.Group( coordinatorPubKey = coordinator, @@ -162,6 +165,38 @@ class CordnBackupTest { assertFailsWith { CordnBackup.seal(archive, "pw", logN = 40) } } + /** + * The id has to hash the contents: `CordnEnvelope.fromJsonObject` checks + * it, so an envelope with a made-up id decodes to null and the archive + * appears to have lost the message. + */ + private fun delivered( + content: String, + cursor: Long, + ): CordnDeliveredMessage { + val unsigned = + CordnEnvelope( + id = "", + pubKey = account, + createdAt = 1_700_000_000L + cursor, + kind = 9, + tags = arrayOf(arrayOf("h", "room-1")), + content = content, + ) + return CordnDeliveredMessage(unsigned.copy(id = unsigned.computedId()), cursor) + } + + @Test + fun `messages survive the round trip`() { + val opened = CordnBackup.open(CordnBackup.seal(archive, "pw", cheap), "pw") + + val room1 = opened.groups.first { it.gid == "room-1" } + assertEquals(listOf("hello", "again"), room1.messages.map { it.envelope.content }) + assertEquals(listOf(7L, 8L), room1.messages.map { it.cursor }) + // The group carrying none still round-trips as none. + assertEquals(emptyList(), opened.groups.first { it.gid == "room-2" }.messages) + } + private companion object { const val MAGIC_LENGTH = 8