From cd5c92bd1bc79b794d47a615923be12f0f9c80ed Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 29 Sep 2026 15:18:34 -0400 Subject: [PATCH] fix(concord): put a new membership in any held List fragment it fits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With part of a fragmented Community List missing, a new membership always went to the lowest held fragment, so once fragment 0 filled every join was refused even with room in another fragment the device held. Take the lowest held fragment it still fits in; refuse only when none does, since opening a new fragment is a repack that needs the complete List (CORD-02 §8). Adds an end-to-end test of the List past one fragment through ConcordChannelListState: 250 heavy memberships split into fragments that each fit the 64 KiB event ceiling (measured on the signed JSON), another device reads every membership and secret back from the published fragments, cross-device leave and join converge, and a partial device neither overflows nor erases the fragments it never held. Co-Authored-By: Claude Opus 5.5 --- .../ConcordChannelListFragmentationTest.kt | 231 ++++++++++++++++++ .../cord02Community/ConcordListFragmentSet.kt | 17 +- 2 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListFragmentationTest.kt diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListFragmentationTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListFragmentationTest.kt new file mode 100644 index 0000000000..8d401335be --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListFragmentationTest.kt @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.model.concord + +import com.vitorpamplona.amethyst.commons.model.AddressableNote +import com.vitorpamplona.amethyst.commons.model.Channel +import com.vitorpamplona.amethyst.commons.model.Note +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.cache.ICacheEventStream +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListFragmentEvent +import com.vitorpamplona.quartz.concord.cord02Community.ConcordListFragments +import com.vitorpamplona.quartz.concord.cord02Community.ConcordListTooLargeException +import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot +import com.vitorpamplona.quartz.concord.cord02Community.PrivateChannelKey +import com.vitorpamplona.quartz.nip01Core.core.Address +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * The Community List past one fragment (CORD-02 §8), driven through the app's own + * [ConcordChannelListState] rather than the fragment math alone: a List that outgrows one event + * must split into several, every published event must fit a relay's event ceiling, and another + * device holding only what was published must read back every membership with its secrets. + */ +class ConcordChannelListFragmentationTest { + private val signer = NostrSignerInternal(KeyPair("0000000000000000000000000000000000000000000000000000000000000009".hexToByteArray())) + + private fun hex( + seed: Int, + salt: Int, + ): HexKey = (seed * 7919 + salt).toString(16).padStart(8, '0').repeat(8) + + /** A realistically heavy membership: staff secrets, two held roots, three private channels. */ + private fun entry(n: Int) = + ConcordCommunityListEntry( + id = hex(n, 1), + owner = hex(n, 2), + ownerSalt = hex(n, 3), + root = hex(n, 4), + rootEpoch = 2, + controlPk = hex(n, 5), + controlRoot = hex(n, 6), + heldRoots = listOf(HeldRoot(0, hex(n, 7)), HeldRoot(1, hex(n, 8), hex(n, 9))), + privateChannels = (0 until 3).map { PrivateChannelKey(hex(n, 20 + it), hex(n, 30 + it), 1, "private-$it") }, + relays = listOf("wss://nos.lol/", "wss://nostr.mom/"), + name = "Community number $n", + addedAt = 1_000L + n, + ) + + /** Holds the List only as published fragments, the way a relay or the offline backup does. */ + private class WireRepository( + var fragments: List = emptyList(), + ) : ConcordListRepository { + override fun concordList(): ConcordCommunityListEvent? = null + + override fun updateConcordListTo(newConcordList: ConcordCommunityListEvent?) = Unit + + override fun concordListFragments() = fragments + + override fun updateConcordListFragmentTo(fragment: ConcordCommunityListFragmentEvent) { + fragments = fragments.filterNot { it.index() == fragment.index() } + fragment + } + } + + private class StubCache : ICacheProvider { + override fun getAnyChannel(note: Note): Channel? = null + + override val relayHints = HintIndexer() + + override fun getUserIfExists(pubkey: HexKey): User? = null + + override fun countUsers(predicate: (String, User) -> Boolean): Int = 0 + + override fun getNoteIfExists(hexKey: HexKey): Note? = null + + override fun checkGetOrCreateNote(hexKey: HexKey): Note? = null + + override fun getOrCreateAddressableNote(address: Address): AddressableNote = AddressableNote(address) + + override fun getEventStream(): ICacheEventStream = error("not used") + + override fun hasBeenDeleted(event: Any): Boolean = false + + override fun getOrCreateUser(pubkey: HexKey): User? = null + + override fun consumeEmbedded(event: Event) = Unit + + override fun justConsumeMyOwnEvent(event: Event): Boolean = false + } + + private fun device(wire: List = emptyList()): Pair { + val repo = WireRepository(wire) + val list = ConcordChannelListState(signer = signer, cache = StubCache(), scope = CoroutineScope(Dispatchers.Unconfined), settings = repo) + list.markRelaysConfirmed() + return list to repo + } + + private suspend fun joinAll( + list: ConcordChannelListState, + count: Int, + ): List { + val published = ArrayList() + for (n in 0 until count) published += list.follow(entry(n)).map { it as ConcordCommunityListFragmentEvent } + return published + } + + @Test + fun aLargeListSplitsIntoFragmentsThatEachFitAnEvent() = + runTest { + val (list, repo) = device() + val published = joinAll(list, 250) + + val wire = repo.fragments + assertTrue(wire.size >= 2, "250 heavy memberships must not fit one fragment; got ${wire.size}") + for (fragment in published) { + val bytes = fragment.toJson().encodeToByteArray().size + assertTrue(bytes <= ConcordListFragments.EVENT_CEILING_BYTES, "fragment ${fragment.index()} is $bytes bytes, over the ${ConcordListFragments.EVENT_CEILING_BYTES} ceiling") + } + assertEquals(250, list.entries().size) + } + + @Test + fun anotherDeviceReadsEveryMembershipFromThePublishedFragments() = + runTest { + val (first, repo) = device() + joinAll(first, 250) + + val (second, _) = device(repo.fragments) + val read = second.entries().associateBy { it.id } + + assertEquals(250, read.size) + for (n in listOf(0, 1, 124, 248, 249)) { + val want = entry(n) + val got = read.getValue(want.id) + assertEquals(want.root, got.root) + assertEquals(want.controlRoot, got.controlRoot) + assertEquals(want.heldRoots.map { it.key }, got.heldRoots.map { it.key }) + assertEquals(want.privateChannels.map { it.key }, got.privateChannels.map { it.key }) + assertEquals(want.name, got.name) + } + } + + @Test + fun editsOnAnotherDeviceSurviveTheRoundTripAcrossFragments() = + runTest { + val (first, repo) = device() + joinAll(first, 250) + + // The second device leaves a membership that lives in a later fragment and joins a new one. + val (second, secondRepo) = device(repo.fragments) + second.unfollow(entry(240).id) + second.follow(entry(900)) + + // The first device, fed what the second published, converges on the same List. + val (third, _) = device(secondRepo.fragments) + val ids = third.entries().map { it.id }.toSet() + assertEquals(250, ids.size) + assertTrue(entry(240).id !in ids, "the leave must hold across fragments") + assertTrue(entry(900).id in ids) + assertTrue(entry(0).id in ids && entry(249).id in ids) + } + + @Test + fun aDeviceMissingFragmentsJoinsIntoAHeldFragmentWithRoom() = + runTest { + val (first, repo) = device() + joinAll(first, 250) + val wire = repo.fragments.sortedBy { it.index() } + assertTrue(wire.size >= 3) + + // Holding the full fragment 0 and the partly filled last one, but not the middle ones + // (they sit on a relay this device can't reach): the join goes where it fits. + val (partial, partialRepo) = device(listOf(wire.first(), wire.last())) + val written = partial.follow(entry(901)).map { it as ConcordCommunityListFragmentEvent } + assertEquals(listOf(wire.last().index()), written.map { it.index() }, "the new membership belongs in the held fragment with room") + + // Nothing it never held was rewritten: once the middle fragments arrive, nothing is lost. + val merged = (partialRepo.fragments + wire).groupBy { it.index() }.map { (_, copies) -> copies.maxBy { it.createdAt } } + val (reader, _) = device(merged) + val ids = reader.entries().map { it.id }.toSet() + assertEquals(251, ids.size, "every membership from every fragment plus the new join") + assertTrue(entry(901).id in ids) + } + + @Test + fun aDeviceHoldingOnlyFullFragmentsRefusesTheJoinRatherThanOverflowing() = + runTest { + val (first, repo) = device() + joinAll(first, 250) + val fragmentZero = repo.fragments.first { it.index() == 0 } + + // Opening a new fragment is a repack, which needs the complete List (CORD-02 §8); an + // oversized event would be refused by relays. Refusing here is the only safe answer, + // and the caller must surface it. + val (partial, partialRepo) = device(listOf(fragmentZero)) + assertFailsWith { partial.follow(entry(902)) } + assertEquals(listOf(fragmentZero), partialRepo.fragments, "a refused write publishes nothing") + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordListFragmentSet.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordListFragmentSet.kt index 21afc71596..231f722fce 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordListFragmentSet.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordListFragmentSet.kt @@ -142,7 +142,22 @@ class ConcordListFragmentSet private constructor( val targets = HashMap>() for (id in changed) { val holders = inRange.filter { id in ConcordListFragments.idsIn(held.getValue(it).fragment.doc) } - for (i in holders.ifEmpty { listOf(inRange.first()) }) targets.getOrPut(i) { HashSet() }.add(id) + if (holders.isNotEmpty()) { + for (i in holders) targets.getOrPut(i) { HashSet() }.add(id) + continue + } + // A new membership may go into any held fragment (CORD-02 §8 scopes the write to the + // fragment it touches). Take the lowest one it still fits in: always taking the lowest + // refused every join once fragment 0 filled, even with room in another held fragment. + // When none fits, the lowest is kept and the size guard below refuses the write, since + // opening a new fragment is a repack and needs the complete List. + val target = + inRange.firstOrNull { i -> + val ids = targets[i].orEmpty() + id + val plaintext = ConcordListFragments.rewriteFragment(held.getValue(i).fragment.doc, newDoc, ids, declared) + ConcordListFragments.projectedEventBytes(plaintext.encodeToByteArray().size) <= ConcordListFragments.EVENT_CEILING_BYTES + } ?: inRange.first() + targets.getOrPut(target) { HashSet() }.add(id) } for ((i, ids) in targets.entries.sortedBy { it.key }) { val plaintext = ConcordListFragments.rewriteFragment(held.getValue(i).fragment.doc, newDoc, ids, declared)