diff --git a/commons/plans/2026-07-17-concord-epoch-walking-backfill.md b/commons/plans/2026-07-17-concord-epoch-walking-backfill.md index ec18d79865..09ea8f2646 100644 --- a/commons/plans/2026-07-17-concord-epoch-walking-backfill.md +++ b/commons/plans/2026-07-17-concord-epoch-walking-backfill.md @@ -247,11 +247,26 @@ Each covered epoch multiplies the subscription/AUTH footprint by ## Suggested sequence -1. Factor `EpochPlaneSet` derivation + make `ConcordCommunitySession` emit - historical addresses/keys/decrypt (commons unit-tested in isolation — no - network). Ship behind a flag defaulting off. -2. Planner + AUTH wiring; commons tests. -3. amethyst `BackwardRelayPager` epoch-stepping + "All caught up" semantics. -4. ~~`amy --epoch/--root` diagnostic~~ **DONE** (§7); still need a real prior - Soapbox root to validate old-epoch decrypt end-to-end. -5. Flip the flag on; on-device verify on a refounded community. +1. ~~Factor derivation + make `ConcordCommunitySession` emit historical + addresses/keys/decrypt~~ **DONE.** `ConcordActions.historicalChannelPlanes` + (bounded by `MAX_BACKFILL_EPOCHS = 8`; 0 disables) + `HistoricalChannelPlane`; + the session keeps a `historicalChannelKeysByAddress` map derived in `refold()`, + folded into `channelAddresses()` (subscribe), `streamKeys()` (AUTH), and + `ingest()` (decrypt with the matching epoch, `isBoundTo` per epoch). Test: + `ConcordCommunitySessionTest.ingestsPriorEpochWrapsFromAHeldRoot`. +2. ~~Planner + AUTH wiring~~ **DONE.** `ConcordSubscriptionPlanner.channelPlaneSubs` + appends the historical planes → the existing `ConcordChannelFilterAssembler` + subscribes to them with no change; AUTH flows through `session.streamKeys()`. + Test: `ConcordSubscriptionPlannerTest.channelSubsAlsoCoverPriorEpochPlanesForHeldRoots`. + The live channel sub now pulls prior-epoch wraps into `LocalCache`, so + pre-Refounding messages appear on channel open (bounded by relay cap / `since`). +3. **TODO** — amethyst `BackwardRelayPager` epoch-stepping + "All caught up" + semantics: "load older" still pages only the current-epoch plane; make it step + to prior epochs so deep scroll reaches the true start (step 2 already surfaces + each prior epoch's recent tail via the live sub). +4. ~~`amy --epoch/--root` diagnostic~~ **DONE** (§7). Cross-validated: the app now + subscribes to the exact prior-epoch plane pubkeys `amy concord read --epoch 0` + proved hold the older Soapbox #nostrhub messages (identical `publicChannel` + derivation). +5. **TODO** — on-device verify on a refounded community (emulator Concord fold is + historically flaky; verify when it cooperates). diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt index 8b6bbb6528..742c689a0b 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState import com.vitorpamplona.quartz.concord.cord02Community.Guestbook import com.vitorpamplona.quartz.concord.cord02Community.GuestbookAction import com.vitorpamplona.quartz.concord.cord02Community.GuestbookEntry +import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot import com.vitorpamplona.quartz.concord.cord02Community.ImagePointer import com.vitorpamplona.quartz.concord.cord02Community.NewConcordCommunity import com.vitorpamplona.quartz.concord.cord03Channels.ChannelChat @@ -62,6 +63,16 @@ data class ConcordChatMessage( val epoch: Long, ) +/** + * One channel's Chat Plane at a prior epoch: the epoch-invariant [channelIdHex], the [epoch] the + * wraps are bound to (for `isBoundTo` validation), and the derived [key] to decrypt them. + */ +data class HistoricalChannelPlane( + val channelIdHex: HexKey, + val epoch: Long, + val key: GroupKey, +) + /** * Concord community verbs — pure builders, plane-key derivation, relay-filter * assembly, and event folding usable from amy CLI, the Android app, and any other @@ -87,6 +98,36 @@ object ConcordActions { rootEpoch: Long, ): GroupKey = ConcordChannelKeys.publicChannel(communityRoot, channelId, rootEpoch) + /** + * How many prior epochs of channel history to backfill. A CORD-06 Refounding rotates the + * `community_root` and bumps the epoch, so pre-refounding messages live under a *different* + * derived Chat Plane per epoch; the client keeps each rotated-out root in + * [ConcordCommunityListEntry.heldRoots]. We re-derive those planes to read the older history + * instead of stopping at the current epoch. Bounded because each covered epoch multiplies the + * subscription + NIP-42 AUTH footprint by (channels); refoundings are rare, so a handful covers + * every real community. Set to 0 to disable historical backfill entirely. + */ + const val MAX_BACKFILL_EPOCHS = 8 + + /** + * The historical Chat Plane keys for [channelIdsHex] across the prior epochs in [heldRoots] + * (newest-held first, bounded to [MAX_BACKFILL_EPOCHS]). The channel id is epoch-invariant, so a + * message decrypted under a held root lands in the same channel as the current-epoch ones. + */ + fun historicalChannelPlanes( + heldRoots: List, + channelIdsHex: Collection, + ): List = + heldRoots + .sortedByDescending { it.epoch } + .take(MAX_BACKFILL_EPOCHS) + .flatMap { held -> + val rootBytes = held.key.hexToByteArray() + channelIdsHex.map { channelIdHex -> + HistoricalChannelPlane(channelIdHex, held.epoch, publicChannel(rootBytes, channelIdHex.hexToByteArray(), held.epoch)) + } + } + /** The Guestbook Plane address for a community at [rootEpoch] — where join/leave motions ride. */ fun guestbookPlane( communityRoot: ByteArray, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlanner.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlanner.kt index 71c50cd625..22024d73f3 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlanner.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlanner.kt @@ -81,21 +81,37 @@ object ConcordSubscriptionPlanner { ) } - /** Chat-plane subscriptions for every live channel in a folded community [state]. */ + /** + * Chat-plane subscriptions for every live channel in a folded community [state] — at the current + * epoch, plus each channel's plane at every prior epoch the account still holds a root for + * ([ConcordCommunityListEntry.heldRoots]). A CORD-06 Refounding rotates the root per epoch, so the + * pre-refounding history lives under those prior-epoch planes; subscribing to them is what lets the + * client fetch messages older than the last Refounding instead of stopping at "All caught up". + */ fun channelPlaneSubs( entry: ConcordCommunityListEntry, state: ConcordCommunityState, ): List { val root = entry.root.hexToByteArray() val relays = normalize(entry.relays) - return state.channels.keys.map { channelIdHex -> - val ch = ConcordActions.publicChannel(root, channelIdHex.hexToByteArray(), entry.rootEpoch) - ConcordPlaneSub( - channelId = ConcordChannelId(entry.id, channelIdHex), - pubKeyHex = ch.publicKeyHex, - relays = relays, - ) - } + val current = + state.channels.keys.map { channelIdHex -> + val ch = ConcordActions.publicChannel(root, channelIdHex.hexToByteArray(), entry.rootEpoch) + ConcordPlaneSub( + channelId = ConcordChannelId(entry.id, channelIdHex), + pubKeyHex = ch.publicKeyHex, + relays = relays, + ) + } + val historical = + ConcordActions.historicalChannelPlanes(entry.heldRoots, state.channels.keys).map { plane -> + ConcordPlaneSub( + channelId = ConcordChannelId(entry.id, plane.channelIdHex), + pubKeyHex = plane.key.publicKeyHex, + relays = relays, + ) + } + return current + historical } /** diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt index 222690950c..3abf03b1c2 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt @@ -124,6 +124,12 @@ class ConcordCommunitySession( // channel plane pubkey -> (channelIdHex, key), refreshed on each control re-fold. private var channelKeysByAddress = HashMap>() + // Prior-epoch channel plane pubkey -> (channelIdHex, key, epoch), for pre-Refounding history. + // A CORD-06 Refounding rotates the root per epoch, so older messages live under a different + // plane per held root; we re-derive those here so historical wraps are subscribed, AUTHed, and + // decrypted alongside the current epoch. Empty when the account holds no prior roots. + private var historicalChannelKeysByAddress = HashMap>() + private val _state = MutableStateFlow(null) val state: StateFlow = _state @@ -185,8 +191,11 @@ class ConcordCommunitySession( /** The size of [allMembers] — the community's true (best-effort) member count. */ fun memberCount(): Int = allMembers().size - /** The current Chat Plane addresses to subscribe to, one per folded channel. */ - fun channelAddresses(): Set = lock.withLock { channelKeysByAddress.keys.toSet() } + /** + * Every Chat Plane address to subscribe to: one per folded channel at the current epoch, plus + * each channel's prior-epoch planes we still hold a root for (pre-Refounding history). + */ + fun channelAddresses(): Set = lock.withLock { channelKeysByAddress.keys + historicalChannelKeysByAddress.keys } /** The Chat Plane stream address for [channelIdHex], once this community has folded that channel (else null). */ fun channelPlaneAddress(channelIdHex: HexKey): HexKey? = lock.withLock { channelKeysByAddress.entries.firstOrNull { it.value.first == channelIdHex }?.key } @@ -212,7 +221,10 @@ class ConcordCommunitySession( */ fun streamKeys(): List = lock.withLock { - listOf(controlPlaneKey) + channelKeysByAddress.values.map { it.second } + listOf(controlPlaneKey) + + channelKeysByAddress.values.map { it.second } + + // Prior-epoch channel stream keys so the gated relays serve their older wraps too. + historicalChannelKeysByAddress.values.map { it.second } } /** The CORD-06 auxiliary plane keys (Guestbook + next base-rekey) for their own isolated AUTH. */ @@ -271,39 +283,58 @@ class ConcordCommunitySession( return ConcordIngestOutcome.STRUCTURAL } else -> { - val channelRef = lock.withLock { channelKeysByAddress[wrap.pubKey] } ?: return ConcordIngestOutcome.NOT_MINE - val (channelIdHex, key) = channelRef - // An ephemeral wrap on a channel plane is a transient signal (typing) — fold it into - // the typing state, never into the stored message buffer or the Note sink. The typing - // UI collects the [typing] StateFlow directly, so this needs no structural revision bump. - if (wrap.kind == ConcordStreamEnvelope.KIND_WRAP_EPHEMERAL) { - ingestTyping(wrap, channelIdHex, key) - return ConcordIngestOutcome.NON_STRUCTURAL + val current = lock.withLock { channelKeysByAddress[wrap.pubKey] } + if (current != null) { + val (channelIdHex, key) = current + return ingestChannelWrap(wrap, channelIdHex, key, entry.rootEpoch, seenOnRelays) } - val isNew = - lock.withLock { - channelWrapsById.getOrPut(channelIdHex) { LinkedHashMap() }.put(wrap.id, wrap) == null - } - // Project only the newly-arrived wrap — the buffer's earlier wraps were already - // emitted when they landed, so re-decrypting the whole history on every message - // would be O(history) per message (quadratic over a channel's lifetime). A duplicate - // re-delivery (isNew == false) is a no-op. A full-history sweep (member-roster harvest) - // relies on this staying O(1) per wrap. - if (isNew) emitChannelRumors(channelIdHex, key, listOf(wrap), seenOnRelays) - // A chat message lands in the feed via [onRumor] → LocalCache, independent of the - // revision; it changes no plane address, so it must NOT bump (see the storm note above). - return ConcordIngestOutcome.NON_STRUCTURAL + // A prior-epoch plane (pre-Refounding history). Decrypt with that epoch's key and + // bind-check against that epoch. Keyed separately from the current buffer so a re-fold + // (which rebuilds only the current-epoch keys) never re-projects the historical ones. + val historical = lock.withLock { historicalChannelKeysByAddress[wrap.pubKey] } ?: return ConcordIngestOutcome.NOT_MINE + val (channelIdHex, key, epoch) = historical + return ingestChannelWrap(wrap, channelIdHex, key, epoch, seenOnRelays) } } } + /** Shared channel-wrap ingest for any epoch: typing → typing state, else buffer-dedup + emit. */ + private fun ingestChannelWrap( + wrap: Event, + channelIdHex: HexKey, + key: GroupKey, + epoch: Long, + seenOnRelays: Set, + ): ConcordIngestOutcome { + // An ephemeral wrap on a channel plane is a transient signal (typing) — fold it into the + // typing state, never the stored buffer or the Note sink. Typing is a current-epoch live + // signal, so a prior-epoch ephemeral (there won't be any — old epochs are frozen) is harmless. + if (wrap.kind == ConcordStreamEnvelope.KIND_WRAP_EPHEMERAL) { + ingestTyping(wrap, channelIdHex, key, epoch) + return ConcordIngestOutcome.NON_STRUCTURAL + } + val isNew = + lock.withLock { + channelWrapsById.getOrPut(channelIdHex) { LinkedHashMap() }.put(wrap.id, wrap) == null + } + // Project only the newly-arrived wrap — the buffer's earlier wraps were already emitted when + // they landed, so re-decrypting the whole history on every message would be O(history) per + // message (quadratic over a channel's lifetime). A duplicate re-delivery (isNew == false) is a + // no-op. A full-history sweep (member-roster harvest) relies on this staying O(1) per wrap. + if (isNew) emitChannelRumors(channelIdHex, key, epoch, listOf(wrap), seenOnRelays) + // A chat message lands in the feed via [onRumor] → LocalCache, independent of the revision; it + // changes no plane address, so it must NOT bump (see the storm note above). + return ConcordIngestOutcome.NON_STRUCTURAL + } + private fun ingestTyping( wrap: Event, channelIdHex: HexKey, key: GroupKey, + epoch: Long, ) { val rumor = ConcordStreamEnvelope.openOrNull(wrap, key)?.rumor ?: return - if (!ChannelChat.isTyping(rumor) || !ChannelChat.isBoundTo(rumor, channelIdHex, entry.rootEpoch)) return + if (!ChannelChat.isTyping(rumor) || !ChannelChat.isBoundTo(rumor, channelIdHex, epoch)) return val who = rumor.pubKey.lowercase() if (who == myPubKey.lowercase()) return // never show my own typing back to me val now = TimeUtils.now() @@ -338,6 +369,16 @@ class ConcordCommunitySession( next[key.publicKeyHex] = channelIdHex to key } channelKeysByAddress = next + + // Re-derive the prior-epoch planes for the same (epoch-invariant) channel ids, so older + // pre-Refounding history is subscribed/AUTHed/decrypted. Channels are known only after a + // fold, hence derived here rather than up front. + val historical = HashMap>() + for (plane in ConcordActions.historicalChannelPlanes(entry.heldRoots, folded.channels.keys)) { + historical[plane.key.publicKeyHex] = Triple(plane.channelIdHex, plane.key, plane.epoch) + } + historicalChannelKeysByAddress = historical + _state.value = folded folded.channels.keys.filterNot { it in prevChannels } } @@ -356,11 +397,13 @@ class ConcordCommunitySession( } } - /** Re-decrypts and re-projects a channel's WHOLE wrap buffer. Only for a re-fold (keys may change). */ + /** Re-decrypts and re-projects a channel's WHOLE wrap buffer at the current epoch. Only for a + * re-fold (keys may change). Prior-epoch wraps in the buffer simply won't open under the current + * key and are skipped — they were already emitted when they landed (the sink dedups by id). */ private fun reprojectChannel(channelIdHex: HexKey) { val key = lock.withLock { channelKeysByAddress.values.firstOrNull { it.first == channelIdHex }?.second } ?: return val wraps = lock.withLock { channelWrapsById[channelIdHex]?.values?.toList() } ?: return - emitChannelRumors(channelIdHex, key, wraps) + emitChannelRumors(channelIdHex, key, entry.rootEpoch, wraps) } /** @@ -371,11 +414,12 @@ class ConcordCommunitySession( private fun emitChannelRumors( channelIdHex: HexKey, key: GroupKey, + epoch: Long, wraps: List, seenOnRelays: Set = emptySet(), ) { val authors = HashSet() - ConcordActions.channelRumors(wraps, key, channelIdHex, entry.rootEpoch).forEach { rumor -> + ConcordActions.channelRumors(wraps, key, channelIdHex, epoch).forEach { rumor -> authors.add(rumor.pubKey.lowercase()) onRumor(entry.id, channelIdHex, rumor, seenOnRelays) } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlannerTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlannerTest.kt index bb43bb818e..7455b8315c 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlannerTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlannerTest.kt @@ -35,6 +35,38 @@ import kotlin.test.assertTrue class ConcordSubscriptionPlannerTest { private val owner = NostrSignerInternal(KeyPair()) + @Test + fun channelSubsAlsoCoverPriorEpochPlanesForHeldRoots() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Nostrichs", createdAt = 1L, relays = listOf("wss://r.example")) + val priorEpoch = 4L + val priorRoot = KeyPair().pubKey + val entry = + com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry( + id = community.communityIdHex, + owner = community.ownerPubKey, + ownerSalt = community.ownerSalt.toHexKey(), + root = community.communityRoot.toHexKey(), + rootEpoch = community.rootEpoch, + heldRoots = + listOf( + com.vitorpamplona.quartz.concord.cord02Community + .HeldRoot(priorEpoch, priorRoot.toHexKey()), + ), + relays = listOf("wss://r.example"), + name = "Nostrichs", + ) + val state = ConcordActions.foldCommunity(community.genesisWraps, community.controlPlane, community.ownerPubKey) + val subs = ConcordSubscriptionPlanner.channelPlaneSubs(entry, state) + + // Both the current-epoch and the prior-epoch #general planes are subscribed. + val currentGeneral = ConcordActions.publicChannel(community.communityRoot, community.generalChannelId, community.rootEpoch).publicKeyHex + val priorGeneral = ConcordActions.publicChannel(priorRoot, community.generalChannelId, priorEpoch).publicKeyHex + assertTrue(subs.any { it.pubKeyHex == currentGeneral }, "current-epoch plane missing") + assertTrue(subs.any { it.pubKeyHex == priorGeneral }, "prior-epoch plane missing") + assertTrue(currentGeneral != priorGeneral) // a Refounding really does move the plane + } + @Test fun controlAndChannelSubsMatchDerivedAddresses() = runTest { diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt index 61ab03a500..c1ceed5613 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.amethyst.commons.model.concord import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityFactory import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry +import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot import com.vitorpamplona.quartz.concord.cord03Channels.ChannelChat import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair @@ -35,6 +36,51 @@ import kotlin.test.assertTrue class ConcordCommunitySessionTest { private val owner = NostrSignerInternal(KeyPair()) + @Test + fun ingestsPriorEpochWrapsFromAHeldRoot() = + runTest { + // A community whose access root has been rotated once (CORD-06 Refounding): the current + // entry is epoch 0/rootA, but the account still holds a prior epoch's root. The prior + // epoch's channel plane is a DIFFERENT stream key; historical backfill must subscribe, + // AUTH, and decrypt it so pre-Refounding messages surface. + val community = ConcordCommunityFactory.create(owner, "Nostrichs", createdAt = 1L, relays = listOf("wss://r.example")) + val priorEpoch = 7L + val priorRoot = KeyPair().pubKey // any 32-byte value is a valid root ikm + val entry = + ConcordCommunityListEntry( + id = community.communityIdHex, + owner = community.ownerPubKey, + ownerSalt = community.ownerSalt.toHexKey(), + root = community.communityRoot.toHexKey(), + rootEpoch = community.rootEpoch, + heldRoots = listOf(HeldRoot(priorEpoch, priorRoot.toHexKey())), + relays = listOf("wss://r.example"), + name = "Nostrichs", + ) + + val captured = mutableListOf() + val session = ConcordCommunitySession(entry, owner.pubKey) { _, _, rumor, _ -> captured += rumor } + + // Fold genesis so #general is known — historical planes are derived off the folded channels. + community.genesisWraps.forEach { session.ingest(it) } + + // The #general channel plane at the PRIOR epoch (derived from the held root) is now a known + // address AND a stream key to AUTH as. + val priorGeneral = ConcordActions.publicChannel(priorRoot, community.generalChannelId, priorEpoch) + assertTrue(session.channelAddresses().contains(priorGeneral.publicKeyHex), "historical plane not subscribed") + assertTrue(session.streamKeys().any { it.publicKeyHex == priorGeneral.publicKeyHex }, "historical stream key not AUTHed") + + // A message authored on the prior-epoch plane, bound to the prior epoch, decrypts + emits. + val oldMsg = ConcordActions.buildChannelMessage(owner, priorGeneral, community.generalChannelIdHex, priorEpoch, "gm from the old epoch", 2L) + assertEquals(ConcordIngestOutcome.NON_STRUCTURAL, session.ingest(oldMsg)) + assertEquals(1, captured.count { it.content == "gm from the old epoch" }) + + // A wrap on the prior plane but bound to the WRONG epoch is rejected (no cross-epoch replay). + val spoofed = ConcordActions.buildChannelMessage(owner, priorGeneral, community.generalChannelIdHex, community.rootEpoch, "wrong epoch", 3L) + session.ingest(spoofed) + assertEquals(0, captured.count { it.content == "wrong epoch" }) + } + @Test fun ingestsControlThenChannelWrapsIntoFlows() = runTest {