diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 00aa30c9ad..e43419f20d 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -221,7 +221,11 @@ class Context( secret = b.connectSecret, // Bunker requires web authorization: surface the URL; the request keeps waiting. onAuthUrl = { url -> System.err.println("[nip46] authorize this request in a browser, then it will continue:\n $url") }, - ) + ).also { + // signer.pubKey must be the USER identity, not the ephemeral transport key, so + // self-encryption/decryption (Concord list, private NIP-51 lists) uses the right peer. + it.bindUserPubkey(identity.pubKeyHex) + } } ?: NostrSignerInternal(identity.keyPair()) /** diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordChannelCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordChannelCommands.kt index 7c0577670b..3769ef8ba3 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordChannelCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordChannelCommands.kt @@ -94,18 +94,37 @@ object ConcordChannelCommands { val limit = args.intFlag("limit", 50) val sc = ConcordStore(dataDir.concordFile).find(handle) ?: return ConcordCommands.notFound(handle) + // Diagnostic overrides (concord-epoch-walking-backfill): read a PRIOR epoch's Chat Plane by + // supplying that epoch's community_root. A Refounding (CORD-06 §3) rotates the root and bumps + // the epoch, so pre-refounding messages live under a different derived stream key that the + // normal read (current epoch only) never fetches. Both derive from the same channel id, which + // is epoch-invariant, so channel resolution stays on the current epoch below. + val epoch = args.longFlag("epoch", sc.rootEpoch) + // Resolve the root for that epoch: explicit --root wins; else the current root if --epoch is + // the current epoch; else a stored heldRoot for that epoch (populated by `amy concord import`). + val rootHex = + args.flag("root") + ?: sc.root.takeIf { epoch == sc.rootEpoch } + ?: sc.heldRoots.firstOrNull { it.epoch == epoch }?.root + ?: return Output + .error("not_found", "no root known for epoch $epoch — pass --root or run `amy concord import` to load heldRoots") + .let { 1 } + if (!HEX64.matches(rootHex)) return Output.error("bad_args", "--root must be a 64-char hex community_root").let { 2 } + Context.open(dataDir).use { ctx -> ctx.prepare() val channelId = resolve(ctx, sc, channelRef) ?: return Output.error("not_found", "no channel '$channelRef'") - val channel = ConcordActions.publicChannel(sc.root.hexToByteArray(), channelId.hexToByteArray(), sc.rootEpoch) + val channel = ConcordActions.publicChannel(rootHex.hexToByteArray(), channelId.hexToByteArray(), epoch) val relays = ConcordCommands.relaysFor(ctx, sc) // The channel plane is NIP-42-gated to its own derived stream key; register it so the drain authenticates. ctx.registerConcordStreamKeys(relays, listOf(channel.secretKey)) val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(channel.publicKeyHex)) }, pendingOnAuthRequired = true).map { it.second } - val msgs = ConcordActions.channelMessages(wraps, channel, channelId, sc.rootEpoch).takeLast(limit) + val msgs = ConcordActions.channelMessages(wraps, channel, channelId, epoch).takeLast(limit) Output.emit( mapOf( "channel" to channelId, + "epoch" to epoch, + "plane" to channel.publicKeyHex, "count" to msgs.size, "messages" to msgs.map { mapOf("id" to it.id, "author" to it.author, "content" to it.content, "created_at" to it.createdAt) }, ), diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index ce377b5992..e0a145c1bf 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -26,8 +26,11 @@ import com.vitorpamplona.amethyst.cli.DataDir import com.vitorpamplona.amethyst.cli.Output import com.vitorpamplona.amethyst.cli.stores.ConcordStore import com.vitorpamplona.amethyst.cli.stores.StoredCommunity +import com.vitorpamplona.amethyst.cli.stores.StoredHeldRoot import com.vitorpamplona.amethyst.commons.actions.ConcordActions +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.utils.TimeUtils @@ -45,10 +48,11 @@ object ConcordCommands { route( "concord", tail, - "concord ", + "concord ", mapOf( "create" to { rest -> create(dataDir, rest) }, "list" to { rest -> list(dataDir, rest) }, + "import" to { rest -> import(dataDir, rest) }, "channels" to { rest -> ConcordChannelCommands.channels(dataDir, rest) }, "send" to { rest -> ConcordChannelCommands.send(dataDir, rest) }, "read" to { rest -> ConcordChannelCommands.read(dataDir, rest) }, @@ -117,6 +121,62 @@ object ConcordCommands { return 0 } + /** + * Fetch this account's own encrypted kind-13302 Concord community list, decrypt it, and + * upsert every community into the local store — crucially carrying each community's + * `heldRoots` (the prior-epoch access roots Amethyst accumulates across Refoundings, CORD-06). + * With those persisted, `amy concord read --epoch ` can re-derive a pre-refounding Chat + * Plane. A fresh account (never lived through a Refounding) simply has empty `heldRoots`. + */ + private suspend fun import( + dataDir: DataDir, + @Suppress("UNUSED_PARAMETER") rest: Array, + ): Int { + Context.open(dataDir).use { ctx -> + ctx.prepare() + val relays = (ctx.outboxRelays() + ctx.bootstrapRelays()) + val filter = Filter(kinds = listOf(ConcordCommunityListEvent.KIND), authors = listOf(ctx.signer.pubKey)) + val events = ctx.drain(relays.associateWith { listOf(filter) }).map { it.second } + val newest = + events.filterIsInstance().maxByOrNull { it.createdAt } + ?: return Output.error("not_found", "no kind-13302 Concord list published by this account").let { 1 } + + val entries = + try { + newest.decrypt(ctx.signer) + } catch (e: Exception) { + return Output.error("decrypt_failed", "could not decrypt kind-13302: ${e.message}").let { 1 } + } + val store = ConcordStore(dataDir.concordFile) + val existing = store.load().associateBy { it.communityId } + val imported = + entries.map { e -> + val prior = existing[e.id] + store.upsert( + StoredCommunity( + name = e.name.ifBlank { prior?.name ?: "" }, + communityId = e.id, + owner = e.owner, + ownerSalt = e.ownerSalt, + root = e.root, + rootEpoch = e.rootEpoch, + generalChannelId = prior?.generalChannelId ?: "", + relays = e.relays, + heldRoots = e.heldRoots.map { StoredHeldRoot(it.epoch, it.key) }, + ), + ) + mapOf( + "name" to e.name, + "community_id" to e.id, + "root_epoch" to e.rootEpoch, + "held_roots" to e.heldRoots.map { mapOf("epoch" to it.epoch, "root" to it.key) }, + ) + } + Output.emit(mapOf("imported" to imported)) + return 0 + } + } + private suspend fun invite( dataDir: DataDir, rest: Array, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/LoginCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/LoginCommand.kt index 7e2f3bce1b..27bc1d274c 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/LoginCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/LoginCommand.kt @@ -21,13 +21,17 @@ package com.vitorpamplona.amethyst.cli.commands import com.vitorpamplona.amethyst.cli.Args +import com.vitorpamplona.amethyst.cli.Context import com.vitorpamplona.amethyst.cli.DataDir import com.vitorpamplona.amethyst.cli.Identity import com.vitorpamplona.amethyst.cli.Output +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip05DnsIdentifiers.Nip05Client import com.vitorpamplona.quartz.nip05DnsIdentifiers.OkHttpNip05Fetcher import com.vitorpamplona.quartz.nip05DnsIdentifiers.resolveUserHexOrNull import com.vitorpamplona.quartz.nip06KeyDerivation.Nip06 +import com.vitorpamplona.quartz.nip19Bech32.toNpub +import com.vitorpamplona.quartz.nip46RemoteSigner.signer.NostrSignerRemote import com.vitorpamplona.quartz.nip49PrivKeyEnc.Nip49 import okhttp3.OkHttpClient @@ -73,19 +77,53 @@ object LoginCommand { ) dataDir.saveIdentity(identity) + + // For a bunker, the pubkey in the URI is the REMOTE SIGNER's key, which for many signer apps + // (Amber, nsec.app) is a per-connection key distinct from the user's identity key. Resolve the + // real identity via the NIP-46 get_public_key RPC and persist THAT (the bunker's remote key is + // kept in Identity.bunker for transport addressing). Best-effort: if the bunker can't answer, + // fall back to the URI pubkey so login still succeeds. + val account = if (identity.bunker != null) resolveBunkerIdentity(dataDir, identity) else identity + Output.emit( mapOf( - "npub" to identity.npub, - "hex" to identity.pubKeyHex, - "read_only" to !identity.canSign, - "signer" to if (identity.bunker != null) "bunker" else "local", - "bunker_relays" to identity.bunker?.relays, + "npub" to account.npub, + "hex" to account.pubKeyHex, + "read_only" to !account.canSign, + "signer" to if (account.bunker != null) "bunker" else "local", + "bunker_relays" to account.bunker?.relays, "data_dir" to dataDir.root.absolutePath, ), ) return 0 } + /** + * Connect the freshly-saved bunker and ask it (NIP-46 `get_public_key`) for the user's real + * identity pubkey, re-persisting the [Identity] when it differs from the bunker's transport key. + * Returns the corrected identity, or [provisional] unchanged if the RPC fails. + */ + private suspend fun resolveBunkerIdentity( + dataDir: DataDir, + provisional: Identity, + ): Identity = + try { + Context.open(dataDir).use { ctx -> + ctx.prepare() + val real = (ctx.signer as NostrSignerRemote).getPublicKey().lowercase() + if (real == provisional.pubKeyHex.lowercase()) { + provisional + } else { + val corrected = provisional.copy(pubKeyHex = real, npub = real.hexToByteArray().toNpub()) + dataDir.saveIdentity(corrected) + corrected + } + } + } catch (e: Exception) { + System.err.println("[nip46] could not resolve identity via get_public_key (${e.message}); using the bunker URI pubkey") + provisional + } + private suspend fun resolveIdentity( key: String, args: Args, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt index 95cfb90f22..379dc44b83 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt @@ -39,6 +39,15 @@ data class StoredCommunity( val rootEpoch: Long = 0, val generalChannelId: String = "", val relays: List = emptyList(), + // Past access roots kept per epoch (CORD-06 Refounding rotates the root). Lets `read --epoch ` + // re-derive a prior epoch's Chat Plane to reach pre-refounding history. Populated by `import`. + val heldRoots: List = emptyList(), +) + +/** A past community_root for a specific epoch, mirroring quartz `HeldRoot`. */ +data class StoredHeldRoot( + val epoch: Long = 0, + val root: String = "", ) /** diff --git a/commons/plans/2026-07-17-concord-epoch-walking-backfill.md b/commons/plans/2026-07-17-concord-epoch-walking-backfill.md new file mode 100644 index 0000000000..ec18d79865 --- /dev/null +++ b/commons/plans/2026-07-17-concord-epoch-walking-backfill.md @@ -0,0 +1,257 @@ +# Concord epoch-walking message backfill + +**Date:** 2026-07-17 +**Module:** `commons` (drivers), with `quartz` primitives (present) + `amethyst` wiring +**Status:** design / not started + +## Problem + +A Concord community that has been **refounded** (CORD-06 §3 — hard removal +rotates `community_root` and bumps the epoch) loses all of its pre-refounding +chat history from the client's view. Every channel message is encrypted under a +plane stream key derived from the *root at the epoch it was authored under* +(`publicChannel(community_root, channel_id, rootEpoch)`), and the message carries +an `["epoch", n]` tag so cross-epoch replay is rejected +(`ChannelChat.isBoundTo`). The client only ever derives, subscribes to, AUTHs +as, and decrypts the plane at the **single current** `entry.rootEpoch`. So after +a refounding the feed shows only messages authored since that refounding, then +reports **"All caught up"** — which is literally correct *for the current +epoch*, while months of history sit on the same relays under the previous +epochs' stream keys, unfetched. + +### Reproduced live (2026-07-17, `amy` vs the real Soapbox Community) + +Joined the Armada invite and probed `relay.ditto.pub` + `relay.dreamith.to`: + +| Fact | Value | +|---|---| +| `rootEpoch` (from invite bundle) | **2** — refounded ≥ once | +| nostrhub @ epoch 2 | 2 messages, both authored **2026-07-17** | +| general @ epoch 2 | 62 messages, **oldest 2026-07-16** | +| ditto @ epoch 2 | 4 messages, **oldest 2026-07-16** | +| agora @ epoch 2 | 3 messages, **oldest 2026-07-16** | + +Every channel's history begins on 2026-07-16 (the refounding date). The app's +"only 2 messages in nostrhub" reproduces exactly through the CLI. Not a relay, +AUTH, or paging-cap issue — a structural epoch gap. + +## Key finding: prior roots are already retained but never consumed + +The hard part — keeping the old keys — **is already done**: + +- `ConcordCommunityListEntry.heldRoots: List` + (`quartz/.../cord02Community/ConcordCommunityList.kt:31,60`, wire field + `held_roots`). `HeldRoot`'s own KDoc: *"A past root key for a specific epoch, + kept so historical channel keys stay derivable."* +- `Account.adoptConcordRoot` (`amethyst/.../model/Account.kt:2513`) appends the + outgoing `(rootEpoch, root)` to `heldRoots` on **every** rotation — both the + refounder path (`refoundConcordCommunity`) and the receive path + (`drainConcordRekeys` → `openBaseRekey`). It is persisted to the account's + Concord list event. + +But a repo-wide grep confirms **no code in the fetch / fold / subscribe path +ever reads `heldRoots`.** `ConcordCommunitySession`, `ConcordSessionRegistry`, +`ConcordSubscriptionPlanner`, and the amethyst filter assemblers all derive +planes solely from `entry.rootEpoch` / `entry.root`. The retained roots are dead +data. **This plan is almost entirely about consuming `heldRoots` on the read +side** — the retention infrastructure the feature needs already exists. + +### Scope boundary — who this helps + +- **Members present across the refoundings** (have populated `heldRoots`): can + fully backfill. Primary target. +- **Fresh joiners via an invite** (empty `heldRoots`; the invite bundle carries + only the current root): **cryptographically cannot** decrypt prior-epoch + history — they were never given those roots, and the relays gate each epoch's + kind-1059 behind AUTH-as-that-epoch's-stream-key. This is by design (a + refounding is meant to sever access) and is **out of scope**. We must not try + to work around it by stuffing old roots into invites — that would hand a + brand-new member the keys a removal was meant to deny. If cross-refounding + history for new joiners is ever wanted, it's a *protocol* change (Armada would + re-publish compacted history under the new root), tracked separately. + +## Design + +`ConcordCommunitySession` is documented as "a pure function of its entry" and is +rebuilt wholesale on rotation (`ConcordSessionRegistry.sync` replaces it when +`root`/`rootEpoch` changes). Two ways to add historical epochs: + +- **(A) Generalize the session to be multi-epoch** — derive a plane key set per + `(epoch, root)` in `{current} ∪ heldRoots`, index buffered wraps and decrypt + attempts across all of them. +- **(B) Keep the current-epoch session as-is (live + write) and attach + read-only "historical epoch readers"** — one lightweight derivation set per + held root, contributing subscribe addresses + AUTH keys + decrypt attempts, + but never used for writing/moderation/rekey. + +**Recommend (B).** Writes, moderation, control-plane folding, rekey adoption, +and Guestbook all must stay strictly on the current epoch — mixing historical +roots into those paths risks authoring under a stale key or re-folding a +superseded control plane. A read-only historical layer keeps the blast radius to +message fetch/decrypt. The channel id is **epoch-invariant** (`ConcordChannelKeys` +KDoc: "stays constant across visibility conversions and epoch rotations"), so a +decrypted historical message lands in the *same* channel feed as current ones — +no feed-merge logic needed; `LocalCache`/the gatherer keys on channel id. + +### New concept: `EpochPlaneSet` + +A small value type: for one `(epoch, root)`, derive the control-plane key and, +given the current folded channel-id list, the per-channel `publicChannel` keys. +The current epoch already computes this inline in `ConcordCommunitySession`; +factor the derivation into a reusable helper so current + historical share it. + +> Channel *membership* comes from folding the **current** control plane (channels +> aren't re-listed per epoch). We assume the channel-id set is stable across the +> covered epochs (channels created after an old epoch simply have no messages +> there → empty, harmless). Deleted/renamed channels: the id persists, so old +> messages still decrypt. Private channels: use `entry.privateChannels` +> (`PrivateChannelKey` already carries a per-`epoch` key) instead of the root. + +## Component-by-component changes + +### 1. `quartz` — none required (primitives already exist) +`ConcordActions.publicChannel/controlPlane/channelRumors/channelMessages` and +`ChannelChat.isBoundTo` already take an explicit `epoch`. Reuse verbatim. + +### 2. `commons/.../model/concord/ConcordCommunitySession.kt` +- Build historical `EpochPlaneSet`s from `entry.heldRoots` (bounded — see + §Bounding) alongside the existing current-epoch derivation. +- `channelAddresses()` → also emit each historical channel plane pubkey so the + planner subscribes to them. +- `streamKeys()` → include historical control + channel `GroupKey`s so the + NIP-42 AUTH set authenticates as each prior epoch's stream key (this is what + unlocks the gated relays for old wraps). Keep the aux (Guestbook / next-rekey) + isolation rule intact. +- `ingest(wrap)` → currently matches a wrap by its plane address against the + current-epoch address map. Extend the address→(channelId, key, **epoch**) map + to include historical entries; on match, decrypt with that epoch's key and + validate `isBoundTo(rumor, channelId, thatEpoch)`. Emit the rumor exactly as + today (same `onRumor` sink → same channel feed). +- Historical wraps feed `observedAuthors` too — a nice side effect: the member + roster harvest gets the full-history posters for free. +- Do **not** route historical wraps into `refold()` (control plane), + `refoldGuestbook()`, or rekey buffers — read path only. + +### 3. `commons/.../model/concord/ConcordSessionRegistry.kt` +`subscribeAddresses()` already unions `session.channelAddresses()`; once the +session emits historical addresses it flows through unchanged. Verify no other +call site assumes one-address-per-channel. + +### 4. `commons/.../actions/ConcordSubscriptionPlanner.kt` +`channelPlaneSubs()` derives `publicChannel(root, channelId, entry.rootEpoch)`. +Generalize to emit a plane sub per `(channel, epoch)` across the covered epoch +set, collapsed into the existing `{kinds:[1059,21059], authors:[…]}` batching in +`relayBasedFilters()`. The historical subs can be one-shot (no live tail needed — +old epochs are frozen), so consider a bounded `until`-less REQ that EOSEs rather +than a standing subscription, to cap connection cost. + +### 5. `amethyst/.../concord/datasource/` filter assemblers +- `ConcordChannelHistoryFilterAssembler` (`BackwardRelayPager`) — today pages one + plane pubkey (`session.channelPlaneAddress(channelId)`, current epoch). The + pager must **step to the previous epoch's plane pubkey when the current epoch + is exhausted** rather than declaring `PagingStatus.exhausted`. Options: + (a) page all epoch planes concurrently and only report exhausted when every + epoch's relays are done; (b) sequential — walk newest→oldest epoch. (a) is + simpler to reason about with the existing per-(uniqueId,relay) EOSE tracking; + (b) gives cleaner "load older" UX. Prefer (a). +- `ConcordChannelFilterAssembler` (live tail) — historical planes need no live + tail; only the current epoch keeps a standing sub. +- **"All caught up"** (`ConcordChannelScreen.kt:199` on `historyStatus.exhausted`) + becomes correct once exhaustion means "all covered epochs drained," not "the + current epoch drained." + +### 6. AUTH — register historical stream keys +`ConcordSessionManager.streamAuthSecretsFor(relay)` derives from +`session.streamKeys()`; once that includes historical keys, `AuthCoordinator` +signs one kind-22242 per prior-epoch stream key and the gated relays serve the +old wraps. Watch the `RelayAuthStatus` LruCache size (widened 10→200 for the +current-epoch multi-identity work) — N epochs × M channels can exceed 200; size +it to `epochs × (channels + 1)` with headroom. + +### 7. `cli` — diagnostics (`ConcordChannelCommands.read`) — **DONE** +`StoredCommunity` has no `heldRoots`, and a fresh `amy concord join` can't obtain +them — so amy can't self-serve a member's history. Landed: +- `amy concord read --epoch --root ` — derives the + Chat Plane at an explicitly supplied `(epoch, root)` and drains it (both flags + default to the stored current epoch/root; channel-id resolution stays on the + current epoch since ids are epoch-invariant). Output now also emits `epoch` and + the derived `plane` pubkey. Verified: explicit `--epoch 2 --root ` + reproduces the stored plane pubkey byte-for-byte; each epoch derives a distinct + plane; a non-hex `--root` errors `bad_args`/exit 2. Confirms old-epoch wraps can + be probed once a prior root is known. +- **DONE:** `StoredCommunity.heldRoots` + `amy concord import` — fetches this + account's own encrypted kind-13302 `ConcordCommunityListEvent`, decrypts it + with the account signer, and upserts every community **including its + `heldRoots`** (the prior-epoch access roots Amethyst persists in that same + published event via `Account.adoptConcordRoot`). `read --epoch ` then + auto-resolves the root for that epoch from the stored `heldRoots` (explicit + `--root` still wins). So a member who lived through the Refoundings can: + `amy concord import` → `amy concord read --epoch ` + and reach pre-refounding history without knowing the raw prior roots. A fresh + account simply imports empty `heldRoots` (nothing to recover — the expected + cryptographic wall). Import + decrypt are **read-only** (no publish). + +## Bounding (cost control) + +Each covered epoch multiplies the subscription/AUTH footprint by +`(channels + 1)` stream keys. Bound it: +- **Config:** `CONCORD_BACKFILL_EPOCHS` (default: all held — the list is small in + practice; refoundings are rare). If a community is refounded often, cap to the + N most recent held epochs. +- **Time window:** the member-roster harvest already bounds to + `now − 90d` (`ConcordMemberHarvest`, `CONCORD_MEMBER_HARVEST_WINDOW_SECS`). + The *interactive* channel backfill should be user-driven (paged on scroll, no + `since` floor) so a member can reach the true beginning; the *background* + harvest keeps its window. +- Historical planes are frozen → prefer one-shot EOSE REQs over standing subs to + avoid holding N× subscriptions open forever. + +## Testing + +- **quartz** — none new (primitives unchanged); existing epoch/`isBoundTo` tests + cover the binding. +- **commons unit** (`ConcordCommunitySessionTest`, `ConcordSubscriptionPlannerTest`, + `ConcordSessionRegistryTest`): + - Build an entry with `heldRoots = [(epoch0,rootA),(epoch1,rootB)]`, current + epoch 2/rootC. Assert `channelAddresses()`/`streamKeys()`/planner subs emit + a plane per `(channel, epoch)` across all three. + - Feed the session wraps authored under each historical key; assert the rumor + is decrypted with the *matching* epoch key, `isBoundTo` passes, and it + reaches the `onRumor` sink; a wrap whose epoch tag ≠ its plane epoch is + dropped. + - Assert historical wraps do **not** enter `refold()`/control state. +- **commons paging** (`BackwardRelayPagerTest`): a multi-epoch channel reports + `exhausted` only after every epoch's relays EOSE on an empty page. +- **Live, via `amy`**: use the new `--epoch/--root` diag against Soapbox once a + prior Soapbox root is available (ask maintainer / capture from an account that + lived through the refounding) → confirm epoch-1 nostrhub wraps decrypt. + +## Risks / open questions + +1. **Fresh joiners still see nothing pre-refounding** — inherent, documented + above. UI could show a "History before requires having been a member" + affordance instead of a bare "All caught up," so it doesn't read as a bug. +2. **AUTH fan-out on gated relays** — N epochs × M channels AUTH events on one + connection. Current-epoch work already accumulates multiple identities on one + connection successfully; validate it scales (LruCache sizing, relay + per-connection AUTH limits). Fall back to bounding epochs if a relay balks. +3. **`heldRoots` completeness** — only populated from the moment the account + started adopting rotations. A member who joined at epoch 2 has no epoch-0/1 + roots even if present later; nothing to do — same cryptographic limit. +4. **Private channels across epochs** — `PrivateChannelKey.epoch` exists, but + verify a private channel's key was actually re-delivered per epoch (rotated on + revocation); if a member missed an epoch's private key, that epoch of that + channel is unreadable (expected). +5. **Standing-sub vs one-shot for history** — decide before wiring the planner; + affects connection budget on the audio-room-heavy relay set. + +## Suggested sequence + +1. Factor `EpochPlaneSet` derivation + make `ConcordCommunitySession` emit + historical addresses/keys/decrypt (commons unit-tested in isolation — no + network). Ship behind a flag defaulting off. +2. Planner + AUTH wiring; commons tests. +3. amethyst `BackwardRelayPager` epoch-stepping + "All caught up" semantics. +4. ~~`amy --epoch/--root` diagnostic~~ **DONE** (§7); still need a real prior + Soapbox root to validate old-epoch decrypt end-to-end. +5. Flip the flag on; on-device verify on a refounded community. diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/account/AccountManager.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/account/AccountManager.kt index 068b5d1770..549847e8b0 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/account/AccountManager.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/account/AccountManager.kt @@ -346,6 +346,11 @@ class AccountManager internal constructor( remoteSigner.getPublicKey() } + // Bind the identity so signer.pubKey is the USER key, not the ephemeral transport key — + // otherwise self-encryption (private NIP-51 lists, drafts, …) keys off the wrong pubkey. + // Idempotent with the getPublicKey() branch above, which already caches the same value. + remoteSigner.bindUserPubkey(pubKeyHex) + val resolvedNpub = npub ?: pubKeyHex.hexToByteArray().toNpub() val state = diff --git a/quartz/plans/2026-07-17-nip46-remote-signer-self-pubkey.md b/quartz/plans/2026-07-17-nip46-remote-signer-self-pubkey.md new file mode 100644 index 0000000000..155ef67186 --- /dev/null +++ b/quartz/plans/2026-07-17-nip46-remote-signer-self-pubkey.md @@ -0,0 +1,151 @@ +# NIP-46 remote signer: `pubKey` must be the user identity, not the transport key + +**Date:** 2026-07-17 +**Module:** `quartz` (core fix) + `desktopApp` / `cli` (affected front ends) +**Status:** design / not started +**Severity:** correctness — private data invisible/corrupted for **bunker (NIP-46) accounts** on desktop + CLI. Android unaffected. + +## The bug + +`NostrSignerRemote` extends `NostrSigner(signer.pubKey)` +(`quartz/.../nip46RemoteSigner/signer/NostrSignerRemote.kt:54-72`), where the +constructor arg `signer: NostrSignerInternal` is the **ephemeral NIP-46 +transport keypair**. `NostrSigner.pubKey` is a plain `val` never reassigned. So +for a bunker account: + +``` +NostrSignerRemote.pubKey == transport key T (ephemeral, per-connection) +the user's real identity == user key U (only via get_public_key RPC) +events the bunker signs have pubKey == U +``` + +Every site that treats `signer.pubKey` as "myself" therefore uses **T instead of +U**. Two failure modes: + +### Mode A — authorship guard short-circuits (unconditional failure) +Sites that gate on `signer.pubKey == event.pubKey`. Event was signed by the +bunker (`pubKey == U`), `signer.pubKey == T`, so the guard blocks decryption +before any key math: +- `nip51Lists/PrivateTagArrayEvent.kt:44-66` — `decrypt()` throws + `UnauthorizedDecryptionException`; `privateTags()` returns `null`. +- `nip51Lists/PrivateReplaceableTagArrayEvent.kt:44-53` — same guard. +- `nip37Drafts/DraftWrapEvent.kt:52` — `canDecrypt = signer.pubKey == pubKey` + → always false. + +Covers **all private NIP-51 lists** (private bookmarks, the private half of the +mute list, private follows/people lists, private hashtag/geohash lists…) and +**NIP-37 drafts**. Fails even for data Amethyst itself wrote through the bunker +(written as U, read back with T). Independent of key-stability nuances. + +### Mode B — wrong self-peer / wrong author filter (no guard) +Sites that self-encrypt directly to `signer.pubKey`: +- `nip51Lists/encryption/PrivateTagsInContent.kt:41,57,64,74` — the crypto under + the guarded lists (`nip44Encrypt(…, signer.pubKey)` / `decrypt(…, signer.pubKey)`). +- `concord/cord02Community/ConcordCommunityListEvent.kt:56,74` + + `ConcordCommunityList.kt:170,222` — Concord list self-encryption. +- `nip60Cashu/token/CashuTokenEvent.kt:74`, `quote/CashuMintQuoteEvent.kt:136`. +- `amethyst/.../nip78AppSpecific/AppSpecificState.kt:60` (Android-only file → moot). + +The bunker derives ECDH(u, T) instead of ECDH(u, U): standard-client content +(peer U) fails MAC; content Amethyst writes is sealed to peer T, unreadable by +every other client and by the same user's local-key login. Author filters like +`authors = listOf(signer.pubKey)` also query T, so the user's own U-authored +events are never fetched (e.g. `Account.importConcordCommunities` filter, the +CLI `concord list`). + +## Impact by front end (verified) + +- **Android `amethyst/`: UNAFFECTED.** No `NostrSignerRemote` is ever + constructed (`AccountCacheState.loadAccount` builds only `NostrSignerInternal` + / `NostrSignerExternal`; the only `NostrSignerRemote` reference is a type-check + in `MeteringNostrSigner.kt:122`). Android has no bunker login, so + `signer.pubKey == account pubkey` always. +- **Desktop `desktopApp/`: AFFECTED.** Bunker login is a first-class path + (`AccountManager.loginWithBunker` / `loginWithNostrConnect`). Confirmed live: + `BookmarksScreen.kt:189,355` → `list.privateBookmarks(account.signer)` → guard + returns null → **private bookmarks always empty** for bunker accounts. Same + guard breaks the private mute section, private follow lists, and drafts. Cashu + self-encryption sealed to the wrong peer. (No Concord feature on desktop.) +- **CLI `cli/`: AFFECTED.** Same failures; `concord list` breaks on both the + author filter and the decrypt. The `amy concord import`/`read` diagnostics + (2026-07-17) already work around it by using `ctx.identity.pubKeyHex` — but + every other self-encryption site in the CLI is still wrong. + +Transport key is **persisted + stable across restarts** on both desktop and CLI, +so Mode-B data *round-trips within one install* (looks fine locally) but is +non-portable and non-standard; a fresh `nostrconnect://` session regenerating +the key makes it permanently unrecoverable. Mode-A fails regardless. + +## Fix + +**Core (quartz):** make `NostrSignerRemote.pubKey` return the **verified user +key U**, not the transport key T. Constraints: +- The user key is known only after the `get_public_key` RPC, so it can't be set + from the raw `bunker://` parts at construction. Resolve it during the + login/connect handshake (desktop `NostrConnectLoginUseCase` and CLI login + already call `getPublicKey()`), and construct/finalize the signer with U as its + identity. +- **Keep the transport key for NIP-46 transport.** Internal sites that legitimately + need T — the response subscription filter `p: signer.pubKey` + (`NostrSignerRemote.kt:91`), request addressing/encryption to the bunker + (`RemoteSignerManager`, `remoteKey = remotePubkey`) — must reference the + transport keypair explicitly (`this.signer.pubKey`), **not** the base-class + `pubKey`. Audit every `signer.pubKey`/`pubKey` use inside `NostrSignerRemote` + and `RemoteSignerManager` and pin transport uses to the transport keypair + before flipping the base `pubKey`. + +Two implementation options: +1. **Explicit identity param** — add `userPubkey: HexKey` to `NostrSignerRemote` + (or a factory that RPCs `get_public_key`, then builds the signer with U as + `NostrSigner(userPubkey)`). Cleanest; makes the contract explicit. Requires + touching every construction site (desktop `AccountManager`, CLI `Context`, + `NostrConnectLoginUseCase`). +2. **Late-resolved pubKey** — allow the base identity to be set once after the + connect handshake. Smaller call-site churn, but `NostrSigner.pubKey` becoming + non-`val` ripples widely; less desirable. + +Prefer **(1)**. + +For local (`NostrSignerInternal`) and external NIP-55 (`NostrSignerExternal`) +signers, `signer.pubKey` already equals the account key, so the change is a +**no-op** for them — only bunker accounts change behavior. That keeps the blast +radius to exactly the broken case. + +Once `NostrSignerRemote.pubKey == U`, all Mode-A guards pass and all Mode-B +self-encryption uses the right peer; **amy can drop its manual-decrypt +workaround** and the `Account.importConcordCommunities` filter/decrypt become +correct for any future bunker use. + +## Migration / data caveat + +Any Mode-B data a bunker user *already wrote* was sealed to peer T. After the +fix (peer U) it becomes unreadable — but it was already unreadable everywhere +except that one install, so the fix trades a hidden-corruption state for a +correct one. Private NIP-51 lists / drafts (Mode-A) were never successfully +written wrong (the guard blocked the write path's read-modify-write too), so +there's nothing to migrate there — they simply start working. Call this out in +the PR; no migration code needed, but a note for affected desktop users is kind. + +## Testing + +- **quartz unit:** construct a `NostrSignerRemote` whose transport key ≠ user + key; assert `pubKey == userKey`; assert `PrivateTagArrayEvent.decrypt` / + `DraftWrapEvent.canDecrypt` succeed against a U-authored event; assert the + NIP-46 response subscription still filters on the transport key. +- **round-trip:** self-encrypt a private list with the remote signer, decrypt + with a *local* `NostrSignerInternal` for U → must match (proves portability). +- **desktop:** bunker login → private bookmarks / private mute / drafts render. +- **CLI:** `amy concord list` (not just `import`) loads communities for a bunker + account; drop the `import` workaround and confirm `newest.decrypt(signer)` + works. + +## Suggested sequence + +1. quartz: audit transport-vs-identity `pubKey` uses inside + `NostrSignerRemote`/`RemoteSignerManager`; pin transport uses to the transport + keypair. +2. quartz: add the explicit-identity construction (option 1) + unit tests. +3. desktop `AccountManager` + `NostrConnectLoginUseCase`: pass the verified U + into the signer; verify private lists on-device. +4. CLI `Context`: pass `identity.pubKeyHex` as the signer identity; drop the amy + Concord decrypt workaround. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/signers/NostrSigner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/signers/NostrSigner.kt index fd5fd2e0d6..70836a44b9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/signers/NostrSigner.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/signers/NostrSigner.kt @@ -27,8 +27,15 @@ import com.vitorpamplona.quartz.nip57Zaps.LnZapPrivateEvent import com.vitorpamplona.quartz.nip57Zaps.LnZapRequestEvent abstract class NostrSigner( - val pubKey: HexKey, + pubKey: HexKey, ) { + /** + * The account's own identity pubkey. `open` because a NIP-46 remote signer resolves it + * from the bunker (`get_public_key`) rather than from a local key it holds — see + * `NostrSignerRemote`, whose transport keypair is deliberately NOT the user identity. + */ + open val pubKey: HexKey = pubKey + abstract fun isWriteable(): Boolean suspend fun sign(ev: EventTemplate): T = sign(ev.createdAt, ev.kind, ev.tags, ev.content) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/signer/NostrSignerRemote.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/signer/NostrSignerRemote.kt index 38ca469718..5ca77a0410 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/signer/NostrSignerRemote.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/signer/NostrSignerRemote.kt @@ -70,6 +70,31 @@ class NostrSignerRemote( */ val onAuthUrl: ((String) -> Unit)? = null, ) : NostrSigner(signer.pubKey) { + // The user's real identity, resolved from the bunker via `get_public_key`. The constructor + // `signer` is the ephemeral NIP-46 TRANSPORT keypair, NOT the user — so until this is bound, + // `pubKey` falls back to the transport key. Every self-encryption / self-authorship site keys + // off `pubKey`, so leaving it as the transport key silently breaks private NIP-51 lists, NIP-37 + // drafts, Concord list decryption, etc. for bunker accounts. Bound either eagerly from a saved + // identity ([bindUserPubkey]) or lazily by the first [getPublicKey] call. + private var resolvedUserPubkey: HexKey? = null + + /** + * The account identity. Returns the bunker-resolved user key once known, else the transport + * key. Internal NIP-46 transport (the response-subscription `p` filter, request addressing) + * deliberately uses `signer.pubKey`/`remotePubkey` directly and is unaffected by this. + */ + override val pubKey: HexKey + get() = resolvedUserPubkey ?: signer.pubKey + + /** + * Bind the user's identity pubkey when it is already known (e.g. a persisted bunker account + * reloaded from disk, or the CLI's stored identity) so `pubKey` is correct without a + * `get_public_key` round-trip. + */ + fun bindUserPubkey(userPubkey: HexKey) { + resolvedUserPubkey = userPubkey + } + private val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) private val manager = @@ -292,6 +317,8 @@ class NostrSignerRemote( ) if (result is SignerResult.RequestAddressed.Successful) { + // Cache it so `pubKey` reflects the real identity from here on (self-encryption etc.). + resolvedUserPubkey = result.result.pubkey return result.result.pubkey } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/signer/NostrSignerRemoteIsolationTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/signer/NostrSignerRemoteIsolationTest.kt index f86af44b03..5ea4738247 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/signer/NostrSignerRemoteIsolationTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/signer/NostrSignerRemoteIsolationTest.kt @@ -119,6 +119,32 @@ class NostrSignerRemoteIsolationTest { private val generalRelay = NormalizedRelayUrl("wss://relay.damus.io/") private val validHex = "a".repeat(64) + @Test + fun pubKeyFallsBackToTransportKeyUntilBoundThenReturnsUserIdentity() { + val trackingClient = TrackingNostrClient() + val ephemeralSigner = NostrSignerInternal(KeyPair()) + val userIdentity = "b".repeat(64) + + val remote = + NostrSignerRemote( + signer = ephemeralSigner, + remotePubkey = validHex, + relays = setOf(bunkerRelay), + client = trackingClient, + ) + + // Before the user identity is known, pubKey is the ephemeral transport key — NOT the + // remotePubkey (which is the bunker's addressing key) and not yet the user's identity. + assertEquals(ephemeralSigner.pubKey, remote.pubKey) + + // Once bound (mirrors a reloaded account / a cached get_public_key), pubKey is the user key. + remote.bindUserPubkey(userIdentity) + assertEquals(userIdentity, remote.pubKey) + + // The transport keypair used for NIP-46 addressing is untouched by the binding. + assertEquals(ephemeralSigner.pubKey, remote.signer.pubKey) + } + @Test fun subscriptionFilterTargetsOnlyBunkerRelays() { val trackingClient = TrackingNostrClient()