From 22a418bea2bc2972a1be3d96441024e552daa12f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Apr 2026 01:40:05 +0000 Subject: [PATCH] feat(cli): make FsEventStore the source of truth for relay events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every event Amy observes — drained from a relay subscription, unwrapped from a NIP-59 gift wrap, or generated locally for publish — now flows through Context.verifyAndStore: NIP-01 id + signature check via Event.verify(), then store.insert(). Bad events are dropped with a stderr log and never reach command code; persistence failures are logged but do not propagate, so a broken store never breaks a relay subscription. - Context.drain() now persists every received event before surfacing it to the caller. Existing callers (FeedCommand, DmCommands, ProfileCommands, Marmot sync) get caching for free. - Context.publish() persists outbound events too, so the local store reflects what Amy has done even when every relay rejects. - Three cache-first read helpers expose the most common lookups without hitting relays: profileOf(pubKey) → kind:0, relaysOf(pubKey) → kind:10002, contactsOf(pubKey) → kind:3. - Class doc on Context spells out the contract; README.md gets a new "Local event store — the source of truth" section pointing at the on-disk layout and the design plans. --- cli/README.md | 54 +++++++++++ .../com/vitorpamplona/amethyst/cli/Context.kt | 92 ++++++++++++++++++- 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/cli/README.md b/cli/README.md index 22879c3255..03841d151b 100644 --- a/cli/README.md +++ b/cli/README.md @@ -48,6 +48,60 @@ change to Amy's public API. --- +## Local event store — the source of truth + +Every Nostr event Amy observes is verified (NIP-01 id + signature +check) and persisted to a file-backed store at +`/events-store/`. That includes: + +- events received from any relay subscription (`amy feed`, `amy dm + list`, `amy keypackage publish`, group sync, …), +- events Amy generates and publishes itself, +- inner events unwrapped from NIP-59 gift wraps. + +Malformed events are dropped before reaching command code. Persistence +is best-effort — if the store fails (full disk, permissions), the +relay subscription still works, but the event is not cached. + +The store is the authoritative cache of everything Amy has seen: +profile metadata, relay lists (NIP-65 and NIP-02), gift wraps, group +events, follow lists, etc. Commands that need any of these should read +from the store first and only fall back to a relay fetch on miss. +Three convenience helpers exist on `Context`: + +```kotlin +ctx.profileOf(pubKey) // latest kind:0 (NIP-01) +ctx.relaysOf(pubKey) // latest kind:10002 (NIP-65) +ctx.contactsOf(pubKey) // latest kind:3 (NIP-02) +``` + +The store implements every feature of the Quartz SQLite store — +NIP-01 replaceable / addressable uniqueness, NIP-09 deletion +tombstones, NIP-40 expiration, NIP-50 search, NIP-62 right-to-vanish, +NIP-91 multi-tag AND. See `cli/plans/2026-04-24-file-event-store-*.md` +for the design and `quartz/.../store/fs/FsEventStore.kt` for the +implementation. The on-disk layout is plain JSON files under shard +directories, intentionally inspectable with `ls`, `cat`, `jq`, +`grep`, `find`, `rsync`, and `git`. + +To manage the store directly: + +```sh +# raw inspection +find $AMY_HOME/events-store/events -name '*.json' | head +jq . $AMY_HOME/events-store/replaceable/0/.json + +# delete a specific event (tombstone NOT installed — see below) +rm $AMY_HOME/events-store/events///.json +``` + +Deleting an event file is treated as a deliberate "I never saw this" +by Amy. The store tolerates external edits: dangling index entries are +skipped at query time and can be cleaned up with `compact()` / +`scrub()` from the API. + +--- + ## Install Until Amy ships as a signed native binary (see diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index d6a98d02d3..096c3f8b72 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -31,6 +31,8 @@ import com.vitorpamplona.quartz.marmot.MarmotFilters import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.crypto.verify +import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.publishAndConfirmDetailed import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener @@ -41,7 +43,9 @@ import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.BasicOkHttpWebSoc import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore +import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent +import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED import kotlinx.coroutines.selects.select @@ -60,6 +64,24 @@ import okhttp3.OkHttpClient * process-incoming, etc). * * Closing flushes run-state to disk and disconnects the client. + * + * # Source of truth — [store] + * + * Every Nostr event Amy observes — whether received from a relay + * subscription, unwrapped from a NIP-59 gift wrap, or generated locally + * before publish — is verified (NIP-01 signature + id check via + * [Event.verify]) and persisted to the file-backed [IEventStore] at + * `/events-store/`. Malformed events are dropped before + * reaching command code. + * + * This makes [store] the authoritative cache of everything Amy has ever + * seen: profile metadata, relay lists, contact lists, gift wraps, + * group events, etc. Persistence is best-effort — an I/O failure on + * the store does not break the relay subscription. + * + * Reads should prefer the local store via the helpers below + * ([profileOf], [relaysOf], [contactsOf]) and only fall back to a + * relay [drain] on cache miss. */ class Context( val dataDir: DataDir, @@ -168,6 +190,10 @@ class Context( relayList: Set, timeoutSecs: Long = 15, ): Map { + // Persist locally before broadcasting. The store is the source of + // truth — even if every relay rejects, we want our own outbound + // event in the local cache. + verifyAndStore(event) if (relayList.isEmpty()) return emptyMap() return client.publishAndConfirmDetailed(event, relayList, timeoutSecs) } @@ -226,7 +252,9 @@ class Context( withTimeoutOrNull(timeoutMs) { while (remaining.isNotEmpty()) { select { - eventChannel.onReceive { collected.add(it) } + eventChannel.onReceive { pair -> + if (verifyAndStore(pair.second)) collected.add(pair) + } doneChannel.onReceive { r -> remaining.remove(r) } } } @@ -234,7 +262,8 @@ class Context( while (true) { val r = eventChannel.tryReceive() if (!r.isSuccess) break - collected.add(r.getOrThrow()) + val pair = r.getOrThrow() + if (verifyAndStore(pair.second)) collected.add(pair) } } } finally { @@ -245,6 +274,65 @@ class Context( return collected } + /** + * Verify [event]'s NIP-01 id+signature and, if valid, persist it + * to [store]. Returns `true` when the event was accepted (and + * therefore should be surfaced to callers). Persistence failures + * (I/O errors, full disk) are logged but do not propagate. + * + * Every event-arrival path in the CLI funnels through this method + * so that [store] is the authoritative cache of what Amy has seen. + */ + fun verifyAndStore(event: Event): Boolean { + if (!event.verify()) { + System.err.println("[cli] dropped event ${event.id.take(8)} kind=${event.kind} — bad signature") + return false + } + try { + store.insert(event) + } catch (t: Throwable) { + System.err.println("[cli] store insert failed for ${event.id.take(8)}: ${t.message}") + } + return true + } + + // ------------------------------------------------------------------ + // Cache-first reads from [store] + // ------------------------------------------------------------------ + + /** + * Latest known kind:0 metadata for [pubKey], read from the local + * store. Returns null if Amy has never observed a profile for + * this user. Callers that need a network fetch on miss should fall + * back to [drain] explicitly — this helper never hits the network. + */ + fun profileOf(pubKey: HexKey): MetadataEvent? = + store + .query( + Filter(authors = listOf(pubKey), kinds = listOf(MetadataEvent.KIND), limit = 1), + ).firstOrNull() as? MetadataEvent + + /** + * Latest known kind:10002 advertised relay list (NIP-65) for + * [pubKey]. `null` when Amy has never seen one. + */ + fun relaysOf(pubKey: HexKey): AdvertisedRelayListEvent? = + store + .query( + Filter(authors = listOf(pubKey), kinds = listOf(AdvertisedRelayListEvent.KIND), limit = 1), + ).firstOrNull() as? AdvertisedRelayListEvent + + /** + * Latest known kind:3 contact list (NIP-02) for [pubKey], or + * `null` if Amy has never observed one. Useful for follow-graph + * lookups without re-hitting relays. + */ + fun contactsOf(pubKey: HexKey): ContactListEvent? = + store + .query( + Filter(authors = listOf(pubKey), kinds = listOf(ContactListEvent.KIND), limit = 1), + ).firstOrNull() as? ContactListEvent + /** * Pull down everything needed to bring local Marmot state current: * - kind:1059 gift wraps on inbox relays → try to unwrap Welcomes