From 0d6f7fd186e209e11706391b99ce05ed5fe09de7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 23:24:10 +0000 Subject: [PATCH] feat(cli): SQLite event-store backend for amy (default), FS opt-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FS event store writes one pretty-printed JSON file per event plus one file per index posting (kind, author, every p-tag value). At crawl scale this explodes: a 96k-event GrapeRank crawl produced 5.6M tiny index files rounding up to 2.8GB on disk — only 457MB of which was actual event data. An 8-hop crawl would blow past available disk. Wire amy's shared store through a new StoreFactory that selects the backend from AMY_STORE (default `sqlite`, opt into the legacy tree with `fs`). Both implement IEventStore, so every command works unchanged. SQLite packs the same postings into shared B-tree pages — several times smaller on disk and the natural fit for large crawls. The two stores live side by side under `/shared/` (events.db vs events-store/) so switching never clobbers the other's data. `amy store` maintenance verbs are now backend-aware: stat reports the total + disk bytes for both (kind histogram/mtime stay fs-only); scrub is a no-op on sqlite (indexes are transactional); compact runs VACUUM on sqlite. Verified end-to-end via the built amy image on both backends: init, notes post round-trip (event persisted + read back), and every store verb. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RWk2ZMrGBSr4WenKgwqmbB --- .../com/vitorpamplona/amethyst/cli/Config.kt | 9 + .../com/vitorpamplona/amethyst/cli/Context.kt | 31 +--- .../com/vitorpamplona/amethyst/cli/Main.kt | 15 +- .../amethyst/cli/StoreFactory.kt | 89 +++++++++ .../amethyst/cli/commands/StoreCommands.kt | 171 +++++++++++++----- 5 files changed, 244 insertions(+), 71 deletions(-) create mode 100644 cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/StoreFactory.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt index dd7c0fa984..ef9c5ed655 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt @@ -213,6 +213,15 @@ class DataDir( val groupsDir = File(marmotDir, "groups") val keyPackageBundleFile = File(marmotDir, "keypackages.bundle") + /** + * SQLite event-store DB file, a sibling of [eventsDir] under + * `/shared/`. Used when the store backend is SQLite (the + * default — see [StoreFactory]); the FS backend uses [eventsDir] + * instead. Kept alongside the FS store so switching backends never + * clobbers the other's data. + */ + val eventsDbFile: File = File(eventsDir.parentFile ?: root, "events.db") + init { SecureFileIO.secureMkdirs(root) SecureFileIO.secureMkdirs(groupsDir) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 6cac69446f..2e6b4dcccd 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -39,7 +39,6 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.crypto.verify -import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.publishAndConfirmDetailed @@ -54,7 +53,6 @@ import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.TcpNoDelaySocketF import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent import com.vitorpamplona.quartz.nip17Dm.settings.ChatMessageRelayListEvent import com.vitorpamplona.quartz.nip46RemoteSigner.signer.NostrSignerRemote @@ -94,9 +92,10 @@ import okhttp3.OkHttpClient * Every Nostr event Amy observes — whether received from a relay * subscription, unwrapped from a NIP-59 gift wrap, or generated locally * before publish — is verified (NIP-01 signature + id check via - * [Event.verify]) and persisted to the file-backed [IEventStore] at - * `/events-store/`. Malformed events are dropped before - * reaching command code. + * [Event.verify]) and persisted to the shared [IEventStore] under + * `/shared/` (a SQLite DB by default, or the FS tree when + * `AMY_STORE=fs` — see [StoreFactory]). Malformed events are dropped + * before reaching command code. * * This makes [store] the authoritative cache of everything Amy has ever * seen: profile metadata, relay lists, contact lists, gift wraps, @@ -159,23 +158,13 @@ class Context( private val messageStore = FileMarmotMessageStore(dataDir.groupsDir) /** - * Filesystem-backed Nostr event store, rooted at [DataDir.eventsDir]. - * Lazy so commands that don't touch persistent event state pay zero - * open cost (no `.lock` file, no seed allocation). Closed by - * [close] when this Context shuts down. - * - * Files are written pretty-printed (not the compact NIP-01 canonical - * form) so `cat`, `jq`, `git diff` are useful out of the box — - * humans inspect these files. Verification always re-canonicalises, - * so the stored bytes never feed back into a signature check. + * Shared Nostr event store for this run, opened via [StoreFactory] + * (SQLite by default, or the FS tree when `AMY_STORE=fs`). Lazy so + * commands that don't touch persistent event state pay zero open cost + * (no DB file / `.lock`, no seed allocation). Closed by [close] when + * this Context shuts down. */ - private val storeDelegate: Lazy = - lazy { - FsEventStore( - root = dataDir.eventsDir.toPath(), - eventToJson = JacksonMapper::toJsonPretty, - ) - } + private val storeDelegate: Lazy = lazy { StoreFactory.open(dataDir) } val store: IEventStore by storeDelegate /** Fully-wired manager. Call [prepare] once before use to load persisted state. */ diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 4c0a746b31..8f4e077eb9 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -321,7 +321,8 @@ private fun printUsage() { | All state lives under ~/.amy/. Per-account directories | ~/.amy// hold identity, cursors, MLS state, and | aliases; every observed Nostr event lands in the shared - | ~/.amy/shared/events-store/. ACCOUNT must match + | store under ~/.amy/shared/ (a SQLite `events.db` by default, or + | the `events-store/` tree when AMY_STORE=fs). ACCOUNT must match | [a-zA-Z0-9_-]{1,64} (no spaces, no slashes). | | Resolution order: @@ -619,11 +620,15 @@ private fun printUsage() { | | marmot reset [--yes] wipe all local MLS/KeyPackage state (destructive) | - |Local event store (`/events-store/`): - | store stat event count, kind histogram, disk usage + |Local event store (shared, under `/shared/`): + | Backend selected by AMY_STORE: sqlite (default; `shared/events.db`) + | or fs (`AMY_STORE=fs`; the `shared/events-store/` tree). SQLite is + | far more compact at scale — the FS tree spends one file per index + | posting, so large crawls balloon on disk. + | store stat event count + disk usage (kind histogram/mtime on fs) | store sweep-expired delete events past their NIP-40 expiration - | store scrub rebuild idx/ from canonical events (after edits / crashes) - | store compact drop dangling idx entries (canonical gone) + | store scrub fs: rebuild idx/ from canonical events; sqlite: no-op + | store compact fs: drop dangling idx entries; sqlite: VACUUM | store reindex-fts rebuild the NIP-50 search index (after a searchable-kinds change) """.trimMargin(), ) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/StoreFactory.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/StoreFactory.kt new file mode 100644 index 0000000000..8c75bbf25f --- /dev/null +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/StoreFactory.kt @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.cli + +import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import kotlin.io.path.Path + +/** On-disk backend for the shared event store. */ +enum class StoreBackend { + /** + * Single SQLite database file at [DataDir.eventsDbFile]. Postings live + * in shared B-tree pages, so an event's kind/author/tag indexes cost a + * handful of rows — not one 4 KB-block file each, the way the FS store + * lays them out. For crawl-scale corpora (hundreds of thousands of + * follow lists) this is several times smaller on disk and the default. + */ + SQLITE, + + /** + * Filesystem tree at [DataDir.eventsDir] — one pretty-printed JSON file + * per event plus one file per index posting. Human-inspectable with + * `cat`/`jq`/`git diff`, but every posting rounds up to a filesystem + * block, so a large corpus balloons. Opt in with `AMY_STORE=fs`. + */ + FS, +} + +/** + * Chooses and opens the event-store backend for `amy`. The backend is + * selected by the `AMY_STORE` environment variable and defaults to + * [StoreBackend.SQLITE]; set `AMY_STORE=fs` for the legacy filesystem + * store. Both backends implement [IEventStore], so every command works + * unchanged regardless of the choice — the only user-visible difference + * is where bytes land ([DataDir.eventsDbFile] vs [DataDir.eventsDir]) and + * how much disk they take. + */ +object StoreFactory { + const val ENV = "AMY_STORE" + + /** Resolve the configured backend. Unrecognised values fall back to the default. */ + fun backend(): StoreBackend = + when (System.getenv(ENV)?.trim()?.lowercase()) { + "fs", "file", "files", "filesystem" -> StoreBackend.FS + else -> StoreBackend.SQLITE + } + + /** + * Open the store for [dataDir] using the configured [backend]. Events + * are written pretty-printed on the FS backend so the on-disk JSON stays + * inspection-friendly; the SQLite backend stores the compact NIP-01 + * form internally. Neither is re-used for signature checks (verification + * always re-canonicalises), so the stored representation is purely an + * implementation detail. Callers own [IEventStore.close]. + */ + fun open(dataDir: DataDir): IEventStore = + when (backend()) { + StoreBackend.SQLITE -> { + // BundledSQLiteDriver won't create parent directories. + dataDir.eventsDbFile.parentFile?.mkdirs() + EventStore(dbName = dataDir.eventsDbFile.absolutePath, relay = null) + } + StoreBackend.FS -> + FsEventStore( + root = Path(dataDir.eventsDir.absolutePath), + eventToJson = JacksonMapper::toJsonPretty, + ) + } +} diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt index bab4c94a55..16a84361d4 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt @@ -22,9 +22,13 @@ package com.vitorpamplona.amethyst.cli.commands import com.vitorpamplona.amethyst.cli.DataDir import com.vitorpamplona.amethyst.cli.Output -import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper +import com.vitorpamplona.amethyst.cli.StoreBackend +import com.vitorpamplona.amethyst.cli.StoreFactory +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import java.io.File import java.io.IOException import java.nio.file.Files import java.nio.file.Path @@ -33,19 +37,24 @@ import kotlin.io.path.exists /** * `amy store ` — direct introspection - * and maintenance of the file-backed event store at - * `/events-store/`. + * and maintenance of the shared event store under `/shared/`. * - * - `stat` total event count, kind histogram, disk bytes, - * mtime range — pure read, no relay traffic. + * The store backend is selected by `AMY_STORE` (SQLite by default, or the + * FS tree with `AMY_STORE=fs` — see [StoreFactory]); each verb adapts to + * whichever is active: + * + * - `stat` total event count, disk bytes, backend, plus (FS only) + * the per-kind histogram and mtime range — pure read, + * no relay traffic. * - `sweep-expired` delete events whose NIP-40 `expiration` tag has * passed (per the store's own sweep logic). Run * from cron / scheduler / `amy` periodically. - * - `scrub` rebuild every `idx/` entry from the canonical - * events. Recovers from partial-write crashes or - * external edits. - * - `compact` drop dangling `idx/` entries whose canonical is - * gone. Cheaper than scrub. + * - `scrub` FS: rebuild every `idx/` entry from the canonical + * events, recovering from partial-write crashes or + * external edits. SQLite: a no-op (indexes are updated + * transactionally and can't drift). + * - `compact` FS: drop dangling `idx/` entries whose canonical is + * gone. SQLite: `VACUUM` the database to reclaim space. * - `reindex-fts` wipe and rebuild only the NIP-50 full-text search * index from the stored events. Run after a quartz * upgrade that changes which kinds are searchable. @@ -68,7 +77,52 @@ object StoreCommands { ), ) - private fun stat(dataDir: DataDir): Int { + private suspend fun stat(dataDir: DataDir): Int = + when (StoreFactory.backend()) { + StoreBackend.SQLITE -> sqliteStat(dataDir) + StoreBackend.FS -> fsStat(dataDir) + } + + /** + * SQLite `stat`: total count via `COUNT(*)` and on-disk bytes from the + * DB file plus its `-wal`/`-shm` sidecars. The per-kind histogram and + * mtime range are FS-store concepts (they read the `idx/kind` tree and + * file mtimes), so they're omitted here. + */ + private suspend fun sqliteStat(dataDir: DataDir): Int { + val dbFile = dataDir.eventsDbFile + if (!dbFile.exists()) { + Output.emit( + mapOf( + "backend" to "sqlite", + "events" to 0, + "disk_bytes" to 0L, + "root" to dbFile.absolutePath, + ), + ) + return 0 + } + val count = + EventStore(dbName = dbFile.absolutePath, relay = null).use { store -> + store.count(Filter()) + } + val diskBytes = + listOf("", "-wal", "-shm").sumOf { suffix -> + val f = File(dbFile.absolutePath + suffix) + if (f.isFile) f.length() else 0L + } + Output.emit( + mapOf( + "backend" to "sqlite", + "events" to count, + "disk_bytes" to diskBytes, + "root" to dbFile.absolutePath, + ), + ) + return 0 + } + + private fun fsStat(dataDir: DataDir): Int { val storeRoot = dataDir.eventsDir.toPath() if (!storeRoot.exists()) { Output.emit( @@ -140,37 +194,65 @@ object StoreCommands { private suspend fun sweepExpired(dataDir: DataDir): Int = withStore(dataDir) { store -> - val expiresAtDir = dataDir.eventsDir.toPath().resolve("idx/expires_at") - val before = countEntries(expiresAtDir) - store.deleteExpiredEvents() - val after = countEntries(expiresAtDir) - Output.emit( - mapOf( - "swept" to (before - after).coerceAtLeast(0L), - "remaining" to after, - ), - ) + if (store is FsEventStore) { + // The FS store exposes its expiration index as a directory, + // so we can report exactly how many entries the sweep cleared. + val expiresAtDir = dataDir.eventsDir.toPath().resolve("idx/expires_at") + val before = countEntries(expiresAtDir) + store.deleteExpiredEvents() + val after = countEntries(expiresAtDir) + Output.emit( + mapOf( + "swept" to (before - after).coerceAtLeast(0L), + "remaining" to after, + ), + ) + } else { + store.deleteExpiredEvents() + Output.emit(mapOf("ok" to true)) + } 0 } - private fun scrub(dataDir: DataDir): Int = + private suspend fun scrub(dataDir: DataDir): Int = withStore(dataDir) { store -> - store.scrub() - Output.emit(mapOf("ok" to true)) + when (store) { + is FsEventStore -> { + store.scrub() + Output.emit(mapOf("ok" to true)) + } + // SQLite indexes are written in the same transaction as the + // event, so they can't drift the way the FS `idx/` tree can — + // there is nothing to rebuild. + else -> + Output.emit( + mapOf( + "ok" to true, + "note" to "scrub is a no-op for the sqlite backend (indexes update transactionally)", + ), + ) + } 0 } - private fun compact(dataDir: DataDir): Int = + private suspend fun compact(dataDir: DataDir): Int = withStore(dataDir) { store -> - store.compact() + when (store) { + // FS: drop dangling idx/ postings. SQLite: VACUUM to rebuild + // the file and hand freed pages back to the OS. + is FsEventStore -> store.compact() + is EventStore -> store.store.vacuum() + else -> Unit + } Output.emit(mapOf("ok" to true)) 0 } private suspend fun reindexFts(dataDir: DataDir): Int = withStore(dataDir) { store -> + val fsBacked = store is FsEventStore val ftsDir = dataDir.eventsDir.toPath().resolve("idx/fts") - val before = countEntries(ftsDir) + val before = if (fsBacked) countEntries(ftsDir) else 0L // Drive the resumable, batched path to completion so a huge // store is processed without holding the writer lock for the // whole pass. A real long-running caller would persist the @@ -184,35 +266,34 @@ object StoreCommands { processed += progress.processedThisBatch batches++ } while (!progress.done) - val after = countEntries(ftsDir) - Output.emit( - mapOf( + val out = + linkedMapOf( "ok" to true, "processed" to processed, "batches" to batches, - "tokens_before" to before, - "tokens_after" to after, - ), - ) + ) + if (fsBacked) { + // Token-file counts are an FS-store notion (idx/fts is a + // directory); the SQLite FTS index doesn't expose one. + out["tokens_before"] = before + out["tokens_after"] = countEntries(ftsDir) + } + Output.emit(out) 0 } /** * Maintenance verbs only need the store — not identity, not relays, - * not the signer. Skip [Context.open] (which throws if no identity - * has been bootstrapped) and construct the [FsEventStore] directly - * from [DataDir.eventsDir]. Pretty formatter matches what the rest - * of the CLI uses for inspection-friendly output. + * not the signer. Skip [Context.open] (which throws if no identity has + * been bootstrapped) and open the configured backend directly via + * [StoreFactory], so `amy store` acts on whichever store the rest of + * the CLI is using. */ - private inline fun withStore( + private suspend fun withStore( dataDir: DataDir, - body: (FsEventStore) -> Int, + body: suspend (IEventStore) -> Int, ): Int { - val store = - FsEventStore( - root = dataDir.eventsDir.toPath(), - eventToJson = JacksonMapper::toJsonPretty, - ) + val store = StoreFactory.open(dataDir) try { return body(store) } finally {