diff --git a/geode/config.example.toml b/geode/config.example.toml index 8ae62edc10..2ba85e0ad4 100644 --- a/geode/config.example.toml +++ b/geode/config.example.toml @@ -41,8 +41,21 @@ path = "/" # call_group_size = 64 [database] +# Which IEventStore implementation backs the relay. Default "sqlite". +# - "sqlite": quartz's SQLite store — honours every knob below. Best for +# real traffic. CLI: --store sqlite. +# - "fs": quartz's filesystem store — one JSON file per event under the +# directory named by `file` (a directory, not a db file, in this mode). +# Human-inspectable with cat/jq; the SQLite-only knobs are ignored, and +# it always needs `file`. CLI: --store fs. +# Any other value is the fully-qualified class name of a custom IEventStore +# on the classpath, instantiated reflectively (needs a public constructor +# `(NormalizedRelayUrl?, IndexingStrategy)`, `(NormalizedRelayUrl?)`, or `()`). +# backend = "sqlite" + # True keeps an in-memory SQLite db (events vanish on restart). Useful -# for tests; set false + `file = "..."` for persistent storage. +# for tests; set false + `file = "..."` for persistent storage. Ignored +# by the "fs" backend, which always writes to the `file` directory. in_memory = false file = "/var/lib/geode/events.db" diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt index 5c30fc2f60..704b5665b7 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt @@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.server.policies.OptionalAuthPoli import com.vitorpamplona.quartz.nip01Core.relay.server.policies.RejectFutureEventsPolicy import com.vitorpamplona.quartz.nip01Core.relay.server.policies.VerifyAuthOnlyPolicy import com.vitorpamplona.quartz.nip01Core.relay.server.policies.VerifyPolicy +import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.NdjsonImportExport import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip77Negentropy.NegentropySettings @@ -94,6 +95,9 @@ import java.io.File * --path

ws path (default from config or /) * --info NIP-11 doc file (overrides [info] section) * --db sqlite db path (overrides [database].file) + * --store event-store backend (overrides [database].backend): + * "sqlite" (default), "fs", or the fully-qualified + * class name of a custom IEventStore on the classpath. * --auth require NIP-42 AUTH (sets options.require_auth = true) * --optional-auth advertise NIP-42 AUTH but don't require it (sets * options.optional_auth = true; ignored when --auth is set) @@ -123,21 +127,34 @@ fun main(args: Array) { * exact store the server would. */ private class StoreContext( + /** Human-readable location of the store's bytes, for the verb's summary line. */ val dbFile: String?, - val store: EventStore, + val store: IEventStore, ) private fun openStore(a: Args): StoreContext { - val config = a.opt(CONFIG_FLAG)?.let { StaticConfig.fromFile(File(it)) } ?: StaticConfig() - val dbFile = a.opt("--db") ?: config.database.file?.takeUnless { config.database.in_memory } + val config = + (a.opt(CONFIG_FLAG)?.let { StaticConfig.fromFile(File(it)) } ?: StaticConfig()) + .withStoreOverride(a.opt(STORE_FLAG)) val fullTextSearch = !a.flag(NO_SEARCH_FLAG) && config.options.full_text_search + // The verbs operate on the same store the server would, honoring the same + // `[database].backend` selection — unscoped (relay = null) because bulk + // load/dump has no single relay to attribute NIP-62 cascades to. val store = - EventStore( - dbName = dbFile, - indexStrategy = relayIndexingStrategy(fullTextSearch, config.negentropy.live_index), - numReaders = config.database.readers ?: 4, + StoreFactory.open( + config = config, + relay = null, + fullTextSearch = fullTextSearch, + dbOverride = a.opt("--db"), ) - return StoreContext(dbFile, store) + // Where the bytes land, for the summary line. Only the SQLite in-memory + // case has no location; the FS store always has a directory. + val sqlite = + config.database.backend + .trim() + .lowercase() in StoreFactory.SQLITE_BACKEND_KEYWORDS + val location = a.opt("--db") ?: config.database.file?.takeUnless { sqlite && config.database.in_memory } + return StoreContext(location, store) } private fun runImport(args: Array) { @@ -189,10 +206,12 @@ private fun serve(args: Array) { val a = parseArgs(args) val config: StaticConfig = - a - .opt(CONFIG_FLAG) - ?.let { StaticConfig.fromFile(File(it)) } - ?: StaticConfig() + ( + a + .opt(CONFIG_FLAG) + ?.let { StaticConfig.fromFile(File(it)) } + ?: StaticConfig() + ).withStoreOverride(a.opt(STORE_FLAG)) config.validate() val host = a.opt("--host") ?: config.network.host @@ -200,7 +219,6 @@ private fun serve(args: Array) { val path = a.opt("--path") ?: config.network.path val cliInfoFile = a.opt("--info")?.let { File(it) } - val dbFile = a.opt("--db") ?: config.database.file?.takeUnless { config.database.in_memory } val requireAuth = a.flag("--auth") || config.options.require_auth // Optional AUTH advertises the challenge without gating commands on it. // Mandatory AUTH already sends the challenge, so it wins when both are set. @@ -239,11 +257,11 @@ private fun serve(args: Array) { if (config.database.temp_store_memory) add("PRAGMA temp_store = MEMORY;") } val store = - EventStore( - dbName = dbFile, + StoreFactory.open( + config = config, relay = advertisedUrl, - indexStrategy = relayIndexingStrategy(fullTextSearch, config.negentropy.live_index), - numReaders = config.database.readers ?: 4, + fullTextSearch = fullTextSearch, + dbOverride = a.opt("--db"), extraPragmas = extraPragmas, ) @@ -371,20 +389,24 @@ private fun serve(args: Array) { // Periodic query-planner statistics refresh (`PRAGMA optimize`). // Incremental and usually a no-op; failures are swallowed — a missed // refresh only means slightly staler planner stats until the next tick. + // SQLite-only: the `optimize()` PRAGMA has no meaning for other backends, + // so the loop only runs when the configured store is the SQLite one. val maintenanceScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - config.database.optimize_interval_seconds?.let { secs -> - maintenanceScope.launch { - while (true) { - delay(secs * 1000) - try { - store.optimize() - } catch (e: CancellationException) { - throw e // shutdown cancelled us; don't swallow it - } catch (e: Exception) { - // A missed refresh only means slightly staler planner - // stats until the next tick — log and keep the loop. - // Errors (OOM, etc.) are NOT swallowed. - println("PRAGMA optimize failed: ${e.message}") + (store as? EventStore)?.let { sqlite -> + config.database.optimize_interval_seconds?.let { secs -> + maintenanceScope.launch { + while (true) { + delay(secs * 1000) + try { + sqlite.optimize() + } catch (e: CancellationException) { + throw e // shutdown cancelled us; don't swallow it + } catch (e: Exception) { + // A missed refresh only means slightly staler planner + // stats until the next tick — log and keep the loop. + // Errors (OOM, etc.) are NOT swallowed. + println("PRAGMA optimize failed: ${e.message}") + } } } } @@ -469,6 +491,14 @@ private class Args( private const val CONFIG_FLAG = "--config" private const val NO_VERIFY_FLAG = "--no-verify" private const val NO_SEARCH_FLAG = "--no-search" +private const val STORE_FLAG = "--store" + +/** + * Apply a `--store ` CLI override on top of the parsed + * `[database].backend`. `null` (flag absent) leaves the config untouched, so + * the TOML value — or its `"sqlite"` default — stands. + */ +private fun StaticConfig.withStoreOverride(backend: String?): StaticConfig = if (backend == null) this else copy(database = database.copy(backend = backend)) /** * Boolean flags that never take a value. Listing them explicitly is what lets a diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/StoreFactory.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/StoreFactory.kt new file mode 100644 index 0000000000..95615cb02c --- /dev/null +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/StoreFactory.kt @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.geode + +import com.vitorpamplona.geode.config.StaticConfig +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy +import java.lang.reflect.InvocationTargetException +import kotlin.io.path.Path + +/** + * Builds the relay's [IEventStore] from the parsed [StaticConfig]. + * + * `[database].backend` selects the implementation (see + * [StaticConfig.DatabaseSection.backend]); everything else the store needs + * — the advertised relay URL for NIP-62 scoping, the relay-tuned + * [relayIndexingStrategy], the SQLite deployment pragmas — is threaded + * through here so both the `serve` path and the `import`/`export` verbs open + * the *same* store from the *same* config. + * + * Two backends ship in quartz and are named by keyword; anything else is + * treated as a fully-qualified class name and loaded reflectively, which is + * what makes "pick any `IEventStore` implementation" true rather than a + * fixed menu of two. + */ +object StoreFactory { + /** `[database].backend` values that select the SQLite [EventStore] (the default). */ + val SQLITE_BACKEND_KEYWORDS = setOf("sqlite", "sqlite3", "") + + /** `[database].backend` values that select the filesystem [FsEventStore]. */ + val FS_BACKEND_KEYWORDS = setOf("fs", "file", "files", "filesystem") + + /** + * Open the configured store. + * + * @param config parsed operator config (owns `[database]`, `[negentropy]`). + * @param relay advertised relay URL, or `null` for an unscoped store + * (the `import`/`export` verbs pass `null`). + * @param fullTextSearch whether to build/maintain the NIP-50 index — + * already resolved from `[options].full_text_search` and `--no-search`. + * @param dbOverride the `--db` CLI flag when present; wins over + * `[database].file`. For SQLite it names the db file (`null` → + * in-memory); for `fs` it names the root directory. + * @param extraPragmas SQLite-only deployment pragmas (mmap/temp_store); + * ignored by non-SQLite backends. + */ + fun open( + config: StaticConfig, + relay: NormalizedRelayUrl?, + fullTextSearch: Boolean, + dbOverride: String? = null, + extraPragmas: List = emptyList(), + ): IEventStore { + val strategy = relayIndexingStrategy(fullTextSearch, config.negentropy.live_index) + val key = + config.database.backend + .trim() + .lowercase() + return when { + key in SQLITE_BACKEND_KEYWORDS -> + EventStore( + // `--db` wins; else the configured file unless the operator + // asked for an in-memory db (events vanish on restart). + dbName = dbOverride ?: config.database.file?.takeUnless { config.database.in_memory }, + relay = relay, + indexStrategy = strategy, + numReaders = config.database.readers ?: 4, + extraPragmas = extraPragmas, + ) + + key in FS_BACKEND_KEYWORDS -> { + // `in_memory` is a SQLite concept — the FS store always needs + // a directory. `--db` overrides the configured path. + val root = + dbOverride ?: config.database.file + ?: throw IllegalArgumentException( + "[database].backend = \"fs\" needs a directory: set [database].file " + + "(or pass --db

). The filesystem store cannot run in-memory.", + ) + FsEventStore( + root = Path(root), + indexingStrategy = strategy, + relay = relay, + ) + } + + else -> loadCustom(config.database.backend.trim(), relay, strategy) + } + } + + /** + * Reflectively instantiate a custom [IEventStore] named by its + * fully-qualified class name. The class must implement `IEventStore` + * and expose one of these public constructors, tried most-specific + * first: + * + * 1. `(NormalizedRelayUrl?, IndexingStrategy)` + * 2. `(NormalizedRelayUrl?)` + * 3. `()` + * + * Enough to hand a custom store the same relay scoping and index + * strategy the built-in backends receive, while still accepting a + * dependency-free no-arg store. + */ + private fun loadCustom( + className: String, + relay: NormalizedRelayUrl?, + strategy: IndexingStrategy, + ): IEventStore { + val clazz = + try { + Class.forName(className) + } catch (e: ClassNotFoundException) { + throw IllegalArgumentException( + "[database].backend = \"$className\" is neither a known backend " + + "(\"sqlite\", \"fs\") nor a class on the classpath. Give the fully-qualified " + + "name of an ${IEventStore::class.simpleName} implementation.", + e, + ) + } + require(IEventStore::class.java.isAssignableFrom(clazz)) { + "[database].backend class $className does not implement ${IEventStore::class.qualifiedName}." + } + + newInstance(clazz, arrayOf(NormalizedRelayUrl::class.java, IndexingStrategy::class.java), arrayOf(relay, strategy)) + ?.let { return it } + newInstance(clazz, arrayOf(NormalizedRelayUrl::class.java), arrayOf(relay)) + ?.let { return it } + newInstance(clazz, emptyArray(), emptyArray()) + ?.let { return it } + + throw IllegalArgumentException( + "$className has no supported public constructor. Provide one of: " + + "(NormalizedRelayUrl?, IndexingStrategy), (NormalizedRelayUrl?), or ().", + ) + } + + /** + * Invoke the `paramTypes` constructor with `args`, or return `null` if + * the class has no such (public) constructor. A throw from inside the + * constructor is unwrapped and rethrown — the store failed to build, + * which is not a signal to fall back to a different constructor shape. + */ + private fun newInstance( + clazz: Class<*>, + paramTypes: Array>, + args: Array, + ): IEventStore? = + try { + clazz.getConstructor(*paramTypes).newInstance(*args) as IEventStore + } catch (e: NoSuchMethodException) { + null + } catch (e: InvocationTargetException) { + throw e.cause ?: e + } +} diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/config/StaticConfig.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/config/StaticConfig.kt index 1145a36a1c..dffe4e50e3 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/config/StaticConfig.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/config/StaticConfig.kt @@ -103,6 +103,31 @@ data class StaticConfig( ) data class DatabaseSection( + /** + * Which [com.vitorpamplona.quartz.nip01Core.store.IEventStore] + * implementation backs the relay. Recognised keywords (case- + * insensitive): + * + * - `"sqlite"` (default): quartz's SQLite-backed + * [com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore] — + * honours every `[database]` knob below ([in_memory]/[file], + * [readers], [mmap_size], …). The right choice for real traffic. + * - `"fs"`: quartz's filesystem + * [com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore], + * one JSON file per event under the directory named by [file] + * (which becomes a directory, not a db file, for this backend). + * Human-inspectable with `cat`/`jq`; the SQLite-only knobs are + * ignored. + * + * Any other value is treated as the fully-qualified class name of a + * custom `IEventStore` on the classpath, instantiated reflectively — + * see [com.vitorpamplona.geode.StoreFactory]. This is the "plug in + * any implementation" escape hatch: the class must implement + * `IEventStore` and expose one of the public constructors + * `(NormalizedRelayUrl?, IndexingStrategy)`, `(NormalizedRelayUrl?)`, + * or `()`. + */ + val backend: String = "sqlite", /** True keeps an in-memory SQLite db (default — events vanish on restart). */ val in_memory: Boolean = true, val file: String? = null, diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/StoreFactoryTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/StoreFactoryTest.kt new file mode 100644 index 0000000000..cf0939a96d --- /dev/null +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/StoreFactoryTest.kt @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.geode + +import com.vitorpamplona.geode.config.StaticConfig +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * A stand-in "any other IEventStore" — the escape hatch's target. It only + * needs to implement [IEventStore] and expose the + * `(NormalizedRelayUrl?, IndexingStrategy)` constructor [StoreFactory] looks + * for; delegation to an in-memory [EventStore] keeps it a couple of lines. + */ +class FakeCustomStore( + relay: NormalizedRelayUrl?, + strategy: IndexingStrategy, +) : IEventStore by EventStore(dbName = null, relay = relay, indexStrategy = strategy) + +class StoreFactoryTest { + private fun config(toml: String) = StaticConfig.fromToml(toml) + + @Test + fun defaultBackendIsSqlite() { + val store = StoreFactory.open(config(""), relay = null, fullTextSearch = true) + store.use { assertTrue(it is EventStore, "default backend should be the SQLite EventStore") } + } + + @Test + fun sqliteKeywordSelectsEventStore() { + val store = StoreFactory.open(config("[database]\nbackend = \"SQLite\""), relay = null, fullTextSearch = true) + store.use { assertTrue(it is EventStore) } + } + + @Test + fun fsBackendSelectsFsEventStore() { + val dir = Files.createTempDirectory("geode-fs-store-").toFile() + try { + val toml = "[database]\nbackend = \"fs\"\nfile = \"${dir.absolutePath}\"" + val store = StoreFactory.open(config(toml), relay = null, fullTextSearch = true) + store.use { assertTrue(it is FsEventStore, "fs backend should be the filesystem FsEventStore") } + } finally { + dir.deleteRecursively() + } + } + + @Test + fun fsBackendViaDbOverride() { + val dir = Files.createTempDirectory("geode-fs-store-").toFile() + try { + // No `file` in the config — the --db override supplies the root. + val store = + StoreFactory.open( + config("[database]\nbackend = \"fs\""), + relay = null, + fullTextSearch = true, + dbOverride = dir.absolutePath, + ) + store.use { assertTrue(it is FsEventStore) } + } finally { + dir.deleteRecursively() + } + } + + @Test + fun fsBackendWithoutPathFailsLoud() { + assertFailsWith { + StoreFactory.open(config("[database]\nbackend = \"fs\""), relay = null, fullTextSearch = true) + } + } + + @Test + fun customClassNameIsLoadedReflectively() { + val toml = "[database]\nbackend = \"com.vitorpamplona.geode.FakeCustomStore\"" + val store = StoreFactory.open(config(toml), relay = null, fullTextSearch = true) + store.use { assertTrue(it is FakeCustomStore, "a FQCN backend should be instantiated reflectively") } + } + + @Test + fun unknownClassNameFailsLoud() { + assertFailsWith { + StoreFactory.open( + config("[database]\nbackend = \"com.example.NoSuchStore\""), + relay = null, + fullTextSearch = true, + ) + } + } + + @Test + fun classThatIsNotAnEventStoreFailsLoud() { + // java.lang.String resolves but does not implement IEventStore. + assertFailsWith { + StoreFactory.open(config("[database]\nbackend = \"java.lang.String\""), relay = null, fullTextSearch = true) + } + } +} diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/config/StaticConfigTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/config/StaticConfigTest.kt index d7698300d7..2ba0823441 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/config/StaticConfigTest.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/config/StaticConfigTest.kt @@ -77,6 +77,19 @@ class StaticConfigTest { assertEquals(null, d.database.optimize_interval_seconds) } + @Test + fun backendDefaultsToSqliteAndParses() { + // Unset → SQLite, so existing configs keep the current backend. + assertEquals("sqlite", StaticConfig.fromToml("").database.backend) + // Explicit values round-trip verbatim (case/keyword resolution is + // StoreFactory's job, not the parser's). + assertEquals("fs", StaticConfig.fromToml("[database]\nbackend = \"fs\"").database.backend) + assertEquals( + "com.example.MyStore", + StaticConfig.fromToml("[database]\nbackend = \"com.example.MyStore\"").database.backend, + ) + } + @Test fun mirrorSectionDefaultsToEmpty() { assertTrue(StaticConfig.fromToml("").mirror.isEmpty())