Merge pull request #3692 from vitorpamplona/claude/geode-quartz-eventstore-selection-zht375

Add pluggable event store backend selection via StoreFactory
This commit is contained in:
Vitor Pamplona
2026-07-24 12:30:49 -04:00
committed by GitHub
6 changed files with 411 additions and 31 deletions
+14 -1
View File
@@ -41,8 +41,21 @@ path = "/"
# call_group_size = 64
[database]
# Which IEventStore implementation backs the relay. Default "sqlite".
# - "sqlite": quartz's SQLite store — honours every knob below. Best for
# real traffic. CLI: --store sqlite.
# - "fs": quartz's filesystem store — one JSON file per event under the
# directory named by `file` (a directory, not a db file, in this mode).
# Human-inspectable with cat/jq; the SQLite-only knobs are ignored, and
# it always needs `file`. CLI: --store fs.
# Any other value is the fully-qualified class name of a custom IEventStore
# on the classpath, instantiated reflectively (needs a public constructor
# `(NormalizedRelayUrl?, IndexingStrategy)`, `(NormalizedRelayUrl?)`, or `()`).
# backend = "sqlite"
# True keeps an in-memory SQLite db (events vanish on restart). Useful
# for tests; set false + `file = "..."` for persistent storage.
# for tests; set false + `file = "..."` for persistent storage. Ignored
# by the "fs" backend, which always writes to the `file` directory.
in_memory = false
file = "/var/lib/geode/events.db"
@@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.server.policies.OptionalAuthPoli
import com.vitorpamplona.quartz.nip01Core.relay.server.policies.RejectFutureEventsPolicy
import com.vitorpamplona.quartz.nip01Core.relay.server.policies.VerifyAuthOnlyPolicy
import com.vitorpamplona.quartz.nip01Core.relay.server.policies.VerifyPolicy
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip01Core.store.NdjsonImportExport
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
import com.vitorpamplona.quartz.nip77Negentropy.NegentropySettings
@@ -94,6 +95,9 @@ import java.io.File
* --path <p> ws path (default from config or /)
* --info <file> NIP-11 doc file (overrides [info] section)
* --db <file> sqlite db path (overrides [database].file)
* --store <backend> event-store backend (overrides [database].backend):
* "sqlite" (default), "fs", or the fully-qualified
* class name of a custom IEventStore on the classpath.
* --auth require NIP-42 AUTH (sets options.require_auth = true)
* --optional-auth advertise NIP-42 AUTH but don't require it (sets
* options.optional_auth = true; ignored when --auth is set)
@@ -123,21 +127,34 @@ fun main(args: Array<String>) {
* exact store the server would.
*/
private class StoreContext(
/** Human-readable location of the store's bytes, for the verb's summary line. */
val dbFile: String?,
val store: EventStore,
val store: IEventStore,
)
private fun openStore(a: Args): StoreContext {
val config = a.opt(CONFIG_FLAG)?.let { StaticConfig.fromFile(File(it)) } ?: StaticConfig()
val dbFile = a.opt("--db") ?: config.database.file?.takeUnless { config.database.in_memory }
val config =
(a.opt(CONFIG_FLAG)?.let { StaticConfig.fromFile(File(it)) } ?: StaticConfig())
.withStoreOverride(a.opt(STORE_FLAG))
val fullTextSearch = !a.flag(NO_SEARCH_FLAG) && config.options.full_text_search
// The verbs operate on the same store the server would, honoring the same
// `[database].backend` selection — unscoped (relay = null) because bulk
// load/dump has no single relay to attribute NIP-62 cascades to.
val store =
EventStore(
dbName = dbFile,
indexStrategy = relayIndexingStrategy(fullTextSearch, config.negentropy.live_index),
numReaders = config.database.readers ?: 4,
StoreFactory.open(
config = config,
relay = null,
fullTextSearch = fullTextSearch,
dbOverride = a.opt("--db"),
)
return StoreContext(dbFile, store)
// Where the bytes land, for the summary line. Only the SQLite in-memory
// case has no location; the FS store always has a directory.
val sqlite =
config.database.backend
.trim()
.lowercase() in StoreFactory.SQLITE_BACKEND_KEYWORDS
val location = a.opt("--db") ?: config.database.file?.takeUnless { sqlite && config.database.in_memory }
return StoreContext(location, store)
}
private fun runImport(args: Array<String>) {
@@ -189,10 +206,12 @@ private fun serve(args: Array<String>) {
val a = parseArgs(args)
val config: StaticConfig =
a
.opt(CONFIG_FLAG)
?.let { StaticConfig.fromFile(File(it)) }
?: StaticConfig()
(
a
.opt(CONFIG_FLAG)
?.let { StaticConfig.fromFile(File(it)) }
?: StaticConfig()
).withStoreOverride(a.opt(STORE_FLAG))
config.validate()
val host = a.opt("--host") ?: config.network.host
@@ -200,7 +219,6 @@ private fun serve(args: Array<String>) {
val path = a.opt("--path") ?: config.network.path
val cliInfoFile = a.opt("--info")?.let { File(it) }
val dbFile = a.opt("--db") ?: config.database.file?.takeUnless { config.database.in_memory }
val requireAuth = a.flag("--auth") || config.options.require_auth
// Optional AUTH advertises the challenge without gating commands on it.
// Mandatory AUTH already sends the challenge, so it wins when both are set.
@@ -239,11 +257,11 @@ private fun serve(args: Array<String>) {
if (config.database.temp_store_memory) add("PRAGMA temp_store = MEMORY;")
}
val store =
EventStore(
dbName = dbFile,
StoreFactory.open(
config = config,
relay = advertisedUrl,
indexStrategy = relayIndexingStrategy(fullTextSearch, config.negentropy.live_index),
numReaders = config.database.readers ?: 4,
fullTextSearch = fullTextSearch,
dbOverride = a.opt("--db"),
extraPragmas = extraPragmas,
)
@@ -371,20 +389,24 @@ private fun serve(args: Array<String>) {
// Periodic query-planner statistics refresh (`PRAGMA optimize`).
// Incremental and usually a no-op; failures are swallowed — a missed
// refresh only means slightly staler planner stats until the next tick.
// SQLite-only: the `optimize()` PRAGMA has no meaning for other backends,
// so the loop only runs when the configured store is the SQLite one.
val maintenanceScope = CoroutineScope(Dispatchers.IO + SupervisorJob())
config.database.optimize_interval_seconds?.let { secs ->
maintenanceScope.launch {
while (true) {
delay(secs * 1000)
try {
store.optimize()
} catch (e: CancellationException) {
throw e // shutdown cancelled us; don't swallow it
} catch (e: Exception) {
// A missed refresh only means slightly staler planner
// stats until the next tick — log and keep the loop.
// Errors (OOM, etc.) are NOT swallowed.
println("PRAGMA optimize failed: ${e.message}")
(store as? EventStore)?.let { sqlite ->
config.database.optimize_interval_seconds?.let { secs ->
maintenanceScope.launch {
while (true) {
delay(secs * 1000)
try {
sqlite.optimize()
} catch (e: CancellationException) {
throw e // shutdown cancelled us; don't swallow it
} catch (e: Exception) {
// A missed refresh only means slightly staler planner
// stats until the next tick — log and keep the loop.
// Errors (OOM, etc.) are NOT swallowed.
println("PRAGMA optimize failed: ${e.message}")
}
}
}
}
@@ -469,6 +491,14 @@ private class Args(
private const val CONFIG_FLAG = "--config"
private const val NO_VERIFY_FLAG = "--no-verify"
private const val NO_SEARCH_FLAG = "--no-search"
private const val STORE_FLAG = "--store"
/**
* Apply a `--store <backend>` CLI override on top of the parsed
* `[database].backend`. `null` (flag absent) leaves the config untouched, so
* the TOML value — or its `"sqlite"` default — stands.
*/
private fun StaticConfig.withStoreOverride(backend: String?): StaticConfig = if (backend == null) this else copy(database = database.copy(backend = backend))
/**
* Boolean flags that never take a value. Listing them explicitly is what lets a
@@ -0,0 +1,177 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.geode
import com.vitorpamplona.geode.config.StaticConfig
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy
import java.lang.reflect.InvocationTargetException
import kotlin.io.path.Path
/**
* Builds the relay's [IEventStore] from the parsed [StaticConfig].
*
* `[database].backend` selects the implementation (see
* [StaticConfig.DatabaseSection.backend]); everything else the store needs
* — the advertised relay URL for NIP-62 scoping, the relay-tuned
* [relayIndexingStrategy], the SQLite deployment pragmas — is threaded
* through here so both the `serve` path and the `import`/`export` verbs open
* the *same* store from the *same* config.
*
* Two backends ship in quartz and are named by keyword; anything else is
* treated as a fully-qualified class name and loaded reflectively, which is
* what makes "pick any `IEventStore` implementation" true rather than a
* fixed menu of two.
*/
object StoreFactory {
/** `[database].backend` values that select the SQLite [EventStore] (the default). */
val SQLITE_BACKEND_KEYWORDS = setOf("sqlite", "sqlite3", "")
/** `[database].backend` values that select the filesystem [FsEventStore]. */
val FS_BACKEND_KEYWORDS = setOf("fs", "file", "files", "filesystem")
/**
* Open the configured store.
*
* @param config parsed operator config (owns `[database]`, `[negentropy]`).
* @param relay advertised relay URL, or `null` for an unscoped store
* (the `import`/`export` verbs pass `null`).
* @param fullTextSearch whether to build/maintain the NIP-50 index —
* already resolved from `[options].full_text_search` and `--no-search`.
* @param dbOverride the `--db` CLI flag when present; wins over
* `[database].file`. For SQLite it names the db file (`null` →
* in-memory); for `fs` it names the root directory.
* @param extraPragmas SQLite-only deployment pragmas (mmap/temp_store);
* ignored by non-SQLite backends.
*/
fun open(
config: StaticConfig,
relay: NormalizedRelayUrl?,
fullTextSearch: Boolean,
dbOverride: String? = null,
extraPragmas: List<String> = emptyList(),
): IEventStore {
val strategy = relayIndexingStrategy(fullTextSearch, config.negentropy.live_index)
val key =
config.database.backend
.trim()
.lowercase()
return when {
key in SQLITE_BACKEND_KEYWORDS ->
EventStore(
// `--db` wins; else the configured file unless the operator
// asked for an in-memory db (events vanish on restart).
dbName = dbOverride ?: config.database.file?.takeUnless { config.database.in_memory },
relay = relay,
indexStrategy = strategy,
numReaders = config.database.readers ?: 4,
extraPragmas = extraPragmas,
)
key in FS_BACKEND_KEYWORDS -> {
// `in_memory` is a SQLite concept — the FS store always needs
// a directory. `--db` overrides the configured path.
val root =
dbOverride ?: config.database.file
?: throw IllegalArgumentException(
"[database].backend = \"fs\" needs a directory: set [database].file " +
"(or pass --db <dir>). The filesystem store cannot run in-memory.",
)
FsEventStore(
root = Path(root),
indexingStrategy = strategy,
relay = relay,
)
}
else -> loadCustom(config.database.backend.trim(), relay, strategy)
}
}
/**
* Reflectively instantiate a custom [IEventStore] named by its
* fully-qualified class name. The class must implement `IEventStore`
* and expose one of these public constructors, tried most-specific
* first:
*
* 1. `(NormalizedRelayUrl?, IndexingStrategy)`
* 2. `(NormalizedRelayUrl?)`
* 3. `()`
*
* Enough to hand a custom store the same relay scoping and index
* strategy the built-in backends receive, while still accepting a
* dependency-free no-arg store.
*/
private fun loadCustom(
className: String,
relay: NormalizedRelayUrl?,
strategy: IndexingStrategy,
): IEventStore {
val clazz =
try {
Class.forName(className)
} catch (e: ClassNotFoundException) {
throw IllegalArgumentException(
"[database].backend = \"$className\" is neither a known backend " +
"(\"sqlite\", \"fs\") nor a class on the classpath. Give the fully-qualified " +
"name of an ${IEventStore::class.simpleName} implementation.",
e,
)
}
require(IEventStore::class.java.isAssignableFrom(clazz)) {
"[database].backend class $className does not implement ${IEventStore::class.qualifiedName}."
}
newInstance(clazz, arrayOf(NormalizedRelayUrl::class.java, IndexingStrategy::class.java), arrayOf(relay, strategy))
?.let { return it }
newInstance(clazz, arrayOf(NormalizedRelayUrl::class.java), arrayOf(relay))
?.let { return it }
newInstance(clazz, emptyArray(), emptyArray())
?.let { return it }
throw IllegalArgumentException(
"$className has no supported public constructor. Provide one of: " +
"(NormalizedRelayUrl?, IndexingStrategy), (NormalizedRelayUrl?), or ().",
)
}
/**
* Invoke the `paramTypes` constructor with `args`, or return `null` if
* the class has no such (public) constructor. A throw from inside the
* constructor is unwrapped and rethrown — the store failed to build,
* which is not a signal to fall back to a different constructor shape.
*/
private fun newInstance(
clazz: Class<*>,
paramTypes: Array<Class<*>>,
args: Array<Any?>,
): IEventStore? =
try {
clazz.getConstructor(*paramTypes).newInstance(*args) as IEventStore
} catch (e: NoSuchMethodException) {
null
} catch (e: InvocationTargetException) {
throw e.cause ?: e
}
}
@@ -103,6 +103,31 @@ data class StaticConfig(
)
data class DatabaseSection(
/**
* Which [com.vitorpamplona.quartz.nip01Core.store.IEventStore]
* implementation backs the relay. Recognised keywords (case-
* insensitive):
*
* - `"sqlite"` (default): quartz's SQLite-backed
* [com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore] —
* honours every `[database]` knob below ([in_memory]/[file],
* [readers], [mmap_size], …). The right choice for real traffic.
* - `"fs"`: quartz's filesystem
* [com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore],
* one JSON file per event under the directory named by [file]
* (which becomes a directory, not a db file, for this backend).
* Human-inspectable with `cat`/`jq`; the SQLite-only knobs are
* ignored.
*
* Any other value is treated as the fully-qualified class name of a
* custom `IEventStore` on the classpath, instantiated reflectively —
* see [com.vitorpamplona.geode.StoreFactory]. This is the "plug in
* any implementation" escape hatch: the class must implement
* `IEventStore` and expose one of the public constructors
* `(NormalizedRelayUrl?, IndexingStrategy)`, `(NormalizedRelayUrl?)`,
* or `()`.
*/
val backend: String = "sqlite",
/** True keeps an in-memory SQLite db (default — events vanish on restart). */
val in_memory: Boolean = true,
val file: String? = null,
@@ -0,0 +1,122 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.geode
import com.vitorpamplona.geode.config.StaticConfig
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip01Core.store.fs.FsEventStore
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertFailsWith
import kotlin.test.assertTrue
/**
* A stand-in "any other IEventStore" — the escape hatch's target. It only
* needs to implement [IEventStore] and expose the
* `(NormalizedRelayUrl?, IndexingStrategy)` constructor [StoreFactory] looks
* for; delegation to an in-memory [EventStore] keeps it a couple of lines.
*/
class FakeCustomStore(
relay: NormalizedRelayUrl?,
strategy: IndexingStrategy,
) : IEventStore by EventStore(dbName = null, relay = relay, indexStrategy = strategy)
class StoreFactoryTest {
private fun config(toml: String) = StaticConfig.fromToml(toml)
@Test
fun defaultBackendIsSqlite() {
val store = StoreFactory.open(config(""), relay = null, fullTextSearch = true)
store.use { assertTrue(it is EventStore, "default backend should be the SQLite EventStore") }
}
@Test
fun sqliteKeywordSelectsEventStore() {
val store = StoreFactory.open(config("[database]\nbackend = \"SQLite\""), relay = null, fullTextSearch = true)
store.use { assertTrue(it is EventStore) }
}
@Test
fun fsBackendSelectsFsEventStore() {
val dir = Files.createTempDirectory("geode-fs-store-").toFile()
try {
val toml = "[database]\nbackend = \"fs\"\nfile = \"${dir.absolutePath}\""
val store = StoreFactory.open(config(toml), relay = null, fullTextSearch = true)
store.use { assertTrue(it is FsEventStore, "fs backend should be the filesystem FsEventStore") }
} finally {
dir.deleteRecursively()
}
}
@Test
fun fsBackendViaDbOverride() {
val dir = Files.createTempDirectory("geode-fs-store-").toFile()
try {
// No `file` in the config — the --db override supplies the root.
val store =
StoreFactory.open(
config("[database]\nbackend = \"fs\""),
relay = null,
fullTextSearch = true,
dbOverride = dir.absolutePath,
)
store.use { assertTrue(it is FsEventStore) }
} finally {
dir.deleteRecursively()
}
}
@Test
fun fsBackendWithoutPathFailsLoud() {
assertFailsWith<IllegalArgumentException> {
StoreFactory.open(config("[database]\nbackend = \"fs\""), relay = null, fullTextSearch = true)
}
}
@Test
fun customClassNameIsLoadedReflectively() {
val toml = "[database]\nbackend = \"com.vitorpamplona.geode.FakeCustomStore\""
val store = StoreFactory.open(config(toml), relay = null, fullTextSearch = true)
store.use { assertTrue(it is FakeCustomStore, "a FQCN backend should be instantiated reflectively") }
}
@Test
fun unknownClassNameFailsLoud() {
assertFailsWith<IllegalArgumentException> {
StoreFactory.open(
config("[database]\nbackend = \"com.example.NoSuchStore\""),
relay = null,
fullTextSearch = true,
)
}
}
@Test
fun classThatIsNotAnEventStoreFailsLoud() {
// java.lang.String resolves but does not implement IEventStore.
assertFailsWith<IllegalArgumentException> {
StoreFactory.open(config("[database]\nbackend = \"java.lang.String\""), relay = null, fullTextSearch = true)
}
}
}
@@ -77,6 +77,19 @@ class StaticConfigTest {
assertEquals(null, d.database.optimize_interval_seconds)
}
@Test
fun backendDefaultsToSqliteAndParses() {
// Unset → SQLite, so existing configs keep the current backend.
assertEquals("sqlite", StaticConfig.fromToml("").database.backend)
// Explicit values round-trip verbatim (case/keyword resolution is
// StoreFactory's job, not the parser's).
assertEquals("fs", StaticConfig.fromToml("[database]\nbackend = \"fs\"").database.backend)
assertEquals(
"com.example.MyStore",
StaticConfig.fromToml("[database]\nbackend = \"com.example.MyStore\"").database.backend,
)
}
@Test
fun mirrorSectionDefaultsToEmpty() {
assertTrue(StaticConfig.fromToml("").mirror.isEmpty())