mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-07-30 19:46:17 +00:00
feat(geode): add import / export NDJSON verbs; drop the benchmark-only server
Bulk NDJSON import/export as first-class geode subcommands, mirroring `strfry import` / `strfry export` (one JSON event per line — the interchange format for seeding a relay, migrating between relays, or taking a backup): geode import [--db …] [--no-verify] [FILE…] # files, or stdin when none geode export [--db …] # NDJSON to stdout Both stream — memory is bounded to one batch (import) / one event (export), so a multi-million-event corpus round-trips in roughly constant memory. `import` verifies signatures by default (same `Event.verify()` the relay's VerifyPolicy uses), upholding the relay's verify-by-default stance rather than trusting the file; `--no-verify` is the trusted-input escape hatch. Verb dispatch is backward-compatible: a bare `geode --port …` (no verb) still serves. This makes the benchmark-only `CorpusServerMain` redundant — a corpus source is now just `geode import` into a DB, then a normal `geode` serve — so it's deleted, removing benchmark-only code from the production geode artifact (the question that started this). The 1M sync-throughput plan is updated to describe sources via `geode import` + serve. Also fixes a native-target CI break: MergeQueryCorrectnessTest used the deprecated `String(CharArray)` (error-level on Kotlin/Native) — switched to `CharArray.concatToString()`. Verified end-to-end through the packaged `geode` binary: import (file + stdin, --no-verify), export round-trip, and verify-on rejecting bad signatures. ImportExportTest covers the counts, duplicate handling, malformed-line skipping, and verify accepting a freshly-signed event while rejecting bad sigs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012EZeWww5TJnzBZKPoc6mvU
This commit is contained in:
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.geode
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.verify
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
|
||||
|
||||
/**
|
||||
* Bulk NDJSON import/export for a geode store — the `geode import` / `geode export`
|
||||
* verbs, geode's equivalent of `strfry import` / `strfry export`. One JSON event
|
||||
* per line (the same on-the-wire event object, no envelope), which is the de-facto
|
||||
* interchange format across relays (strfry dumps, corpus files, backups).
|
||||
*
|
||||
* Both directions stream: memory is bounded to one batch (import) or one event
|
||||
* (export) regardless of corpus size, so a multi-million-event dump round-trips in
|
||||
* roughly constant memory.
|
||||
*/
|
||||
object ImportExport {
|
||||
/** Events per [IEventStore.batchInsert]; one transaction per batch. */
|
||||
const val BATCH = 10_000
|
||||
|
||||
class ImportStats(
|
||||
/** Non-blank lines read. */
|
||||
val read: Long,
|
||||
/** Events newly stored. */
|
||||
val imported: Long,
|
||||
/** Events the store rejected — overwhelmingly duplicates (unique-id). */
|
||||
val rejected: Long,
|
||||
/** Events dropped for a bad signature (only when verifying). */
|
||||
val invalid: Long,
|
||||
/** Lines that didn't parse as a NIP-01 event. */
|
||||
val malformed: Long,
|
||||
) {
|
||||
operator fun plus(o: ImportStats) = ImportStats(read + o.read, imported + o.imported, rejected + o.rejected, invalid + o.invalid, malformed + o.malformed)
|
||||
|
||||
companion object {
|
||||
val ZERO = ImportStats(0, 0, 0, 0, 0)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads one JSON event per line from [lines] and batch-inserts them into
|
||||
* [store]. When [verify], each event's Schnorr signature is checked with the
|
||||
* same `Event.verify()` the relay's `VerifyPolicy` uses, and a bad signature
|
||||
* is counted ([ImportStats.invalid]) and skipped — so `import` upholds the
|
||||
* relay's verify-by-default stance rather than trusting the file. Duplicates
|
||||
* are dropped by the store's unique-id constraint and counted as
|
||||
* [ImportStats.rejected].
|
||||
*/
|
||||
suspend fun import(
|
||||
store: IEventStore,
|
||||
lines: Sequence<String>,
|
||||
verify: Boolean,
|
||||
batchSize: Int = BATCH,
|
||||
): ImportStats {
|
||||
var read = 0L
|
||||
var imported = 0L
|
||||
var rejected = 0L
|
||||
var invalid = 0L
|
||||
var malformed = 0L
|
||||
val batch = ArrayList<Event>(batchSize)
|
||||
|
||||
suspend fun flush() {
|
||||
if (batch.isEmpty()) return
|
||||
for (outcome in store.batchInsert(batch)) {
|
||||
when (outcome) {
|
||||
IEventStore.InsertOutcome.Accepted -> imported++
|
||||
is IEventStore.InsertOutcome.Rejected -> rejected++
|
||||
}
|
||||
}
|
||||
batch.clear()
|
||||
}
|
||||
|
||||
for (line in lines) {
|
||||
if (line.isBlank()) continue
|
||||
read++
|
||||
val event = runCatching { OptimizedJsonMapper.fromJson(line) }.getOrNull()
|
||||
if (event == null) {
|
||||
malformed++
|
||||
continue
|
||||
}
|
||||
if (verify && !event.verify()) {
|
||||
invalid++
|
||||
continue
|
||||
}
|
||||
batch.add(event)
|
||||
if (batch.size >= batchSize) flush()
|
||||
}
|
||||
flush()
|
||||
return ImportStats(read, imported, rejected, invalid, malformed)
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams every stored event as NDJSON (one compact JSON object per line, no
|
||||
* trailing whitespace) to [out], newest-first. Returns the count written.
|
||||
*/
|
||||
suspend fun export(
|
||||
store: IEventStore,
|
||||
out: Appendable,
|
||||
): Long {
|
||||
var n = 0L
|
||||
store.query<Event>(Filter()) { event ->
|
||||
out.append(event.toJson()).append('\n')
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
}
|
||||
@@ -49,15 +49,30 @@ import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* Standalone entry point.
|
||||
* Standalone entry point. The first argument may be a verb:
|
||||
*
|
||||
* geode [relay] [flags] serve the relay (default when no verb is given)
|
||||
* geode import [flags] [FILE…] bulk-load NDJSON events into the store
|
||||
* geode export [flags] dump the store as NDJSON to stdout
|
||||
*
|
||||
* `import`/`export` are geode's equivalent of `strfry import` / `strfry export`:
|
||||
* one JSON event per line, the interchange format for seeding a relay, migrating
|
||||
* between relays, or taking a backup. `import` reads the given files (or stdin
|
||||
* when none are named), verifies signatures by default (same as the relay;
|
||||
* `--no-verify` to skip), and prints a read/imported/rejected summary to stderr.
|
||||
* `export` streams every stored event, newest-first, to stdout. Both stream, so
|
||||
* a multi-million-event corpus round-trips in roughly constant memory. They share
|
||||
* the relay's `--db`/`--config`/`--no-search` flags so the same store is targeted.
|
||||
*
|
||||
* Run with:
|
||||
* ./gradlew :geode:run --args="--config /etc/geode.toml"
|
||||
* ./gradlew :geode:run --args="import --db relay.sqlite corpus.ndjson"
|
||||
* or
|
||||
* java -cp ... com.vitorpamplona.geode.MainKt --port 7447 --verify
|
||||
* java -cp ... com.vitorpamplona.geode.MainKt --port 7447
|
||||
*
|
||||
* Configuration precedence (highest to lowest):
|
||||
* 1. CLI flags (`--host`, `--port`, …)
|
||||
@@ -91,6 +106,85 @@ import java.io.File
|
||||
* per-event tokenization cost on ingest.
|
||||
*/
|
||||
fun main(args: Array<String>) {
|
||||
when (args.firstOrNull()?.takeUnless { it.startsWith("--") }) {
|
||||
"import" -> runImport(args.copyOfRange(1, args.size))
|
||||
"export" -> runExport(args.copyOfRange(1, args.size))
|
||||
// Explicit `relay` verb or no verb at all → serve. A bare `geode --port …`
|
||||
// (no verb) stays valid so existing invocations don't change.
|
||||
"relay" -> serve(args.copyOfRange(1, args.size))
|
||||
else -> serve(args)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the store the `import`/`export` verbs operate on, honoring the same
|
||||
* `--db`/`--config`/`--no-search` selection the relay uses so a verb targets the
|
||||
* exact store the server would.
|
||||
*/
|
||||
private class StoreContext(
|
||||
val dbFile: String?,
|
||||
val store: EventStore,
|
||||
)
|
||||
|
||||
private fun openStore(a: Args): StoreContext {
|
||||
val config = a.opt("--config")?.let { StaticConfig.fromFile(File(it)) } ?: StaticConfig()
|
||||
val dbFile = a.opt("--db") ?: config.database.file?.takeUnless { config.database.in_memory }
|
||||
val fullTextSearch = !a.flag("--no-search") && config.options.full_text_search
|
||||
val store =
|
||||
EventStore(
|
||||
dbName = dbFile,
|
||||
indexStrategy = relayIndexingStrategy(fullTextSearch, config.negentropy.live_index),
|
||||
numReaders = config.database.readers ?: 4,
|
||||
)
|
||||
return StoreContext(dbFile, store)
|
||||
}
|
||||
|
||||
private fun runImport(args: Array<String>) {
|
||||
val a = parseArgs(args)
|
||||
val config = a.opt("--config")?.let { StaticConfig.fromFile(File(it)) } ?: StaticConfig()
|
||||
// Verify by default, matching the relay's stance — `import` won't trust a
|
||||
// file's signatures any more than the relay trusts a client's. `--no-verify`
|
||||
// is the trusted-input escape hatch (fixture replay, a dump from a relay you
|
||||
// already trust).
|
||||
val verify = !a.flag("--no-verify") && config.options.verify_signatures
|
||||
val ctx = openStore(a)
|
||||
try {
|
||||
val stats =
|
||||
runBlocking {
|
||||
if (a.positionals.isEmpty()) {
|
||||
System.`in`.bufferedReader().useLines { ImportExport.import(ctx.store, it, verify) }
|
||||
} else {
|
||||
var acc = ImportExport.ImportStats.ZERO
|
||||
for (file in a.positionals) {
|
||||
acc += File(file).bufferedReader().useLines { ImportExport.import(ctx.store, it, verify) }
|
||||
}
|
||||
acc
|
||||
}
|
||||
}
|
||||
System.err.println(
|
||||
"geode import: read=${stats.read} imported=${stats.imported} " +
|
||||
"rejected=${stats.rejected} invalid-sig=${stats.invalid} malformed=${stats.malformed} " +
|
||||
"→ ${ctx.dbFile ?: "(in-memory — not persisted; pass --db)"}",
|
||||
)
|
||||
} finally {
|
||||
ctx.store.close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun runExport(args: Array<String>) {
|
||||
val a = parseArgs(args)
|
||||
val ctx = openStore(a)
|
||||
try {
|
||||
val out = System.out.bufferedWriter()
|
||||
val n = runBlocking { ImportExport.export(ctx.store, out) }
|
||||
out.flush()
|
||||
System.err.println("geode export: $n events from ${ctx.dbFile ?: "(in-memory — empty)"}")
|
||||
} finally {
|
||||
ctx.store.close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun serve(args: Array<String>) {
|
||||
val a = parseArgs(args)
|
||||
|
||||
val config: StaticConfig =
|
||||
@@ -363,15 +457,26 @@ private fun composePolicy(
|
||||
private class Args(
|
||||
private val opts: Map<String, String>,
|
||||
private val flags: Set<String>,
|
||||
/** Non-`--` operands, in order (e.g. the NDJSON files for `import`). */
|
||||
val positionals: List<String>,
|
||||
) {
|
||||
fun opt(k: String) = opts[k]
|
||||
|
||||
fun flag(k: String) = k in flags
|
||||
}
|
||||
|
||||
/**
|
||||
* Boolean flags that never take a value. Listing them explicitly is what lets a
|
||||
* trailing positional survive after a flag — `import --no-verify corpus.ndjson`
|
||||
* must read `corpus.ndjson` as a file, not as `--no-verify`'s value.
|
||||
*/
|
||||
private val BOOLEAN_FLAGS =
|
||||
setOf("--auth", "--optional-auth", "--no-verify", "--no-parallel-verify", "--no-search")
|
||||
|
||||
private fun parseArgs(args: Array<String>): Args {
|
||||
val opts = mutableMapOf<String, String>()
|
||||
val flags = mutableSetOf<String>()
|
||||
val positionals = mutableListOf<String>()
|
||||
var i = 0
|
||||
while (i < args.size) {
|
||||
val a = args[i]
|
||||
@@ -381,22 +486,30 @@ private fun parseArgs(args: Array<String>): Args {
|
||||
// happen to contain `=` (e.g. NIP-11 contact emails) by
|
||||
// using the space-separated form.
|
||||
val eq = a.indexOf('=')
|
||||
if (eq > 0) {
|
||||
opts[a.substring(0, eq)] = a.substring(eq + 1)
|
||||
i += 1
|
||||
} else {
|
||||
val next = args.getOrNull(i + 1)
|
||||
if (next != null && !next.startsWith("--")) {
|
||||
opts[a] = next
|
||||
i += 2
|
||||
} else {
|
||||
when {
|
||||
eq > 0 -> {
|
||||
opts[a.substring(0, eq)] = a.substring(eq + 1)
|
||||
i += 1
|
||||
}
|
||||
a in BOOLEAN_FLAGS -> {
|
||||
flags += a
|
||||
i += 1
|
||||
}
|
||||
else -> {
|
||||
val next = args.getOrNull(i + 1)
|
||||
if (next != null && !next.startsWith("--")) {
|
||||
opts[a] = next
|
||||
i += 2
|
||||
} else {
|
||||
flags += a
|
||||
i += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
positionals += a
|
||||
i += 1
|
||||
}
|
||||
}
|
||||
return Args(opts, flags)
|
||||
return Args(opts, flags, positionals)
|
||||
}
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.geode.tools
|
||||
|
||||
import com.vitorpamplona.geode.KtorRelay
|
||||
import com.vitorpamplona.geode.RelayEngine
|
||||
import com.vitorpamplona.geode.RelayIndexingStrategy
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import java.io.File
|
||||
import java.util.concurrent.CountDownLatch
|
||||
|
||||
/**
|
||||
* Boots a real geode relay (geode's default indexing: FTS + live negentropy
|
||||
* index) preloaded with a corpus of nostr events read from an NDJSON file, then
|
||||
* serves forever so external clients — `strfry sync`, another geode, the
|
||||
* relayBench negentropy sink — can reconcile against it.
|
||||
*
|
||||
* A benchmark-only source: it exists so the negentropy sync comparison has a
|
||||
* geode relay holding the same corpus a strfry source does, reachable over the
|
||||
* production WebSocket transport.
|
||||
*
|
||||
* Usage: `CorpusServerMain <port> <corpus.ndjson> [maxCount]`
|
||||
*/
|
||||
fun main(args: Array<String>) {
|
||||
if (args.size < 2) {
|
||||
System.err.println("usage: CorpusServerMain <port> <corpus.ndjson> [maxCount]")
|
||||
return
|
||||
}
|
||||
val port = args[0].toInt()
|
||||
val corpus = File(args[1])
|
||||
val maxCount = args.getOrNull(2)?.toInt() ?: Int.MAX_VALUE
|
||||
|
||||
// File-backed so a 1M in-memory corpus here doesn't compete for RAM with an
|
||||
// in-memory sink in the same box during the comparison.
|
||||
val dbFile = "/tmp/geode-source-$port.sqlite"
|
||||
// Reuse an already-loaded DB (serve-only, skipping the multi-minute reload)
|
||||
// ONLY when a completion sentinel proves it holds *this* corpus, fully loaded.
|
||||
// The sentinel is written only after a full load and is keyed on corpus
|
||||
// identity (path + byte length) and maxCount, so a different/rebuilt corpus, a
|
||||
// different cap, or a load interrupted mid-way (row count > 0 but incomplete)
|
||||
// all fail the check and force a clean reload — never silently serving the
|
||||
// wrong events under a run that reports success.
|
||||
val sentinel = File("$dbFile.done")
|
||||
val signature = "${corpus.absolutePath}\t${corpus.length()}\tmax=$maxCount"
|
||||
val reusable = sentinel.takeIf { it.exists() }?.readText() == signature
|
||||
if (!reusable) {
|
||||
sentinel.delete()
|
||||
listOf(dbFile, "$dbFile-wal", "$dbFile-shm", "$dbFile-journal").forEach { File(it).delete() }
|
||||
}
|
||||
val store = EventStore(dbName = dbFile, indexStrategy = RelayIndexingStrategy)
|
||||
val engine = RelayEngine(url = "ws://127.0.0.1:$port/".normalizeRelayUrl(), store = store)
|
||||
|
||||
var loaded = runBlocking { store.count(Filter()) }
|
||||
if (reusable && loaded > 0) {
|
||||
println("CorpusServerMain: reusing existing DB with $loaded events (serve-only)")
|
||||
} else {
|
||||
println("CorpusServerMain: loading up to $maxCount events from ${corpus.name}…")
|
||||
loaded =
|
||||
runBlocking {
|
||||
var total = 0
|
||||
val batch = ArrayList<Event>(10_000)
|
||||
corpus.bufferedReader().useLines { lines ->
|
||||
for (line in lines) {
|
||||
if (total >= maxCount) break
|
||||
if (line.isBlank()) continue
|
||||
val event = runCatching { OptimizedJsonMapper.fromJson(line) }.getOrNull() ?: continue
|
||||
batch.add(event)
|
||||
if (batch.size == 10_000) {
|
||||
store.batchInsert(batch)
|
||||
total += batch.size
|
||||
batch.clear()
|
||||
if (total % 200_000 == 0) println(" …loaded $total")
|
||||
}
|
||||
}
|
||||
}
|
||||
if (batch.isNotEmpty()) {
|
||||
store.batchInsert(batch)
|
||||
total += batch.size
|
||||
}
|
||||
total
|
||||
}
|
||||
// Mark the load complete only now — a crash before this leaves no
|
||||
// sentinel, so the next run rebuilds instead of serving a partial DB.
|
||||
sentinel.writeText(signature)
|
||||
}
|
||||
val count = runBlocking { store.count(Filter()) }
|
||||
|
||||
val server = KtorRelay(engine, host = "127.0.0.1", port = port).start()
|
||||
println("CorpusServerMain: READY port=$port loaded=$loaded distinct=$count")
|
||||
|
||||
// Serve until the JVM is killed.
|
||||
Runtime.getRuntime().addShutdownHook(
|
||||
Thread {
|
||||
server.stop(gracePeriodMillis = 0, timeoutMillis = 500)
|
||||
engine.close()
|
||||
},
|
||||
)
|
||||
CountDownLatch(1).await()
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.geode
|
||||
|
||||
import com.vitorpamplona.geode.fixtures.SyntheticEvents
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
|
||||
import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
|
||||
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlin.test.AfterTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
/**
|
||||
* Guards the `geode import` / `geode export` NDJSON round-trip: import counts,
|
||||
* duplicate handling, malformed-line skipping, and — the security-relevant part —
|
||||
* that verification actually gates a bad signature while still admitting a good one.
|
||||
*/
|
||||
class ImportExportTest {
|
||||
private val store =
|
||||
EventStore(
|
||||
dbName = null,
|
||||
indexStrategy =
|
||||
DefaultIndexingStrategy(
|
||||
indexEventsByCreatedAtAlone = true,
|
||||
indexEventsByPubkeyAlone = true,
|
||||
indexFullTextSearch = false,
|
||||
),
|
||||
)
|
||||
|
||||
@AfterTest
|
||||
fun tearDown() = store.close()
|
||||
|
||||
private fun ndjson(events: List<Event>): Sequence<String> = events.asSequence().map { it.toJson() }
|
||||
|
||||
@Test
|
||||
fun importThenExport_roundTrips() =
|
||||
runBlocking {
|
||||
val events = SyntheticEvents.batch(count = 25)
|
||||
val stats = ImportExport.import(store, ndjson(events), verify = false)
|
||||
|
||||
assertEquals(25L, stats.read)
|
||||
assertEquals(25L, stats.imported)
|
||||
assertEquals(0L, stats.rejected)
|
||||
assertEquals(0L, stats.invalid)
|
||||
assertEquals(0L, stats.malformed)
|
||||
|
||||
val out = StringBuilder()
|
||||
val exported = ImportExport.export(store, out)
|
||||
assertEquals(25L, exported)
|
||||
|
||||
val backIds =
|
||||
out
|
||||
.trim()
|
||||
.lineSequence()
|
||||
.map { OptimizedJsonMapper.fromJson(it).id }
|
||||
.toSet()
|
||||
assertEquals(events.map { it.id }.toSet(), backIds, "every imported event must round-trip through export")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun reimport_countsDuplicatesAsRejected() =
|
||||
runBlocking {
|
||||
val events = SyntheticEvents.batch(count = 10)
|
||||
ImportExport.import(store, ndjson(events), verify = false)
|
||||
|
||||
val second = ImportExport.import(store, ndjson(events), verify = false)
|
||||
assertEquals(10L, second.read)
|
||||
assertEquals(0L, second.imported)
|
||||
assertEquals(10L, second.rejected, "the store's unique-id constraint drops the re-import")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun malformedAndBlankLines_areSkipped() =
|
||||
runBlocking {
|
||||
val good = SyntheticEvents.batch(count = 3)
|
||||
val lines =
|
||||
sequenceOf(
|
||||
good[0].toJson(),
|
||||
"",
|
||||
"{not a valid event}",
|
||||
good[1].toJson(),
|
||||
" ",
|
||||
"[\"NOTANEVENT\"]",
|
||||
good[2].toJson(),
|
||||
)
|
||||
val stats = ImportExport.import(store, lines, verify = false)
|
||||
|
||||
assertEquals(5L, stats.read, "blank lines are not counted as read")
|
||||
assertEquals(3L, stats.imported)
|
||||
assertEquals(2L, stats.malformed)
|
||||
assertEquals(0L, stats.invalid)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun verify_rejectsBadSignaturesButKeepsValidOnes() =
|
||||
runBlocking {
|
||||
// Fake events carry a syntactically-valid but cryptographically-wrong
|
||||
// signature — verification must drop them all.
|
||||
val fakes = SyntheticEvents.batch(count = 8)
|
||||
val fakeStats = ImportExport.import(store, ndjson(fakes), verify = true)
|
||||
assertEquals(8L, fakeStats.read)
|
||||
assertEquals(0L, fakeStats.imported)
|
||||
assertEquals(8L, fakeStats.invalid, "bad signatures must be rejected under verify")
|
||||
|
||||
// Genuinely-signed events (a fresh key, real Schnorr signatures) must
|
||||
// pass verification and land in the store.
|
||||
val signer = NostrSignerSync(KeyPair())
|
||||
val real = (1..5).map { signer.sign(TextNoteEvent.build("import verify $it", createdAt = it.toLong())) }
|
||||
val realStats = ImportExport.import(store, ndjson(real), verify = true)
|
||||
assertEquals(5L, realStats.read)
|
||||
assertEquals(0L, realStats.invalid, "correctly-signed events must not be flagged invalid")
|
||||
assertEquals(5L, realStats.imported, "valid events must be admitted under verify")
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -65,7 +65,7 @@ class MergeQueryCorrectnessTest {
|
||||
out[(w * 8 + b) * 2 + 1] = hex[byte and 0xF]
|
||||
}
|
||||
}
|
||||
return String(out)
|
||||
return out.concatToString()
|
||||
}
|
||||
|
||||
private val sig = "0".repeat(128)
|
||||
|
||||
@@ -8,8 +8,9 @@ All four source→sink pairings, **all NIP-77 negentropy**, empty sink pulls the
|
||||
same ~1M damus.io corpus, run sequentially (no contention). geode sink
|
||||
configured like strfry: **verify ON, FTS OFF** (`-DsyncVerify=true -DsyncFts=false`);
|
||||
`strfry sync` verifies too and has no FTS. Sinks: geode = its negentropy client;
|
||||
strfry = `strfry sync --dir down`. Sources: `CorpusServerMain` (geode) + strfry
|
||||
relay, each holding the corpus.
|
||||
strfry = `strfry sync --dir down`. Sources: a geode relay seeded with
|
||||
`geode import <corpus.ndjson> --db …` then served normally, and a strfry relay
|
||||
(`strfry import` + serve) — each holding the corpus.
|
||||
|
||||
| # | source → sink | sink engine | synced | time | throughput |
|
||||
|---|---------------|-------------|--------|------|------------|
|
||||
|
||||
Reference in New Issue
Block a user