From 5db2543cfc14afe42d764ba7108872d711c42b89 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 13:56:11 +0000 Subject: [PATCH] feat(geode): add `import` / `export` NDJSON verbs; drop the benchmark-only server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bulk NDJSON import/export as first-class geode subcommands, mirroring `strfry import` / `strfry export` (one JSON event per line — the interchange format for seeding a relay, migrating between relays, or taking a backup): geode import [--db …] [--no-verify] [FILE…] # files, or stdin when none geode export [--db …] # NDJSON to stdout Both stream — memory is bounded to one batch (import) / one event (export), so a multi-million-event corpus round-trips in roughly constant memory. `import` verifies signatures by default (same `Event.verify()` the relay's VerifyPolicy uses), upholding the relay's verify-by-default stance rather than trusting the file; `--no-verify` is the trusted-input escape hatch. Verb dispatch is backward-compatible: a bare `geode --port …` (no verb) still serves. This makes the benchmark-only `CorpusServerMain` redundant — a corpus source is now just `geode import` into a DB, then a normal `geode` serve — so it's deleted, removing benchmark-only code from the production geode artifact (the question that started this). The 1M sync-throughput plan is updated to describe sources via `geode import` + serve. Also fixes a native-target CI break: MergeQueryCorrectnessTest used the deprecated `String(CharArray)` (error-level on Kotlin/Native) — switched to `CharArray.concatToString()`. Verified end-to-end through the packaged `geode` binary: import (file + stdin, --no-verify), export round-trip, and verify-on rejecting bad signatures. ImportExportTest covers the counts, duplicate handling, malformed-line skipping, and verify accepting a freshly-signed event while rejecting bad sigs. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012EZeWww5TJnzBZKPoc6mvU --- .../com/vitorpamplona/geode/ImportExport.kt | 129 +++++++++++++++++ .../kotlin/com/vitorpamplona/geode/Main.kt | 137 ++++++++++++++++-- .../geode/tools/CorpusServerMain.kt | 122 ---------------- .../vitorpamplona/geode/ImportExportTest.kt | 137 ++++++++++++++++++ .../store/sqlite/MergeQueryCorrectnessTest.kt | 2 +- .../plans/2026-07-04-sync-throughput-1m.md | 5 +- 6 files changed, 395 insertions(+), 137 deletions(-) create mode 100644 geode/src/main/kotlin/com/vitorpamplona/geode/ImportExport.kt delete mode 100644 geode/src/main/kotlin/com/vitorpamplona/geode/tools/CorpusServerMain.kt create mode 100644 geode/src/test/kotlin/com/vitorpamplona/geode/ImportExportTest.kt diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/ImportExport.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/ImportExport.kt new file mode 100644 index 0000000000..62bf6b10c7 --- /dev/null +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/ImportExport.kt @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.geode + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper +import com.vitorpamplona.quartz.nip01Core.crypto.verify +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.IEventStore + +/** + * Bulk NDJSON import/export for a geode store — the `geode import` / `geode export` + * verbs, geode's equivalent of `strfry import` / `strfry export`. One JSON event + * per line (the same on-the-wire event object, no envelope), which is the de-facto + * interchange format across relays (strfry dumps, corpus files, backups). + * + * Both directions stream: memory is bounded to one batch (import) or one event + * (export) regardless of corpus size, so a multi-million-event dump round-trips in + * roughly constant memory. + */ +object ImportExport { + /** Events per [IEventStore.batchInsert]; one transaction per batch. */ + const val BATCH = 10_000 + + class ImportStats( + /** Non-blank lines read. */ + val read: Long, + /** Events newly stored. */ + val imported: Long, + /** Events the store rejected — overwhelmingly duplicates (unique-id). */ + val rejected: Long, + /** Events dropped for a bad signature (only when verifying). */ + val invalid: Long, + /** Lines that didn't parse as a NIP-01 event. */ + val malformed: Long, + ) { + operator fun plus(o: ImportStats) = ImportStats(read + o.read, imported + o.imported, rejected + o.rejected, invalid + o.invalid, malformed + o.malformed) + + companion object { + val ZERO = ImportStats(0, 0, 0, 0, 0) + } + } + + /** + * Reads one JSON event per line from [lines] and batch-inserts them into + * [store]. When [verify], each event's Schnorr signature is checked with the + * same `Event.verify()` the relay's `VerifyPolicy` uses, and a bad signature + * is counted ([ImportStats.invalid]) and skipped — so `import` upholds the + * relay's verify-by-default stance rather than trusting the file. Duplicates + * are dropped by the store's unique-id constraint and counted as + * [ImportStats.rejected]. + */ + suspend fun import( + store: IEventStore, + lines: Sequence, + verify: Boolean, + batchSize: Int = BATCH, + ): ImportStats { + var read = 0L + var imported = 0L + var rejected = 0L + var invalid = 0L + var malformed = 0L + val batch = ArrayList(batchSize) + + suspend fun flush() { + if (batch.isEmpty()) return + for (outcome in store.batchInsert(batch)) { + when (outcome) { + IEventStore.InsertOutcome.Accepted -> imported++ + is IEventStore.InsertOutcome.Rejected -> rejected++ + } + } + batch.clear() + } + + for (line in lines) { + if (line.isBlank()) continue + read++ + val event = runCatching { OptimizedJsonMapper.fromJson(line) }.getOrNull() + if (event == null) { + malformed++ + continue + } + if (verify && !event.verify()) { + invalid++ + continue + } + batch.add(event) + if (batch.size >= batchSize) flush() + } + flush() + return ImportStats(read, imported, rejected, invalid, malformed) + } + + /** + * Streams every stored event as NDJSON (one compact JSON object per line, no + * trailing whitespace) to [out], newest-first. Returns the count written. + */ + suspend fun export( + store: IEventStore, + out: Appendable, + ): Long { + var n = 0L + store.query(Filter()) { event -> + out.append(event.toJson()).append('\n') + n++ + } + return n + } +} diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt index 9c6acab5f1..7dcf218097 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt @@ -49,15 +49,30 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking import java.io.File /** - * Standalone entry point. + * Standalone entry point. The first argument may be a verb: + * + * geode [relay] [flags] serve the relay (default when no verb is given) + * geode import [flags] [FILE…] bulk-load NDJSON events into the store + * geode export [flags] dump the store as NDJSON to stdout + * + * `import`/`export` are geode's equivalent of `strfry import` / `strfry export`: + * one JSON event per line, the interchange format for seeding a relay, migrating + * between relays, or taking a backup. `import` reads the given files (or stdin + * when none are named), verifies signatures by default (same as the relay; + * `--no-verify` to skip), and prints a read/imported/rejected summary to stderr. + * `export` streams every stored event, newest-first, to stdout. Both stream, so + * a multi-million-event corpus round-trips in roughly constant memory. They share + * the relay's `--db`/`--config`/`--no-search` flags so the same store is targeted. * * Run with: * ./gradlew :geode:run --args="--config /etc/geode.toml" + * ./gradlew :geode:run --args="import --db relay.sqlite corpus.ndjson" * or - * java -cp ... com.vitorpamplona.geode.MainKt --port 7447 --verify + * java -cp ... com.vitorpamplona.geode.MainKt --port 7447 * * Configuration precedence (highest to lowest): * 1. CLI flags (`--host`, `--port`, …) @@ -91,6 +106,85 @@ import java.io.File * per-event tokenization cost on ingest. */ fun main(args: Array) { + when (args.firstOrNull()?.takeUnless { it.startsWith("--") }) { + "import" -> runImport(args.copyOfRange(1, args.size)) + "export" -> runExport(args.copyOfRange(1, args.size)) + // Explicit `relay` verb or no verb at all → serve. A bare `geode --port …` + // (no verb) stays valid so existing invocations don't change. + "relay" -> serve(args.copyOfRange(1, args.size)) + else -> serve(args) + } +} + +/** + * Opens the store the `import`/`export` verbs operate on, honoring the same + * `--db`/`--config`/`--no-search` selection the relay uses so a verb targets the + * exact store the server would. + */ +private class StoreContext( + val dbFile: String?, + val store: EventStore, +) + +private fun openStore(a: Args): StoreContext { + val config = a.opt("--config")?.let { StaticConfig.fromFile(File(it)) } ?: StaticConfig() + val dbFile = a.opt("--db") ?: config.database.file?.takeUnless { config.database.in_memory } + val fullTextSearch = !a.flag("--no-search") && config.options.full_text_search + val store = + EventStore( + dbName = dbFile, + indexStrategy = relayIndexingStrategy(fullTextSearch, config.negentropy.live_index), + numReaders = config.database.readers ?: 4, + ) + return StoreContext(dbFile, store) +} + +private fun runImport(args: Array) { + val a = parseArgs(args) + val config = a.opt("--config")?.let { StaticConfig.fromFile(File(it)) } ?: StaticConfig() + // Verify by default, matching the relay's stance — `import` won't trust a + // file's signatures any more than the relay trusts a client's. `--no-verify` + // is the trusted-input escape hatch (fixture replay, a dump from a relay you + // already trust). + val verify = !a.flag("--no-verify") && config.options.verify_signatures + val ctx = openStore(a) + try { + val stats = + runBlocking { + if (a.positionals.isEmpty()) { + System.`in`.bufferedReader().useLines { ImportExport.import(ctx.store, it, verify) } + } else { + var acc = ImportExport.ImportStats.ZERO + for (file in a.positionals) { + acc += File(file).bufferedReader().useLines { ImportExport.import(ctx.store, it, verify) } + } + acc + } + } + System.err.println( + "geode import: read=${stats.read} imported=${stats.imported} " + + "rejected=${stats.rejected} invalid-sig=${stats.invalid} malformed=${stats.malformed} " + + "→ ${ctx.dbFile ?: "(in-memory — not persisted; pass --db)"}", + ) + } finally { + ctx.store.close() + } +} + +private fun runExport(args: Array) { + val a = parseArgs(args) + val ctx = openStore(a) + try { + val out = System.out.bufferedWriter() + val n = runBlocking { ImportExport.export(ctx.store, out) } + out.flush() + System.err.println("geode export: $n events from ${ctx.dbFile ?: "(in-memory — empty)"}") + } finally { + ctx.store.close() + } +} + +private fun serve(args: Array) { val a = parseArgs(args) val config: StaticConfig = @@ -363,15 +457,26 @@ private fun composePolicy( private class Args( private val opts: Map, private val flags: Set, + /** Non-`--` operands, in order (e.g. the NDJSON files for `import`). */ + val positionals: List, ) { fun opt(k: String) = opts[k] fun flag(k: String) = k in flags } +/** + * Boolean flags that never take a value. Listing them explicitly is what lets a + * trailing positional survive after a flag — `import --no-verify corpus.ndjson` + * must read `corpus.ndjson` as a file, not as `--no-verify`'s value. + */ +private val BOOLEAN_FLAGS = + setOf("--auth", "--optional-auth", "--no-verify", "--no-parallel-verify", "--no-search") + private fun parseArgs(args: Array): Args { val opts = mutableMapOf() val flags = mutableSetOf() + val positionals = mutableListOf() var i = 0 while (i < args.size) { val a = args[i] @@ -381,22 +486,30 @@ private fun parseArgs(args: Array): Args { // happen to contain `=` (e.g. NIP-11 contact emails) by // using the space-separated form. val eq = a.indexOf('=') - if (eq > 0) { - opts[a.substring(0, eq)] = a.substring(eq + 1) - i += 1 - } else { - val next = args.getOrNull(i + 1) - if (next != null && !next.startsWith("--")) { - opts[a] = next - i += 2 - } else { + when { + eq > 0 -> { + opts[a.substring(0, eq)] = a.substring(eq + 1) + i += 1 + } + a in BOOLEAN_FLAGS -> { flags += a i += 1 } + else -> { + val next = args.getOrNull(i + 1) + if (next != null && !next.startsWith("--")) { + opts[a] = next + i += 2 + } else { + flags += a + i += 1 + } + } } } else { + positionals += a i += 1 } } - return Args(opts, flags) + return Args(opts, flags, positionals) } diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/tools/CorpusServerMain.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/tools/CorpusServerMain.kt deleted file mode 100644 index 0f92f60a62..0000000000 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/tools/CorpusServerMain.kt +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.geode.tools - -import com.vitorpamplona.geode.KtorRelay -import com.vitorpamplona.geode.RelayEngine -import com.vitorpamplona.geode.RelayIndexingStrategy -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore -import kotlinx.coroutines.runBlocking -import java.io.File -import java.util.concurrent.CountDownLatch - -/** - * Boots a real geode relay (geode's default indexing: FTS + live negentropy - * index) preloaded with a corpus of nostr events read from an NDJSON file, then - * serves forever so external clients — `strfry sync`, another geode, the - * relayBench negentropy sink — can reconcile against it. - * - * A benchmark-only source: it exists so the negentropy sync comparison has a - * geode relay holding the same corpus a strfry source does, reachable over the - * production WebSocket transport. - * - * Usage: `CorpusServerMain [maxCount]` - */ -fun main(args: Array) { - if (args.size < 2) { - System.err.println("usage: CorpusServerMain [maxCount]") - return - } - val port = args[0].toInt() - val corpus = File(args[1]) - val maxCount = args.getOrNull(2)?.toInt() ?: Int.MAX_VALUE - - // File-backed so a 1M in-memory corpus here doesn't compete for RAM with an - // in-memory sink in the same box during the comparison. - val dbFile = "/tmp/geode-source-$port.sqlite" - // Reuse an already-loaded DB (serve-only, skipping the multi-minute reload) - // ONLY when a completion sentinel proves it holds *this* corpus, fully loaded. - // The sentinel is written only after a full load and is keyed on corpus - // identity (path + byte length) and maxCount, so a different/rebuilt corpus, a - // different cap, or a load interrupted mid-way (row count > 0 but incomplete) - // all fail the check and force a clean reload — never silently serving the - // wrong events under a run that reports success. - val sentinel = File("$dbFile.done") - val signature = "${corpus.absolutePath}\t${corpus.length()}\tmax=$maxCount" - val reusable = sentinel.takeIf { it.exists() }?.readText() == signature - if (!reusable) { - sentinel.delete() - listOf(dbFile, "$dbFile-wal", "$dbFile-shm", "$dbFile-journal").forEach { File(it).delete() } - } - val store = EventStore(dbName = dbFile, indexStrategy = RelayIndexingStrategy) - val engine = RelayEngine(url = "ws://127.0.0.1:$port/".normalizeRelayUrl(), store = store) - - var loaded = runBlocking { store.count(Filter()) } - if (reusable && loaded > 0) { - println("CorpusServerMain: reusing existing DB with $loaded events (serve-only)") - } else { - println("CorpusServerMain: loading up to $maxCount events from ${corpus.name}…") - loaded = - runBlocking { - var total = 0 - val batch = ArrayList(10_000) - corpus.bufferedReader().useLines { lines -> - for (line in lines) { - if (total >= maxCount) break - if (line.isBlank()) continue - val event = runCatching { OptimizedJsonMapper.fromJson(line) }.getOrNull() ?: continue - batch.add(event) - if (batch.size == 10_000) { - store.batchInsert(batch) - total += batch.size - batch.clear() - if (total % 200_000 == 0) println(" …loaded $total") - } - } - } - if (batch.isNotEmpty()) { - store.batchInsert(batch) - total += batch.size - } - total - } - // Mark the load complete only now — a crash before this leaves no - // sentinel, so the next run rebuilds instead of serving a partial DB. - sentinel.writeText(signature) - } - val count = runBlocking { store.count(Filter()) } - - val server = KtorRelay(engine, host = "127.0.0.1", port = port).start() - println("CorpusServerMain: READY port=$port loaded=$loaded distinct=$count") - - // Serve until the JVM is killed. - Runtime.getRuntime().addShutdownHook( - Thread { - server.stop(gracePeriodMillis = 0, timeoutMillis = 500) - engine.close() - }, - ) - CountDownLatch(1).await() -} diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/ImportExportTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/ImportExportTest.kt new file mode 100644 index 0000000000..3e12cedea4 --- /dev/null +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/ImportExportTest.kt @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.geode + +import com.vitorpamplona.geode.fixtures.SyntheticEvents +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import kotlinx.coroutines.runBlocking +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Guards the `geode import` / `geode export` NDJSON round-trip: import counts, + * duplicate handling, malformed-line skipping, and — the security-relevant part — + * that verification actually gates a bad signature while still admitting a good one. + */ +class ImportExportTest { + private val store = + EventStore( + dbName = null, + indexStrategy = + DefaultIndexingStrategy( + indexEventsByCreatedAtAlone = true, + indexEventsByPubkeyAlone = true, + indexFullTextSearch = false, + ), + ) + + @AfterTest + fun tearDown() = store.close() + + private fun ndjson(events: List): Sequence = events.asSequence().map { it.toJson() } + + @Test + fun importThenExport_roundTrips() = + runBlocking { + val events = SyntheticEvents.batch(count = 25) + val stats = ImportExport.import(store, ndjson(events), verify = false) + + assertEquals(25L, stats.read) + assertEquals(25L, stats.imported) + assertEquals(0L, stats.rejected) + assertEquals(0L, stats.invalid) + assertEquals(0L, stats.malformed) + + val out = StringBuilder() + val exported = ImportExport.export(store, out) + assertEquals(25L, exported) + + val backIds = + out + .trim() + .lineSequence() + .map { OptimizedJsonMapper.fromJson(it).id } + .toSet() + assertEquals(events.map { it.id }.toSet(), backIds, "every imported event must round-trip through export") + } + + @Test + fun reimport_countsDuplicatesAsRejected() = + runBlocking { + val events = SyntheticEvents.batch(count = 10) + ImportExport.import(store, ndjson(events), verify = false) + + val second = ImportExport.import(store, ndjson(events), verify = false) + assertEquals(10L, second.read) + assertEquals(0L, second.imported) + assertEquals(10L, second.rejected, "the store's unique-id constraint drops the re-import") + } + + @Test + fun malformedAndBlankLines_areSkipped() = + runBlocking { + val good = SyntheticEvents.batch(count = 3) + val lines = + sequenceOf( + good[0].toJson(), + "", + "{not a valid event}", + good[1].toJson(), + " ", + "[\"NOTANEVENT\"]", + good[2].toJson(), + ) + val stats = ImportExport.import(store, lines, verify = false) + + assertEquals(5L, stats.read, "blank lines are not counted as read") + assertEquals(3L, stats.imported) + assertEquals(2L, stats.malformed) + assertEquals(0L, stats.invalid) + } + + @Test + fun verify_rejectsBadSignaturesButKeepsValidOnes() = + runBlocking { + // Fake events carry a syntactically-valid but cryptographically-wrong + // signature — verification must drop them all. + val fakes = SyntheticEvents.batch(count = 8) + val fakeStats = ImportExport.import(store, ndjson(fakes), verify = true) + assertEquals(8L, fakeStats.read) + assertEquals(0L, fakeStats.imported) + assertEquals(8L, fakeStats.invalid, "bad signatures must be rejected under verify") + + // Genuinely-signed events (a fresh key, real Schnorr signatures) must + // pass verification and land in the store. + val signer = NostrSignerSync(KeyPair()) + val real = (1..5).map { signer.sign(TextNoteEvent.build("import verify $it", createdAt = it.toLong())) } + val realStats = ImportExport.import(store, ndjson(real), verify = true) + assertEquals(5L, realStats.read) + assertEquals(0L, realStats.invalid, "correctly-signed events must not be flagged invalid") + assertEquals(5L, realStats.imported, "valid events must be admitted under verify") + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryCorrectnessTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryCorrectnessTest.kt index 1b9d4b1001..f347779cd7 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryCorrectnessTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/MergeQueryCorrectnessTest.kt @@ -65,7 +65,7 @@ class MergeQueryCorrectnessTest { out[(w * 8 + b) * 2 + 1] = hex[byte and 0xF] } } - return String(out) + return out.concatToString() } private val sig = "0".repeat(128) diff --git a/relayBench/plans/2026-07-04-sync-throughput-1m.md b/relayBench/plans/2026-07-04-sync-throughput-1m.md index 5230a2f4fe..c0fca6c830 100644 --- a/relayBench/plans/2026-07-04-sync-throughput-1m.md +++ b/relayBench/plans/2026-07-04-sync-throughput-1m.md @@ -8,8 +8,9 @@ All four source→sink pairings, **all NIP-77 negentropy**, empty sink pulls the same ~1M damus.io corpus, run sequentially (no contention). geode sink configured like strfry: **verify ON, FTS OFF** (`-DsyncVerify=true -DsyncFts=false`); `strfry sync` verifies too and has no FTS. Sinks: geode = its negentropy client; -strfry = `strfry sync --dir down`. Sources: `CorpusServerMain` (geode) + strfry -relay, each holding the corpus. +strfry = `strfry sync --dir down`. Sources: a geode relay seeded with +`geode import --db …` then served normally, and a strfry relay +(`strfry import` + serve) — each holding the corpus. | # | source → sink | sink engine | synced | time | throughput | |---|---------------|-------------|--------|------|------------|