diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt index 7dcf218097..3ccb80af07 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt @@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.server.policies.OptionalAuthPoli import com.vitorpamplona.quartz.nip01Core.relay.server.policies.RejectFutureEventsPolicy import com.vitorpamplona.quartz.nip01Core.relay.server.policies.VerifyAuthOnlyPolicy import com.vitorpamplona.quartz.nip01Core.relay.server.policies.VerifyPolicy +import com.vitorpamplona.quartz.nip01Core.store.NdjsonImportExport import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip77Negentropy.NegentropySettings import kotlinx.coroutines.CancellationException @@ -152,11 +153,11 @@ private fun runImport(args: Array) { val stats = runBlocking { if (a.positionals.isEmpty()) { - System.`in`.bufferedReader().useLines { ImportExport.import(ctx.store, it, verify) } + System.`in`.bufferedReader().useLines { NdjsonImportExport.import(ctx.store, it, verify) } } else { - var acc = ImportExport.ImportStats.ZERO + var acc = NdjsonImportExport.ImportStats.ZERO for (file in a.positionals) { - acc += File(file).bufferedReader().useLines { ImportExport.import(ctx.store, it, verify) } + acc += File(file).bufferedReader().useLines { NdjsonImportExport.import(ctx.store, it, verify) } } acc } @@ -176,7 +177,7 @@ private fun runExport(args: Array) { val ctx = openStore(a) try { val out = System.out.bufferedWriter() - val n = runBlocking { ImportExport.export(ctx.store, out) } + val n = runBlocking { NdjsonImportExport.export(ctx.store, out) } out.flush() System.err.println("geode export: $n events from ${ctx.dbFile ?: "(in-memory — empty)"}") } finally { diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/ImportExport.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExport.kt similarity index 74% rename from geode/src/main/kotlin/com/vitorpamplona/geode/ImportExport.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExport.kt index 62bf6b10c7..d7c85a742a 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/ImportExport.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExport.kt @@ -18,26 +18,31 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.geode +package com.vitorpamplona.quartz.nip01Core.store import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.crypto.verify import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.store.IEventStore /** - * Bulk NDJSON import/export for a geode store — the `geode import` / `geode export` - * verbs, geode's equivalent of `strfry import` / `strfry export`. One JSON event - * per line (the same on-the-wire event object, no envelope), which is the de-facto - * interchange format across relays (strfry dumps, corpus files, backups). + * Bulk NDJSON import/export for any [IEventStore] — one JSON event per line (the + * on-the-wire event object, no envelope), which is the de-facto interchange format + * across the Nostr ecosystem (`strfry import`/`export` dumps, corpus files, relay + * backups, migrations). * - * Both directions stream: memory is bounded to one batch (import) or one event - * (export) regardless of corpus size, so a multi-million-event dump round-trips in - * roughly constant memory. + * Protocol/store logic only: it operates purely on the [IEventStore] interface and + * Quartz event types, so any Quartz consumer — a relay (geode's `import`/`export` + * verbs), the `amy` CLI, a desktop backup/restore — can reuse it. The caller owns + * the byte plumbing (files, stdin/stdout, compression): [import] takes a line + * [Sequence] and [export] writes to an [Appendable]. + * + * Both directions stream: memory is bounded to one batch ([import]) or one event + * ([export]) regardless of corpus size, so a multi-million-event dump round-trips + * in roughly constant memory. */ -object ImportExport { - /** Events per [IEventStore.batchInsert]; one transaction per batch. */ +object NdjsonImportExport { + /** Events per [IEventStore.batchInsert]; one store transaction per batch. */ const val BATCH = 10_000 class ImportStats( @@ -62,11 +67,10 @@ object ImportExport { /** * Reads one JSON event per line from [lines] and batch-inserts them into * [store]. When [verify], each event's Schnorr signature is checked with the - * same `Event.verify()` the relay's `VerifyPolicy` uses, and a bad signature - * is counted ([ImportStats.invalid]) and skipped — so `import` upholds the - * relay's verify-by-default stance rather than trusting the file. Duplicates - * are dropped by the store's unique-id constraint and counted as - * [ImportStats.rejected]. + * same `Event.verify()` a relay's `VerifyPolicy` uses, and a bad signature is + * counted ([ImportStats.invalid]) and skipped — so an import upholds a relay's + * verify-by-default stance rather than trusting the file. Duplicates are dropped + * by the store's unique-id constraint and counted as [ImportStats.rejected]. */ suspend fun import( store: IEventStore, diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/ImportExportTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExportTest.kt similarity index 75% rename from geode/src/test/kotlin/com/vitorpamplona/geode/ImportExportTest.kt rename to quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExportTest.kt index 3e12cedea4..fa8693a59f 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/ImportExportTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExportTest.kt @@ -18,9 +18,8 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.geode +package com.vitorpamplona.quartz.nip01Core.store -import com.vitorpamplona.geode.fixtures.SyntheticEvents import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair @@ -28,17 +27,18 @@ import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.utils.EventFactory import kotlinx.coroutines.runBlocking import kotlin.test.AfterTest import kotlin.test.Test import kotlin.test.assertEquals /** - * Guards the `geode import` / `geode export` NDJSON round-trip: import counts, + * Guards [NdjsonImportExport]: the NDJSON import/export round-trip, its counts, * duplicate handling, malformed-line skipping, and — the security-relevant part — - * that verification actually gates a bad signature while still admitting a good one. + * that verification gates a bad signature while still admitting a good one. */ -class ImportExportTest { +class NdjsonImportExportTest { private val store = EventStore( dbName = null, @@ -53,13 +53,20 @@ class ImportExportTest { @AfterTest fun tearDown() = store.close() + private val fakeSig = "0".repeat(128) + + private fun hex64(n: Int): String = n.toString(16).padStart(64, '0') + + /** A structurally-valid kind-1 event with a fake (cryptographically-invalid) sig. */ + private fun fake(i: Int): Event = EventFactory.create(hex64(i), hex64(1_000_000 + i), i.toLong(), 1, emptyArray(), "note-$i", fakeSig) + private fun ndjson(events: List): Sequence = events.asSequence().map { it.toJson() } @Test fun importThenExport_roundTrips() = runBlocking { - val events = SyntheticEvents.batch(count = 25) - val stats = ImportExport.import(store, ndjson(events), verify = false) + val events = (1..25).map { fake(it) } + val stats = NdjsonImportExport.import(store, ndjson(events), verify = false) assertEquals(25L, stats.read) assertEquals(25L, stats.imported) @@ -68,7 +75,7 @@ class ImportExportTest { assertEquals(0L, stats.malformed) val out = StringBuilder() - val exported = ImportExport.export(store, out) + val exported = NdjsonImportExport.export(store, out) assertEquals(25L, exported) val backIds = @@ -83,10 +90,10 @@ class ImportExportTest { @Test fun reimport_countsDuplicatesAsRejected() = runBlocking { - val events = SyntheticEvents.batch(count = 10) - ImportExport.import(store, ndjson(events), verify = false) + val events = (1..10).map { fake(it) } + NdjsonImportExport.import(store, ndjson(events), verify = false) - val second = ImportExport.import(store, ndjson(events), verify = false) + val second = NdjsonImportExport.import(store, ndjson(events), verify = false) assertEquals(10L, second.read) assertEquals(0L, second.imported) assertEquals(10L, second.rejected, "the store's unique-id constraint drops the re-import") @@ -95,7 +102,7 @@ class ImportExportTest { @Test fun malformedAndBlankLines_areSkipped() = runBlocking { - val good = SyntheticEvents.batch(count = 3) + val good = (1..3).map { fake(it) } val lines = sequenceOf( good[0].toJson(), @@ -106,7 +113,7 @@ class ImportExportTest { "[\"NOTANEVENT\"]", good[2].toJson(), ) - val stats = ImportExport.import(store, lines, verify = false) + val stats = NdjsonImportExport.import(store, lines, verify = false) assertEquals(5L, stats.read, "blank lines are not counted as read") assertEquals(3L, stats.imported) @@ -119,17 +126,16 @@ class ImportExportTest { runBlocking { // Fake events carry a syntactically-valid but cryptographically-wrong // signature — verification must drop them all. - val fakes = SyntheticEvents.batch(count = 8) - val fakeStats = ImportExport.import(store, ndjson(fakes), verify = true) + val fakes = (1..8).map { fake(it) } + val fakeStats = NdjsonImportExport.import(store, ndjson(fakes), verify = true) assertEquals(8L, fakeStats.read) assertEquals(0L, fakeStats.imported) assertEquals(8L, fakeStats.invalid, "bad signatures must be rejected under verify") - // Genuinely-signed events (a fresh key, real Schnorr signatures) must - // pass verification and land in the store. + // Genuinely-signed events (fresh key, real Schnorr signatures) must pass. val signer = NostrSignerSync(KeyPair()) val real = (1..5).map { signer.sign(TextNoteEvent.build("import verify $it", createdAt = it.toLong())) } - val realStats = ImportExport.import(store, ndjson(real), verify = true) + val realStats = NdjsonImportExport.import(store, ndjson(real), verify = true) assertEquals(5L, realStats.read) assertEquals(0L, realStats.invalid, "correctly-signed events must not be flagged invalid") assertEquals(5L, realStats.imported, "valid events must be admitted under verify")