mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-09 08:04:45 +00:00
refactor(quartz): move NDJSON import/export into Quartz as store logic
The `import`/`export` engine is pure protocol/store logic — it operates only on
the `IEventStore` interface and Quartz event types (Event, OptimizedJsonMapper,
verify, Filter), with zero geode dependency — so per the sharing philosophy
("quartz = Nostr business logic, protocol, data") it belongs in Quartz, not in
the geode app. Any Quartz consumer (a relay, the `amy` CLI, a desktop
backup/restore) can now reuse it.
- move `com.vitorpamplona.geode.ImportExport` →
`com.vitorpamplona.quartz.nip01Core.store.NdjsonImportExport` (commonMain,
next to IEventStore); rename for a clear library-level name.
- geode keeps only the CLI glue (verb dispatch, arg parsing, file/stdin/stdout,
the stderr summary) in Main.kt, delegating to the Quartz engine.
- move the test into quartz jvmTest, rebuilt on Quartz's own EventFactory +
NostrSignerSync (real Schnorr signing) instead of geode fixtures.
No behavior change — `geode import`/`export` work exactly as before (verified
end-to-end previously); this is purely where the code lives.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012EZeWww5TJnzBZKPoc6mvU
This commit is contained in:
Claude
@@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.server.policies.OptionalAuthPoli
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.server.policies.RejectFutureEventsPolicy
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.server.policies.VerifyAuthOnlyPolicy
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.server.policies.VerifyPolicy
|
||||
import com.vitorpamplona.quartz.nip01Core.store.NdjsonImportExport
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
|
||||
import com.vitorpamplona.quartz.nip77Negentropy.NegentropySettings
|
||||
import kotlinx.coroutines.CancellationException
|
||||
@@ -152,11 +153,11 @@ private fun runImport(args: Array<String>) {
|
||||
val stats =
|
||||
runBlocking {
|
||||
if (a.positionals.isEmpty()) {
|
||||
System.`in`.bufferedReader().useLines { ImportExport.import(ctx.store, it, verify) }
|
||||
System.`in`.bufferedReader().useLines { NdjsonImportExport.import(ctx.store, it, verify) }
|
||||
} else {
|
||||
var acc = ImportExport.ImportStats.ZERO
|
||||
var acc = NdjsonImportExport.ImportStats.ZERO
|
||||
for (file in a.positionals) {
|
||||
acc += File(file).bufferedReader().useLines { ImportExport.import(ctx.store, it, verify) }
|
||||
acc += File(file).bufferedReader().useLines { NdjsonImportExport.import(ctx.store, it, verify) }
|
||||
}
|
||||
acc
|
||||
}
|
||||
@@ -176,7 +177,7 @@ private fun runExport(args: Array<String>) {
|
||||
val ctx = openStore(a)
|
||||
try {
|
||||
val out = System.out.bufferedWriter()
|
||||
val n = runBlocking { ImportExport.export(ctx.store, out) }
|
||||
val n = runBlocking { NdjsonImportExport.export(ctx.store, out) }
|
||||
out.flush()
|
||||
System.err.println("geode export: $n events from ${ctx.dbFile ?: "(in-memory — empty)"}")
|
||||
} finally {
|
||||
|
||||
+20
-16
@@ -18,26 +18,31 @@
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.geode
|
||||
package com.vitorpamplona.quartz.nip01Core.store
|
||||
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.verify
|
||||
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
|
||||
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
|
||||
|
||||
/**
|
||||
* Bulk NDJSON import/export for a geode store — the `geode import` / `geode export`
|
||||
* verbs, geode's equivalent of `strfry import` / `strfry export`. One JSON event
|
||||
* per line (the same on-the-wire event object, no envelope), which is the de-facto
|
||||
* interchange format across relays (strfry dumps, corpus files, backups).
|
||||
* Bulk NDJSON import/export for any [IEventStore] — one JSON event per line (the
|
||||
* on-the-wire event object, no envelope), which is the de-facto interchange format
|
||||
* across the Nostr ecosystem (`strfry import`/`export` dumps, corpus files, relay
|
||||
* backups, migrations).
|
||||
*
|
||||
* Both directions stream: memory is bounded to one batch (import) or one event
|
||||
* (export) regardless of corpus size, so a multi-million-event dump round-trips in
|
||||
* roughly constant memory.
|
||||
* Protocol/store logic only: it operates purely on the [IEventStore] interface and
|
||||
* Quartz event types, so any Quartz consumer — a relay (geode's `import`/`export`
|
||||
* verbs), the `amy` CLI, a desktop backup/restore — can reuse it. The caller owns
|
||||
* the byte plumbing (files, stdin/stdout, compression): [import] takes a line
|
||||
* [Sequence] and [export] writes to an [Appendable].
|
||||
*
|
||||
* Both directions stream: memory is bounded to one batch ([import]) or one event
|
||||
* ([export]) regardless of corpus size, so a multi-million-event dump round-trips
|
||||
* in roughly constant memory.
|
||||
*/
|
||||
object ImportExport {
|
||||
/** Events per [IEventStore.batchInsert]; one transaction per batch. */
|
||||
object NdjsonImportExport {
|
||||
/** Events per [IEventStore.batchInsert]; one store transaction per batch. */
|
||||
const val BATCH = 10_000
|
||||
|
||||
class ImportStats(
|
||||
@@ -62,11 +67,10 @@ object ImportExport {
|
||||
/**
|
||||
* Reads one JSON event per line from [lines] and batch-inserts them into
|
||||
* [store]. When [verify], each event's Schnorr signature is checked with the
|
||||
* same `Event.verify()` the relay's `VerifyPolicy` uses, and a bad signature
|
||||
* is counted ([ImportStats.invalid]) and skipped — so `import` upholds the
|
||||
* relay's verify-by-default stance rather than trusting the file. Duplicates
|
||||
* are dropped by the store's unique-id constraint and counted as
|
||||
* [ImportStats.rejected].
|
||||
* same `Event.verify()` a relay's `VerifyPolicy` uses, and a bad signature is
|
||||
* counted ([ImportStats.invalid]) and skipped — so an import upholds a relay's
|
||||
* verify-by-default stance rather than trusting the file. Duplicates are dropped
|
||||
* by the store's unique-id constraint and counted as [ImportStats.rejected].
|
||||
*/
|
||||
suspend fun import(
|
||||
store: IEventStore,
|
||||
+24
-18
@@ -18,9 +18,8 @@
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.geode
|
||||
package com.vitorpamplona.quartz.nip01Core.store
|
||||
|
||||
import com.vitorpamplona.geode.fixtures.SyntheticEvents
|
||||
import com.vitorpamplona.quartz.nip01Core.core.Event
|
||||
import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
|
||||
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
|
||||
@@ -28,17 +27,18 @@ import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
|
||||
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
|
||||
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
|
||||
import com.vitorpamplona.quartz.utils.EventFactory
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlin.test.AfterTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
/**
|
||||
* Guards the `geode import` / `geode export` NDJSON round-trip: import counts,
|
||||
* Guards [NdjsonImportExport]: the NDJSON import/export round-trip, its counts,
|
||||
* duplicate handling, malformed-line skipping, and — the security-relevant part —
|
||||
* that verification actually gates a bad signature while still admitting a good one.
|
||||
* that verification gates a bad signature while still admitting a good one.
|
||||
*/
|
||||
class ImportExportTest {
|
||||
class NdjsonImportExportTest {
|
||||
private val store =
|
||||
EventStore(
|
||||
dbName = null,
|
||||
@@ -53,13 +53,20 @@ class ImportExportTest {
|
||||
@AfterTest
|
||||
fun tearDown() = store.close()
|
||||
|
||||
private val fakeSig = "0".repeat(128)
|
||||
|
||||
private fun hex64(n: Int): String = n.toString(16).padStart(64, '0')
|
||||
|
||||
/** A structurally-valid kind-1 event with a fake (cryptographically-invalid) sig. */
|
||||
private fun fake(i: Int): Event = EventFactory.create(hex64(i), hex64(1_000_000 + i), i.toLong(), 1, emptyArray(), "note-$i", fakeSig)
|
||||
|
||||
private fun ndjson(events: List<Event>): Sequence<String> = events.asSequence().map { it.toJson() }
|
||||
|
||||
@Test
|
||||
fun importThenExport_roundTrips() =
|
||||
runBlocking {
|
||||
val events = SyntheticEvents.batch(count = 25)
|
||||
val stats = ImportExport.import(store, ndjson(events), verify = false)
|
||||
val events = (1..25).map { fake(it) }
|
||||
val stats = NdjsonImportExport.import(store, ndjson(events), verify = false)
|
||||
|
||||
assertEquals(25L, stats.read)
|
||||
assertEquals(25L, stats.imported)
|
||||
@@ -68,7 +75,7 @@ class ImportExportTest {
|
||||
assertEquals(0L, stats.malformed)
|
||||
|
||||
val out = StringBuilder()
|
||||
val exported = ImportExport.export(store, out)
|
||||
val exported = NdjsonImportExport.export(store, out)
|
||||
assertEquals(25L, exported)
|
||||
|
||||
val backIds =
|
||||
@@ -83,10 +90,10 @@ class ImportExportTest {
|
||||
@Test
|
||||
fun reimport_countsDuplicatesAsRejected() =
|
||||
runBlocking {
|
||||
val events = SyntheticEvents.batch(count = 10)
|
||||
ImportExport.import(store, ndjson(events), verify = false)
|
||||
val events = (1..10).map { fake(it) }
|
||||
NdjsonImportExport.import(store, ndjson(events), verify = false)
|
||||
|
||||
val second = ImportExport.import(store, ndjson(events), verify = false)
|
||||
val second = NdjsonImportExport.import(store, ndjson(events), verify = false)
|
||||
assertEquals(10L, second.read)
|
||||
assertEquals(0L, second.imported)
|
||||
assertEquals(10L, second.rejected, "the store's unique-id constraint drops the re-import")
|
||||
@@ -95,7 +102,7 @@ class ImportExportTest {
|
||||
@Test
|
||||
fun malformedAndBlankLines_areSkipped() =
|
||||
runBlocking {
|
||||
val good = SyntheticEvents.batch(count = 3)
|
||||
val good = (1..3).map { fake(it) }
|
||||
val lines =
|
||||
sequenceOf(
|
||||
good[0].toJson(),
|
||||
@@ -106,7 +113,7 @@ class ImportExportTest {
|
||||
"[\"NOTANEVENT\"]",
|
||||
good[2].toJson(),
|
||||
)
|
||||
val stats = ImportExport.import(store, lines, verify = false)
|
||||
val stats = NdjsonImportExport.import(store, lines, verify = false)
|
||||
|
||||
assertEquals(5L, stats.read, "blank lines are not counted as read")
|
||||
assertEquals(3L, stats.imported)
|
||||
@@ -119,17 +126,16 @@ class ImportExportTest {
|
||||
runBlocking {
|
||||
// Fake events carry a syntactically-valid but cryptographically-wrong
|
||||
// signature — verification must drop them all.
|
||||
val fakes = SyntheticEvents.batch(count = 8)
|
||||
val fakeStats = ImportExport.import(store, ndjson(fakes), verify = true)
|
||||
val fakes = (1..8).map { fake(it) }
|
||||
val fakeStats = NdjsonImportExport.import(store, ndjson(fakes), verify = true)
|
||||
assertEquals(8L, fakeStats.read)
|
||||
assertEquals(0L, fakeStats.imported)
|
||||
assertEquals(8L, fakeStats.invalid, "bad signatures must be rejected under verify")
|
||||
|
||||
// Genuinely-signed events (a fresh key, real Schnorr signatures) must
|
||||
// pass verification and land in the store.
|
||||
// Genuinely-signed events (fresh key, real Schnorr signatures) must pass.
|
||||
val signer = NostrSignerSync(KeyPair())
|
||||
val real = (1..5).map { signer.sign(TextNoteEvent.build("import verify $it", createdAt = it.toLong())) }
|
||||
val realStats = ImportExport.import(store, ndjson(real), verify = true)
|
||||
val realStats = NdjsonImportExport.import(store, ndjson(real), verify = true)
|
||||
assertEquals(5L, realStats.read)
|
||||
assertEquals(0L, realStats.invalid, "correctly-signed events must not be flagged invalid")
|
||||
assertEquals(5L, realStats.imported, "valid events must be admitted under verify")
|
||||
Reference in New Issue
Block a user