From 29041ef77715b3a91afcc58211de0ee3c36d2a3f Mon Sep 17 00:00:00 2001 From: franzap <_@franzap.com> Date: Sun, 27 Sep 2026 20:07:26 -0300 Subject: [PATCH] Replace the Quartz cache with a verified local catalog store. --- gradle/libs.versions.toml | 12 +- iolite/build.gradle.kts | 13 +- iolite/consumer-rules.pro | 8 +- .../QuartzCompatibilityInstrumentedTest.kt | 199 --- .../main/kotlin/dev/zapstore/iolite/Bech32.kt | 89 + .../main/kotlin/dev/zapstore/iolite/Bolt11.kt | 17 + .../dev/zapstore/iolite/CatalogArtifacts.kt | 115 ++ .../dev/zapstore/iolite/CatalogImporter.kt | 215 +++ .../kotlin/dev/zapstore/iolite/ChaCha20.kt | 66 + .../main/kotlin/dev/zapstore/iolite/Crypto.kt | 45 + .../main/kotlin/dev/zapstore/iolite/Event.kt | 231 +++ .../kotlin/dev/zapstore/iolite/EventRows.kt | 365 ++++ .../main/kotlin/dev/zapstore/iolite/Hex.kt | 33 + .../main/kotlin/dev/zapstore/iolite/Iolite.kt | 1473 ++++++----------- .../dev/zapstore/iolite/IoliteConfig.kt | 36 +- .../kotlin/dev/zapstore/iolite/IoliteStore.kt | 752 +++++++++ .../zapstore/iolite/JdbcSqliteConnection.kt | 100 ++ .../kotlin/dev/zapstore/iolite/ListingRows.kt | 325 ++++ .../main/kotlin/dev/zapstore/iolite/Models.kt | 296 ++++ .../main/kotlin/dev/zapstore/iolite/Nip42.kt | 22 + .../main/kotlin/dev/zapstore/iolite/Nip44.kt | 129 ++ .../dev/zapstore/iolite/OutboxRouter.kt | 119 -- .../main/kotlin/dev/zapstore/iolite/Query.kt | 79 + .../dev/zapstore/iolite/QueryFingerprint.kt | 5 +- .../dev/zapstore/iolite/QueryRefreshCache.kt | 149 -- .../kotlin/dev/zapstore/iolite/QueryTypes.kt | 109 +- .../kotlin/dev/zapstore/iolite/RelayPool.kt | 286 ++++ .../kotlin/dev/zapstore/iolite/RelayUrl.kt | 45 + .../main/kotlin/dev/zapstore/iolite/Schema.kt | 160 ++ .../main/kotlin/dev/zapstore/iolite/Search.kt | 119 ++ .../main/kotlin/dev/zapstore/iolite/Signer.kt | 57 + .../kotlin/dev/zapstore/iolite/TarZstd.kt | 162 ++ .../zapstore/iolite/CatalogArtifactsTest.kt | 180 ++ .../zapstore/iolite/CatalogImporterTest.kt | 492 ++++++ .../dev/zapstore/iolite/EventCryptoTest.kt | 58 + .../zapstore/iolite/IoliteContractsTest.kt | 441 +++++ .../zapstore/iolite/LegacyQueryTestAdapter.kt | 53 - .../kotlin/dev/zapstore/iolite/Nip44Test.kt | 89 + .../dev/zapstore/iolite/ObservationTest.kt | 397 ----- .../dev/zapstore/iolite/OutboxRouterTest.kt | 276 --- .../zapstore/iolite/PublicContractsTest.kt | 70 +- .../dev/zapstore/iolite/QueryInvariantTest.kt | 819 --------- .../zapstore/iolite/QueryRefreshCacheTest.kt | 72 - .../dev/zapstore/iolite/QueryWiringTest.kt | 228 --- .../dev/zapstore/iolite/RelayOutboxTest.kt | 252 +++ .../test/resources/golden/from-0-to-1.tar.zst | Bin 0 -> 1049 bytes .../test/resources/golden/from-0-to-2.tar.zst | Bin 0 -> 1056 bytes .../test/resources/golden/from-1-to-2.tar.zst | Bin 0 -> 1052 bytes iolite/src/test/resources/nip44.vectors.json | 648 ++++++++ 49 files changed, 6475 insertions(+), 3431 deletions(-) delete mode 100644 iolite/src/androidTest/kotlin/dev/zapstore/iolite/QuartzCompatibilityInstrumentedTest.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Bech32.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Bolt11.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/CatalogArtifacts.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/CatalogImporter.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/ChaCha20.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Crypto.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Event.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/EventRows.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Hex.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/IoliteStore.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/JdbcSqliteConnection.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/ListingRows.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Models.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Nip42.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Nip44.kt delete mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/OutboxRouter.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Query.kt delete mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/QueryRefreshCache.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/RelayPool.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/RelayUrl.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Schema.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Search.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/Signer.kt create mode 100644 iolite/src/main/kotlin/dev/zapstore/iolite/TarZstd.kt create mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/CatalogArtifactsTest.kt create mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/CatalogImporterTest.kt create mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/EventCryptoTest.kt create mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/IoliteContractsTest.kt delete mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/LegacyQueryTestAdapter.kt create mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/Nip44Test.kt delete mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/ObservationTest.kt delete mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/OutboxRouterTest.kt delete mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/QueryInvariantTest.kt delete mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/QueryRefreshCacheTest.kt delete mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/QueryWiringTest.kt create mode 100644 iolite/src/test/kotlin/dev/zapstore/iolite/RelayOutboxTest.kt create mode 100644 iolite/src/test/resources/golden/from-0-to-1.tar.zst create mode 100644 iolite/src/test/resources/golden/from-0-to-2.tar.zst create mode 100644 iolite/src/test/resources/golden/from-1-to-2.tar.zst create mode 100644 iolite/src/test/resources/nip44.vectors.json diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 87d23c8..5a6d88c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,7 +1,6 @@ [versions] agp = "9.2.1" kotlin = "2.4.0" -quartz = "1.13.1" coroutines = "1.11.0" coil = "3.5.0" okhttp = "5.4.0" @@ -12,9 +11,15 @@ lifecycle = "2.11.0" navigation = "2.9.8" androidx-test = "1.7.0" junit = "4.13.2" +secp256k1 = "0.23.0" +zstd = "1.5.7-4" +onnxruntime = "1.23.2" [libraries] -quartz = { module = "com.vitorpamplona.quartz:quartz", version.ref = "quartz" } +secp256k1 = { module = "fr.acinq.secp256k1:secp256k1-kmp", version.ref = "secp256k1" } +secp256k1-jni-android = { module = "fr.acinq.secp256k1:secp256k1-kmp-jni-android", version.ref = "secp256k1" } +secp256k1-jni-jvm = { module = "fr.acinq.secp256k1:secp256k1-kmp-jni-jvm", version.ref = "secp256k1" } +zstd-jni = { module = "com.github.luben:zstd-jni", version.ref = "zstd" } coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-android", version.ref = "coroutines" } coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "coroutines" } coil = { module = "io.coil-kt.coil3:coil", version.ref = "coil" } @@ -26,7 +31,6 @@ sqlite = { module = "androidx.sqlite:sqlite", version.ref = "sqlite" } sqlite-bundled = { module = "androidx.sqlite:sqlite-bundled", version.ref = "sqlite" } compose-bom = { module = "androidx.compose:compose-bom", version.ref = "compose-bom" } compose-foundation = { module = "androidx.compose.foundation:foundation" } -compose-material-icons-extended = { module = "androidx.compose.material:material-icons-extended" } compose-material3 = { module = "androidx.compose.material3:material3" } compose-ui = { module = "androidx.compose.ui:ui" } compose-ui-tooling = { module = "androidx.compose.ui:ui-tooling" } @@ -40,6 +44,8 @@ lifecycle-viewmodel-compose = { module = "androidx.lifecycle:lifecycle-viewmodel navigation-compose = { module = "androidx.navigation:navigation-compose", version.ref = "navigation" } navigation-testing = { module = "androidx.navigation:navigation-testing", version.ref = "navigation" } junit = { module = "junit:junit", version.ref = "junit" } +onnxruntime-android = { module = "com.microsoft.onnxruntime:onnxruntime-android", version.ref = "onnxruntime" } +onnxruntime = { module = "com.microsoft.onnxruntime:onnxruntime", version.ref = "onnxruntime" } androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test" } androidx-test-junit = { module = "androidx.test.ext:junit", version = "1.3.0" } diff --git a/iolite/build.gradle.kts b/iolite/build.gradle.kts index 41fcec0..ca64023 100644 --- a/iolite/build.gradle.kts +++ b/iolite/build.gradle.kts @@ -22,13 +22,22 @@ kotlin { } dependencies { - api(libs.quartz) + api(libs.secp256k1) + implementation(libs.secp256k1.jni.android) implementation(libs.coroutines.android) + implementation(libs.sqlite) + implementation(libs.sqlite.bundled) + implementation("com.github.luben:zstd-jni:${libs.versions.zstd.get()}@aar") + testImplementation("org.json:json:20250517") testImplementation(libs.junit) testImplementation(libs.coroutines.test) + testImplementation(libs.secp256k1.jni.jvm) + testImplementation(libs.sqlite.bundled) + testImplementation(libs.zstd.jni) + testImplementation("org.xerial:sqlite-jdbc:3.50.3.0") + androidTestImplementation(libs.androidx.test.junit) androidTestImplementation(libs.androidx.test.runner) androidTestImplementation(libs.coroutines.test) - androidTestImplementation(libs.okhttp) } diff --git a/iolite/consumer-rules.pro b/iolite/consumer-rules.pro index 6cf31aa..966ffc4 100644 --- a/iolite/consumer-rules.pro +++ b/iolite/consumer-rules.pro @@ -1 +1,7 @@ -# Iolite requires no consumer-specific shrinking rules. +# JNI libraries Iolite loads by class / native name. +-keepclasseswithmembernames,includedescriptorclasses class * { + native ; +} +-keep class fr.acinq.secp256k1.** { *; } +# Native GetFieldID("srcPos"|"dstPos", "J") on the stream classes. +-keep class com.github.luben.zstd.** { *; } diff --git a/iolite/src/androidTest/kotlin/dev/zapstore/iolite/QuartzCompatibilityInstrumentedTest.kt b/iolite/src/androidTest/kotlin/dev/zapstore/iolite/QuartzCompatibilityInstrumentedTest.kt deleted file mode 100644 index 2ddfc37..0000000 --- a/iolite/src/androidTest/kotlin/dev/zapstore/iolite/QuartzCompatibilityInstrumentedTest.kt +++ /dev/null @@ -1,199 +0,0 @@ -package dev.zapstore.iolite - -import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.filters.SdkSuppress -import androidx.test.platform.app.InstrumentationRegistry -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair -import com.vitorpamplona.quartz.nip01Core.crypto.verifyId -import com.vitorpamplona.quartz.nip01Core.crypto.verifySignature -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.BasicOkHttpWebSocket -import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate -import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync -import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore -import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore -import java.io.File -import java.util.UUID -import kotlin.time.Duration.Companion.hours -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.CoroutineStart -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.async -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.take -import kotlinx.coroutines.flow.toList -import kotlinx.coroutines.runBlocking -import okhttp3.OkHttpClient -import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -@SdkSuppress(minSdkVersion = 29) -class QuartzCompatibilityInstrumentedTest { - @Test - fun queryRefreshMetadataPersistsAndRotatesWithDatabaseIdentity() { - val context = InstrumentationRegistry.getInstrumentation().targetContext - val path = context.getDatabasePath("refresh-cache-${UUID.randomUUID()}.db").absolutePath - val fingerprint = "profile-query" - val refreshedAt = System.currentTimeMillis() - val database = File(path) - val replacement = File("$path.replacement") - database.parentFile?.mkdirs() - database.writeText("first database") - - try { - val first = SharedPreferencesQueryRefreshCache.create(context, path, databaseExisted = false) - assertTrue(first.recordRefresh(fingerprint, refreshedAt)) - - val reopened = SharedPreferencesQueryRefreshCache.create(context, path, databaseExisted = true) - assertEquals(refreshedAt, reopened.lastRefresh(fingerprint)) - - replacement.writeText("replacement database") - assertTrue(database.delete()) - assertTrue(replacement.renameTo(database)) - - val recreated = SharedPreferencesQueryRefreshCache.create(context, path, databaseExisted = true) - assertEquals(null, recreated.lastRefresh(fingerprint)) - } finally { - database.delete() - replacement.delete() - } - } - - @Test - fun publishedStoreEmitsAfterCommitAndSurvivesReopen() = runBlocking { - val context = InstrumentationRegistry.getInstrumentation().targetContext - val databaseName = "quartz-compat-${UUID.randomUUID()}.db" - val path = context.getDatabasePath(databaseName).absolutePath - val event = knownValidEvent() - context.deleteDatabase(databaseName) - - val observable = ObservableEventStore(EventStore(dbName = path, relay = null)) - try { - val change = async(start = CoroutineStart.UNDISPATCHED) { - observable.changes.first() - } - observable.insert(event) - - assertTrue(change.await() is ObservableEventStore.StoreChange.Insert) - assertEquals(listOf(event.id), observable.query(Filter(ids = listOf(event.id))).map(Event::id)) - runCatching { observable.insert(event) } - assertEquals(1, observable.count(Filter(ids = listOf(event.id)))) - } finally { - observable.close() - } - - val reopened = EventStore(dbName = path, relay = null) - try { - assertEquals(listOf(event.id), reopened.query(Filter(ids = listOf(event.id))).map(Event::id)) - assertEquals(listOf(event.id), reopened.query(Filter(search = "test")).map(Event::id)) - } finally { - reopened.close() - context.deleteDatabase(databaseName) - } - } - - @Test - fun facadeRecreationReadsPersistedRowsWithoutNetwork() = runBlocking { - val context = InstrumentationRegistry.getInstrumentation().targetContext - val databaseName = "iolite-offline-${UUID.randomUUID()}.db" - val path = context.getDatabasePath(databaseName).absolutePath - val event = knownValidEvent() - context.deleteDatabase(databaseName) - EventStore(dbName = path, relay = null).also { store -> - try { - store.insert(event) - } finally { - store.close() - } - } - val okHttpClient = OkHttpClient() - - try { - repeat(2) { - val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - val purpleQuartz = Iolite.create( - context = context, - websocketBuilder = BasicOkHttpWebSocket.Builder { okHttpClient }, - parentScope = scope, - config = IoliteConfig(databaseName = databaseName), - ) - try { - val state = purpleQuartz.query(Filter(ids = listOf(event.id))).first() - assertEquals(QueryPhase.LocalOnly, state.phase) - assertEquals(listOf(event.id), state.items.map(Event::id)) - } finally { - purpleQuartz.close() - scope.cancel() - } - } - } finally { - okHttpClient.dispatcher.executorService.shutdown() - okHttpClient.connectionPool.evictAll() - context.deleteDatabase(databaseName) - } - } - - @Test - fun api29ConsumerConstructsQueriesAllSourcesAndCloses() = runBlocking { - val context = InstrumentationRegistry.getInstrumentation().targetContext - val databaseName = "iolite-consumer-${UUID.randomUUID()}.db" - val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - val okHttpClient = OkHttpClient() - val relay = "ws://127.0.0.1:1".normalizeRelayUrl() - val purpleQuartz = Iolite.create( - context = context, - websocketBuilder = BasicOkHttpWebSocket.Builder { okHttpClient }, - parentScope = scope, - config = IoliteConfig(databaseName = databaseName), - ) - - try { - assertEquals(QueryPhase.LocalOnly, purpleQuartz.query(Filter(kinds = listOf(1))).first().phase) - assertEquals( - QueryPhase.Connecting, - purpleQuartz.query( - Filter(kinds = listOf(1)), - QueryOptions.localAndRemote( - relays = setOf(relay), - remoteMode = RemoteMode.Stream, - cachedFor = 6.hours, - ), - ).take(1).toList().single().phase, - ) - assertEquals( - QueryPhase.Connecting, - purpleQuartz.query( - Filter(kinds = listOf(1)), - QueryOptions.remote(setOf(relay), RemoteMode.Stream), - ).take(1).toList().single().phase, - ) - } finally { - purpleQuartz.close() - scope.cancel() - okHttpClient.dispatcher.executorService.shutdown() - okHttpClient.connectionPool.evictAll() - context.deleteDatabase(databaseName) - } - } - - private fun knownValidEvent(): Event { - val event = NostrSignerSync(KeyPair()).sign( - EventTemplate( - createdAt = 1_753_988_264, - kind = 1, - tags = emptyArray(), - content = "test", - ), - ) - assertTrue(event.verifyId()) - assertTrue(event.verifySignature()) - return event - } -} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Bech32.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Bech32.kt new file mode 100644 index 0000000..8d77209 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Bech32.kt @@ -0,0 +1,89 @@ +package dev.zapstore.iolite + +private const val BECH32_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" +private val BECH32_GENERATOR = longArrayOf(0x3b6a57b2L, 0x26508e6dL, 0x1ea119faL, 0x3d4233ddL, 0x2a1462b3L) + +/** NIP-19 `npub` for a 64-char hex pubkey; other strings are returned unchanged. */ +fun String.toNpub(): String { + if (!Hex.isHex(lowercase(), 64)) return this + val data = convertBits(Hex.decode(this).map { it.toInt() and 0xff }, 8, 5) + val checksum = bech32Checksum("npub", data) + return buildString { + append("npub1") + (data + checksum).forEach { append(BECH32_CHARSET[it]) } + } +} + +private fun convertBits(data: List, fromBits: Int, toBits: Int): List { + var accumulator = 0 + var bits = 0 + val result = mutableListOf() + val maxValue = (1 shl toBits) - 1 + data.forEach { value -> + accumulator = (accumulator shl fromBits) or value + bits += fromBits + while (bits >= toBits) { + bits -= toBits + result += (accumulator shr bits) and maxValue + } + } + if (bits > 0) result += (accumulator shl (toBits - bits)) and maxValue + return result +} + +/** Hex pubkey for an `npub`, or null when the string is not a valid NIP-19 public key. */ +fun String.decodeNpub(): String? { + val value = lowercase() + if (!value.startsWith("npub1")) return null + val dataChars = value.substring(5) + if (dataChars.length < 6) return null + val data = dataChars.map { char -> + val index = BECH32_CHARSET.indexOf(char) + if (index < 0) return null + index + } + if (polymod(hrpExpand("npub") + data) != 1L) return null + val bytes = convertBitsDown(data.dropLast(6)) ?: return null + if (bytes.size != 32) return null + return Hex.encode(bytes) +} + +private fun hrpExpand(hrp: String): List = + hrp.map { it.code shr 5 } + listOf(0) + hrp.map { it.code and 31 } + +private fun polymod(values: List): Long { + var checksum = 1L + values.forEach { value -> + val top = checksum shr 25 + checksum = ((checksum and 0x1ffffffL) shl 5) xor value.toLong() + BECH32_GENERATOR.forEachIndexed { index, generator -> + if (((top shr index) and 1L) != 0L) checksum = checksum xor generator + } + } + return checksum +} + +private fun bech32Checksum(hrp: String, data: List): List { + val polymod = polymod(hrpExpand(hrp) + data + List(6) { 0 }) xor 1L + return (0 until 6).map { shift -> ((polymod shr (5 * (5 - shift))) and 31).toInt() } +} + +/** 5-bit bech32 payload to bytes. Leftover padding must be zero. */ +private fun convertBitsDown(data: List): ByteArray? { + var accumulator = 0 + var bits = 0 + val out = ArrayList(data.size * 5 / 8) + val maxValue = (1 shl 8) - 1 + val maxAccumulator = (1 shl (5 + 8 - 1)) - 1 + for (value in data) { + if (value < 0 || value shr 5 != 0) return null + accumulator = ((accumulator shl 5) or value) and maxAccumulator + bits += 5 + while (bits >= 8) { + bits -= 8 + out += ((accumulator shr bits) and maxValue).toByte() + } + } + if (bits >= 5 || ((accumulator shl (8 - bits)) and maxValue) != 0) return null + return out.toByteArray() +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Bolt11.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Bolt11.kt new file mode 100644 index 0000000..0521a9c --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Bolt11.kt @@ -0,0 +1,17 @@ +package dev.zapstore.iolite + +object Bolt11 { + fun amountSats(invoice: String): Long { + val lower = invoice.lowercase() + val match = Regex("^lnbc(\\d+)([munp]?)").find(lower) ?: return 0 + val amount = match.groupValues[1].toLongOrNull() ?: return 0 + return when (match.groupValues[2]) { + "m" -> amount * 100_000 + "u" -> amount * 100 + "n" -> amount / 10 + "p" -> amount / 10_000 + "" -> amount * 100_000_000 + else -> 0 + } + } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/CatalogArtifacts.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/CatalogArtifacts.kt new file mode 100644 index 0000000..9d199d4 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/CatalogArtifacts.kt @@ -0,0 +1,115 @@ +package dev.zapstore.iolite + +/** + * Artifact members of a catalog bundle. App files are `/about`, `security`, `facts`, + * `vector`, and `icon.webp`. Avatars are `<64 hex characters>.webp`. + * A missing file leaves the previous local value in place. + */ +internal data class CatalogArtifact( + val apps: List = emptyList(), + val avatars: List = emptyList(), +) { + companion object { + val EMPTY = CatalogArtifact() + } +} + +internal data class AppArtifact( + val appId: String, + val webp: ByteArray?, + val embedding: ByteArray?, + val facts: String?, + val about: String?, + val security: String?, +) + +internal data class AvatarArtifact( + val pubkey: String, + val webp: ByteArray, +) + +internal object CatalogArtifacts { + private const val MAX_APP_ID = 255 + + fun fromMembers(members: Map, maxIconBytes: Long): CatalogArtifact { + val apps = LinkedHashMap() + val avatars = ArrayList() + for ((name, member) in members) { + if (name == "manifest.json" || name == "diff.jsonl" || name == "index") continue + val pubkey = avatarPubkey(name) + if (pubkey != null) { + if (member.data.isEmpty() || member.data.size.toLong() > maxIconBytes) { + fail("bad avatar $name") + } + avatars += AvatarArtifact(pubkey, member.data) + continue + } + val slash = name.indexOf('/') + if (slash <= 0 || slash != name.lastIndexOf('/')) fail("unexpected catalog member $name") + val appId = name.substring(0, slash) + val file = name.substring(slash + 1) + if (!validAppId(appId) || file !in APP_FILES) fail("unexpected catalog member $name") + val app = apps.getOrPut(appId) { Partial(appId) } + when (file) { + "about", "security", "facts" -> applyText(app, file, member.data) + "vector" -> { + if (member.data.size != EMBEDDING_DIMS) fail("vector $appId") + app.embedding = member.data + } + "icon.webp" -> { + if (member.data.isEmpty() || member.data.size.toLong() > maxIconBytes) fail("icon $appId exceeds size limit") + app.webp = member.data + } + } + } + return CatalogArtifact( + apps = apps.values.map { it.toArtifact() }, + avatars = avatars, + ) + } + + fun allowedMember(name: String): Boolean { + if (name == "diff.jsonl" || name == "index") return true + if (avatarPubkey(name) != null) return true + val slash = name.indexOf('/') + if (slash <= 0 || slash != name.lastIndexOf('/')) return false + val appId = name.substring(0, slash) + val file = name.substring(slash + 1) + return validAppId(appId) && file in APP_FILES + } + + private fun applyText(app: Partial, file: String, raw: ByteArray) { + val text = raw.toString(Charsets.UTF_8) + if (text.any { it == '\u0000' }) fail("$file ${app.appId} contains NUL") + when (file) { + "about" -> app.about = text + "security" -> app.security = text + "facts" -> app.facts = text + } + } + + private fun avatarPubkey(name: String): String? { + if (name.contains('/')) return null + val pubkey = name.removeSuffix(".webp") + if (pubkey.length == name.length) return null + if (!Hex.isHex(pubkey, 64)) return null + return pubkey + } + + private fun validAppId(id: String): Boolean = + id.length in 1..MAX_APP_ID && id.none { it == '\u0000' || it == '/' || it == '\\' } && ".." !in id + + private fun fail(reason: String): Nothing = throw CatalogImportException(reason) + + private class Partial(val appId: String) { + var webp: ByteArray? = null + var embedding: ByteArray? = null + var facts: String? = null + var about: String? = null + var security: String? = null + + fun toArtifact() = AppArtifact(appId, webp, embedding, facts, about, security) + } + + private val APP_FILES = setOf("about", "security", "facts", "vector", "icon.webp") +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/CatalogImporter.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/CatalogImporter.kt new file mode 100644 index 0000000..a56dcdb --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/CatalogImporter.kt @@ -0,0 +1,215 @@ +package dev.zapstore.iolite + +import java.io.ByteArrayInputStream +import java.io.File +import org.json.JSONObject + +data class CatalogImportResult(val from: Long, val to: Long) + +/** Outcome of `GET /deltas`. [imported] is null when the relay answered 304. */ +data class CatalogSyncResult( + val imported: CatalogImportResult?, + val bytes: Long, +) { + val notModified: Boolean get() = imported == null +} + +/** + * Verifies a catalog-sync bundle (signed manifest, member hashes, event signatures) and hands the + * verified events and artifact files to [IoliteStore.importCatalog]. + */ +class CatalogImporter( + private val store: IoliteStore, + private val iconsDir: File, + private val config: IoliteConfig, + private val device: DeviceProfile, + private val nowSeconds: () -> Long = { System.currentTimeMillis() / 1_000 }, +) { + fun importBundle(catalogId: Long, compressed: ByteArray, endpoint: RelayUrl? = null): CatalogImportResult { + val existing = store.catalog(catalogId) + val members = try { + TarZstd.read( + ByteArrayInputStream(compressed), + maxCompressedBytes = config.maxCompressedBytes, + maxUncompressedBytes = config.maxUncompressedBytes, + maxMembers = config.maxMembers, + maxMemberBytes = config.maxUncompressedBytes, + ) + } catch (failure: Exception) { + throw CatalogImportException(failure.message ?: "invalid catalog bundle") + } + val byName = members.associateBy { it.name } + val manifest = parseAndVerifyManifest(members.first().data, existing?.manifestPubkey, existing?.epoch ?: 0L) + val names = members.drop(1).map { it.name } + if (names != names.sorted()) throw CatalogImportException("tar members after manifest.json must be lexicographic") + + val fileTags = manifest.event.tags.filter { it.firstOrNull() == "file" } + verifyFileTags(fileTags, byName) + val index = byName["index"] ?: throw CatalogImportException("index is missing") + verifyIndex(index.data, members) + + val unexpected = names.filter { !CatalogArtifacts.allowedMember(it) } + if (unexpected.isNotEmpty()) throw CatalogImportException("unexpected catalog member ${unexpected.first()}") + + val (events, deletes) = parseDiff(byName["diff.jsonl"], config.maxJsonLineBytes) + validateOperations(events, deletes) + validateStacks(events, deletes, config.catalogStackPubkey) + events.filter { it.kind == Kinds.IdentityProof }.forEach(ListingRows::validateProof) + val artifacts = CatalogArtifacts.fromMembers(byName, config.maxIconBytes) + + if (existing == null || existing.relays.isEmpty()) { + val relay = endpoint ?: throw CatalogImportException("catalog relay is missing") + store.upsertCatalogRelays(catalogId, listOf(relay), replaceIdentity = true) + } + if (existing?.manifestPubkey == null) store.pinManifestPubkeyIfAbsent(catalogId, manifest.event.pubkey) + val catalog = store.catalog(catalogId) ?: throw CatalogImportException("unknown catalog $catalogId") + val removed = try { + store.importCatalog( + catalog = catalog, + events = events, + deletes = deletes, + replaceAll = manifest.fromEpoch == 0L, + toEpoch = manifest.toEpoch, + device = device, + now = nowSeconds(), + artifacts = artifacts, + stackPubkey = config.catalogStackPubkey, + ) + } catch (failure: Throwable) { + throw if (failure is CatalogImportException) failure else CatalogImportException(failure.message ?: "import failed") + } + removed.forEach { appId -> File(iconsDir, "$catalogId/$appId.webp").delete() } + return CatalogImportResult(manifest.fromEpoch, manifest.toEpoch) + } + + private fun verifyFileTags(fileTags: List>, byName: Map) { + val seenFiles = HashSet() + for (tag in fileTags) { + if (tag.size < 3) throw CatalogImportException("malformed file tag") + val name = tag[1] + if (!seenFiles.add(name)) throw CatalogImportException("duplicate file tag $name") + val member = byName[name] ?: throw CatalogImportException("manifest file $name is missing") + if (member.data.sha256Hex() != tag[2]) throw CatalogImportException("hash mismatch for $name") + } + } + + private fun verifyIndex(raw: ByteArray, members: List) { + val lines = raw.toString(Charsets.UTF_8).split('\n').filter { it.isNotEmpty() } + val listed = HashMap() + for (line in lines) { + if (line.length < 66 || line[64] != ' ') throw CatalogImportException("malformed index line") + val hash = line.substring(0, 64) + val name = line.substring(65) + if (listed.put(name, hash) != null) throw CatalogImportException("duplicate index entry $name") + } + for (member in members) { + if (member.name == "manifest.json" || member.name == "index") continue + val hash = listed.remove(member.name) ?: throw CatalogImportException("index missing ${member.name}") + if (member.data.sha256Hex() != hash) throw CatalogImportException("hash mismatch for ${member.name}") + } + if (listed.isNotEmpty()) throw CatalogImportException("index entry ${listed.keys.first()} is missing") + } + + private fun parseAndVerifyManifest(raw: ByteArray, expectedPubkey: String?, localEpoch: Long): Manifest { + val event = Event.parse(raw.toString(Charsets.UTF_8)) + if (event.kind != Kinds.Preferences) throw CatalogImportException("manifest kind must be 30078") + if (!event.verify()) throw CatalogImportException("manifest signature is invalid") + if (expectedPubkey != null && event.pubkey != expectedPubkey) { + throw CatalogImportException("manifest pubkey does not match the pinned key") + } + val from = event.tagValue("from")?.toLongOrNull() ?: throw CatalogImportException("manifest from is missing") + val to = event.tagValue("to")?.toLongOrNull() ?: throw CatalogImportException("manifest to is missing") + val version = event.tagValue("v")?.toIntOrNull() ?: throw CatalogImportException("manifest v is missing") + if (from != localEpoch) throw CatalogImportException("manifest from $from does not match catalog epoch $localEpoch") + if (to < from) throw CatalogImportException("manifest to is behind from") + if (version != 1) throw CatalogImportException("unsupported catalog-sync version $version") + return Manifest(from, to, event) + } + + private data class Manifest( + val fromEpoch: Long, + val toEpoch: Long, + val event: Event, + ) + + companion object { + private fun parseDiff(member: TarMember?, maxLine: Long): Pair, List> { + if (member == null) return emptyList() to emptyList() + val text = member.data.toString(Charsets.UTF_8) + if (text.isEmpty()) return emptyList() to emptyList() + val events = ArrayList() + val deletes = ArrayList() + for (line in text.split('\n')) { + if (line.isEmpty()) continue + if (line.length.toLong() > maxLine) throw CatalogImportException("JSONL line exceeds limit") + val json = JSONObject(line) + when { + json.has("id") -> { + val event = Event.parse(line) + if (!event.verify()) throw CatalogImportException("event ${event.id} failed verification") + events += event + } + json.has("delete") -> { + if (json.has("kind") || json.has("pubkey") || json.has("d") || json.has("app_id")) { + throw CatalogImportException("delete line has mixed forms") + } + val ids = json.optJSONArray("delete") ?: throw CatalogImportException("invalid delete line") + for (index in 0 until ids.length()) { + val appId = ids.optString(index) + if (appId.isBlank()) throw CatalogImportException("invalid delete line") + deletes += CatalogDelete.Listing(appId) + } + } + else -> { + val kind = json.optInt("kind", -1) + val pubkey = json.optString("pubkey") + val d = json.optString("d") + if (kind < 0 || pubkey.isBlank() || d.isBlank() || json.has("app_id")) { + throw CatalogImportException("invalid delete line") + } + deletes += CatalogDelete.Coordinate(kind, pubkey, d) + } + } + } + return events to deletes + } + + private fun validateStacks(events: List, deletes: List, catalogPubkey: String) { + for (event in events) { + if (event.kind != Kinds.AppStack) continue + if (event.pubkey != catalogPubkey) throw CatalogImportException("stack is not signed by the curator") + if (event.dTag().isNullOrBlank()) throw CatalogImportException("stack missing d") + } + for (delete in deletes) { + if (delete is CatalogDelete.Coordinate && delete.kind == Kinds.AppStack && delete.pubkey != catalogPubkey) { + throw CatalogImportException("stack is not signed by the curator") + } + } + } + + private fun validateOperations(events: List, deletes: List) { + val listingKeys = HashSet() + val eventKeys = HashSet() + for (event in events) { + if (ListingRows.isListingKind(event.kind)) { + listingKeys += ListingRows.listingAppId(event) ?: continue + } else { + val key = "${event.kind}:${event.pubkey}:${event.dTag().orEmpty()}" + if (!eventKeys.add(key)) throw CatalogImportException("duplicate logical key $key") + } + } + for (delete in deletes) { + when (delete) { + is CatalogDelete.Listing -> if (!listingKeys.add(delete.appId)) { + throw CatalogImportException("event and delete for ${delete.appId}") + } + is CatalogDelete.Coordinate -> { + val key = "${delete.kind}:${delete.pubkey}:${delete.d}" + if (!eventKeys.add(key)) throw CatalogImportException("event and delete for $key") + } + } + } + } + + } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/ChaCha20.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/ChaCha20.kt new file mode 100644 index 0000000..f00d6da --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/ChaCha20.kt @@ -0,0 +1,66 @@ +package dev.zapstore.iolite + +internal object ChaCha20 { + fun xor(key: ByteArray, nonce: ByteArray, data: ByteArray): ByteArray { + require(key.size == 32) { "ChaCha20 key must be 32 bytes" } + require(nonce.size == 12) { "ChaCha20 nonce must be 12 bytes" } + val out = data.copyOf() + val state = IntArray(16) + state[0] = 0x61707865 + state[1] = 0x3320646e + state[2] = 0x79622d32 + state[3] = 0x6b206574 + for (i in 0 until 8) state[4 + i] = key.leInt(i * 4) + for (i in 0 until 3) state[13 + i] = nonce.leInt(i * 4) + val block = ByteArray(64) + var offset = 0 + var counter = 0 + while (offset < out.size) { + state[12] = counter + block(state, block) + counter += 1 + val n = minOf(64, out.size - offset) + for (i in 0 until n) { + out[offset + i] = (out[offset + i].toInt() xor (block[i].toInt() and 0xff)).toByte() + } + offset += n + } + return out + } + + private fun block(input: IntArray, out: ByteArray) { + val x = input.copyOf() + repeat(10) { + quarter(x, 0, 4, 8, 12) + quarter(x, 1, 5, 9, 13) + quarter(x, 2, 6, 10, 14) + quarter(x, 3, 7, 11, 15) + quarter(x, 0, 5, 10, 15) + quarter(x, 1, 6, 11, 12) + quarter(x, 2, 7, 8, 13) + quarter(x, 3, 4, 9, 14) + } + for (i in x.indices) x[i] += input[i] + for (i in x.indices) { + val word = x[i] + val o = i * 4 + out[o] = word.toByte() + out[o + 1] = (word ushr 8).toByte() + out[o + 2] = (word ushr 16).toByte() + out[o + 3] = (word ushr 24).toByte() + } + } + + private fun quarter(x: IntArray, a: Int, b: Int, c: Int, d: Int) { + x[a] += x[b]; x[d] = (x[d] xor x[a]).rotateLeft(16) + x[c] += x[d]; x[b] = (x[b] xor x[c]).rotateLeft(12) + x[a] += x[b]; x[d] = (x[d] xor x[a]).rotateLeft(8) + x[c] += x[d]; x[b] = (x[b] xor x[c]).rotateLeft(7) + } + + private fun ByteArray.leInt(offset: Int): Int = + (this[offset].toInt() and 0xff) or + ((this[offset + 1].toInt() and 0xff) shl 8) or + ((this[offset + 2].toInt() and 0xff) shl 16) or + ((this[offset + 3].toInt() and 0xff) shl 24) +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Crypto.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Crypto.kt new file mode 100644 index 0000000..c0a2b55 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Crypto.kt @@ -0,0 +1,45 @@ +package dev.zapstore.iolite + +import fr.acinq.secp256k1.Secp256k1 + +object Crypto { + private val secp: Secp256k1 = Secp256k1.get() + + fun xOnlyPubkey(secretKey: ByteArray): ByteArray { + require(secretKey.size == 32) { "secret key must be 32 bytes" } + val uncompressed = secp.pubkeyCreate(secretKey) + val compressed = secp.pubKeyCompress(uncompressed) + return compressed.copyOfRange(1, 33) + } + + fun signSchnorr(message: ByteArray, secretKey: ByteArray, auxRand: ByteArray? = ByteArray(32)): ByteArray { + require(message.size == 32) { "message must be 32 bytes" } + require(secretKey.size == 32) { "secret key must be 32 bytes" } + return secp.signSchnorr(message, secretKey, auxRand) + } + + fun verifySchnorr(signature: ByteArray, message: ByteArray, xOnlyPubkey: ByteArray): Boolean { + if (signature.size != 64 || message.size != 32 || xOnlyPubkey.size != 32) return false + return runCatching { secp.verifySchnorr(signature, message, xOnlyPubkey) }.getOrDefault(false) + } + + fun sha256(data: ByteArray): ByteArray = + java.security.MessageDigest.getInstance("SHA-256").digest(data) + + fun randomBytes(size: Int): ByteArray = ByteArray(size).also { java.security.SecureRandom().nextBytes(it) } + + fun sharedX(secretKey: ByteArray, xOnlyPubkey: ByteArray): ByteArray { + require(secretKey.size == 32) { "secret key must be 32 bytes" } + require(xOnlyPubkey.size == 32) { "public key must be 32 bytes" } + val compressed = ByteArray(33) + xOnlyPubkey.copyInto(compressed, 1) + val uncompressed = runCatching { + compressed[0] = 0x02 + secp.pubkeyParse(compressed) + }.recoverCatching { + compressed[0] = 0x03 + secp.pubkeyParse(compressed) + }.getOrElse { throw IllegalArgumentException("invalid public key") } + return secp.pubKeyTweakMul(uncompressed, secretKey).copyOfRange(1, 33) + } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Event.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Event.kt new file mode 100644 index 0000000..bde3d4e --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Event.kt @@ -0,0 +1,231 @@ +package dev.zapstore.iolite + +import java.security.MessageDigest +import org.json.JSONArray +import org.json.JSONObject + +data class Event( + val id: String, + val pubkey: String, + val createdAt: Long, + val kind: Int, + val tags: List>, + val content: String, + val sig: String, +) { + fun tagValue(name: String): String? = tags.firstOrNull { it.firstOrNull() == name }?.getOrNull(1) + + fun tagValues(name: String): List = + tags.mapNotNull { tag -> tag.getOrNull(1)?.takeIf { tag.firstOrNull() == name } } + + fun tagCount(name: String): Int = tags.count { it.firstOrNull() == name } + + fun dTag(): String? = tagValue("d") + + fun address(): Address? = when { + kind.isAddressable() -> Address(kind, pubkey, dTag().orEmpty()) + kind.isReplaceable() -> Address(kind, pubkey, "") + else -> null + } + + fun toJsonObject(): JSONObject = JSONObject() + .put("id", id) + .put("pubkey", pubkey) + .put("created_at", createdAt) + .put("kind", kind) + .put("tags", tags.toJsonArray()) + .put("content", content) + .put("sig", sig) + + fun toJson(): String = toJsonObject().toString() + + fun verify(): Boolean = id == computeId(pubkey, createdAt, kind, tags, content) && + Crypto.verifySchnorr(Hex.decode(sig), Hex.decode(id), Hex.decode(pubkey)) + + companion object { + fun parse(raw: String): Event = parse(JSONObject(raw)) + + fun parse(json: JSONObject): Event { + val tags = json.getJSONArray("tags").toStringLists() + return Event( + id = json.getString("id"), + pubkey = json.getString("pubkey"), + createdAt = json.getLong("created_at"), + kind = json.getInt("kind"), + tags = tags, + content = json.optString("content", ""), + sig = json.getString("sig"), + ) + } + + fun computeId( + pubkey: String, + createdAt: Long, + kind: Int, + tags: List>, + content: String, + ): String { + val digest = MessageDigest.getInstance("SHA-256") + digest.update(canonicalBytes(pubkey, createdAt, kind, tags, content)) + return Hex.encode(digest.digest()) + } + + fun sign( + secretKey: ByteArray, + createdAt: Long, + kind: Int, + tags: List>, + content: String, + ): Event { + val pubkey = Hex.encode(Crypto.xOnlyPubkey(secretKey)) + val id = computeId(pubkey, createdAt, kind, tags, content) + val sig = Hex.encode(Crypto.signSchnorr(Hex.decode(id), secretKey)) + return Event(id, pubkey, createdAt, kind, tags, content, sig) + } + + internal fun canonicalBytes( + pubkey: String, + createdAt: Long, + kind: Int, + tags: List>, + content: String, + ): ByteArray { + // NIP-01 id bytes must match go-nostr / JSON.stringify: compact JSON, + // UTF-8, no escaped solidus. Android org.json escapes '/' and breaks verify. + val json = StringBuilder(64 + pubkey.length + content.length + tags.sumOf { it.sumOf { value -> value.length + 3 } }) + json.append("[0,") + appendJsonString(json, pubkey) + json.append(',').append(createdAt).append(',').append(kind).append(',') + json.append('[') + tags.forEachIndexed { tagIndex, tag -> + if (tagIndex > 0) json.append(',') + json.append('[') + tag.forEachIndexed { valueIndex, value -> + if (valueIndex > 0) json.append(',') + appendJsonString(json, value) + } + json.append(']') + } + json.append("],") + appendJsonString(json, content) + json.append(']') + return json.toString().toByteArray(Charsets.UTF_8) + } + + private fun appendJsonString(out: StringBuilder, value: String) { + out.append('"') + for (ch in value) { + when (ch) { + '"' -> out.append("\\\"") + '\\' -> out.append("\\\\") + '\b' -> out.append("\\b") + '\u000C' -> out.append("\\f") + '\n' -> out.append("\\n") + '\r' -> out.append("\\r") + '\t' -> out.append("\\t") + else -> if (ch.code < 0x20) { + out.append("\\u") + val hex = ch.code.toString(16) + repeat(4 - hex.length) { out.append('0') } + out.append(hex) + } else { + out.append(ch) + } + } + } + out.append('"') + } + } +} + +data class Address(val kind: Int, val pubkey: String, val d: String) { + override fun toString(): String = "$kind:$pubkey:$d" +} + +data class Filter( + val ids: List? = null, + val authors: List? = null, + val kinds: List? = null, + val tags: Map>? = null, + val since: Long? = null, + val until: Long? = null, + val limit: Int? = null, + val search: String? = null, +) { + fun toJsonObject(): JSONObject { + val json = JSONObject() + ids?.let { json.put("ids", JSONArray(it)) } + authors?.let { json.put("authors", JSONArray(it)) } + kinds?.let { kinds -> + val values = JSONArray() + kinds.forEach(values::put) + json.put("kinds", values) + } + tags?.forEach { (key, values) -> json.put("#$key", JSONArray(values)) } + since?.let { json.put("since", it) } + until?.let { json.put("until", it) } + limit?.let { json.put("limit", it) } + search?.let { json.put("search", it) } + return json + } +} + +fun Int.isReplaceable(): Boolean = this == 0 || this == 3 || this in 10_000..19_999 + +fun Int.isEphemeral(): Boolean = this in 20_000..29_999 + +fun Int.isAddressable(): Boolean = this in 30_000..39_999 + +/** Replaceable-event precedence: newer `created_at` wins, then the lower event ID. */ +fun supersedes(nextCreatedAt: Long, nextId: String, currentCreatedAt: Long, currentId: String): Boolean = when { + nextCreatedAt != currentCreatedAt -> nextCreatedAt > currentCreatedAt + else -> nextId < currentId +} + +internal fun List>.toJsonArray(): JSONArray { + val tags = JSONArray() + forEach { tag -> + val values = JSONArray() + tag.forEach(values::put) + tags.put(values) + } + return tags +} + +internal fun JSONArray?.toStringLists(): List> { + if (this == null) return emptyList() + return List(length()) { index -> + val tag = getJSONArray(index) + List(tag.length()) { tag.getString(it) } + } +} + +object Kinds { + const val Profile = 0 + const val Comment = 1_111 + const val Auth = 22_242 + const val Asset = 3_063 + const val Zap = 9_735 + const val RelayList = 10_002 + const val CatalogRelayList = 10_067 + const val AppStack = 30_267 + const val Preferences = 30_078 + const val Release = 30_063 + const val IdentityProof = 30_509 + const val App = 32_267 +} + +/** Signs the Zapstore curated stacks (kind 30267). Not the catalog manifest key. */ +const val CatalogStackPubkey = "acfeaea6e51420e8068fac446ca9d17d7a9ef6a5d20d93894e50fee3d4902a84" + +/** NIP-65: unmarked or `read`-marked `r` tags. */ +fun List>.readRelayUrls(): Set = markedRelayUrls("read") + +private fun List>.markedRelayUrls(marker: String): Set = + asSequence() + .filter { tag -> tag.getOrNull(0) == "r" } + .filter { tag -> tag.getOrNull(2).let { it == null || it == marker } } + .mapNotNull { tag -> tag.getOrNull(1) } + .filter { it.startsWith("ws://") || it.startsWith("wss://") } + .mapNotNull { runCatching { it.normalizeRelayUrl() }.getOrNull() } + .toSet() diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/EventRows.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/EventRows.kt new file mode 100644 index 0000000..3375e9f --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/EventRows.kt @@ -0,0 +1,365 @@ +package dev.zapstore.iolite + +import androidx.sqlite.SQLiteConnection +import androidx.sqlite.SQLiteStatement +import androidx.sqlite.execSQL +import org.json.JSONArray +import org.json.JSONObject + +/** Per-process facts every ingest needs: which device this is and how to decrypt device-private payloads. */ +internal class IngestContext( + val device: DeviceProfile, + val deviceSigner: LocalSigner?, +) + +/** + * Derives SQLite rows from verified relay events (kinds 0, 10002, 30267, 9735, 1111, 10067, 30078). + * Event JSON is not retained beyond what rehydration needs. + */ +internal object EventRows { + /** Key of the `preferences` row that stores the device's kind 10067. */ + const val CATALOG_RELAY_LIST_KEY = "\u000010067" + val STACK_MODELED_TAGS = setOf("d", "name", "description", "a", "f") + + /** Writes [event] if it is admissible and newer than the stored row. Returns whether a row changed. */ + fun persist(tx: Tx, event: Event, context: IngestContext): Boolean { + val db = tx.db + return when (event.kind) { + Kinds.Profile -> ifNewer(event, profileVersion(db, event.pubkey)) { + upsertProfile(db, event) + tx.touch(Table.Profiles) + } + Kinds.RelayList -> ifNewer(event, relayListVersion(db, event.pubkey)) { + upsertRelayList(db, event) + tx.touch(Table.Profiles) + } + Kinds.AppStack -> { + val identifier = event.dTag() ?: return false + if (!stackMatchesDevice(event, context.device)) return false + ifNewer(event, stackVersion(db, event.pubkey, identifier)) { + upsertStack(db, event, identifier, context.deviceSigner) + tx.touch(Table.Stacks) + } + } + Kinds.Zap -> { + val appId = zapAppId(event) ?: return false + val amount = zapAmountSats(event) + if (amount <= 0) return false + insertZap(db, event, appId, amount) + tx.touch(Table.Zaps) + true + } + Kinds.Comment -> { + insertComment(db, event) + tx.touch(Table.Comments) + true + } + Kinds.CatalogRelayList -> { + if (context.deviceSigner?.publicKey != event.pubkey) return false + ifNewer(event, preferenceVersion(db, CATALOG_RELAY_LIST_KEY)) { + upsertCatalogRelayList(db, event, context.deviceSigner) + tx.touch(Table.Catalogs, Table.Preferences) + } + } + Kinds.Preferences -> { + val identifier = event.dTag() ?: return false + if (isCatalogManifest(event)) return false + ifNewer(event, preferenceVersion(db, identifier)) { + upsertPreference(db, event, identifier) + tx.touch(Table.Preferences) + } + } + else -> false + } + } + + fun insertOutbox(tx: Tx, event: Event) { + tx.db.prepare("INSERT OR REPLACE INTO outbox (event_id, event_json) VALUES (?, ?)").use { statement -> + statement.bindBlob(1, Hex.decode(event.id)) + statement.bindText(2, event.toJson()) + statement.step() + } + tx.touch(Table.Outbox) + } + + // --- replaceable-event versions ------------------------------------------------------------- + + private class Version(val createdAt: Long, val id: String) + + private inline fun ifNewer(event: Event, current: Version?, write: () -> Unit): Boolean { + if (current != null && !supersedes(event.createdAt, event.id, current.createdAt, current.id)) return false + write() + return true + } + + private fun profileVersion(db: SQLiteConnection, pubkey: String): Version? = + profileMetadata(db, pubkey)?.let { (updatedAt, metadata) -> + metadata.string("id")?.let { Version(updatedAt, it) } + } + + private fun relayListVersion(db: SQLiteConnection, pubkey: String): Version? = + profileMetadata(db, pubkey)?.let { (_, metadata) -> + metadata.string("nip65_id")?.let { Version(metadata.optLong("nip65_created_at"), it) } + } + + private fun stackVersion(db: SQLiteConnection, pubkey: String, identifier: String): Version? = + db.prepare("SELECT updated_at, event_id FROM stacks WHERE pubkey = ? AND identifier = ?").use { statement -> + statement.bindBlob(1, Hex.decode(pubkey)) + statement.bindText(2, identifier) + if (statement.step()) Version(statement.getLong(0), Hex.encode(statement.getBlob(1))) else null + } + + private fun preferenceVersion(db: SQLiteConnection, identifier: String): Version? = + db.prepare("SELECT updated_at, metadata FROM preferences WHERE identifier = ?").use { statement -> + statement.bindText(1, identifier) + if (!statement.step()) return null + JSONObject(statement.getText(1)).string("id")?.let { Version(statement.getLong(0), it) } + } + + /** `(updated_at, metadata)` of a profile row, or null. */ + fun profileMetadata(db: SQLiteConnection, pubkey: String): Pair? = + db.prepare("SELECT updated_at, metadata FROM profiles WHERE pubkey = ?").use { statement -> + statement.bindBlob(1, Hex.decode(pubkey)) + if (statement.step()) statement.getLong(0) to JSONObject(statement.getText(1)) else null + } + + // --- writers ----------------------------------------------------------------------------------- + + private fun upsertProfile(db: SQLiteConnection, event: Event) { + val content = runCatching { JSONObject(event.content) }.getOrElse { JSONObject() } + val metadata = profileMetadata(db, event.pubkey)?.second ?: JSONObject() + metadata.put("id", event.id) + metadata.put("content", event.content) + db.prepare( + """ + INSERT INTO profiles (pubkey, updated_at, name, picture_url, metadata) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(pubkey) DO UPDATE SET + updated_at = excluded.updated_at, + name = excluded.name, + picture_url = excluded.picture_url, + metadata = excluded.metadata + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, Hex.decode(event.pubkey)) + statement.bindLong(2, event.createdAt) + statement.bindTextOrNull(3, content.string("name")) + statement.bindTextOrNull(4, content.string("picture") ?: content.string("image")) + statement.bindText(5, metadata.toString()) + statement.step() + } + } + + private fun upsertRelayList(db: SQLiteConnection, event: Event) { + val existing = profileMetadata(db, event.pubkey) + val metadata = existing?.second ?: JSONObject() + metadata.put("nip65_id", event.id) + metadata.put("nip65_created_at", event.createdAt) + metadata.put("nip65_tags", event.tags.toJsonArray()) + db.prepare( + """ + INSERT INTO profiles (pubkey, updated_at, name, picture_url, metadata) + VALUES (?, ?, NULL, NULL, ?) + ON CONFLICT(pubkey) DO UPDATE SET metadata = excluded.metadata + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, Hex.decode(event.pubkey)) + statement.bindLong(2, existing?.first ?: event.createdAt) + statement.bindText(3, metadata.toString()) + statement.step() + } + } + + private fun insertZap(db: SQLiteConnection, event: Event, appId: String, amountSats: Long) { + db.prepare( + """ + INSERT OR IGNORE INTO zaps (event_id, pubkey, amount_sats, created_at, app_id, metadata) + VALUES (?, ?, ?, ?, ?, ?) + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, Hex.decode(event.id)) + statement.bindBlob(2, Hex.decode(event.pubkey)) + statement.bindLong(3, amountSats) + statement.bindLong(4, event.createdAt) + statement.bindText(5, appId) + statement.bindText(6, JSONObject().put("tags", event.tags.toJsonArray()).toString()) + statement.step() + } + } + + private fun insertComment(db: SQLiteConnection, event: Event) { + val appId = event.tagValue("i") + ?: event.tagValue("a")?.let { AppCoordinate.parse(it)?.appId } + ?: event.tagValue("A")?.let { AppCoordinate.parse(it)?.appId } + db.prepare( + """ + INSERT OR IGNORE INTO comments ( + event_id, pubkey, created_at, content, app_id, stack, parent_event_id, metadata + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, Hex.decode(event.id)) + statement.bindBlob(2, Hex.decode(event.pubkey)) + statement.bindLong(3, event.createdAt) + statement.bindText(4, event.content) + statement.bindTextOrNull(5, appId) + statement.bindTextOrNull(6, event.tagValue("A")) + statement.bindBlobOrNull(7, event.tagValue("e")?.let(Hex::decode)) + statement.bindText(8, JSONObject().put("tags", event.tags.toJsonArray()).toString()) + statement.step() + } + } + + private fun upsertStack(db: SQLiteConnection, event: Event, identifier: String, deviceSigner: LocalSigner?) { + val metadata = JSONObject() + .put("tags", event.tags.toJsonArray()) + .put("content", event.content) + .put("private_apps", JSONArray(decryptJsonArray(event.content, deviceSigner))) + db.prepare( + """ + INSERT INTO stacks (pubkey, identifier, event_id, name, description, public_apps, updated_at, metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(pubkey, identifier) DO UPDATE SET + event_id = excluded.event_id, + name = excluded.name, + description = excluded.description, + public_apps = excluded.public_apps, + updated_at = excluded.updated_at, + metadata = excluded.metadata + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, Hex.decode(event.pubkey)) + statement.bindText(2, identifier) + statement.bindBlob(3, Hex.decode(event.id)) + statement.bindText(4, event.tagValue("name") ?: identifier) + statement.bindTextOrNull(5, event.tagValue("description")) + statement.bindText(6, JSONArray(event.tagValues("a")).toString()) + statement.bindLong(7, event.createdAt) + statement.bindText(8, metadata.toString()) + statement.step() + } + } + + private fun upsertPreference(db: SQLiteConnection, event: Event, identifier: String) { + upsertPreferenceRow(db, identifier, event) + } + + private fun upsertCatalogRelayList(db: SQLiteConnection, event: Event, deviceSigner: LocalSigner?) { + val publicUrls = event.tagValues("r").mapNotNull { runCatching { it.normalizeRelayUrl() }.getOrNull() } + val privateUrls = decryptJsonArray(event.content, deviceSigner) + .mapNotNull { runCatching { it.normalizeRelayUrl() }.getOrNull() } + db.execSQL("UPDATE catalogs SET position = NULL") + val seen = HashSet() + var position = 0 + fun select(url: RelayUrl, isPrivate: Boolean) { + if (!seen.add(url.url)) return + db.prepare( + """ + INSERT INTO catalogs (relay_url, position, is_private, epoch) + VALUES (?, ?, ?, 0) + ON CONFLICT(relay_url) DO UPDATE SET + position = excluded.position, + is_private = excluded.is_private + """.trimIndent(), + ).use { statement -> + statement.bindText(1, url.url) + statement.bindLong(2, position.toLong()) + statement.bindLong(3, if (isPrivate) 1 else 0) + statement.step() + } + position++ + } + publicUrls.forEach { select(it, false) } + privateUrls.forEach { select(it, true) } + upsertPreferenceRow(db, CATALOG_RELAY_LIST_KEY, event) + } + + private fun upsertPreferenceRow(db: SQLiteConnection, identifier: String, event: Event) { + val metadata = JSONObject() + .put("id", event.id) + .put("pubkey", event.pubkey) + .put("content", event.content) + .put("tags", event.tags.toJsonArray()) + db.prepare( + """ + INSERT INTO preferences (identifier, updated_at, metadata) + VALUES (?, ?, ?) + ON CONFLICT(identifier) DO UPDATE SET + updated_at = excluded.updated_at, + metadata = excluded.metadata + """.trimIndent(), + ).use { statement -> + statement.bindText(1, identifier) + statement.bindLong(2, event.createdAt) + statement.bindText(3, metadata.toString()) + statement.step() + } + } + + // --- event helpers ----------------------------------------------------------------------------- + + fun stackMatchesDevice(event: Event, device: DeviceProfile): Boolean { + val platforms = event.tagValues("f").filter { it.startsWith("android-") } + return platforms.isEmpty() || platforms.any { it in device.platforms } + } + + fun deleteStack(tx: Tx, pubkey: String, identifier: String) { + tx.db.prepare("DELETE FROM stacks WHERE pubkey = ? AND identifier = ?").use { statement -> + statement.bindBlob(1, Hex.decode(pubkey)) + statement.bindText(2, identifier) + statement.step() + } + tx.touch(Table.Stacks) + } + + /** Drops catalog stacks whose identifier is not in [keep]. Other authors stay. */ + fun retainCatalogStacks(tx: Tx, pubkey: String, keep: Set) { + val stale = tx.db.prepare("SELECT identifier FROM stacks WHERE pubkey = ?").use { statement -> + statement.bindBlob(1, Hex.decode(pubkey)) + buildList { + while (statement.step()) { + val identifier = statement.getText(0) + if (identifier !in keep) add(identifier) + } + } + } + stale.forEach { deleteStack(tx, pubkey, it) } + } + + private fun zapAppId(event: Event): String? = + event.tagValue("i") ?: event.tagValue("a")?.let { AppCoordinate.parse(it)?.appId } + + private fun zapAmountSats(event: Event): Long { + event.tagValue("bolt11")?.let { invoice -> + val amount = Bolt11.amountSats(invoice) + if (amount > 0) return amount + } + return (event.tagValue("amount")?.toLongOrNull() ?: 0) / 1_000 + } + + fun isCatalogManifest(event: Event): Boolean = + event.kind == Kinds.Preferences && + event.tagValue("from") != null && + event.tagValue("to") != null && + event.tagValue("v") != null + + fun decryptJsonArray(content: String, deviceSigner: LocalSigner?): List { + if (content.isBlank() || deviceSigner == null) return emptyList() + return runCatching { JSONArray(deviceSigner.decryptFromSelf(content)).toStringList() }.getOrDefault(emptyList()) + } + + fun leftoverTags(stored: List>, modeled: Set): List> = + stored.filter { tag -> tag.firstOrNull() !in modeled } +} + +internal fun SQLiteStatement.bindTextOrNull(index: Int, value: String?) { + if (value == null) bindNull(index) else bindText(index, value) +} + +internal fun SQLiteStatement.bindBlobOrNull(index: Int, value: ByteArray?) { + if (value == null) bindNull(index) else bindBlob(index, value) +} + +internal fun SQLiteStatement.textOrNull(index: Int): String? = if (isNull(index)) null else getText(index) + +internal fun SQLiteStatement.blobHexOrNull(index: Int): String? = if (isNull(index)) null else Hex.encode(getBlob(index)) diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Hex.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Hex.kt new file mode 100644 index 0000000..9f40be3 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Hex.kt @@ -0,0 +1,33 @@ +package dev.zapstore.iolite + +object Hex { + fun encode(bytes: ByteArray): String = buildString(bytes.size * 2) { + for (byte in bytes) { + val value = byte.toInt() and 0xff + append("0123456789abcdef"[value ushr 4]) + append("0123456789abcdef"[value and 0x0f]) + } + } + + fun decode(value: String): ByteArray { + require(value.length % 2 == 0) { "hex length must be even" } + require(value.all { it in '0'..'9' || it in 'a'..'f' || it in 'A'..'F' }) { "invalid hex" } + return ByteArray(value.length / 2) { index -> + ((digit(value[index * 2]) shl 4) or digit(value[index * 2 + 1])).toByte() + } + } + + fun isHex(value: String, length: Int): Boolean = + value.length == length && value.all { it in '0'..'9' || it in 'a'..'f' } + + private fun digit(char: Char): Int = when (char) { + in '0'..'9' -> char - '0' + in 'a'..'f' -> char - 'a' + 10 + in 'A'..'F' -> char - 'A' + 10 + else -> error("invalid hex") + } +} + +fun ByteArray.toHex(): String = Hex.encode(this) + +fun String.decodeHex(): ByteArray = Hex.decode(this) diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Iolite.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Iolite.kt index 0e2ef72..994a5c2 100644 --- a/iolite/src/main/kotlin/dev/zapstore/iolite/Iolite.kt +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Iolite.kt @@ -1,1074 +1,559 @@ package dev.zapstore.iolite import android.content.Context -import android.net.ConnectivityManager -import android.net.Network -import com.vitorpamplona.quartz.nip01Core.cache.projection.EventStoreProjection -import com.vitorpamplona.quartz.nip01Core.core.Address -import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.isEphemeral -import com.vitorpamplona.quartz.nip01Core.core.isReplaceable -import com.vitorpamplona.quartz.nip01Core.crypto.verifyId -import com.vitorpamplona.quartz.nip01Core.crypto.verifySignature -import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener -import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener -import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient -import com.vitorpamplona.quartz.nip01Core.relay.client.single.newSubId -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.sockets.WebsocketBuilder -import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore -import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore.StoreChange -import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore -import com.vitorpamplona.quartz.nip40Expiration.isExpired -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.CoroutineDispatcher +import java.io.File +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.ProducerScope import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.buffer import kotlinx.coroutines.flow.callbackFlow -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.onSubscription +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.map import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicLong -import kotlin.time.Duration +/** + * The local-first Nostr client. Every event, whether it arrives in a catalog bundle or over a relay + * socket, is verified, turned into rows by [IoliteStore], and committed; commits notify the observers + * reading those tables. UI code reads records and never touches event JSON. + */ class Iolite private constructor( - private val store: ObservableEventStore, - private val client: INostrClient, + private val store: IoliteStore, + private val importer: CatalogImporter, private val scope: CoroutineScope, private val job: Job, private val databasePath: String, + private val http: HttpTransport?, + private val pool: RelayPool?, private val config: IoliteConfig, - private val eventVerifier: (Event) -> Boolean, - private val connectivityManager: ConnectivityManager?, - private val refreshCache: QueryRefreshCache, - private val clock: EpochMillisClock, + private val device: DeviceProfile, + private val deviceSigner: LocalSigner?, + private val writeRelays: Set, + private val nowMillis: () -> Long, ) : AutoCloseable { private val closed = AtomicBoolean(false) - private val networkCallbackRegistered = AtomicBoolean(false) - private val relayTrafficEnabled = AtomicBoolean(true) - private val sessions = mutableSetOf() - private val sessionsLock = Any() - private val refreshLock = Any() - private val relayTrafficLock = Any() - private val closeResult = CompletableDeferred>() + private val ingestContext = IngestContext(device, deviceSigner) + private val outboxAcks = ConcurrentHashMap>() + private val outboxLock = Mutex() + private val pendingIngest = ArrayList() - private val networkCallback = object : ConnectivityManager.NetworkCallback() { - override fun onAvailable(network: Network) { - // While relay traffic is suspended (app backgrounded) a network flap - // must not wake the pool. - if (client.isActive()) reconnectWithoutBackoff() - } - } + // --- catalog listings (local only; catalogs are synced, never queried over sockets) ----------- - init { - registerNetworkCallback() - } + fun observeApps(filter: AppFilter = AppFilter()): Flow> = observe(APP_TABLES) { it.apps(filter) } - private sealed interface Inbound { - val relay: NormalizedRelayUrl - val generation: Long + fun observeApp(appId: String): Flow = observe(APP_TABLES) { it.app(appId) } - data class Started(override val relay: NormalizedRelayUrl, override val generation: Long) : Inbound - data class EventReceived(override val relay: NormalizedRelayUrl, override val generation: Long, val event: Event) : Inbound - data class Eose(override val relay: NormalizedRelayUrl, override val generation: Long) : Inbound - data class Closed(override val relay: NormalizedRelayUrl, override val generation: Long, val message: String) : Inbound - data class CannotConnect(override val relay: NormalizedRelayUrl, override val generation: Long, val message: String) : Inbound - data class Connecting(override val relay: NormalizedRelayUrl, override val generation: Long) : Inbound - data class Disconnected(override val relay: NormalizedRelayUrl, override val generation: Long) : Inbound - } + fun apps(filter: AppFilter = AppFilter()): List = store.apps(filter) - private data class IngestOutcome( - val batch: List? = null, - val eoseAccepted: Boolean = false, - ) + fun app(appId: String): AppRecord? = store.app(appId) - private val expirationJob = scope.launch { - while (isActive) { - delay(config.expirationSweepInterval) - if (!closed.get()) runCatching { withStoreOperation { sweepStore() } } - } - } + fun catalogs(): List = store.selectedCatalogs() - private suspend fun sweepStore() { - store.deleteExpiredEvents() - if (config.pruneRules.isNotEmpty()) { - val nowSeconds = clock.now() / 1_000 - config.pruneRules.forEach { (kind, maxAge) -> - store.delete(Filter(kinds = listOf(kind), until = nowSeconds - maxAge.inWholeSeconds)) - } - } - // Keeps the SQLite query planner honest as the corpus grows; no-op for other stores. - (store.inner as? EventStore)?.optimize() - } + fun allCatalogs(): List = store.allCatalogs() - fun query(filter: Filter, options: QueryOptions = QueryOptions.local()): Flow = - query(listOf(filter), options) + fun defaultCatalog(): CatalogRecord = selectedCatalog() ?: error("default catalog is missing") - fun query(filters: List, options: QueryOptions = QueryOptions.local()): Flow = callbackFlow { - if (closed.get()) { - trySend(QueryState(emptyList(), QueryPhase.Failed, error = QueryError.Lifecycle("Iolite is closed"))) - close() - return@callbackFlow - } - if (filters.isEmpty()) { - trySend(QueryState(emptyList(), QueryPhase.Failed, error = QueryError.InvalidQuery("At least one filter is required"))) - close() - return@callbackFlow - } + fun selectedCatalog(): CatalogRecord? = store.selectedCatalogs().firstOrNull() - val snapshot = filters.map(::copyFilter) - val session = QuerySession(snapshot, options, this) - synchronized(sessionsLock) { - if (closed.get()) { - trySend(QueryState(emptyList(), QueryPhase.Failed, error = QueryError.Lifecycle("Iolite is closed"))) - close() - return@callbackFlow - } - sessions += session - } - session.start() - awaitClose { - session.stop() - synchronized(sessionsLock) { sessions -= session } - } - }.buffer(Channel.CONFLATED) + fun observeCatalogs(): Flow> = observe(setOf(Table.Catalogs)) { it.selectedCatalogs() } - override fun close() { - if (closed.compareAndSet(false, true)) { - val result = runCatching(::performClose) - closeResult.complete(result) - result.getOrThrow() - } else { - runBlocking { - closeResult.await() - }.getOrThrow() - } + fun importCatalogUpdate(catalogId: Long, bundle: ByteArray, endpoint: RelayUrl? = null): CatalogImportResult { + checkOpen() + return importer.importBundle(catalogId, bundle, endpoint) } /** - * Notifies the facade that its host application has returned to the - * foreground. Active relay subscriptions are retried immediately. + * Imports [bundled] when the catalog is missing or still at epoch 0, then fetches the next epoch. + * With no bundle and no stored catalog, fetches `GET /deltas?from=0` from [endpoint] and imports + * that body. Both bodies go through [importCatalogUpdate]. [CatalogSyncResult.imported] is null on + * HTTP 304 when nothing was imported. [endpoint] is stored when the catalog has none. [useOnion] + * selects a stored Tor endpoint when one is present. */ - fun refreshConnections() { - reconnectWithoutBackoff() - } - - /** - * Reloads every active local projection from the store. Use after a catalog - * delta or snapshot commits so screens see the new epoch without thousands - * of per-row store-change events. - */ - fun invalidateLocalProjections() { - if (closed.get()) return - synchronized(sessionsLock) { sessions.toList() }.forEach(QuerySession::reseedLocal) - } - - /** - * Enables or suspends all relay traffic for app foreground/background. - * Suspending closes every relay socket and stops the keep-alive reconnector; - * local data remains fully queryable. Re-enabling re-dials stale connections - * immediately, bypassing retry backoff. - */ - fun setRelayTrafficEnabled(enabled: Boolean) { - if (closed.get()) return - if (enabled) { - synchronized(relayTrafficLock) { - relayTrafficEnabled.set(true) - client.connect() - } - synchronized(sessionsLock) { sessions.toList() } - .forEach(QuerySession::resumeDeferredRemote) - synchronized(relayTrafficLock) { - if (relayTrafficEnabled.get()) { - client.reconnect(onlyIfChanged = true, ignoreRetryDelays = true) - } - } - } else { - synchronized(relayTrafficLock) { - relayTrafficEnabled.set(false) - client.disconnect() - } - } - } - - private fun performClose() { - var firstFailure: Throwable? = null - - fun attempt(block: () -> Unit) { - try { - block() - } catch (failure: Throwable) { - if (firstFailure == null) { - firstFailure = failure - } else { - firstFailure.addSuppressed(failure) - } - } - } - - val active = synchronized(sessionsLock) { sessions.toList().also { sessions.clear() } } - active.forEach { session -> attempt(session::closeFromFacade) } - expirationJob.cancel() - attempt(::unregisterNetworkCallback) - attempt(client::close) - job.cancel() - attempt { runBlocking(Dispatchers.IO) { job.join() } } - attempt(store::close) - synchronized(openDatabases) { openDatabases.remove(databasePath) } - firstFailure?.let { throw it } - } - - private fun registerNetworkCallback() { - val manager = connectivityManager ?: return - try { - manager.registerDefaultNetworkCallback(networkCallback) - networkCallbackRegistered.set(true) - } catch (_: SecurityException) { - // Connectivity callbacks are an enhancement; relay keep-alive - // reconnects remain available if the host denies registration. - } - } - - private fun unregisterNetworkCallback() { - val manager = connectivityManager ?: return - if (networkCallbackRegistered.compareAndSet(true, false)) { - manager.unregisterNetworkCallback(networkCallback) - } - } - - private fun reconnectWithoutBackoff() { - if (!closed.get()) { - client.reconnect(onlyIfChanged = true, ignoreRetryDelays = true) - } - } - - private suspend fun withStoreOperation(block: suspend () -> T): T { - check(!closed.get()) { "Iolite is closing" } - return block() - } - - private inner class QuerySession( - private val filters: List, - private val options: QueryOptions, - private val producer: kotlinx.coroutines.channels.ProducerScope, - ) { - private val stopped = AtomicBoolean(false) - private val acceptingCallbacks = AtomicBoolean(true) - private val timeoutRequested = AtomicBoolean(false) - private val remotePrepared = AtomicBoolean(false) - private val remoteStarted = AtomicBoolean(false) - private val remoteSubscribed = AtomicBoolean(false) - private val remoteDeferred = AtomicBoolean(false) - private val trafficDeferred = AtomicBoolean(false) - private val stateMutex = Mutex() - private val emitMutex = Mutex() - private val batchMutex = Mutex() - private val flushIoMutex = Mutex() - private val subId = newSubId() - private val relays = options.relays.toSet() - private val mode = options.remoteMode - private val cachedFor = options.cachedFor - private val queryFingerprint = if (options.sourceMode == SourceMode.LocalAndRemote) { - runCatching { QueryFingerprint.create(filters, relays) }.getOrNull() + fun syncCatalog( + catalogId: Long = defaultCatalog().id, + transport: HttpTransport? = http, + useOnion: Boolean = false, + bundled: ByteArray = ByteArray(0), + endpoint: RelayUrl? = null, + ): CatalogSyncResult { + checkOpen() + val httpClient = transport ?: throw IllegalStateException("HTTP transport is required for catalog sync") + val existing = store.catalog(catalogId) + val seeded = if (bundled.isNotEmpty() && (existing == null || existing.epoch == 0L)) { + importCatalogUpdate(catalogId, bundled, endpoint) } else { null } - private val relayStates = mutableMapOf() - private val callbackGenerations = mutableMapOf() - private val callbackGenerationStarted = mutableMapOf() - private var items: List = emptyList() - private var error: QueryError? = null - @Volatile - private var lastState = QueryState(emptyList(), QueryPhase.Connecting, relayStates.toMap()) - private var emittedPhase: QueryPhase? = null - private val remoteItemsById = LinkedHashMap() - private val remoteMatchesSinceBarrier = AtomicBoolean(false) - private val messages = Channel(config.ingestionCapacity) - private var observerJob: Job? = null - private var workerJob: Job? = null + val catalog = store.catalog(catalogId) + if (catalog == null || catalog.relays.isEmpty()) { + val relay = endpoint ?: throw CatalogImportException("catalog relay is missing") + return applyCatalogResponse( + catalogId, + httpClient.get(relay.deltasUrl(0), emptyMap()), + seeded = null, + seededBytes = 0, + endpoint = relay, + ) + } + return applyCatalogResponse( + catalogId, + httpClient.get(catalog.syncRelay(useOnion).deltasUrl(catalog.epoch), emptyMap()), + seeded = seeded, + seededBytes = bundled.size.toLong(), + endpoint = null, + ) + } + + private fun applyCatalogResponse( + catalogId: Long, + response: HttpResponse, + seeded: CatalogImportResult?, + seededBytes: Long, + endpoint: RelayUrl?, + ): CatalogSyncResult { + val body = response.body + val bytes = (body?.size ?: 0).toLong() + return when (response.status) { + 304 -> CatalogSyncResult(imported = seeded, bytes = if (seeded != null) seededBytes else bytes) + 200 -> CatalogSyncResult( + imported = importCatalogUpdate( + catalogId, + body ?: throw CatalogImportException("empty catalog update"), + endpoint, + ), + bytes = bytes, + ) + 400 -> throw CatalogImportException(response.headers["X-Reason"] ?: "catalog protocol error") + else -> throw CatalogImportException("catalog update failed with HTTP ${response.status}") + } + } + + /** Avatar WebP from the catalog bundle, keyed by hex pubkey. Absent until that profile is sealed. */ + fun avatarFile(pubkey: String): File = store.avatarFile(pubkey) + + fun wipeLocalData() { + checkOpen() + store.wipe() + val parent = File(databasePath).parentFile + listOf("icons", "avatars").forEach { name -> + val dir = File(parent, name) + dir.deleteRecursively() + dir.mkdirs() + } + } + + // --- typed queries ----------------------------------------------------------------------------- + + /** + * Runs [query] under [options]. The flow re-emits after every commit touching the query's tables and + * completes only when the collector cancels, so scope it to the screen that shows it. + */ + fun query(query: Query, options: QueryOptions = QueryOptions.local()): Flow> = callbackFlow { + if (closed.get()) { + close() + return@callbackFlow + } + when (options.sourceMode) { + SourceMode.Local -> { + emitOnChange(query, QueryPhase.LocalOnly) + awaitClose() + } + SourceMode.LocalAndRemote, SourceMode.Remote -> RemoteQuery(query, options, this).run() + } + }.buffer(Channel.CONFLATED).flowOn(Dispatchers.IO) + + /** Relay path for events obtained outside the pool (push, share sheet, tests). Returns rows changed. */ + fun ingest(events: List): Int { + checkOpen() + return store.ingest(events, ingestContext) + } + + // --- publishing -------------------------------------------------------------------------------- + + suspend fun publish( + kind: Int, + tags: List>, + content: String, + signer: Signer, + createdAt: Long = nowMillis() / 1_000, + relays: Set = writeRelays, + ): Event { + checkOpen() + val event = signer.sign(createdAt, kind, tags, content) + store.commitPublish(event, ingestContext) + publishOutbox(event, relays) + return event + } + + suspend fun rehydrateCatalogRelayList(): Event { + val signer = deviceSigner ?: error("device signer is required to rehydrate kind 10067") + val (tags, content) = store.catalogRelayListTemplate(signer) + return publish(Kinds.CatalogRelayList, tags, content, signer) + } + + suspend fun rehydratePreferences(identifier: String, signer: Signer): Event { + val (tags, content) = store.preferenceTemplate(identifier) + return publish(Kinds.Preferences, tags, content, signer) + } + + suspend fun rehydrateStack(identifier: String, signer: Signer): Event { + val (tags, content) = store.stackTemplate(signer.publicKey, identifier, device, deviceSigner) + return publish(Kinds.AppStack, tags, content, signer) + } + + fun pendingOutbox(): List = store.pendingOutbox() + + // --- connectivity ------------------------------------------------------------------------------ + + fun refreshConnections() { + pool?.refreshConnections() + scope.launch { flushOutbox() } + } + + fun setRelayTrafficEnabled(enabled: Boolean) { + pool?.setTrafficEnabled(enabled) + if (enabled) scope.launch { flushOutbox() } + } + + override fun close() { + if (!closed.compareAndSet(false, true)) return + flushIngest() + pool?.close() + job.cancel() + store.close() + synchronized(openDatabases) { openDatabases.remove(databasePath) } + } + + // --- internals --------------------------------------------------------------------------------- + + private fun checkOpen() = check(!closed.get()) { "Iolite is closed" } + + private fun observe(tables: Set, read: (IoliteStore) -> T): Flow = + store.changes(tables).map { read(store) }.flowOn(Dispatchers.IO) + + private fun ProducerScope>.emitOnChange(query: Query, phase: QueryPhase) { + launch { + store.changes(query.tables).collect { + if (!closed.get()) trySend(QueryState(query.read(store, deviceSigner), phase)) + } + } + } + + /** One remote-enabled collection: relay lifecycle, phase transitions, and re-reads on commit. */ + private inner class RemoteQuery( + private val query: Query, + private val options: QueryOptions, + private val producer: ProducerScope>, + ) { + private val lock = Any() + @Volatile private var phase: QueryPhase? = null + @Volatile private var error: QueryError? = null + private val relayStates = ConcurrentHashMap() + private var unsubscribe: (() -> Unit)? = null + private var graceJob: Job? = null private var timeoutJob: Job? = null - private var remoteDelayJob: Job? = null - private var flushJob: Job? = null - private var remotePublishJob: Job? = null - private val pendingInserts = ArrayList() - @Volatile - private var emittedIds: List = emptyList() - @Volatile - private var emittedAddresses: Set
= emptySet() - private val subscriptionListener = object : SubscriptionListener { - override fun onSubscriptionStarted(relay: String, forFilters: List) { - relay.normalizeRelayUrl().let { normalized -> - callbackGenerationStarted[normalized]?.set(true) - enqueue(Inbound.Started(normalized, generationOf(normalized))) - } - } - - override fun onEvent(event: Event, isLive: Boolean, relay: NormalizedRelayUrl, forFilters: List?) { - enqueue(Inbound.EventReceived(relay, generationOf(relay), event)) - } - - override fun onEose(relay: NormalizedRelayUrl, forFilters: List?) { - enqueue(Inbound.Eose(relay, generationOf(relay))) - } - - override fun onClosed(message: String, relay: NormalizedRelayUrl, forFilters: List?) { - enqueue(Inbound.Closed(relay, generationOf(relay), "Relay closed the request")) - } - - override fun onCannotConnect(relay: NormalizedRelayUrl, message: String, forFilters: List?) { - enqueue(Inbound.CannotConnect(relay, generationOf(relay), "Unable to connect to relay")) - } - } - - private val connectionListener = object : RelayConnectionListener { - override fun onConnecting(relay: IRelayClient) { - val normalized = relay.url - if (normalized !in relays) return - val generation = callbackGenerations.getValue(normalized) - if (callbackGenerationStarted.getValue(normalized).compareAndSet(true, false)) generation.incrementAndGet() - enqueue(Inbound.Connecting(normalized, generation.get())) - } - - override fun onDisconnected(relay: IRelayClient) { - val normalized = relay.url - enqueue(Inbound.Disconnected(normalized, generationOf(normalized))) - } - } - - fun start() { - when (options.sourceMode) { - SourceMode.Local -> startLocalOnly() - SourceMode.LocalAndRemote -> startLocalAndRemote() - SourceMode.Remote -> startRemoteOnly() - } - } - - fun reseedLocal() { - if (stopped.get() || options.sourceMode == SourceMode.Remote) return - observerJob?.cancel() - observerJob = startObserver(initialPhase = { lastState.phase }) - } - - private fun startLocalOnly() { - observerJob = startObserver(initialPhase = { QueryPhase.LocalOnly }) - } - - private fun startLocalAndRemote() { - observerJob = startObserver( - initialPhase = { localItems -> - val fresh = localItems.isNotEmpty() && cachedFor?.let(::freshness)?.isFresh == true - remoteDeferred.set(fresh) - if (!fresh) prepareRemote() - when { - !fresh -> QueryPhase.Connecting - else -> QueryPhase.Cached - } - }, - afterSeed = { seedPhase -> - when (seedPhase) { - QueryPhase.Cached -> { - if (mode is RemoteMode.OneShot) finish() else scheduleRemoteAfterCache() - } - else -> startRemote() - } - }, - ) - } - - private fun scheduleRemoteAfterCache() { - val cacheDuration = cachedFor ?: return - remoteDelayJob = scope.launch { - while (!stopped.get()) { - val status = freshness(cacheDuration) - if (status.isFresh) { - delay(status.remainingMillis.coerceIn(1, CACHE_RECHECK_INTERVAL_MILLIS)) - continue - } - var shouldStart = false - stateMutex.withLock { - shouldStart = synchronized(refreshLock) { - !freshnessUnlocked(cacheDuration).isFresh && - remoteDeferred.compareAndSet(true, false) - } - if (!stopped.get() && shouldStart) { - prepareRemote() - emitState(QueryPhase.Connecting) - startRemote() - } - } - if (shouldStart || stopped.get()) return@launch - } - } - } - - private fun freshness(cacheDuration: Duration): QueryFreshness = synchronized(refreshLock) { - freshnessUnlocked(cacheDuration) - } - - private fun freshnessUnlocked(cacheDuration: Duration): QueryFreshness { - val fingerprint = queryFingerprint - ?: return QueryFreshness(isFresh = false, remainingMillis = 0) - return runCatching { - refreshCache.freshness(fingerprint, cacheDuration, clock.now()) - }.getOrDefault(QueryFreshness(isFresh = false, remainingMillis = 0)) - } - - private fun recordRefresh() { - if (options.sourceMode != SourceMode.LocalAndRemote) return - if (!remoteMatchesSinceBarrier.getAndSet(false)) return - val fingerprint = queryFingerprint ?: return - synchronized(refreshLock) { - runCatching { refreshCache.recordRefresh(fingerprint, clock.now()) } - } - } - - private fun startRemoteOnly() { - scope.launch { - stateMutex.withLock { - prepareRemote() - emitState(QueryPhase.Connecting) - startRemote() - } - } - } - - /** - * Subscribes to the store change feed before seeding the projection, so a change - * that races the seed is buffered by the SharedFlow and applied right after it. - * The projection applies each change in memory; collectors only re-emit when the - * visible id set actually changes (plus in-place updates of visible replaceables). - */ - private fun startObserver( - initialPhase: (List) -> QueryPhase, - afterSeed: ((QueryPhase) -> Unit)? = null, - ): Job = - scope.launch(start = CoroutineStart.UNDISPATCHED) { - val projection = EventStoreProjection(store, filters) - try { - store.changes - .onSubscription { - projection.seed() - val seededItems = projection.snapshotItems() - val seedPhase = initialPhase(seededItems) - stateMutex.withLock { - publishLocked(seededItems, seedPhase, force = true) - } - afterSeed?.invoke(seedPhase) - if (stopped.get()) throw CancellationException("query completed from cache") - } - .collect { change -> - if (stopped.get()) throw CancellationException("query session stopped") - val applied = projection.apply(change) - val inPlaceUpdate = change.isVisibleInPlaceUpdate() - if (!applied && !inPlaceUpdate) return@collect - val fresh = projection.snapshotItems() - stateMutex.withLock { - if ( - fresh.isEmpty() && - options.sourceMode == SourceMode.LocalAndRemote && - remoteDeferred.compareAndSet(true, false) - ) { - remoteDelayJob?.cancel() - prepareRemote() - publishLocked(fresh, QueryPhase.Connecting) - startRemote() - } else { - publishLocked(fresh, sync(), force = inPlaceUpdate) - } - } - } - } catch (cancelled: CancellationException) { - throw cancelled - } catch (failure: Throwable) { - fail(QueryError.UnsupportedLocalProjection("Local store projection failed")) - } - } - - private fun StoreChange.isVisibleInPlaceUpdate(): Boolean { - val event = (this as? StoreChange.Insert)?.event ?: return false - val address = addressOf(event) ?: return false - return address in emittedAddresses - } - - private suspend fun publishLocked(fresh: List, phase: QueryPhase, force: Boolean = false) { - val ids = fresh.map { it.id } - if (!force && ids == emittedIds) return - emittedIds = ids - emittedAddresses = fresh.mapNotNullTo(HashSet(), Companion::addressOf) - items = fresh - emitState(phase) - } - - private fun startRemote() { - synchronized(relayTrafficLock) { - if (stopped.get()) return - prepareRemote() - if (!relayTrafficEnabled.get()) { - trafficDeferred.set(true) + suspend fun run() { + val fingerprint = QueryFingerprint.create(query.filters, options.relays) + val cachedFor = options.cachedFor + if (cachedFor != null) { + val last = store.lastRefresh(fingerprint) + if (last != null && nowMillis() - last < cachedFor.inWholeMilliseconds) { + producer.emitOnChange(query, QueryPhase.Cached) + producer.awaitClose() return } - if (!remoteStarted.compareAndSet(false, true)) return - trafficDeferred.set(false) - workerJob = scope.launch { - try { - for (message in messages) { - val outcome = stateMutex.withLock { - if (!stopped.get()) processLocked(message) else IngestOutcome() - } - outcome.batch?.let { flushBatch(it) } - if (outcome.eoseAccepted) { - flushPendingAndWait() - stateMutex.withLock { - if (!stopped.get()) postEoseLocked(message as Inbound.Eose) + } + + transition(QueryPhase.Connecting) { if (options.sourceMode == SourceMode.Remote) query.empty else read() } + producer.launch { + // The first emission is the subscription itself; the snapshot above already covered it. + store.changes(query.tables).drop(1).collect { emitCurrent() } + } + + val relayPool = pool + if (relayPool == null) { + error = QueryError.IncompatibleDependency("WebSocket transport is required") + transition(QueryPhase.Failed) + producer.awaitClose() + return + } + + val relays = if (options.useAuthorRelays) options.relays + resolveReadRelays(query.authors, options.relays) else options.relays + val timeout = (options.remoteMode as? RemoteMode.OneShot)?.timeout ?: config.oneShotTimeout + timeoutJob = producer.launch { + delay(timeout) + if (phase != QueryPhase.Connecting && phase != QueryPhase.CatchingUp) return@launch + error = QueryError.Timeout(timeout, "one-shot query timed out after $timeout") + transition(QueryPhase.TimedOut) + stopRelayWork() + } + unsubscribe = relayPool.subscribe( + id = "q" + Hex.encode(Crypto.randomBytes(4)), + filters = query.filters, + relays = relays, + onEvent = { relay, event -> + if (!event.verify()) { + error = QueryError.VerificationRejected(relay, event.id, "event failed verification") + return@subscribe + } + enqueueIngest(event) + if (phase == QueryPhase.Connecting) transition(QueryPhase.CatchingUp) + }, + onEose = { _ -> + synchronized(lock) { + if (graceJob != null) return@subscribe + graceJob = producer.launch { + delay(config.eoseGrace) + flushIngest() + when (options.remoteMode) { + is RemoteMode.OneShot -> { + store.recordRefresh(fingerprint, nowMillis()) + transition(QueryPhase.Complete) + stopRelayWork() } + RemoteMode.Stream -> { + timeoutJob?.cancel() + transition(QueryPhase.Live) + } + null -> Unit } } - } finally { - flushPendingAndWait() } - } - try { - remoteSubscribed.set(true) - client.addConnectionListener(connectionListener) - // Generation 1 has already been allocated in relayStates before this call. - client.subscribe(subId, relays.associateWith { filters }, subscriptionListener) - } catch (cancelled: CancellationException) { - throw cancelled - } catch (_: Throwable) { - fail(QueryError.IncompatibleDependency("Unable to start the Quartz subscription")) - return - } - val oneShot = mode as? RemoteMode.OneShot - if (oneShot != null) { - val timeout = oneShot.timeout ?: config.oneShotTimeout - timeoutJob = scope.launch { - delay(timeout) - timeout(timeout) + }, + onState = { relay, state -> + relayStates[relay] = state + if (state.lastError != null && phase == QueryPhase.Connecting) { + error = QueryError.RelayFailure(relay, state.lastError) } - } - } - } - - fun resumeDeferredRemote() { - if (!trafficDeferred.compareAndSet(true, false) || stopped.get()) return - scope.launch { - stateMutex.withLock { - if (!stopped.get()) startRemote() - } - } - } - - private fun prepareRemote() { - if (!remotePrepared.compareAndSet(false, true)) return - relays.forEach { relay -> - relayStates[relay] = RelayQueryState(1, RelayConnectionState.Connecting, eose = false) - callbackGenerations[relay] = AtomicLong(1) - callbackGenerationStarted[relay] = AtomicBoolean(false) - } - } - - private fun enqueue(message: Inbound) { - if (message.relay !in relays || stopped.get() || !acceptingCallbacks.get()) return - val result = messages.trySend(message) - if (result.isFailure && !result.isClosed) { - acceptingCallbacks.set(false) - fail(QueryError.IngestionSaturated(message.relay, "Remote ingestion queue is full")) - } - } - - private suspend fun processLocked(message: Inbound): IngestOutcome { - val currentGeneration = relayStates[message.relay]?.generation ?: return IngestOutcome() - if (message is Inbound.Connecting) { - if (message.generation < currentGeneration) return IngestOutcome() - } else if (message.generation != currentGeneration) { - return IngestOutcome() - } - return when (message) { - is Inbound.Started -> { - updateRelay(message.relay) { it.copy(connection = RelayConnectionState.Connected, eose = false, lastError = null) } - emitState(sync()) - IngestOutcome() - } - is Inbound.Connecting -> { - updateRelay(message.relay) { - it.copy(generation = message.generation, connection = RelayConnectionState.Connecting, eose = false) - } - emitState(sync()) - IngestOutcome() - } - is Inbound.Disconnected -> { - updateRelay(message.relay) { it.copy(connection = RelayConnectionState.Disconnected, eose = false) } - emitState(sync()) - IngestOutcome() - } - is Inbound.CannotConnect -> { - updateRelay(message.relay) { it.copy(connection = RelayConnectionState.Disconnected, lastError = message.message) } - error = QueryError.RelayFailure(message.relay, message.message) - emitState(sync()) - IngestOutcome() - } - is Inbound.Closed -> { - updateRelay(message.relay) { - it.copy(connection = RelayConnectionState.Closed, eose = true, lastError = message.message) - } - if (relayStates.values.all { it.connection == RelayConnectionState.Closed }) { - fail(QueryError.RelayFailure(message.relay, message.message)) - } else { - emitState(sync()) - } - IngestOutcome() - } - is Inbound.EventReceived -> IngestOutcome(batch = ingestLocked(message.relay, message.event)) - is Inbound.Eose -> { - updateRelay(message.relay) { it.copy(eose = true) } - if (timeoutRequested.get()) return IngestOutcome() - IngestOutcome(batch = drainPendingInserts(), eoseAccepted = true) - } - } - } - - private suspend fun postEoseLocked(message: Inbound.Eose) { - if (timeoutRequested.get()) return - snapshotRemoteItemsLocked() - error = null - val allRelaysCaughtUp = relayStates.values.all { it.eose } - if (allRelaysCaughtUp) { - if (mode is RemoteMode.OneShot) { - if (options.sourceMode == SourceMode.LocalAndRemote) { - val fresh = try { - withStoreOperation { store.query(filters) } - } catch (cancelled: CancellationException) { - throw cancelled - } catch (_: Throwable) { - fail(QueryError.UnsupportedLocalProjection("Local store projection failed")) - return - } - recordRefresh() - try { - publishLocked(fresh, QueryPhase.Complete, force = true) - } catch (cancelled: CancellationException) { - throw cancelled - } catch (_: Throwable) { - fail(QueryError.UnsupportedLocalProjection("Local store projection failed")) - return - } - } else { - emitState(QueryPhase.Complete) - } - finish() - } else { - if (options.sourceMode == SourceMode.LocalAndRemote) { - // EOSE is a sync barrier: the final ingest batch was just flushed, - // but the observer coroutine may not have applied it yet. Requery - // so the terminal-at-EOSE state cannot lag the store. - val fresh = try { - withStoreOperation { store.query(filters) } - } catch (cancelled: CancellationException) { - throw cancelled - } catch (_: Throwable) { - fail(QueryError.UnsupportedLocalProjection("Local store projection failed")) - return - } - recordRefresh() - try { - publishLocked(fresh, sync(), force = true) - } catch (cancelled: CancellationException) { - throw cancelled - } catch (_: Throwable) { - fail(QueryError.UnsupportedLocalProjection("Local store projection failed")) - return - } - } else { - emitState(sync()) - } - } - } else { - emitState(sync()) - } - } - - private suspend fun ingestLocked(relay: NormalizedRelayUrl, event: Event): List? { - val eventId = validatedId(event.id) - val valid = try { - eventVerifier(event) - } catch (cancelled: CancellationException) { - throw cancelled - } catch (_: Throwable) { - false - } - if (!valid) { - error = QueryError.VerificationRejected(relay, eventId, "Relay event failed verification") - updateRelay(relay) { it.copy(lastError = error?.message) } - emitState(sync()) - return null - } - if (!filters.any { it.match(event) }) { - error = QueryError.ProtocolViolation(relay, eventId, "Relay event does not match this query") - updateRelay(relay) { it.copy(lastError = error?.message) } - emitState(sync()) - return null - } - if (event.isExpired()) return null - - remoteMatchesSinceBarrier.set(true) - if (options.sourceMode == SourceMode.Remote && event.id !in remoteItemsById) { - remoteItemsById[event.id] = event - error = null - scheduleRemotePublish() - } - if (event.kind.isEphemeral()) return null - - return batchMutex.withLock { - pendingInserts += event - if (pendingInserts.size >= config.ingestBatchSize) { - drainPendingInsertsLocked() - } else { - scheduleFlush() - null - } - } - } - - private fun scheduleFlush() { - if (stopped.get() || flushJob?.isActive == true) return - flushJob = scope.launch { - delay(config.ingestFlushInterval) - val batch = drainPendingInserts() - if (batch != null) flushBatch(batch) - } - } - - private fun scheduleRemotePublish() { - if (stopped.get() || remotePublishJob?.isActive == true) return - remotePublishJob = scope.launch { - delay(REMOTE_EMIT_INTERVAL_MILLIS) - stateMutex.withLock { - if (!stopped.get() && snapshotRemoteItemsLocked()) emitState(sync()) - } - } - } - - private fun snapshotRemoteItemsLocked(): Boolean { - if (options.sourceMode != SourceMode.Remote || remoteItemsById.size == items.size) return false - remotePublishJob?.cancel() - remotePublishJob = null - items = remoteItemsById.values.toList() - return true - } - - private fun drainPendingInsertsLocked(): List? { - if (pendingInserts.isEmpty()) return null - flushJob?.cancel() - return pendingInserts.toList().also { pendingInserts.clear() } - } - - private suspend fun drainPendingInserts(): List? = - batchMutex.withLock { drainPendingInsertsLocked() } - - private suspend fun flushBatch(batch: List) { - val outcomes = try { - flushIoMutex.withLock { - withStoreOperation { store.batchInsert(batch) } - } - } catch (cancelled: CancellationException) { - throw cancelled - } catch (_: Throwable) { - null - } - val rejected = when { - outcomes == null -> batch - else -> batch.filterIndexed { index, _ -> - outcomes.getOrNull(index) is IEventStore.InsertOutcome.Rejected - } - } - val relevantRejected = rejected.filterNot(Event::isExpired) - // Quartz may reject after concurrent duplicate commits. Verify the whole - // rejected set in bounded chunks instead of issuing one query per event. - val existingIds = relevantRejected - .map(Event::id) - .chunked(REJECTED_ID_QUERY_CHUNK_SIZE) - .flatMapTo(HashSet()) { ids -> - runCatching { - withStoreOperation { store.query(Filter(ids = ids)) } - }.getOrDefault(emptyList()).map(Event::id) - } - val genuinelyMissing = relevantRejected.firstOrNull { it.id !in existingIds } - if (genuinelyMissing != null) { - fail(QueryError.PersistenceFailure(validatedId(genuinelyMissing.id), "Unable to persist relay event")) - } else { - stateMutex.withLock { if (!stopped.get()) error = null } - } - } - - private suspend fun flushPendingAndWait() { - val batch = drainPendingInserts() - if (batch != null) flushBatch(batch) - flushIoMutex.withLock { } - } - - private fun sync(): QueryPhase = when { - remoteDeferred.get() -> QueryPhase.Cached - relays.isEmpty() -> QueryPhase.LocalOnly - relayStates.values - .filter { it.connection != RelayConnectionState.Closed } - .any { it.connection != RelayConnectionState.Connected } -> QueryPhase.Connecting - relayStates.values - .filter { it.connection != RelayConnectionState.Closed } - .any { !it.eose } -> QueryPhase.CatchingUp - mode is RemoteMode.Stream -> QueryPhase.Live - else -> QueryPhase.Complete - } - - private suspend fun emitState(phase: QueryPhase) { - emitMutex.withLock { - if (!stopped.get()) { - check(isLegalPhaseTransition(emittedPhase, phase)) { - "Illegal query phase transition: $emittedPhase -> $phase" - } - emittedPhase = phase - val visibleRelays = if (remoteDeferred.get()) emptyMap() else relayStates.toMap() - val state = QueryState(items, phase, visibleRelays, error) - lastState = state - producer.send(state) - } - } - } - - private fun updateRelay(relay: NormalizedRelayUrl, transform: (RelayQueryState) -> RelayQueryState) { - relayStates[relay]?.let { relayStates[relay] = transform(it) } - } - - private fun generationOf(relay: NormalizedRelayUrl): Long = callbackGenerations[relay]?.get() ?: 0 - - private fun timeout(duration: Duration) { - if (stopped.get()) return - timeoutRequested.set(true) - if (!acceptingCallbacks.compareAndSet(true, false)) return - unsubscribeRemote() - messages.close() - scope.launch { - workerJob?.join() - stateMutex.withLock { - if (!stopped.get()) { - snapshotRemoteItemsLocked() - error = QueryError.Timeout(duration, "Remote query timed out") - emitState(QueryPhase.TimedOut) - finish() - } - } - } - } - - private fun fail(queryError: QueryError) { - if (!stopped.compareAndSet(false, true)) return - acceptingCallbacks.set(false) - cleanup() - scope.launch { - stateMutex.withLock { - snapshotRemoteItemsLocked() - error = queryError - check(isLegalPhaseTransition(emittedPhase, QueryPhase.Failed)) { - "Illegal query phase transition: $emittedPhase -> ${QueryPhase.Failed}" - } - emittedPhase = QueryPhase.Failed - val state = QueryState(items, QueryPhase.Failed, relayStates.toMap(), error) - lastState = state - producer.send(state) - producer.close() - } - } - } - - private fun finish() { - if (!stopped.compareAndSet(false, true)) return - producer.close() - cleanup() - } - - fun stop() { - if (!stopped.compareAndSet(false, true)) return - acceptingCallbacks.set(false) - cleanup() - } - - fun closeFromFacade() { - stopped.set(true) - acceptingCallbacks.set(false) - cleanup() - val terminal = lastState.copy( - phase = QueryPhase.Failed, - error = QueryError.Lifecycle("Iolite is closed"), + emitCurrent() + }, ) - lastState = terminal - producer.trySend(terminal) - producer.close() + producer.awaitClose { stopRelayWork() } } - private fun cleanup() { - acceptingCallbacks.set(false) + private fun read(): T = query.read(store, deviceSigner) + + private fun emitCurrent() { + if (closed.get()) return + val current = phase ?: return + producer.trySend(QueryState(read(), current, relayStates.toMap(), error)) + } + + private fun transition(next: QueryPhase, items: () -> T = ::read) { + synchronized(lock) { + if (!isLegalPhaseTransition(phase, next)) return + phase = next + } + if (closed.get()) return + producer.trySend(QueryState(items(), next, relayStates.toMap(), error)) + } + + private fun stopRelayWork() { timeoutJob?.cancel() - remoteDelayJob?.cancel() - trafficDeferred.set(false) - observerJob?.cancel() - flushJob?.cancel() - remotePublishJob?.cancel() - messages.close() - workerJob?.cancel() - unsubscribeRemote() + graceJob?.cancel() + unsubscribe?.invoke() + unsubscribe = null } + } - private fun unsubscribeRemote() { - if (!remoteSubscribed.compareAndSet(true, false)) return - runCatching { client.unsubscribe(subId) } - runCatching { client.removeConnectionListener(connectionListener) } + /** NIP-65 read relays of [authors], from SQLite when fresh, else asked of [bootstrap]. */ + private suspend fun resolveReadRelays(authors: List, bootstrap: Set): Set { + if (authors.isEmpty()) return emptySet() + return coroutineScope { + authors.distinct().map { author -> + async { + runCatching { + query( + Query.relayList(author), + QueryOptions.localAndRemote( + relays = bootstrap, + remoteMode = RemoteMode.OneShot(config.relayListTimeout), + cachedFor = config.relayListCacheFor, + ), + ).first { it.isTerminal }.items.orEmpty() + }.getOrDefault(emptySet()) + } + }.awaitAll().flatten().toSet() + } + } + + private fun enqueueIngest(event: Event) { + val flushNow = synchronized(pendingIngest) { + pendingIngest += event + pendingIngest.size >= config.ingestBatchSize + } + if (flushNow) flushIngest() + } + + private fun flushIngest() { + val batch = synchronized(pendingIngest) { + if (pendingIngest.isEmpty()) return + pendingIngest.toList().also { pendingIngest.clear() } + } + runCatching { store.ingest(batch, ingestContext) } + } + + private fun start() { + scope.launch { flushOutbox() } + scope.launch { + while (isActive) { + delay(config.ingestFlushInterval) + flushIngest() + } + } + scope.launch { + while (isActive) { + delay(config.expirationSweepInterval) + pruneExpired() + } + } + scope.launch { + while (isActive) { + val now = nowMillis() / 1_000 + val next = store.nextProofExpiry(now) + val sweep = config.expirationSweepInterval.inWholeMilliseconds + val waitMs = if (next == null) sweep else minOf(((next - now) * 1_000L + 50L).coerceAtLeast(0L), sweep) + delay(waitMs) + store.recomputeProofs(nowMillis() / 1_000) + } + } + } + + private fun pruneExpired() { + val now = nowMillis() + config.pruneRules.forEach { (kind, maxAge) -> + store.pruneOlderThan(kind, (now - maxAge.inWholeMilliseconds) / 1_000L) + } + } + + private suspend fun flushOutbox() { + outboxLock.withLock { + store.pendingOutbox().forEach { publishOutbox(it, writeRelays) } + } + } + + private fun publishOutbox(event: Event, relays: Set) { + val dest = relays.ifEmpty { writeRelays } + if (dest.isEmpty() || pool == null) return + val needed = okThreshold(dest.size) + pool.publish(event, dest) { relay, accepted, _ -> + if (!accepted) return@publish + val acks = outboxAcks.getOrPut(event.id) { mutableSetOf() } + synchronized(acks) { + acks += relay + if (acks.size >= needed) { + store.deleteOutbox(event.id) + outboxAcks.remove(event.id) + } + } } } companion object { + private val APP_TABLES = setOf(Table.Apps, Table.Proofs, Table.Catalogs) private val openDatabases = mutableSetOf() fun create( context: Context, - websocketBuilder: WebsocketBuilder, parentScope: CoroutineScope, config: IoliteConfig = IoliteConfig(), - eventStore: (path: String) -> IEventStore = { path -> EventStore(dbName = path, relay = null) }, + http: HttpTransport? = null, + webSocket: WebSocketFactory? = null, + signer: Signer? = null, + deviceSigner: LocalSigner? = signer as? LocalSigner, + writeRelays: Set = emptySet(), + device: DeviceProfile = DeviceProfile(abis = listOf("arm64-v8a"), sdk = 29), + ): Iolite = open( + databasePath = context.applicationContext.getDatabasePath(config.databaseName).canonicalPath, + parentScope = parentScope, + config = config, + device = device, + http = http, + webSocket = webSocket, + signer = signer, + deviceSigner = deviceSigner, + writeRelays = writeRelays, + nowMillis = { System.currentTimeMillis() }, + ) + + fun createForTesting( + databasePath: String, + parentScope: CoroutineScope, + config: IoliteConfig = IoliteConfig(), + device: DeviceProfile = DeviceProfile(abis = listOf("arm64-v8a"), sdk = 34), + http: HttpTransport? = null, + webSocket: WebSocketFactory? = null, + signer: Signer? = null, + deviceSigner: LocalSigner? = signer as? LocalSigner, + writeRelays: Set = emptySet(), + nowMillis: () -> Long = { System.currentTimeMillis() }, + ): Iolite = open(databasePath, parentScope, config, device, http, webSocket, signer, deviceSigner, writeRelays, nowMillis) + + private fun open( + databasePath: String, + parentScope: CoroutineScope, + config: IoliteConfig, + device: DeviceProfile, + http: HttpTransport?, + webSocket: WebSocketFactory?, + signer: Signer?, + deviceSigner: LocalSigner?, + writeRelays: Set, + nowMillis: () -> Long, ): Iolite { config.validate() - val parentJob = requireActiveParentJob(parentScope) - val appContext = context.applicationContext - val databaseFile = appContext.getDatabasePath(config.databaseName).absoluteFile - val databaseExisted = databaseFile.exists() - val path = databaseFile.canonicalPath + val parentJob = requireNotNull(parentScope.coroutineContext[Job]) { "parentScope must contain an active Job" } + require(parentJob.isActive) { "parentScope must be active" } synchronized(openDatabases) { - check(path !in openDatabases) { "A Iolite instance already owns this database" } - openDatabases += path + check(databasePath !in openDatabases) { "An Iolite instance already owns this database" } + openDatabases += databasePath } - val job = SupervisorJob(parentJob) - val scope = CoroutineScope(parentScope.coroutineContext + job + Dispatchers.IO) - var store: ObservableEventStore? = null - var client: INostrClient? = null try { - store = ObservableEventStore(eventStore(path)) - val refreshCache = SharedPreferencesQueryRefreshCache.create( - context = appContext, - databasePath = path, - databaseExisted = databaseExisted, - ) - client = NostrClient(websocketBuilder, scope) + val job = SupervisorJob(parentJob) + val scope = CoroutineScope(parentScope.coroutineContext + job + Dispatchers.IO) + val nowSeconds = { nowMillis() / 1_000 } + val store = IoliteStore(databasePath) + try { + store.recomputeProofs(nowSeconds()) + } catch (failure: Throwable) { + store.close() + throw failure + } + val importer = CatalogImporter(store, File(File(databasePath).parentFile, "icons"), config, device, nowSeconds) return Iolite( - store, - client, - scope, - job, - path, - config, - DEFAULT_EVENT_VERIFIER, - appContext.getSystemService(Context.CONNECTIVITY_SERVICE) as? ConnectivityManager, - refreshCache, - SYSTEM_CLOCK, - ) + store = store, + importer = importer, + scope = scope, + job = job, + databasePath = databasePath, + http = http, + pool = webSocket?.let { RelayPool(it, scope, signer, nowSeconds) }, + config = config, + device = device, + deviceSigner = deviceSigner, + writeRelays = writeRelays, + nowMillis = nowMillis, + ).also { it.start() } } catch (failure: Throwable) { - runCatching { client?.close() }.exceptionOrNull()?.let(failure::addSuppressed) - job.cancel() - runCatching { runBlocking(Dispatchers.IO) { job.join() } }.exceptionOrNull()?.let(failure::addSuppressed) - runCatching { store?.close() }.exceptionOrNull()?.let(failure::addSuppressed) - synchronized(openDatabases) { openDatabases.remove(path) } + synchronized(openDatabases) { openDatabases.remove(databasePath) } throw failure } } - - /** - * Test construction seam. Production code should pass [eventStore] to [create] - * instead of calling this. - */ - internal fun createForTesting( - eventStore: IEventStore, - client: INostrClient, - parentScope: CoroutineScope, - config: IoliteConfig = IoliteConfig(), - databasePath: String = "test-${System.nanoTime()}", - eventVerifier: (Event) -> Boolean = DEFAULT_EVENT_VERIFIER, - refreshCache: QueryRefreshCache = InMemoryQueryRefreshCache(), - clock: EpochMillisClock = SYSTEM_CLOCK, - dispatcher: CoroutineDispatcher = Dispatchers.IO, - ): Iolite { - config.validate() - val job = SupervisorJob(requireActiveParentJob(parentScope)) - val scope = CoroutineScope(parentScope.coroutineContext + job + dispatcher) - return Iolite( - store = ObservableEventStore(eventStore), - client = client, - scope = scope, - job = job, - databasePath = databasePath, - config = config, - eventVerifier = eventVerifier, - connectivityManager = null, - refreshCache = refreshCache, - clock = clock, - ) - } - - private fun copyFilter(filter: Filter): Filter = Filter( - ids = filter.ids?.toList(), - authors = filter.authors?.toList(), - kinds = filter.kinds?.toList(), - tags = filter.tags?.mapValues { it.value.toList() }, - tagsAll = filter.tagsAll?.mapValues { it.value.toList() }, - since = filter.since, - until = filter.until, - limit = filter.limit, - search = filter.search, - ) - - private fun validatedId(value: String?): String? = - value?.takeIf { it.length == 64 && it.all { char -> char in '0'..'9' || char in 'a'..'f' } } - - private fun requireActiveParentJob(parentScope: CoroutineScope): Job = - requireNotNull(parentScope.coroutineContext[Job]) { - "parentScope must contain an active Job" - }.also { parentJob -> - require(parentJob.isActive) { "parentScope must be active" } - } - - private fun addressOf(event: Event): Address? = when { - event is AddressableEvent -> event.address() - event.kind.isReplaceable() -> Address(event.kind, event.pubKey, "") - else -> null - } - - private fun EventStoreProjection.snapshotItems(): List = - snapshot().items.map { it.value } - - private val DEFAULT_EVENT_VERIFIER: (Event) -> Boolean = { event -> - event.verifyId() && event.verifySignature() - } - - private val SYSTEM_CLOCK = EpochMillisClock(System::currentTimeMillis) - private const val CACHE_RECHECK_INTERVAL_MILLIS = 60_000L - private const val REMOTE_EMIT_INTERVAL_MILLIS = 16L - private const val REJECTED_ID_QUERY_CHUNK_SIZE = 500 } } diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/IoliteConfig.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/IoliteConfig.kt index 0d1d908..74a60e9 100644 --- a/iolite/src/main/kotlin/dev/zapstore/iolite/IoliteConfig.kt +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/IoliteConfig.kt @@ -1,6 +1,7 @@ package dev.zapstore.iolite import kotlin.time.Duration +import kotlin.time.Duration.Companion.hours import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds @@ -8,17 +9,25 @@ import kotlin.time.Duration.Companion.seconds data class IoliteConfig( val databaseName: String = "iolite.db", val oneShotTimeout: Duration = 30.seconds, + val eoseGrace: Duration = 200.milliseconds, val ingestionCapacity: Int = 1_024, val expirationSweepInterval: Duration = 1.minutes, val ingestBatchSize: Int = 100, val ingestFlushInterval: Duration = 250.milliseconds, - /** - * Retention policy for unbounded regular kinds, as `kind -> maxAge`. - * Each expiration sweep deletes events of these kinds whose - * `created_at` is older than the configured age. Addressable kinds - * need no rule — supersession already bounds them. - */ val pruneRules: Map = emptyMap(), + /** How long a resolved NIP-65 relay list is trusted before relays are asked again. */ + val relayListCacheFor: Duration = 6.hours, + val relayListTimeout: Duration = 10.seconds, + val maxCompressedBytes: Long = 32L * 1024 * 1024, + /** Whole tar after zstd. Epoch 0 is every current listing; 20k apps is tens of MB. */ + val maxUncompressedBytes: Long = 512L * 1024 * 1024, + /** Tar members in one bundle. A from-0 catalog is about three files per app, plus one avatar each. */ + val maxMembers: Int = 100_000, + /** One JSONL line (one event or delete). Matches the relay's 0.5 MiB event cap. */ + val maxJsonLineBytes: Long = 512L * 1024, + val maxIconBytes: Long = 256L * 1024, + /** Kind 30267 author included in catalog bundles. */ + val catalogStackPubkey: String = CatalogStackPubkey, ) { internal fun validate() { require(databaseName.isNotBlank() && databaseName.none { it == '/' || it == '\\' || it == '\u0000' }) { @@ -27,14 +36,29 @@ data class IoliteConfig( require(oneShotTimeout.isFinite() && oneShotTimeout.isPositive()) { "oneShotTimeout must be finite and positive" } + require(eoseGrace.isFinite() && eoseGrace.isPositive()) { + "eoseGrace must be finite and positive" + } require(ingestionCapacity >= 1) { "ingestionCapacity must be at least one" } require(expirationSweepInterval.isFinite() && expirationSweepInterval.isPositive()) { "expirationSweepInterval must be finite and positive" } require(ingestBatchSize >= 1) { "ingestBatchSize must be at least one" } + require(relayListCacheFor.isFinite() && relayListCacheFor.isPositive()) { + "relayListCacheFor must be finite and positive" + } + require(relayListTimeout.isFinite() && relayListTimeout.isPositive()) { + "relayListTimeout must be finite and positive" + } require(ingestFlushInterval.isFinite() && ingestFlushInterval.isPositive()) { "ingestFlushInterval must be finite and positive" } + require(maxCompressedBytes > 0 && maxUncompressedBytes > 0 && maxMembers > 0) { + "catalog bundle limits must be positive" + } + require(maxJsonLineBytes > 0 && maxJsonLineBytes <= maxUncompressedBytes) { + "maxJsonLineBytes must be positive and at most maxUncompressedBytes" + } pruneRules.forEach { (kind, maxAge) -> require(kind >= 0) { "pruneRules kinds must be non-negative" } require(maxAge.isFinite() && maxAge.isPositive()) { diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/IoliteStore.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/IoliteStore.kt new file mode 100644 index 0000000..2f91de7 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/IoliteStore.kt @@ -0,0 +1,752 @@ +package dev.zapstore.iolite + +import androidx.sqlite.SQLiteConnection +import androidx.sqlite.SQLiteStatement +import androidx.sqlite.driver.bundled.BundledSQLiteDriver +import androidx.sqlite.execSQL +import java.io.File +import java.util.EnumSet +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.callbackFlow +import org.json.JSONArray +import org.json.JSONObject + +/** Selects visible catalog listings. All fields are optional and combine with AND. */ +data class AppFilter( + val appIds: Collection? = null, + /** Matches the proof owner or the app event signer. */ + val author: String? = null, + /** + * Case-insensitive exact or prefix match on the name or app ID. + * With [queryVector], listings are also ranked by embedding similarity. + */ + val search: String? = null, + /** leaf-ir-v1 query embedding, [EMBEDDING_DIMS] int8 bytes. Ignored unless [search] is set. */ + val queryVector: ByteArray? = null, + val limit: Int? = null, +) + +/** One open transaction. Row writers record which tables they touch so observers can be notified precisely. */ +class Tx internal constructor(val db: SQLiteConnection) { + internal val touched: EnumSet
= EnumSet.noneOf(Table::class.java) + + fun touch(vararg tables: Table) { + touched += tables + } +} + +/** + * The SQLite file. Every write is one transaction; every commit notifies listeners of the touched tables. + * Reads return typed records. No event JSON is rehydrated for reads. + */ +class IoliteStore internal constructor( + path: String, + private val iconsDir: File = File(File(path).parentFile, "icons"), + private val avatarsDir: File = File(File(path).parentFile, "avatars"), + openConnection: (String) -> SQLiteConnection = ::openBundledOrJdbc, +) : AutoCloseable { + private class Listener(val tables: Set
, val notify: () -> Unit) + + private val lock = ReentrantLock() + private val connection: SQLiteConnection = open(path, openConnection) + private val listeners = CopyOnWriteArrayList() + + // --- change notification ----------------------------------------------------------------------- + + /** Emits once on subscription and after every commit that touched any of [tables]. Conflated. */ + fun changes(tables: Set
): Flow = callbackFlow { + val listener = Listener(tables) { trySend(Unit) } + listeners += listener + trySend(Unit) + awaitClose { listeners -= listener } + }.buffer(Channel.CONFLATED) + + fun read(block: (SQLiteConnection) -> T): T = lock.withLock { block(connection) } + + fun write(block: (Tx) -> Unit) { + val touched = lock.withLock { + val tx = Tx(connection) + connection.execSQL("BEGIN IMMEDIATE") + try { + block(tx) + connection.execSQL("COMMIT") + } catch (failure: Throwable) { + runCatching { connection.execSQL("ROLLBACK") } + throw failure + } + tx.touched + } + if (touched.isEmpty()) return + listeners.forEach { listener -> + if (listener.tables.any(touched::contains)) runCatching(listener.notify) + } + } + + override fun close() { + lock.withLock { + checkpointWalLocked() + connection.close() + } + } + + /** Folds the WAL into the main file so a large import does not leave a second copy on disk. */ + private fun checkpointWal() { + lock.withLock { checkpointWalLocked() } + } + + private fun checkpointWalLocked() { + runCatching { + connection.prepare("PRAGMA wal_checkpoint(TRUNCATE)").use { it.step() } + } + } + + // --- ingest ------------------------------------------------------------------------------------ + + /** Relay path: verified non-catalog events, one transaction. Returns how many rows changed. */ + internal fun ingest(events: List, context: IngestContext): Int { + val admissible = events.filter { it.kind !in ListingRows.KINDS && it.verify() } + if (admissible.isEmpty()) return 0 + var written = 0 + write { tx -> admissible.forEach { if (EventRows.persist(tx, it, context)) written++ } } + return written + } + + /** + * Catalog path: one epoch's verified events and deletes, one transaction. When [replaceAll] is set, + * listings absent from [events] are removed. Returns the app IDs whose listings were deleted. + */ + internal fun importCatalog( + catalog: CatalogRecord, + events: List, + deletes: List, + replaceAll: Boolean, + toEpoch: Long, + device: DeviceProfile, + now: Long, + artifacts: CatalogArtifact = CatalogArtifact.EMPTY, + stackPubkey: String = CatalogStackPubkey, + ): Set { + val removed = LinkedHashSet() + write { tx -> + for (event in events) { + if (event.kind != Kinds.AppStack) continue + if (event.pubkey != stackPubkey) { + throw CatalogImportException("stack is not signed by the curator") + } + if (event.dTag().isNullOrBlank()) throw CatalogImportException("stack missing d") + } + for (delete in deletes) { + if (delete is CatalogDelete.Coordinate && delete.kind == Kinds.AppStack && delete.pubkey != stackPubkey) { + throw CatalogImportException("stack is not signed by the curator") + } + ListingRows.delete(tx, catalog, delete) + if (delete is CatalogDelete.Listing) removed += delete.appId + } + val listings = events.filter { ListingRows.isListingKind(it.kind) }.groupBy(ListingRows::listingAppId) + for ((appId, listing) in listings) { + if (appId == null) continue + ListingRows.persistListing(tx, catalog, appId, listing, device, now) + } + events.filter { it.kind == Kinds.IdentityProof }.forEach { ListingRows.persistProof(tx, catalog, it) } + val keptStacks = LinkedHashSet() + val context = IngestContext(device, null) + for (event in events) { + if (event.kind == Kinds.Profile) EventRows.persist(tx, event, context) + } + for (event in events) { + if (event.kind != Kinds.AppStack || !EventRows.stackMatchesDevice(event, device)) continue + val identifier = event.dTag() ?: continue + keptStacks += identifier + EventRows.persist(tx, event, context) + } + if (replaceAll) EventRows.retainCatalogStacks(tx, stackPubkey, keptStacks) + if (replaceAll) { + for (appId in ListingRows.staleAppIds(tx.db, catalog, listings.keys.filterNotNull().toSet())) { + ListingRows.delete(tx, catalog, CatalogDelete.Listing(appId)) + removed += appId + } + } + applyArtifacts(tx, catalog, artifacts) + ListingRows.recomputeAppPubkeys(tx, catalog.id, now) + tx.db.prepare("UPDATE catalogs SET epoch = ? WHERE id = ?").use { statement -> + statement.bindLong(1, toEpoch) + statement.bindLong(2, catalog.id) + statement.step() + } + tx.touch(Table.Catalogs) + } + checkpointWal() + return removed + } + + /** Local avatar WebP for [pubkey], whether or not the file has been written yet. */ + fun avatarFile(pubkey: String): File { + require(Hex.isHex(pubkey, 64)) { "pubkey is not 32-byte hex" } + return File(avatarsDir, "${pubkey.lowercase()}.webp") + } + + /** + * Writes the vector and about, security, and facts for apps that already have a listing row. + * A record that omits one of those keeps the value stored earlier. + * Icon and avatar bytes land on disk for Coil. + */ + private fun applyArtifacts(tx: Tx, catalog: CatalogRecord, artifacts: CatalogArtifact) { + if (artifacts.apps.isEmpty() && artifacts.avatars.isEmpty()) return + val db = tx.db + for (artifact in artifacts.apps) { + val appRow = db.prepare("SELECT id FROM apps WHERE catalog_id = ? AND app_id = ?").use { statement -> + statement.bindLong(1, catalog.id) + statement.bindText(2, artifact.appId) + if (statement.step()) statement.getBlob(0) else null + } ?: continue + artifact.about?.let { about -> + db.prepare("UPDATE apps SET about = ? WHERE catalog_id = ? AND app_id = ?").use { statement -> + statement.bindText(1, about) + statement.bindLong(2, catalog.id) + statement.bindText(3, artifact.appId) + statement.step() + } + } + artifact.security?.let { security -> + db.prepare("UPDATE apps SET security = ? WHERE catalog_id = ? AND app_id = ?").use { statement -> + statement.bindText(1, security) + statement.bindLong(2, catalog.id) + statement.bindText(3, artifact.appId) + statement.step() + } + } + artifact.facts?.let { facts -> + db.prepare("UPDATE apps SET facts = ? WHERE catalog_id = ? AND app_id = ?").use { statement -> + statement.bindText(1, facts) + statement.bindLong(2, catalog.id) + statement.bindText(3, artifact.appId) + statement.step() + } + } + val embedding = artifact.embedding + if (embedding != null) { + db.prepare( + """ + INSERT INTO apps_search (id, embedding) + VALUES (?, ?) + ON CONFLICT(id) DO UPDATE SET + embedding = excluded.embedding + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, appRow) + statement.bindBlob(2, embedding) + statement.step() + } + } + artifact.webp?.let { writeBytes(File(iconsDir, "${catalog.id}/${artifact.appId}.webp"), it) } + } + for (avatar in artifacts.avatars) { + writeBytes(avatarFile(avatar.pubkey), avatar.webp) + } + if (artifacts.apps.isNotEmpty()) tx.touch(Table.Apps) + } + + /** Re-derives `apps.pubkey` for every catalog; run at open and at each proof expiry. */ + fun recomputeProofs(now: Long) { + write { tx -> + allCatalogs().forEach { ListingRows.recomputeAppPubkeys(tx, it.id, now) } + } + } + + /** Signing path: derived rows and the outbox row commit together. */ + internal fun commitPublish(event: Event, context: IngestContext) { + require(event.kind != Kinds.Auth) { "NIP-42 AUTH is never added to the outbox" } + require(event.verify()) { "published event failed verification" } + write { tx -> + EventRows.persist(tx, event, context) + EventRows.insertOutbox(tx, event) + } + } + + fun wipe() { + write { tx -> + listOf("comments", "zaps", "stacks", "profiles", "outbox", "preferences", "certificate_proofs", "apps", "query_refresh") + .forEach { tx.db.execSQL("DELETE FROM $it") } + tx.db.execSQL("UPDATE catalogs SET epoch = 0") + tx.db.execSQL("UPDATE catalogs SET manifest_pubkey = NULL WHERE id = 1") + tx.touch(*Table.values()) + } + checkpointWal() + } + + fun pruneOlderThan(kind: Int, createdBefore: Long) { + val (table, touched) = when (kind) { + Kinds.Zap -> "zaps" to Table.Zaps + Kinds.Comment -> "comments" to Table.Comments + else -> return + } + write { tx -> + tx.db.prepare("DELETE FROM $table WHERE created_at < ?").use { statement -> + statement.bindLong(1, createdBefore) + statement.step() + } + tx.touch(touched) + } + } + + // --- catalogs ---------------------------------------------------------------------------------- + + fun catalog(id: Long): CatalogRecord? = read { db -> + db.prepare("$SELECT_CATALOG WHERE id = ?").use { statement -> + statement.bindLong(1, id) + statement.rowOrNull { it.toCatalog() } + } + } + + fun selectedCatalogs(): List = read { db -> + db.prepare("$SELECT_CATALOG WHERE position IS NOT NULL ORDER BY position ASC").use { it.rows { toCatalog() } } + } + + fun allCatalogs(): List = read { db -> + db.prepare("$SELECT_CATALOG ORDER BY COALESCE(position, 999999), id").use { it.rows { toCatalog() } } + } + + /** Writes the catalog endpoint. [replaceIdentity] updates the stable catalog URL used in app ids. */ + fun upsertCatalogRelays(catalogId: Long, relays: List, replaceIdentity: Boolean) { + val canonical = relays.firstOrNull { !it.isOnion } ?: relays.first() + val encoded = JSONArray().apply { relays.forEach { put(it.url) } }.toString() + write { tx -> + val exists = tx.db.prepare("SELECT 1 FROM catalogs WHERE id = ?").use { statement -> + statement.bindLong(1, catalogId) + statement.step() + } + if (!exists) { + tx.db.prepare( + """ + INSERT INTO catalogs (id, relay_url, position, is_private, epoch, endpoints) + VALUES (?, ?, (SELECT COALESCE(MAX(position), -1) + 1 FROM catalogs), 0, 0, ?) + """.trimIndent(), + ).use { statement -> + statement.bindLong(1, catalogId) + statement.bindText(2, canonical.url) + statement.bindText(3, encoded) + statement.step() + } + } else if (replaceIdentity) { + tx.db.prepare("UPDATE catalogs SET relay_url = ?, endpoints = ? WHERE id = ?").use { statement -> + statement.bindText(1, canonical.url) + statement.bindText(2, encoded) + statement.bindLong(3, catalogId) + statement.step() + } + } else { + tx.db.prepare("UPDATE catalogs SET endpoints = ? WHERE id = ?").use { statement -> + statement.bindText(1, encoded) + statement.bindLong(2, catalogId) + statement.step() + } + } + tx.touch(Table.Catalogs) + } + } + + fun pinManifestPubkeyIfAbsent(catalogId: Long, pubkey: String) { + write { tx -> + tx.db.prepare("UPDATE catalogs SET manifest_pubkey = ? WHERE id = ? AND manifest_pubkey IS NULL").use { statement -> + statement.bindBlob(1, Hex.decode(pubkey)) + statement.bindLong(2, catalogId) + statement.step() + } + tx.touch(Table.Catalogs) + } + } + + // --- apps -------------------------------------------------------------------------------------- + + /** Visible listings: for each app ID, the row from the first selected catalog. */ + fun apps(filter: AppFilter = AppFilter()): List = read { db -> + val where = StringBuilder() + val binds = mutableListOf<(SQLiteStatement, Int) -> Unit>() + filter.appIds?.let { ids -> + if (ids.isEmpty()) return@read emptyList() + where.append(" AND a.app_id IN (").append(ids.joinToString(",") { "?" }).append(")") + ids.forEach { id -> binds += { s, i -> s.bindText(i, id) } } + } + filter.author?.let { author -> + val bytes = Hex.decode(author) + where.append(" AND (a.pubkey = ? OR a.event_pubkey = ?)") + binds += { s, i -> s.bindBlob(i, bytes) } + binds += { s, i -> s.bindBlob(i, bytes) } + } + val query = filter.search?.let(::normalizeSearchQuery)?.takeIf { it.isNotEmpty() } + if (query == null) { + val limit = filter.limit?.let { " LIMIT $it" } ?: "" + return@read db.prepare("$SELECT_VISIBLE_APP$where ORDER BY a.app_event_created_at DESC, a.app_id ASC$limit").use { statement -> + binds.forEachIndexed { index, bind -> bind(statement, index + 1) } + statement.rows { toApp(iconsDir) } + } + } + val ranker = SearchRanker(query, filter.queryVector, filter.limit) + db.prepare("$SELECT_SEARCH_KEY$where").use { statement -> + binds.forEachIndexed { index, bind -> bind(statement, index + 1) } + while (statement.step()) { + ranker.consider( + id = statement.getBlob(0), + appId = statement.getText(1), + name = statement.getText(2), + embedding = if (statement.isNull(3)) null else statement.getBlob(3), + ) + } + } + val ids = ranker.ids() + if (ids.isEmpty()) return@read emptyList() + val sql = "$SELECT_VISIBLE_APP$where AND a.id IN (${ids.joinToString(",") { "?" }})" + val byId = db.prepare(sql).use { statement -> + binds.forEachIndexed { index, bind -> bind(statement, index + 1) } + ids.forEachIndexed { index, id -> statement.bindBlob(binds.size + index + 1, id) } + statement.rows { toApp(iconsDir) }.associateBy { Hex.encode(it.id) } + } + ids.mapNotNull { byId[Hex.encode(it)] } + } + + fun app(appId: String): AppRecord? = apps(AppFilter(appIds = listOf(appId), limit = 1)).firstOrNull() + + fun nextProofExpiry(nowSeconds: Long): Long? = read { db -> + db.prepare("SELECT MIN(expiry) FROM certificate_proofs WHERE revoked = 0 AND expiry > ?").use { statement -> + statement.bindLong(1, nowSeconds) + if (!statement.step() || statement.isNull(0)) null else statement.getLong(0) + } + } + + // --- profiles ---------------------------------------------------------------------------------- + + /** Stored kind 0 projections for [pubkeys]; rows that only carry a relay list are omitted. */ + fun profiles(pubkeys: Collection): Map = read { db -> + if (pubkeys.isEmpty()) return@read emptyMap() + val sql = "SELECT pubkey, updated_at, name, picture_url, metadata FROM profiles WHERE pubkey IN (" + + pubkeys.joinToString(",") { "?" } + ")" + db.prepare(sql).use { statement -> + pubkeys.forEachIndexed { index, pubkey -> statement.bindBlob(index + 1, Hex.decode(pubkey)) } + buildMap { + while (statement.step()) { + val metadata = JSONObject(statement.getText(4)) + val eventId = metadata.string("id") ?: continue + val content = runCatching { JSONObject(metadata.optString("content")) }.getOrElse { JSONObject() } + val pubkey = Hex.encode(statement.getBlob(0)) + put( + pubkey, + ProfileRecord( + pubkey = pubkey, + updatedAt = statement.getLong(1), + eventId = eventId, + name = statement.textOrNull(2), + displayName = content.string("display_name"), + about = content.string("about"), + picture = statement.textOrNull(3), + banner = content.string("banner"), + website = content.string("website"), + nip05 = content.string("nip05"), + ), + ) + } + } + } + } + + /** NIP-65 read relays for [pubkey], or null when no relay list is stored. */ + fun readRelays(pubkey: String): Set? = read { db -> + EventRows.profileMetadata(db, pubkey)?.second + ?.takeIf { it.has("nip65_id") } + ?.optJSONArray("nip65_tags").toStringLists() + .takeIf { it.isNotEmpty() } + ?.readRelayUrls() + } + + // --- stacks, zaps, comments -------------------------------------------------------------------- + + fun stacks(author: String, identifier: String? = null, limit: Int? = null, deviceSigner: LocalSigner? = null): List = + read { db -> + val where = if (identifier == null) "" else " AND identifier = ?" + val limitSql = limit?.let { " LIMIT $it" } ?: "" + db.prepare( + "SELECT pubkey, identifier, event_id, name, description, public_apps, updated_at, metadata FROM stacks " + + "WHERE pubkey = ?$where ORDER BY updated_at DESC$limitSql", + ).use { statement -> + statement.bindBlob(1, Hex.decode(author)) + if (identifier != null) statement.bindText(2, identifier) + statement.rows { + val metadata = JSONObject(getText(7)) + val privateApps = if (deviceSigner != null) metadata.stringList("private_apps") else emptyList() + StackRecord( + pubkey = Hex.encode(getBlob(0)), + identifier = getText(1), + eventId = Hex.encode(getBlob(2)), + name = getText(3), + description = textOrNull(4).orEmpty(), + apps = (JSONArray(getText(5)).toStringList() + privateApps).mapNotNull(AppCoordinate::parse), + updatedAt = getLong(6), + ) + } + } + } + + fun zaps(appId: String, limit: Int? = null): List = read { db -> + val limitSql = limit?.let { " LIMIT $it" } ?: "" + db.prepare( + "SELECT event_id, pubkey, amount_sats, created_at FROM zaps WHERE app_id = ? ORDER BY created_at DESC$limitSql", + ).use { statement -> + statement.bindText(1, appId) + statement.rows { + ZapRecord(Hex.encode(getBlob(0)), Hex.encode(getBlob(1)), getLong(2), getLong(3), appId) + } + } + } + + fun comments(appId: String, limit: Int? = null): List = read { db -> + val limitSql = limit?.let { " LIMIT $it" } ?: "" + db.prepare( + "SELECT event_id, pubkey, created_at, content, stack, parent_event_id FROM comments " + + "WHERE app_id = ? ORDER BY created_at DESC$limitSql", + ).use { statement -> + statement.bindText(1, appId) + statement.rows { + CommentRecord( + eventId = Hex.encode(getBlob(0)), + pubkey = Hex.encode(getBlob(1)), + createdAt = getLong(2), + content = getText(3), + appId = appId, + stack = textOrNull(4), + parentEventId = blobHexOrNull(5), + ) + } + } + } + + // --- outbox ------------------------------------------------------------------------------------ + + fun pendingOutbox(): List = read { db -> + db.prepare("SELECT event_json FROM outbox").use { it.rows { Event.parse(getText(0)) } } + } + + fun deleteOutbox(eventId: String) { + write { tx -> + tx.db.prepare("DELETE FROM outbox WHERE event_id = ?").use { statement -> + statement.bindBlob(1, Hex.decode(eventId)) + statement.step() + } + tx.touch(Table.Outbox) + } + } + + // --- query refresh cache ----------------------------------------------------------------------- + + fun lastRefresh(fingerprint: String): Long? = read { db -> + db.prepare("SELECT refreshed_at FROM query_refresh WHERE fingerprint = ?").use { statement -> + statement.bindText(1, fingerprint) + if (statement.step()) statement.getLong(0) else null + } + } + + fun recordRefresh(fingerprint: String, epochMillis: Long) { + if (epochMillis <= 0) return + // No listener cares about this table, so no notification is needed. + read { db -> + db.prepare("INSERT OR REPLACE INTO query_refresh (fingerprint, refreshed_at) VALUES (?, ?)").use { statement -> + statement.bindText(1, fingerprint) + statement.bindLong(2, epochMillis) + statement.step() + } + } + } + + // --- rehydration templates --------------------------------------------------------------------- + + fun catalogRelayListTemplate(deviceSigner: LocalSigner): Pair>, String> { + val selected = selectedCatalogs() + val tags = selected.filter { !it.isPrivate }.map { listOf("r", it.relayUrl.url) } + val privateUrls = selected.filter { it.isPrivate }.map { it.relayUrl.url } + val content = if (privateUrls.isEmpty()) "" else deviceSigner.encryptToSelf(JSONArray(privateUrls).toString()) + return tags to content + } + + fun preferenceTemplate(identifier: String): Pair>, String> { + val row = read { db -> + db.prepare("SELECT metadata FROM preferences WHERE identifier = ?").use { statement -> + statement.bindText(1, identifier) + statement.rowOrNull { JSONObject(it.getText(0)) } + } + } ?: error("missing preference $identifier") + val leftovers = EventRows.leftoverTags(row.optJSONArray("tags").toStringLists(), setOf("d")) + return (listOf(listOf("d", identifier)) + leftovers) to row.optString("content") + } + + fun stackTemplate(pubkey: String, identifier: String, device: DeviceProfile, deviceSigner: LocalSigner?): Pair>, String> { + val (stack, storedTags) = read { db -> + db.prepare("SELECT metadata FROM stacks WHERE pubkey = ? AND identifier = ?").use { statement -> + statement.bindBlob(1, Hex.decode(pubkey)) + statement.bindText(2, identifier) + statement.rowOrNull { JSONObject(it.getText(0)).optJSONArray("tags").toStringLists() } + } + }?.let { tags -> stacks(pubkey, identifier, deviceSigner = deviceSigner).first() to tags } + ?: error("missing stack $identifier") + val privateApps = if (deviceSigner == null) emptyList() else read { db -> + db.prepare("SELECT metadata FROM stacks WHERE pubkey = ? AND identifier = ?").use { statement -> + statement.bindBlob(1, Hex.decode(pubkey)) + statement.bindText(2, identifier) + statement.rowOrNull { JSONObject(it.getText(0)).stringList("private_apps") } + } + }.orEmpty() + val privateSet = privateApps.toSet() + val tags = buildList { + add(listOf("d", identifier)) + add(listOf("name", stack.name)) + stack.description.takeIf { it.isNotBlank() }?.let { add(listOf("description", it)) } + stack.apps.map(AppCoordinate::toString).filter { it !in privateSet }.forEach { add(listOf("a", it)) } + device.platforms.forEach { add(listOf("f", it)) } + addAll(EventRows.leftoverTags(storedTags, EventRows.STACK_MODELED_TAGS)) + } + val content = if (privateApps.isEmpty() || deviceSigner == null) { + "" + } else { + deviceSigner.encryptToSelf(JSONArray(privateApps).toString()) + } + return tags to content + } + + companion object { + private const val SELECT_CATALOG = + "SELECT id, relay_url, position, is_private, manifest_pubkey, epoch, endpoints FROM catalogs" + + /** + * Visible listings: join the catalog for its manifest key, keep only selected catalogs, and for each + * app ID keep the row whose catalog comes first in selection order. + */ + private val SELECT_VISIBLE_APP = """ + SELECT a.id, a.catalog_id, a.app_id, a.certificate_hash, a.app_event_created_at, a.pubkey, a.event_pubkey, + c.manifest_pubkey, a.name, a.summary, a.repository, a.version, a.version_code, a.channel, a.metadata, + a.about, a.security, a.facts + FROM apps a + JOIN catalogs c ON c.id = a.catalog_id + WHERE c.position IS NOT NULL + AND a.catalog_id = ( + SELECT a2.catalog_id FROM apps a2 JOIN catalogs c2 ON c2.id = a2.catalog_id + WHERE a2.app_id = a.app_id AND c2.position IS NOT NULL + ORDER BY c2.position ASC LIMIT 1 + ) + """.trimIndent() + + /** Columns [SearchRanker] needs. Full listings are loaded only for the ids it keeps. */ + private val SELECT_SEARCH_KEY = """ + SELECT a.id, a.app_id, a.name, s.embedding + FROM apps a + JOIN catalogs c ON c.id = a.catalog_id + LEFT JOIN apps_search s ON s.id = a.id + WHERE c.position IS NOT NULL + AND a.catalog_id = ( + SELECT a2.catalog_id FROM apps a2 JOIN catalogs c2 ON c2.id = a2.catalog_id + WHERE a2.app_id = a.app_id AND c2.position IS NOT NULL + ORDER BY c2.position ASC LIMIT 1 + ) + """.trimIndent() + + fun open( + path: String, + openConnection: (String) -> SQLiteConnection = ::openBundledOrJdbc, + ): SQLiteConnection { + File(path).parentFile?.mkdirs() + val connection = openConnection(path) + connection.execSQL("PRAGMA foreign_keys = ON") + connection.execSQL("PRAGMA journal_mode = WAL") + val version = connection.prepare("PRAGMA user_version").use { statement -> + statement.step() + statement.getLong(0).toInt() + } + when (version) { + Schema.USER_VERSION -> Unit + 0 -> { + connection.execSQL("BEGIN IMMEDIATE") + try { + Schema.statements.forEach(connection::execSQL) + connection.execSQL("PRAGMA user_version = ${Schema.USER_VERSION}") + connection.execSQL("COMMIT") + } catch (failure: Throwable) { + runCatching { connection.execSQL("ROLLBACK") } + throw failure + } + } + else -> error("unsupported Iolite schema version $version") + } + runCatching { + connection.prepare("PRAGMA wal_checkpoint(TRUNCATE)").use { it.step() } + } + return connection + } + } +} + +internal fun openBundledOrJdbc(path: String): SQLiteConnection { + if (runCatching { Class.forName("org.sqlite.JDBC") }.isSuccess) { + return JdbcSqliteConnection(path) + } + return BundledSQLiteDriver().open(path) +} + +private fun SQLiteStatement.toCatalog(): CatalogRecord { + val relayUrl = getText(1).normalizeRelayUrl() + val relays = decodeRelays(if (isNull(6)) "[]" else getText(6)).ifEmpty { listOf(relayUrl) } + return CatalogRecord( + id = getLong(0), + relayUrl = relayUrl, + relays = relays, + position = if (isNull(2)) null else getInt(2), + isPrivate = getLong(3) == 1L, + manifestPubkey = blobHexOrNull(4), + epoch = getLong(5), + ) +} + +private fun decodeRelays(raw: String): List { + if (raw.isBlank() || raw == "[]") return emptyList() + val array = JSONArray(raw) + return List(array.length()) { index -> array.getString(index).normalizeRelayUrl() } +} + +private fun writeBytes(file: File, bytes: ByteArray) { + file.parentFile?.mkdirs() + val tmp = File(file.parentFile, "${file.name}.tmp") + tmp.writeBytes(bytes) + if (!tmp.renameTo(file)) { + tmp.copyTo(file, overwrite = true) + tmp.delete() + } +} + +private fun SQLiteStatement.toApp(iconsDir: File): AppRecord = AppRecord( + id = getBlob(0), + catalogId = getLong(1), + appId = getText(2), + certificateHash = Hex.encode(getBlob(3)), + createdAt = getLong(4), + proofPubkey = blobHexOrNull(5), + eventPubkey = Hex.encode(getBlob(6)), + catalogManifestPubkey = blobHexOrNull(7), + name = getText(8), + summary = getText(9), + repository = textOrNull(10), + version = getText(11), + versionCode = getLong(12), + channel = textOrNull(13), + metadata = JSONObject(getText(14)), + about = getText(15), + security = getText(16), + facts = getText(17), + iconFile = File(iconsDir, "${getLong(1)}/${getText(2)}.webp").takeIf { it.isFile }, +) + +private fun SQLiteStatement.rowOrNull(map: (SQLiteStatement) -> T): T? = if (step()) map(this) else null + +private fun SQLiteStatement.rows(map: SQLiteStatement.() -> T): List = buildList { + while (step()) add(map()) +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/JdbcSqliteConnection.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/JdbcSqliteConnection.kt new file mode 100644 index 0000000..7dd4791 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/JdbcSqliteConnection.kt @@ -0,0 +1,100 @@ +package dev.zapstore.iolite + +import androidx.sqlite.SQLiteConnection +import androidx.sqlite.SQLiteStatement +import java.sql.Connection +import java.sql.DriverManager +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.sql.Types + +internal class JdbcSqliteConnection(path: String) : SQLiteConnection { + private val connection: Connection + + init { + Class.forName("org.sqlite.JDBC") + connection = DriverManager.getConnection("jdbc:sqlite:$path") + connection.autoCommit = true + } + + override fun prepare(sql: String): SQLiteStatement = JdbcSqliteStatement(connection.prepareStatement(sql)) + + override fun inTransaction(): Boolean = !connection.autoCommit + + override fun close() { + connection.close() + } +} + +private class JdbcSqliteStatement( + private val statement: PreparedStatement, +) : SQLiteStatement { + private var result: ResultSet? = null + private var executed = false + + override fun bindBlob(index: Int, value: ByteArray) { + statement.setBytes(index, value) + } + + override fun bindDouble(index: Int, value: Double) { + statement.setDouble(index, value) + } + + override fun bindLong(index: Int, value: Long) { + statement.setLong(index, value) + } + + override fun bindText(index: Int, value: String) { + statement.setString(index, value) + } + + override fun bindNull(index: Int) { + statement.setNull(index, Types.NULL) + } + + override fun getBlob(index: Int): ByteArray = result!!.getBytes(index + 1) + + override fun getDouble(index: Int): Double = result!!.getDouble(index + 1) + + override fun getLong(index: Int): Long = result!!.getLong(index + 1) + + override fun getText(index: Int): String = result!!.getString(index + 1) + + override fun isNull(index: Int): Boolean { + result!!.getObject(index + 1) + return result!!.wasNull() + } + + override fun getColumnCount(): Int = result!!.metaData.columnCount + + override fun getColumnName(index: Int): String = result!!.metaData.getColumnName(index + 1) + + override fun getColumnType(index: Int): Int = result!!.metaData.getColumnType(index + 1) + + override fun step(): Boolean { + if (!executed) { + executed = true + val hasResult = statement.execute() + result = if (hasResult) statement.resultSet else statement.resultSet + val rs = result ?: return false + return rs.next() + } + return result?.next() == true + } + + override fun reset() { + result?.close() + result = null + executed = false + statement.clearParameters() + } + + override fun clearBindings() { + statement.clearParameters() + } + + override fun close() { + result?.close() + statement.close() + } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/ListingRows.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/ListingRows.kt new file mode 100644 index 0000000..1277368 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/ListingRows.kt @@ -0,0 +1,325 @@ +package dev.zapstore.iolite + +import androidx.sqlite.SQLiteConnection +import java.io.ByteArrayInputStream +import java.security.Signature +import java.security.cert.CertificateFactory +import java.security.interfaces.ECPublicKey +import java.security.interfaces.RSAPublicKey +import java.util.Base64 +import org.json.JSONArray +import org.json.JSONObject + +class CatalogImportException(message: String) : Exception(message) + +/** A catalog delete operation from `deletes.jsonl`. */ +sealed interface CatalogDelete { + data class Listing(val appId: String) : CatalogDelete + data class Coordinate(val kind: Int, val pubkey: String, val d: String) : CatalogDelete +} + +/** + * Derives `apps` and `certificate_proofs` rows from verified catalog events (kinds 32267, 30063, 3063, 30509). + * Each listing event writes its own columns. Sibling events are not required. + */ +internal object ListingRows { + val KINDS = setOf(Kinds.App, Kinds.Release, Kinds.Asset, Kinds.IdentityProof) + private val LISTING_KINDS = setOf(Kinds.App, Kinds.Release, Kinds.Asset) + private const val ANDROID_APK = "application/vnd.android.package-archive" + + fun listingAppId(event: Event): String? = when (event.kind) { + Kinds.App -> event.dTag() + Kinds.Release, Kinds.Asset -> event.tagValue("i") + else -> null + } + + fun isListingKind(kind: Int): Boolean = kind in LISTING_KINDS + + /** + * Applies whatever listing events arrived for [appId]. + * Each event writes its own columns. A missing sibling leaves those columns as stored. + */ + fun persistListing(tx: Tx, catalog: CatalogRecord, appId: String, events: List, device: DeviceProfile, now: Long) { + val db = tx.db + val app = events.firstOrNull { it.kind == Kinds.App } + val release = events.firstOrNull { it.kind == Kinds.Release && it.tagValue("c") == "main" } + val asset = events + .filter { it.kind == Kinds.Asset && it.tagValue("i") == appId && assetMatchesDevice(it, device) } + .maxWithOrNull(compareBy { it.tagValue("version_code")?.toLongOrNull() ?: -1 }.thenBy { it.id }) + if (app == null && release == null && asset == null) return + val stored = storedListing(db, catalog.id, appId) + val certificate = asset?.tagValue("apk_certificate_hash")?.takeIf { Hex.isHex(it, 64) } + ?: stored?.certificate + ?: return + val eventPubkey = app?.pubkey ?: stored?.eventPubkey ?: release?.pubkey ?: asset?.pubkey ?: return + if (!Hex.isHex(eventPubkey, 64)) return + val metadata = JSONObject(stored?.metadata ?: "{}").apply { + if (app != null) putAppMetadata(app) + if (release != null) putReleaseMetadata(release) + if (asset != null) putAssetMetadata(asset) + } + db.prepare( + """ + INSERT INTO apps ( + id, catalog_id, app_id, variant_id, certificate_hash, app_event_created_at, pubkey, event_pubkey, + name, summary, repository, version, version_code, channel, metadata + ) VALUES (?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'main', ?) + ON CONFLICT(catalog_id, app_id) DO UPDATE SET + certificate_hash = excluded.certificate_hash, + app_event_created_at = excluded.app_event_created_at, + pubkey = excluded.pubkey, + event_pubkey = excluded.event_pubkey, + name = excluded.name, + summary = excluded.summary, + repository = excluded.repository, + version = excluded.version, + version_code = excluded.version_code, + channel = excluded.channel, + metadata = excluded.metadata + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, identity(catalog.relayUrl.url, appId)) + statement.bindLong(2, catalog.id) + statement.bindText(3, appId) + statement.bindBlob(4, Hex.decode(certificate)) + statement.bindLong(5, app?.createdAt ?: stored?.createdAt ?: 0) + statement.bindBlobOrNull(6, currentProofPubkey(db, catalog.id, certificate, now)?.let(Hex::decode)) + statement.bindBlob(7, Hex.decode(eventPubkey)) + statement.bindText(8, app?.tagValue("name")?.takeIf { it.isNotBlank() } ?: stored?.name ?: appId) + statement.bindText(9, if (app != null) app.tagValue("summary") ?: "" else stored?.summary ?: "") + statement.bindTextOrNull(10, if (app != null) app.tagValue("repository") else stored?.repository) + statement.bindText( + 11, + asset?.tagValue("version")?.takeIf { it.isNotBlank() } + ?: stored?.version + ?: "", + ) + statement.bindLong(12, asset?.tagValue("version_code")?.toLongOrNull() ?: stored?.versionCode ?: 0) + statement.bindText(13, metadata.toString()) + statement.step() + } + tx.touch(Table.Apps) + } + + private data class StoredListing( + val certificate: String, + val createdAt: Long, + val eventPubkey: String, + val name: String, + val summary: String, + val repository: String?, + val version: String, + val versionCode: Long, + val metadata: String, + ) + + private fun storedListing(db: SQLiteConnection, catalogId: Long, appId: String): StoredListing? = + db.prepare( + """ + SELECT certificate_hash, app_event_created_at, event_pubkey, name, summary, repository, + version, version_code, metadata + FROM apps WHERE catalog_id = ? AND app_id = ? + """.trimIndent(), + ).use { statement -> + statement.bindLong(1, catalogId) + statement.bindText(2, appId) + if (!statement.step()) return null + StoredListing( + certificate = Hex.encode(statement.getBlob(0)), + createdAt = statement.getLong(1), + eventPubkey = Hex.encode(statement.getBlob(2)), + name = statement.getText(3), + summary = statement.getText(4), + repository = if (statement.isNull(5)) null else statement.getText(5), + version = statement.getText(6), + versionCode = statement.getLong(7), + metadata = statement.getText(8), + ) + } + + private fun JSONObject.putAppMetadata(app: Event) { + put("description", app.content) + put("icon", app.tagValue("icon") ?: "") + put("url", app.tagValue("url") ?: "") + put("license", app.tagValue("license") ?: "") + put("image", JSONArray(app.tagValues("image"))) + put("app_event_id", app.id) + } + + private fun JSONObject.putReleaseMetadata(release: Event) { + put("release_event_id", release.id) + put("release_created_at", release.createdAt) + put("release_notes", release.content) + } + + private fun JSONObject.putAssetMetadata(asset: Event) { + put("asset_event_id", asset.id) + asset.tagValue("url")?.let { put("download_url", it) } + asset.tagValue("x")?.takeIf { it.isNotBlank() }?.let { put("apk_hash", it) } + asset.tagValue("filename")?.let { put("filename", it) } + asset.tagValue("m")?.let { put("mime", it) } + } + + fun persistProof(tx: Tx, catalog: CatalogRecord, event: Event) { + validateProof(event) + val hash = event.tagValue("d")!! + val revoked = event.tagCount("revoked") > 0 + val expiry = if (revoked) 0 else event.tagValue("expiry")!!.toLong() + tx.db.prepare( + """ + INSERT INTO certificate_proofs ( + id, catalog_id, certificate_hash, pubkey, event_id, created_at, expiry, revoked, delegations + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(catalog_id, certificate_hash, pubkey) DO UPDATE SET + event_id = excluded.event_id, + created_at = excluded.created_at, + expiry = excluded.expiry, + revoked = excluded.revoked, + delegations = excluded.delegations + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, identity(catalog.relayUrl.url, hash, event.pubkey)) + statement.bindLong(2, catalog.id) + statement.bindBlob(3, Hex.decode(hash)) + statement.bindBlob(4, Hex.decode(event.pubkey)) + statement.bindBlob(5, Hex.decode(event.id)) + statement.bindLong(6, event.createdAt) + statement.bindLong(7, expiry) + statement.bindLong(8, if (revoked) 1 else 0) + statement.bindText(9, JSONArray(event.tagValues("delegation")).toString()) + statement.step() + } + tx.touch(Table.Proofs) + } + + fun delete(tx: Tx, catalog: CatalogRecord, delete: CatalogDelete) { + when (delete) { + is CatalogDelete.Listing -> { + tx.db.prepare("DELETE FROM apps WHERE catalog_id = ? AND app_id = ?").use { statement -> + statement.bindLong(1, catalog.id) + statement.bindText(2, delete.appId) + statement.step() + } + tx.touch(Table.Apps) + } + is CatalogDelete.Coordinate -> { + if (delete.kind == Kinds.AppStack) { + EventRows.deleteStack(tx, delete.pubkey, delete.d) + return + } + if (delete.kind != Kinds.IdentityProof) return + tx.db.prepare( + "DELETE FROM certificate_proofs WHERE catalog_id = ? AND pubkey = ? AND certificate_hash = ?", + ).use { statement -> + statement.bindLong(1, catalog.id) + statement.bindBlob(2, Hex.decode(delete.pubkey)) + statement.bindBlob(3, Hex.decode(delete.d)) + statement.step() + } + tx.touch(Table.Proofs) + } + } + } + + /** App IDs stored for [catalog] that are not in [keep]. */ + fun staleAppIds(db: SQLiteConnection, catalog: CatalogRecord, keep: Set): List = + db.prepare("SELECT app_id FROM apps WHERE catalog_id = ?").use { statement -> + statement.bindLong(1, catalog.id) + buildList { + while (statement.step()) { + val appId = statement.getText(0) + if (appId !in keep) add(appId) + } + } + } + + /** Re-derives `apps.pubkey` from the currently active proofs. */ + fun recomputeAppPubkeys(tx: Tx, catalogId: Long, now: Long) { + val db = tx.db + val rows = db.prepare("SELECT id, certificate_hash, pubkey FROM apps WHERE catalog_id = ?").use { apps -> + apps.bindLong(1, catalogId) + buildList { + while (apps.step()) add(Triple(apps.getBlob(0), Hex.encode(apps.getBlob(1)), apps.blobHexOrNull(2))) + } + } + var changed = false + for ((id, certificate, stored) in rows) { + val pubkey = currentProofPubkey(db, catalogId, certificate, now) + if (pubkey == stored) continue + db.prepare("UPDATE apps SET pubkey = ? WHERE id = ?").use { update -> + update.bindBlobOrNull(1, pubkey?.let(Hex::decode)) + update.bindBlob(2, id) + update.step() + } + changed = true + } + if (changed) tx.touch(Table.Apps) + } + + private fun currentProofPubkey(db: SQLiteConnection, catalogId: Long, certificateHash: String, now: Long): String? = + db.prepare( + """ + SELECT pubkey FROM certificate_proofs + WHERE catalog_id = ? AND certificate_hash = ? AND revoked = 0 AND expiry > ? + ORDER BY created_at DESC, event_id ASC LIMIT 1 + """.trimIndent(), + ).use { statement -> + statement.bindLong(1, catalogId) + statement.bindBlob(2, Hex.decode(certificateHash)) + statement.bindLong(3, now) + if (statement.step()) Hex.encode(statement.getBlob(0)) else null + } + + private fun identity(vararg parts: String): ByteArray = + Crypto.sha256(parts.joinToString("\u0000").toByteArray(Charsets.UTF_8)) + + // --- validation -------------------------------------------------------------------------------- + + private fun assetMatchesDevice(event: Event, device: DeviceProfile): Boolean { + if (!event.tagValue("variant").isNullOrBlank()) return false + if (event.tagValue("m") != ANDROID_APK) return false + val platforms = event.tagValues("f").filter { it.startsWith("android-") } + if (platforms.isNotEmpty() && platforms.none { it in device.platforms }) return false + val minSdk = event.tagValue("min_platform_version")?.toIntOrNull() + return minSdk == null || device.sdk >= minSdk + } + + fun validateProof(event: Event) { + if (event.kind != Kinds.IdentityProof) throw CatalogImportException("expected proof") + if (event.tagCount("d") != 1) throw CatalogImportException("proof must have one d tag") + val hash = event.tagValue("d") ?: throw CatalogImportException("proof missing d") + if (!Hex.isHex(hash, 64)) throw CatalogImportException("proof d is not a sha256") + if (event.tagCount("revoked") > 0) { + if (event.tagCount("signature") != 0 || event.tagCount("expiry") != 0 || + event.tagCount("cert") != 0 || event.tagCount("delegation") != 0 + ) { + throw CatalogImportException("revocation must omit active tags") + } + return + } + if (event.tagCount("signature") != 1 || event.tagCount("expiry") != 1) { + throw CatalogImportException("active proof requires signature and expiry") + } + val expiry = event.tagValue("expiry")?.toLongOrNull() ?: throw CatalogImportException("invalid expiry") + if (expiry <= event.createdAt) throw CatalogImportException("expiry must be after created_at") + val cert = event.tagValue("cert") ?: return + val der = Base64.getDecoder().decode(cert) + if (Hex.encode(Crypto.sha256(der)) != hash) throw CatalogImportException("embedded certificate hash does not match d") + verifyCertificateKey(der, event, expiry) + } + + private fun verifyCertificateKey(der: ByteArray, event: Event, expiry: Long) { + val certificate = CertificateFactory.getInstance("X.509").generateCertificate(ByteArrayInputStream(der)) + val message = "Verifying at ${event.createdAt} until $expiry that I control the following Nostr public key: ${event.pubkey}" + val signature = Base64.getDecoder().decode(event.tagValue("signature")) + val algorithm = when (certificate.publicKey) { + is RSAPublicKey -> "SHA256withRSA" + is ECPublicKey -> "SHA256withECDSA" + else -> throw CatalogImportException("unsupported certificate key type") + } + val verifier = Signature.getInstance(algorithm) + verifier.initVerify(certificate.publicKey) + verifier.update(message.toByteArray(Charsets.UTF_8)) + if (!verifier.verify(signature)) throw CatalogImportException("invalid certificate-key signature") + } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Models.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Models.kt new file mode 100644 index 0000000..77d9fe2 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Models.kt @@ -0,0 +1,296 @@ +package dev.zapstore.iolite + +import java.io.File +import org.json.JSONArray +import org.json.JSONObject + +/** Which SQLite tables a commit touched. Observers subscribe to the tables they read. */ +enum class Table { + Catalogs, + Apps, + Proofs, + Profiles, + Stacks, + Zaps, + Comments, + Preferences, + Outbox, +} + +data class CatalogRecord( + val id: Long, + /** Stable catalog identity: the first clearnet endpoint, or the first endpoint when every one is onion. */ + val relayUrl: RelayUrl, + /** Endpoints from the signed manifest, clearnet and Tor. */ + val relays: List, + val position: Int?, + val isPrivate: Boolean, + val manifestPubkey: String?, + val epoch: Long, +) { + /** Onion when Tor is in use and one is published; otherwise the first clearnet endpoint. */ + fun syncRelay(useOnion: Boolean): RelayUrl { + val onion = relays.firstOrNull { it.isOnion } + if (useOnion && onion != null) return onion + return relays.firstOrNull { !it.isOnion } ?: relays.first() + } +} + +/** `32267::` as carried by stack `a` tags and release `a` tags. */ +data class AppCoordinate(val pubkey: String, val appId: String) { + override fun toString(): String = "${Kinds.App}:$pubkey:$appId" + + companion object { + fun parse(value: String): AppCoordinate? { + val parts = value.split(":", limit = 3) + if (parts.size != 3 || parts[0] != Kinds.App.toString()) return null + if (parts[1].isBlank() || parts[2].isBlank()) return null + return AppCoordinate(parts[1], parts[2]) + } + } +} + +/** + * One visible catalog listing: the `apps` row for the main channel and no-variant asset. + * All values come from SQLite; nothing is re-derived from event JSON. + */ +class AppRecord( + val id: ByteArray, + val catalogId: Long, + val appId: String, + val certificateHash: String, + /** Kind 32267 timestamp; sorts recently updated lists. */ + val createdAt: Long, + /** Active NIP-C1 proof owner; null when the certificate has no current proof. */ + val proofPubkey: String?, + /** Kind 32267 signer. */ + val eventPubkey: String, + /** The catalog manifest signer; an author equal to it is hidden. */ + private val catalogManifestPubkey: String?, + val name: String, + val summary: String, + val repository: String?, + val version: String, + val versionCode: Long, + val channel: String?, + private val metadata: JSONObject, + /** Generated paragraph of what the app does. Bundle member `about`. */ + val about: String = "", + /** Security paragraph, then one warning per line. Bundle member `security`. */ + val security: String = "", + /** Scanner fact sheet. Bundle member `facts`. CSV columns are fact, value, and reason. */ + val facts: String = "", + /** Local WebP from the bundle, when the file is present. */ + val iconFile: File? = null, +) { + val coordinate: AppCoordinate get() = AppCoordinate(eventPubkey, appId) + val isVerified: Boolean get() = proofPubkey != null + + /** Publisher to show: proof owner, else app signer, hidden when it is the catalog relay itself. */ + val authorPubkey: String? get() = (proofPubkey ?: eventPubkey).takeUnless { it == catalogManifestPubkey } + + val description: String get() = metadata.optString("description").ifBlank { summary } + + /** Warning lines folded into [security]. Each line starts with a warning mark. */ + val securityWarnings: String + get() = security.lineSequence().map { it.trim() }.filter { it.startsWith("⚠") }.joinToString("\n") + + /** Security paragraph, without the warning lines. */ + val securityBody: String + get() = security.lineSequence().filter { !it.trim().startsWith("⚠") }.joinToString("\n").trim() + val iconUrl: String? get() = metadata.string("icon") + val screenshots: List get() = metadata.stringList("image") + val website: String? get() = metadata.string("url") + val license: String? get() = metadata.string("license") + val downloadUrl: String? get() = metadata.string("download_url") + val apkHash: String? get() = metadata.string("apk_hash") + val releaseNotes: String get() = metadata.optString("release_notes") + + /** + * [facts] in file order, one row per key. + * The file is a CSV with columns fact, value, and reason. + * The sheet repeats a key once per library. A yes wins over a no. + */ + val factRows: List + get() = parseFactRows(facts) + /** Kind 30063 timestamp, in Unix seconds. */ + val releasedAt: Long get() = metadata.optLong("release_created_at", createdAt) + + override fun equals(other: Any?): Boolean = other is AppRecord && id.contentEquals(other.id) + override fun hashCode(): Int = id.contentHashCode() + override fun toString(): String = "AppRecord($appId@$version)" +} + +/** Reads a facts CSV. A header is required. Repeated keys collapse to one row; a yes wins. */ +internal fun parseFactRows(facts: String): List { + val table = parseCsv(facts) + if (table.isEmpty()) return emptyList() + val header = table.first().map { it.trim() } + val factIdx = header.indexOf("fact") + val valueIdx = header.indexOf("value") + val reasonIdx = header.indexOf("reason") + if (factIdx < 0 || valueIdx < 0) return emptyList() + val rows = LinkedHashMap() + for (record in table.drop(1)) { + val key = record.getOrNull(factIdx)?.trim().orEmpty() + if (key.isEmpty()) continue + val yes = when (record.getOrNull(valueIdx)?.trim()) { + "yes" -> true + "no" -> false + else -> continue + } + val reason = if (reasonIdx < 0) "" else record.getOrNull(reasonIdx)?.trim().orEmpty() + val previous = rows[key] + rows[key] = AppFact( + key = key, + yes = previous?.yes == true || yes, + reason = reason.ifBlank { previous?.reason.orEmpty() }, + ) + } + return rows.values.toList() +} + +internal fun parseCsv(text: String): List> { + val rows = mutableListOf>() + var row = mutableListOf() + val field = StringBuilder() + var quoted = false + var i = 0 + while (i < text.length) { + val char = text[i] + if (quoted) { + if (char == '"') { + if (i + 1 < text.length && text[i + 1] == '"') { + field.append('"') + i += 2 + continue + } + quoted = false + } else { + field.append(char) + } + i++ + continue + } + when (char) { + '"' -> quoted = true + ',' -> { + row.add(field.toString()) + field.clear() + } + '\n' -> { + row.add(field.toString()) + field.clear() + rows.add(row) + row = mutableListOf() + } + '\r' -> Unit + else -> field.append(char) + } + i++ + } + if (field.isNotEmpty() || row.isNotEmpty()) { + row.add(field.toString()) + rows.add(row) + } + return rows +} + +/** One scanner fact. [reason] is empty when the facts file has no phrase for this key. */ +data class AppFact( + val key: String, + val yes: Boolean, + val reason: String = "", +) + +/** Kind 0 projection. `name`/`displayName` are already trimmed to non-blank. */ +data class ProfileRecord( + val pubkey: String, + val updatedAt: Long, + val eventId: String, + val name: String?, + val displayName: String?, + val about: String?, + val picture: String?, + val banner: String?, + val website: String?, + val nip05: String?, +) { + val displayNameOrNpub: String get() = displayName ?: name ?: pubkey.toNpub() +} + +/** Kind 30267 projection. Private (device-encrypted) coordinates are already decrypted. */ +data class StackRecord( + val pubkey: String, + val identifier: String, + val eventId: String, + val name: String, + val description: String, + val apps: List, + val updatedAt: Long, +) { + val appIds: List get() = apps.map(AppCoordinate::appId) +} + +/** Kind 9735 projection. */ +data class ZapRecord( + val eventId: String, + val pubkey: String, + val amountSats: Long, + val createdAt: Long, + val appId: String, +) + +/** Kind 1111 projection. */ +data class CommentRecord( + val eventId: String, + val pubkey: String, + val createdAt: Long, + val content: String, + val appId: String?, + val stack: String?, + val parentEventId: String?, +) + +data class DeviceProfile( + val abis: List, + val sdk: Int, +) { + /** `android-` platform tags as used by NIP-82 `f` tags. */ + val platforms: List get() = abis.map { if (it.startsWith("android-")) it else "android-$it" } +} + +data class HttpResponse( + val status: Int, + val headers: Map, + val body: ByteArray?, +) + +fun interface HttpTransport { + fun get(url: String, headers: Map): HttpResponse +} + +interface WebSocketSession { + fun send(text: String) + fun close() +} + +interface WebSocketListener { + fun onOpen() + fun onMessage(text: String) + fun onClosing(code: Int, reason: String) + fun onFailure(error: Throwable) +} + +fun interface WebSocketFactory { + fun open(url: String, listener: WebSocketListener): WebSocketSession +} + +internal fun JSONObject.string(key: String): String? = optString(key).takeIf { it.isNotBlank() } + +internal fun JSONObject.stringList(key: String): List = optJSONArray(key).toStringList() + +internal fun JSONArray?.toStringList(): List { + if (this == null) return emptyList() + return List(length()) { index -> optString(index) }.filter { it.isNotBlank() } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Nip42.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Nip42.kt new file mode 100644 index 0000000..246d962 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Nip42.kt @@ -0,0 +1,22 @@ +package dev.zapstore.iolite + +import org.json.JSONArray + +object Nip42 { + suspend fun auth(signer: Signer, relay: RelayUrl, challenge: String, createdAt: Long): Event { + require(challenge.isNotEmpty()) { "NIP-42 challenge must not be empty" } + val event = signer.sign( + createdAt = createdAt, + kind = Kinds.Auth, + tags = listOf( + listOf("relay", relay.url), + listOf("challenge", challenge), + ), + content = "", + ) + require(event.verify()) { "NIP-42 AUTH signature is invalid" } + return event + } + + fun wire(event: Event): String = JSONArray().put("AUTH").put(event.toJsonObject()).toString() +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Nip44.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Nip44.kt new file mode 100644 index 0000000..dc66b87 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Nip44.kt @@ -0,0 +1,129 @@ +package dev.zapstore.iolite + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import java.util.Base64 +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec + +object Nip44 { + private const val VERSION: Byte = 0x02 + private val salt = "nip44-v2".toByteArray(Charsets.UTF_8) + + fun conversationKey(secretKey: ByteArray, publicKey: ByteArray): ByteArray { + require(secretKey.size == 32) { "secret key must be 32 bytes" } + require(publicKey.size == 32) { "public key must be 32 bytes" } + return hkdfExtract(salt, Crypto.sharedX(secretKey, publicKey)) + } + + fun encrypt(plaintext: String, conversationKey: ByteArray, nonce: ByteArray = Crypto.randomBytes(32)): String { + require(conversationKey.size == 32) { "invalid conversation_key length" } + require(nonce.size == 32) { "invalid nonce length" } + val (chachaKey, chachaNonce, hmacKey) = messageKeys(conversationKey, nonce) + val ciphertext = ChaCha20.xor(chachaKey, chachaNonce, pad(plaintext)) + val mac = hmacAad(hmacKey, ciphertext, nonce) + val payload = ByteArray(1 + nonce.size + ciphertext.size + mac.size) + payload[0] = VERSION + nonce.copyInto(payload, 1) + ciphertext.copyInto(payload, 33) + mac.copyInto(payload, 33 + ciphertext.size) + return Base64.getEncoder().encodeToString(payload) + } + + fun decrypt(payload: String, conversationKey: ByteArray): String { + require(conversationKey.size == 32) { "invalid conversation_key length" } + if (payload.isEmpty() || payload[0] == '#') throw IllegalArgumentException("unknown version") + if (payload.length < 132) throw IllegalArgumentException("invalid payload size") + val data = try { + Base64.getDecoder().decode(payload) + } catch (_: IllegalArgumentException) { + throw IllegalArgumentException("invalid payload") + } + if (data.size < 99) throw IllegalArgumentException("invalid data size") + if (data[0] != VERSION) throw IllegalArgumentException("unknown version ${data[0].toInt() and 0xff}") + val nonce = data.copyOfRange(1, 33) + val ciphertext = data.copyOfRange(33, data.size - 32) + val mac = data.copyOfRange(data.size - 32, data.size) + val (chachaKey, chachaNonce, hmacKey) = messageKeys(conversationKey, nonce) + val expected = hmacAad(hmacKey, ciphertext, nonce) + if (!MessageDigest.isEqual(expected, mac)) throw IllegalArgumentException("invalid MAC") + return unpad(ChaCha20.xor(chachaKey, chachaNonce, ciphertext)) + } + + internal fun messageKeys(conversationKey: ByteArray, nonce: ByteArray): Triple { + require(conversationKey.size == 32) { "invalid conversation_key length" } + require(nonce.size == 32) { "invalid nonce length" } + val keys = hkdfExpand(conversationKey, nonce, 76) + return Triple(keys.copyOfRange(0, 32), keys.copyOfRange(32, 44), keys.copyOfRange(44, 76)) + } + + internal fun paddedLength(unpadded: Int): Int { + require(unpadded in 1..Int.MAX_VALUE) + if (unpadded <= 32) return 32 + val nextPower = 1 shl (32 - Integer.numberOfLeadingZeros(unpadded - 1)) + val chunk = if (nextPower <= 256) 32 else nextPower / 8 + return chunk * (((unpadded - 1) / chunk) + 1) + } + + private fun pad(plaintext: String): ByteArray { + val unpadded = plaintext.toByteArray(Charsets.UTF_8) + if (unpadded.isEmpty()) throw IllegalArgumentException("invalid plaintext length") + val prefix = if (unpadded.size >= 65_536) { + ByteBuffer.allocate(6).order(ByteOrder.BIG_ENDIAN).putShort(0).putInt(unpadded.size).array() + } else { + ByteBuffer.allocate(2).order(ByteOrder.BIG_ENDIAN).putShort(unpadded.size.toShort()).array() + } + val paddedLen = paddedLength(unpadded.size) + val out = ByteArray(prefix.size + paddedLen) + prefix.copyInto(out) + unpadded.copyInto(out, prefix.size) + return out + } + + private fun unpad(padded: ByteArray): String { + if (padded.size < 2) throw IllegalArgumentException("invalid padding") + val firstTwo = ((padded[0].toInt() and 0xff) shl 8) or (padded[1].toInt() and 0xff) + val (unpaddedLen, prefixLen) = if (firstTwo == 0) { + if (padded.size < 6) throw IllegalArgumentException("invalid padding") + val len = ByteBuffer.wrap(padded, 2, 4).order(ByteOrder.BIG_ENDIAN).int + if (len < 65_536) throw IllegalArgumentException("invalid padding") + len to 6 + } else { + firstTwo to 2 + } + if (unpaddedLen <= 0 || padded.size != prefixLen + paddedLength(unpaddedLen)) { + throw IllegalArgumentException("invalid padding") + } + val unpadded = padded.copyOfRange(prefixLen, prefixLen + unpaddedLen) + if (unpadded.size != unpaddedLen) throw IllegalArgumentException("invalid padding") + return String(unpadded, Charsets.UTF_8) + } + + private fun hmacAad(key: ByteArray, message: ByteArray, aad: ByteArray): ByteArray { + require(aad.size == 32) { "AAD associated data must be 32 bytes" } + return hmac(key, aad + message) + } + + private fun hkdfExtract(salt: ByteArray, ikm: ByteArray): ByteArray = hmac(salt, ikm) + + private fun hkdfExpand(prk: ByteArray, info: ByteArray, length: Int): ByteArray { + val hashLen = 32 + val n = (length + hashLen - 1) / hashLen + val out = ByteArray(n * hashLen) + var previous = ByteArray(0) + var offset = 0 + for (i in 1..n) { + previous = hmac(prk, previous + info + byteArrayOf(i.toByte())) + previous.copyInto(out, offset) + offset += hashLen + } + return out.copyOf(length) + } + + private fun hmac(key: ByteArray, message: ByteArray): ByteArray { + val mac = Mac.getInstance("HmacSHA256") + mac.init(SecretKeySpec(key, "HmacSHA256")) + return mac.doFinal(message) + } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/OutboxRouter.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/OutboxRouter.kt deleted file mode 100644 index a22d278..0000000 --- a/iolite/src/main/kotlin/dev/zapstore/iolite/OutboxRouter.kt +++ /dev/null @@ -1,119 +0,0 @@ -package dev.zapstore.iolite - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import kotlinx.coroutines.async -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.channelFlow -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.launch -import kotlin.time.Duration -import kotlin.time.Duration.Companion.hours -import kotlin.time.Duration.Companion.seconds - -/** - * NIP-65 outbox routing on top of [Iolite]. - * - * Resolution is local-first: kind-10002 relay lists are read from the local - * store and only fetched from [bootstrapRelays] when the TTL'd freshness cache - * ([QueryOptions.cachedFor]) reports a miss or staleness. A fetch - * that cannot complete within [resolveTimeout] resolves to whatever the local - * store holds. - */ -class OutboxRouter( - private val client: Iolite, - private val bootstrapRelays: Set, - private val cacheDuration: Duration = 6.hours, - private val resolveTimeout: Duration = 10.seconds, -) { - init { - require(bootstrapRelays.isNotEmpty()) { "bootstrapRelays must not be empty" } - } - - /** - * Read relays advertised by [pubkey] (unmarked `r` tags count as read+write, - * per NIP-65). Empty when no relay list is known or reachable. - */ - suspend fun resolveReadRelays(pubkey: String): Set { - val state = client.query( - Filter(authors = listOf(pubkey), kinds = listOf(RELAY_LIST_KIND), limit = 1), - QueryOptions.localAndRemote( - relays = bootstrapRelays, - remoteMode = RemoteMode.OneShot(resolveTimeout), - cachedFor = cacheDuration, - ), - ).first { it.phase.isTerminal } - return state.items.maxByOrNull(Event::createdAt)?.readRelayUrls().orEmpty() - } - - /** - * Runs [filters] against [fallbackRelays] immediately so local data flows without - * waiting, resolves the [authors]' read relays in parallel, then transparently - * restarts the remote side on the expanded relay set when resolution lands. - * - * With [unionWithFallback] the resolved relays are added to the fallback set; - * otherwise the resolved set replaces it whenever it is non-empty. - */ - fun queryWithOutbox( - filters: List, - authors: List, - fallbackRelays: Set, - cachedFor: Duration? = null, - remoteMode: RemoteMode = RemoteMode.Stream, - unionWithFallback: Boolean = true, - ): Flow = channelFlow { - fun options(relays: Set) = - QueryOptions.localAndRemote( - relays = relays, - remoteMode = remoteMode, - cachedFor = cachedFor, - ) - - if (authors.isEmpty()) { - client.query(filters, options(fallbackRelays)).collect { send(it) } - return@channelFlow - } - - val fallbackJob = launch { - client.query(filters, options(fallbackRelays)).collect { send(it) } - } - val resolved = authors - .map { async { resolveReadRelays(it) } } - .flatMap { it.await() } - .toSet() - val expanded = when { - resolved.isEmpty() -> fallbackRelays - unionWithFallback -> fallbackRelays + resolved - else -> resolved - } - if (expanded == fallbackRelays) { - fallbackJob.join() - } else { - fallbackJob.cancel() - fallbackJob.join() - client.query(filters, options(expanded)).collect { send(it) } - } - } - - companion object { - const val RELAY_LIST_KIND = 10_002 - - private val QueryPhase.isTerminal: Boolean - get() = this == QueryPhase.Cached || - this == QueryPhase.Complete || - this == QueryPhase.TimedOut || - this == QueryPhase.Failed - - private fun Event.readRelayUrls(): Set = - tags.asSequence() - .filter { tag -> tag.getOrNull(0) == "r" } - .filter { tag -> tag.getOrNull(2).let { marker -> marker == null || marker == "read" } } - .mapNotNull { tag -> tag.getOrNull(1) } - .filter { it.startsWith("wss://") } - .mapNotNull { runCatching { it.normalizeRelayUrl() }.getOrNull() } - .toSet() - } -} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Query.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Query.kt new file mode 100644 index 0000000..fe2b339 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Query.kt @@ -0,0 +1,79 @@ +package dev.zapstore.iolite + +/** + * A typed query: the relay filters that fetch it, the tables whose commits re-run it, and the SQLite read + * that produces its items. Relay events flow through the store; the read never sees event JSON. + */ +class Query internal constructor( + val filters: List, + internal val tables: Set
, + /** Authors whose NIP-65 read relays are added when `QueryOptions.useAuthorRelays` is set. */ + internal val authors: List, + /** Items shown by a `Remote` query before the relays answer. */ + internal val empty: T, + internal val read: (IoliteStore, LocalSigner?) -> T, +) { + companion object { + fun profiles(pubkeys: Collection): Query> { + val distinct = pubkeys.distinct() + return Query( + filters = listOf(Filter(authors = distinct, kinds = listOf(Kinds.Profile))), + tables = setOf(Table.Profiles), + authors = distinct, + empty = emptyMap(), + read = { store, _ -> store.profiles(distinct) }, + ) + } + + fun profile(pubkey: String): Query = profiles(listOf(pubkey)).map { it[pubkey] } + + fun relayList(pubkey: String): Query?> = Query( + filters = listOf(Filter(authors = listOf(pubkey), kinds = listOf(Kinds.RelayList), limit = 1)), + tables = setOf(Table.Profiles), + authors = emptyList(), + empty = null, + read = { store, _ -> store.readRelays(pubkey) }, + ) + + fun stacks(author: String, limit: Int? = null): Query> = Query( + filters = listOf(Filter(authors = listOf(author), kinds = listOf(Kinds.AppStack), limit = limit)), + tables = setOf(Table.Stacks), + authors = listOf(author), + empty = emptyList(), + read = { store, signer -> store.stacks(author, limit = limit, deviceSigner = signer) }, + ) + + fun stack(author: String, identifier: String): Query = Query( + filters = listOf( + Filter(authors = listOf(author), kinds = listOf(Kinds.AppStack), tags = mapOf("d" to listOf(identifier)), limit = 1), + ), + tables = setOf(Table.Stacks), + authors = listOf(author), + empty = null, + read = { store, signer -> store.stacks(author, identifier, limit = 1, deviceSigner = signer).firstOrNull() }, + ) + + fun zaps(app: AppCoordinate, limit: Int? = null): Query> = Query( + filters = listOf( + Filter(kinds = listOf(Kinds.Zap), tags = mapOf("a" to listOf(app.toString())), limit = limit), + ), + tables = setOf(Table.Zaps), + authors = emptyList(), + empty = emptyList(), + read = { store, _ -> store.zaps(app.appId, limit) }, + ) + + fun comments(app: AppCoordinate, limit: Int? = null): Query> = Query( + filters = listOf( + Filter(kinds = listOf(Kinds.Comment), tags = mapOf("A" to listOf(app.toString())), limit = limit), + ), + tables = setOf(Table.Comments), + authors = listOf(app.pubkey), + empty = emptyList(), + read = { store, _ -> store.comments(app.appId) }, + ) + } + + fun map(transform: (T) -> R): Query = + Query(filters, tables, authors, transform(empty)) { store, signer -> transform(read(store, signer)) } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/QueryFingerprint.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/QueryFingerprint.kt index 0814dca..8fa19c4 100644 --- a/iolite/src/main/kotlin/dev/zapstore/iolite/QueryFingerprint.kt +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/QueryFingerprint.kt @@ -1,7 +1,5 @@ package dev.zapstore.iolite -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import java.io.ByteArrayOutputStream import java.io.DataOutputStream import java.security.MessageDigest @@ -11,7 +9,7 @@ internal object QueryFingerprint { fun create( filters: List, - relays: Set, + relays: Set, ): String { val encodedFilters = filters.map(::encodeFilter).sortedWith(::compareBytes) val canonical = ByteArrayOutputStream().use { bytes -> @@ -38,7 +36,6 @@ internal object QueryFingerprint { output.writeNullableStrings(filter.authors) output.writeNullableInts(filter.kinds) output.writeNullableStringMap(filter.tags) - output.writeNullableStringMap(filter.tagsAll) output.writeNullableLong(filter.since) output.writeNullableLong(filter.until) output.writeNullableInt(filter.limit) diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/QueryRefreshCache.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/QueryRefreshCache.kt deleted file mode 100644 index 2b7324f..0000000 --- a/iolite/src/main/kotlin/dev/zapstore/iolite/QueryRefreshCache.kt +++ /dev/null @@ -1,149 +0,0 @@ -package dev.zapstore.iolite - -import android.content.Context -import android.content.SharedPreferences -import java.io.File -import java.nio.file.Files -import java.nio.file.attribute.BasicFileAttributes -import java.security.MessageDigest -import java.util.UUID -import kotlin.time.Duration - -internal fun interface EpochMillisClock { - fun now(): Long -} - -internal interface QueryRefreshCache { - fun lastRefresh(fingerprint: String): Long? - - fun recordRefresh(fingerprint: String, epochMillis: Long): Boolean -} - -internal data class QueryFreshness( - val isFresh: Boolean, - val remainingMillis: Long, -) - -internal fun QueryRefreshCache.freshness( - fingerprint: String, - cachedFor: Duration, - now: Long, -): QueryFreshness { - val refreshedAt = lastRefresh(fingerprint) - ?: return QueryFreshness(isFresh = false, remainingMillis = 0) - if (refreshedAt <= 0 || now < refreshedAt) { - return QueryFreshness(isFresh = false, remainingMillis = 0) - } - - val cachedForMillis = maxOf(1, cachedFor.inWholeMilliseconds) - val age = now - refreshedAt - val remaining = cachedForMillis - age - return QueryFreshness( - isFresh = remaining > 0, - remainingMillis = remaining.coerceAtLeast(0), - ) -} - -internal class InMemoryQueryRefreshCache : QueryRefreshCache { - private val entries = mutableMapOf() - - override fun lastRefresh(fingerprint: String): Long? = synchronized(entries) { - entries[fingerprint] - } - - override fun recordRefresh(fingerprint: String, epochMillis: Long): Boolean = synchronized(entries) { - if (epochMillis <= 0) return@synchronized false - entries[fingerprint] = epochMillis - true - } -} - -internal class SharedPreferencesQueryRefreshCache private constructor( - private val preferences: SharedPreferences, - private val entryPrefix: String, -) : QueryRefreshCache { - private val lock = Any() - - override fun lastRefresh(fingerprint: String): Long? = runCatching { - preferences.takeIf { it.contains(entryPrefix + fingerprint) } - ?.getLong(entryPrefix + fingerprint, 0L) - ?.takeIf { it > 0 } - }.getOrNull() - - override fun recordRefresh(fingerprint: String, epochMillis: Long): Boolean { - if (epochMillis <= 0) return false - synchronized(lock) { - val key = entryPrefix + fingerprint - val currentEntries = preferences.all.asSequence() - .filter { (entryKey, value) -> entryKey.startsWith(entryPrefix) && value is Long } - .sortedBy { (_, value) -> value as Long } - .toList() - val removals = if (preferences.contains(key)) { - 0 - } else { - (currentEntries.size - MAX_ENTRIES + 1).coerceAtLeast(0) - } - val editor = preferences.edit() - currentEntries.take(removals).forEach { (entryKey, _) -> editor.remove(entryKey) } - val committed = editor.putLong(key, epochMillis).commit() - if (!committed) { - // commit updates SharedPreferences memory before attempting disk I/O. - // Remove the optimistic value so this process also fails open. - preferences.edit().remove(key).apply() - } - return committed - } - } - - companion object { - private const val PREFERENCES_NAME = "dev.zapstore.iolite.query-refresh-v1" - private const val MAX_ENTRIES = 1_024 - - fun create( - context: Context, - databasePath: String, - databaseExisted: Boolean = File(databasePath).exists(), - ): SharedPreferencesQueryRefreshCache { - val preferences = context.getSharedPreferences(PREFERENCES_NAME, Context.MODE_PRIVATE) - val pathHash = sha256(databasePath) - val generationKey = "database:$pathHash:generation" - val identityKey = "database:$pathHash:identity" - val previousGeneration = runCatching { preferences.getString(generationKey, null) }.getOrNull() - val previousIdentity = runCatching { preferences.getString(identityKey, null) }.getOrNull() - val currentIdentity = databaseIdentity(databasePath) - val canReuseGeneration = - databaseExisted && - currentIdentity != null && - currentIdentity == previousIdentity && - previousGeneration != null - val generation = previousGeneration?.takeIf { canReuseGeneration } ?: UUID.randomUUID().toString() - - if (!canReuseGeneration) { - val oldPrefix = "refresh:$pathHash:" - val editor = preferences.edit() - .putString(generationKey, generation) - .putString(identityKey, currentIdentity) - preferences.all.keys.filter { it.startsWith(oldPrefix) }.forEach(editor::remove) - editor.commit() - } - - return SharedPreferencesQueryRefreshCache( - preferences = preferences, - entryPrefix = "refresh:$pathHash:$generation:", - ) - } - - private fun sha256(value: String): String = - MessageDigest.getInstance("SHA-256") - .digest(value.toByteArray(Charsets.UTF_8)) - .joinToString(separator = "") { byte -> "%02x".format(byte.toInt() and 0xff) } - - private fun databaseIdentity(databasePath: String): String? = runCatching { - val attributes = Files.readAttributes(File(databasePath).toPath(), BasicFileAttributes::class.java) - val fileKey = attributes.fileKey()?.toString().orEmpty() - val createdAt = attributes.creationTime().toMillis() - if (fileKey.isEmpty() && createdAt <= 0) return@runCatching null - sha256("$fileKey:$createdAt") - }.getOrNull() - } -} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/QueryTypes.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/QueryTypes.kt index 444cd40..7f106b8 100644 --- a/iolite/src/main/kotlin/dev/zapstore/iolite/QueryTypes.kt +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/QueryTypes.kt @@ -1,20 +1,38 @@ package dev.zapstore.iolite -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import kotlin.time.Duration enum class SourceMode { + /** SQLite only. Emits on every commit touching the observed tables. */ Local, + /** SQLite first, then relays; re-emits after each commit. */ LocalAndRemote, + /** Empty first, then relays; re-emits after each commit. */ Remote, } +sealed interface RemoteMode { + /** Ends 200 ms after the first EOSE, or at [timeout]. */ + data class OneShot(val timeout: Duration? = null) : RemoteMode { + init { + require(timeout == null || (timeout.isFinite() && timeout.isPositive())) { + "One-shot timeout must be finite and positive" + } + } + } + + /** Keeps the subscription open until the collector is cancelled. */ + data object Stream : RemoteMode +} + data class QueryOptions( val sourceMode: SourceMode = SourceMode.Local, val remoteMode: RemoteMode? = null, - val relays: Set = emptySet(), + val relays: Set = emptySet(), + /** Skip relay work when the same query completed within this duration. Local-and-remote only. */ val cachedFor: Duration? = null, + /** Also query the NIP-65 read relays of the observed authors. */ + val useAuthorRelays: Boolean = false, ) { init { when (sourceMode) { @@ -22,6 +40,7 @@ data class QueryOptions( require(remoteMode == null) { "Local queries cannot have a remote mode" } require(relays.isEmpty()) { "Local queries cannot have relays" } require(cachedFor == null) { "Local queries cannot be cached" } + require(!useAuthorRelays) { "Local queries cannot use author relays" } } SourceMode.LocalAndRemote, SourceMode.Remote, @@ -38,42 +57,41 @@ data class QueryOptions( } } + val isRemote: Boolean get() = sourceMode != SourceMode.Local + companion object { fun local(): QueryOptions = QueryOptions() fun localAndRemote( - relays: Set, + relays: Set, remoteMode: RemoteMode, cachedFor: Duration? = null, - ): QueryOptions = QueryOptions(SourceMode.LocalAndRemote, remoteMode, relays, cachedFor) + useAuthorRelays: Boolean = false, + ): QueryOptions = QueryOptions(SourceMode.LocalAndRemote, remoteMode, relays, cachedFor, useAuthorRelays) fun remote( - relays: Set, + relays: Set, remoteMode: RemoteMode, - ): QueryOptions = QueryOptions(SourceMode.Remote, remoteMode, relays) + useAuthorRelays: Boolean = false, + ): QueryOptions = QueryOptions(SourceMode.Remote, remoteMode, relays, useAuthorRelays = useAuthorRelays) } } -sealed interface RemoteMode { - data class OneShot( - val timeout: Duration? = null, - ) : RemoteMode { - init { - require(timeout == null || (timeout.isFinite() && timeout.isPositive())) { - "One-shot timeout must be finite and positive" - } - } - } - - data object Stream : RemoteMode -} - -data class QueryState( - val items: List, +/** One emission of a collected query: the current SQLite read plus relay progress. */ +data class QueryState( + val items: T, val phase: QueryPhase, - val relays: Map = emptyMap(), + val relays: Map = emptyMap(), val error: QueryError? = null, -) +) { + /** True while the first remote request is still unfinished. */ + val isLoading: Boolean get() = phase == QueryPhase.Connecting || phase == QueryPhase.CatchingUp + + /** True once no further relay work will happen for this session. */ + val isTerminal: Boolean get() = phase.isTerminal + + fun map(transform: (T) -> R): QueryState = QueryState(transform(items), phase, relays, error) +} sealed interface QueryPhase { data object LocalOnly : QueryPhase @@ -84,6 +102,9 @@ sealed interface QueryPhase { data object Complete : QueryPhase data object TimedOut : QueryPhase data object Failed : QueryPhase + + val isTerminal: Boolean + get() = this == LocalOnly || this == Cached || this == Complete || this == TimedOut || this == Failed } internal fun isLegalPhaseTransition(from: QueryPhase?, to: QueryPhase): Boolean = when { @@ -123,40 +144,8 @@ enum class RelayConnectionState { sealed interface QueryError { val message: String - data class InvalidQuery(override val message: String) : QueryError - data class UnsupportedLocalProjection(override val message: String) : QueryError - data class RelayFailure( - val relay: NormalizedRelayUrl, - override val message: String, - ) : QueryError - - data class VerificationRejected( - val relay: NormalizedRelayUrl, - val eventId: String?, - override val message: String, - ) : QueryError - - data class ProtocolViolation( - val relay: NormalizedRelayUrl, - val eventId: String?, - override val message: String, - ) : QueryError - - data class PersistenceFailure( - val eventId: String?, - override val message: String, - ) : QueryError - - data class IngestionSaturated( - val relay: NormalizedRelayUrl, - override val message: String, - ) : QueryError - - data class Timeout( - val duration: Duration, - override val message: String, - ) : QueryError - + data class RelayFailure(val relay: RelayUrl, override val message: String) : QueryError + data class VerificationRejected(val relay: RelayUrl, val eventId: String?, override val message: String) : QueryError + data class Timeout(val duration: Duration, override val message: String) : QueryError data class IncompatibleDependency(override val message: String) : QueryError - data class Lifecycle(override val message: String) : QueryError } diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/RelayPool.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/RelayPool.kt new file mode 100644 index 0000000..37198f7 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/RelayPool.kt @@ -0,0 +1,286 @@ +package dev.zapstore.iolite + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import org.json.JSONArray +import org.json.JSONObject + +internal class RelayPool( + private val factory: WebSocketFactory, + private val scope: CoroutineScope, + private val signer: Signer?, + private val nowSeconds: () -> Long, +) { + private val connections = ConcurrentHashMap() + private val trafficEnabled = AtomicBoolean(true) + + fun setTrafficEnabled(enabled: Boolean) { + trafficEnabled.set(enabled) + if (!enabled) { + connections.values.forEach { it.close("traffic disabled") } + } else { + connections.values.forEach { it.ensureOpen() } + } + } + + fun refreshConnections() { + connections.values.forEach { it.reconnect("refresh") } + } + + fun subscribe( + id: String, + filters: List, + relays: Set, + onEvent: (RelayUrl, Event) -> Unit, + onEose: (RelayUrl) -> Unit, + onState: (RelayUrl, RelayQueryState) -> Unit, + ): () -> Unit { + val subscription = Subscription(id, filters, onEvent, onEose, onState) + relays.forEach { url -> connection(url).add(subscription) } + return { + relays.forEach { url -> connections[url]?.remove(id) } + } + } + + fun publish(event: Event, relays: Set, onOk: (RelayUrl, Boolean, String) -> Unit) { + relays.forEach { url -> connection(url).publish(event, onOk) } + } + + fun close() { + trafficEnabled.set(false) + connections.values.forEach { it.close("closed") } + connections.clear() + } + + private fun connection(url: RelayUrl): RelayConnection = + connections.getOrPut(url) { + RelayConnection( + url = url, + factory = factory, + scope = scope, + signer = signer, + nowSeconds = nowSeconds, + trafficEnabled = trafficEnabled, + ) + }.also { it.ensureOpen() } +} + +private class Subscription( + val id: String, + val filters: List, + val onEvent: (RelayUrl, Event) -> Unit, + val onEose: (RelayUrl) -> Unit, + val onState: (RelayUrl, RelayQueryState) -> Unit, +) + +private class RelayConnection( + private val url: RelayUrl, + private val factory: WebSocketFactory, + private val scope: CoroutineScope, + private val signer: Signer?, + private val nowSeconds: () -> Long, + private val trafficEnabled: AtomicBoolean, +) { + private val lock = Any() + private val subscriptions = ConcurrentHashMap() + private val okWaiters = ConcurrentHashMap Unit>>() + private val pendingEvents = ConcurrentHashMap() + private val generation = AtomicLong(0) + private var session: WebSocketSession? = null + private var reconnectJob: Job? = null + private var authEventId: String? = null + private var lastError: String? = null + private var connected = false + + fun add(subscription: Subscription) { + subscriptions[subscription.id] = subscription + ensureOpen() + synchronized(lock) { + if (connected) sendReq(subscription) + } + emitState(RelayConnectionState.Connecting, eose = false) + } + + fun remove(id: String) { + subscriptions.remove(id) ?: return + synchronized(lock) { + if (connected) session?.send(JSONArray().put("CLOSE").put(id).toString()) + } + if (subscriptions.isEmpty() && pendingEvents.isEmpty()) { + close("idle") + } + } + + fun publish(event: Event, onOk: (RelayUrl, Boolean, String) -> Unit) { + pendingEvents[event.id] = event + okWaiters.getOrPut(event.id) { CopyOnWriteArrayList() }.add(onOk) + ensureOpen() + synchronized(lock) { + if (connected) sendEvent(event) + } + } + + fun ensureOpen() { + if (!trafficEnabled.get()) return + synchronized(lock) { + if (session != null || reconnectJob?.isActive == true) return + openLocked() + } + } + + fun reconnect(reason: String) { + close(reason) + ensureOpen() + } + + fun close(reason: String) { + synchronized(lock) { + reconnectJob?.cancel() + reconnectJob = null + runCatching { session?.close() } + session = null + connected = false + authEventId = null + } + emitState(RelayConnectionState.Disconnected, eose = false, error = reason) + } + + private fun openLocked() { + emitState(RelayConnectionState.Connecting, eose = false) + val listener = object : WebSocketListener { + override fun onOpen() { + synchronized(lock) { + connected = true + lastError = null + generation.incrementAndGet() + subscriptions.values.forEach(::sendReq) + pendingEvents.values.forEach(::sendEvent) + } + emitState(RelayConnectionState.Connected, eose = false) + } + + override fun onMessage(text: String) = handle(text) + + override fun onClosing(code: Int, reason: String) { + synchronized(lock) { + connected = false + session = null + } + emitState(RelayConnectionState.Disconnected, eose = false, error = reason) + scheduleReconnect() + } + + override fun onFailure(error: Throwable) { + synchronized(lock) { + connected = false + session = null + lastError = error.message + } + emitState(RelayConnectionState.Disconnected, eose = false, error = error.message) + scheduleReconnect() + } + } + session = try { + factory.open(url.url, listener) + } catch (failure: Throwable) { + lastError = failure.message + emitState(RelayConnectionState.Disconnected, eose = false, error = failure.message) + scheduleReconnect() + null + } + } + + private fun handle(text: String) { + val message = try { + JSONArray(text) + } catch (_: Exception) { + return + } + if (message.length() < 2) return + when (message.optString(0)) { + "EVENT" -> { + if (message.length() < 3) return + val subId = message.optString(1) + val raw = message.opt(2) as? JSONObject ?: return + val event = runCatching { Event.parse(raw) }.getOrNull() ?: return + val subscription = subscriptions[subId] ?: return + subscription.onEvent(url, event) + } + "EOSE" -> subscriptions[message.optString(1)]?.onEose(url) + "OK" -> { + val eventId = message.optString(1) + val accepted = message.optBoolean(2) + val reason = message.optString(3) + if (eventId == authEventId && accepted) { + synchronized(lock) { subscriptions.values.forEach(::sendReq) } + } + val waiters = okWaiters.remove(eventId) + if (accepted) pendingEvents.remove(eventId) + waiters?.forEach { it(url, accepted, reason) } + } + "AUTH" -> { + val challenge = message.optString(1) + val current = signer + if (current == null) { + lastError = "auth required" + emitState(RelayConnectionState.Connected, eose = false, error = lastError) + return + } + scope.launch { + runCatching { + val event = Nip42.auth(current, url, challenge, nowSeconds()) + authEventId = event.id + synchronized(lock) { session?.send(Nip42.wire(event)) } + }.onFailure { failure -> + lastError = failure.message + emitState(RelayConnectionState.Connected, eose = false, error = lastError) + } + } + } + "CLOSED" -> { + val subId = message.optString(1) + lastError = message.optString(2).ifBlank { "closed" } + subscriptions[subId]?.onEose(url) + emitState(RelayConnectionState.Connected, eose = true, error = lastError) + } + } + } + + private fun sendReq(subscription: Subscription) { + val payload = JSONArray().put("REQ").put(subscription.id) + subscription.filters.forEach { payload.put(it.toJsonObject()) } + session?.send(payload.toString()) + } + + private fun sendEvent(event: Event) { + session?.send(JSONArray().put("EVENT").put(event.toJsonObject()).toString()) + } + + private fun scheduleReconnect() { + if (!trafficEnabled.get()) return + if (subscriptions.isEmpty() && pendingEvents.isEmpty()) return + synchronized(lock) { + if (reconnectJob?.isActive == true || session != null) return + reconnectJob = scope.launch { + delay(500) + synchronized(lock) { + reconnectJob = null + if (session == null && trafficEnabled.get()) openLocked() + } + } + } + } + + private fun emitState(connection: RelayConnectionState, eose: Boolean, error: String? = lastError) { + val state = RelayQueryState(generation.get(), connection, eose, error) + subscriptions.values.forEach { it.onState(url, state) } + } +} + +internal fun okThreshold(relayCount: Int): Int = if (relayCount <= 2) 1 else 2 diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/RelayUrl.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/RelayUrl.kt new file mode 100644 index 0000000..51e8678 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/RelayUrl.kt @@ -0,0 +1,45 @@ +package dev.zapstore.iolite + +@JvmInline +value class RelayUrl(val url: String) { + override fun toString(): String = url +} + +fun String.normalizeRelayUrl(): RelayUrl { + val trimmed = trim() + require(trimmed.startsWith("ws://") || trimmed.startsWith("wss://")) { + "relay URL must be ws:// or wss://" + } + val withoutTrailing = trimmed.trimEnd('/') + return RelayUrl(withoutTrailing) +} + +fun String.httpOriginToRelayUrl(): RelayUrl { + val trimmed = trim().trimEnd('/') + val ws = when { + trimmed.startsWith("https://", ignoreCase = true) -> "wss://" + trimmed.substring(8) + trimmed.startsWith("http://", ignoreCase = true) -> "ws://" + trimmed.substring(7) + else -> trimmed + } + return ws.normalizeRelayUrl() +} + +fun RelayUrl.originHttpUrl(): String = + url.replaceFirst("wss://", "https://").replaceFirst("ws://", "http://") + +/** `GET /deltas` on the relay origin; path, query, and fragment are ignored. */ +fun RelayUrl.deltasUrl(from: Long): String = origin() + "/deltas?from=$from" + +/** True when the host is a Tor onion service. */ +val RelayUrl.isOnion: Boolean + get() = url.substringAfter("://").substringBefore('/').substringBefore(':').endsWith(".onion") + +private fun RelayUrl.origin(): String { + val http = originHttpUrl().substringBefore('#').substringBefore('?') + val schemeEnd = http.indexOf("://") + require(schemeEnd > 0) { "invalid relay URL $url" } + val afterScheme = http.substring(schemeEnd + 3) + val slash = afterScheme.indexOf('/') + val authority = (if (slash == -1) afterScheme else afterScheme.substring(0, slash)).trimEnd('/') + return http.substring(0, schemeEnd + 3) + authority +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Schema.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Schema.kt new file mode 100644 index 0000000..dc6a07d --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Schema.kt @@ -0,0 +1,160 @@ +package dev.zapstore.iolite + +internal object Schema { + const val USER_VERSION = 1 + + const val APPS_SEARCH = """ + CREATE TABLE apps_search ( + id BLOB PRIMARY KEY REFERENCES apps(id) ON DELETE CASCADE, + embedding BLOB, + CHECK (embedding IS NULL OR length(embedding) = 768) + ) + """ + + val statements: List = listOf( + // Last successful relay refresh per query fingerprint; backs QueryOptions.cachedFor. + """ + CREATE TABLE query_refresh ( + fingerprint TEXT PRIMARY KEY, + refreshed_at INTEGER NOT NULL + ) + """.trimIndent(), + """ + CREATE TABLE catalogs ( + id INTEGER PRIMARY KEY, + relay_url TEXT NOT NULL UNIQUE, + position INTEGER UNIQUE, + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + manifest_pubkey BLOB, + epoch INTEGER NOT NULL DEFAULT 0, + endpoints TEXT NOT NULL DEFAULT '[]' + ) + """.trimIndent(), + """ + CREATE TABLE apps ( + id BLOB PRIMARY KEY, + catalog_id INTEGER NOT NULL + REFERENCES catalogs(id) ON DELETE CASCADE, + app_id TEXT NOT NULL, + variant_id BLOB, + certificate_hash BLOB NOT NULL, + app_event_created_at INTEGER NOT NULL, + pubkey BLOB, + event_pubkey BLOB NOT NULL, + name TEXT NOT NULL, + summary TEXT NOT NULL DEFAULT '', + repository TEXT, + version TEXT NOT NULL, + version_code INTEGER NOT NULL, + channel TEXT, + metadata TEXT NOT NULL DEFAULT '{}', + about TEXT NOT NULL DEFAULT '', + security TEXT NOT NULL DEFAULT '', + facts TEXT NOT NULL DEFAULT '' + ) + """.trimIndent(), + "CREATE UNIQUE INDEX apps_identity_idx ON apps(catalog_id, app_id)", + "CREATE INDEX apps_pubkey_idx ON apps(pubkey)", + "CREATE INDEX apps_event_pubkey_idx ON apps(event_pubkey)", + "CREATE INDEX apps_updated_idx ON apps(app_event_created_at DESC)", + "CREATE INDEX apps_visible_idx ON apps(app_id)", + "CREATE INDEX apps_version_idx ON apps(app_id, version_code DESC)", + """ + CREATE TABLE certificate_proofs ( + id BLOB PRIMARY KEY, + catalog_id INTEGER NOT NULL + REFERENCES catalogs(id) ON DELETE CASCADE, + certificate_hash BLOB NOT NULL, + pubkey BLOB NOT NULL, + event_id BLOB NOT NULL, + created_at INTEGER NOT NULL, + expiry INTEGER NOT NULL, + revoked INTEGER NOT NULL DEFAULT 0 + CHECK (revoked IN (0, 1)), + delegations TEXT NOT NULL DEFAULT '[]', + UNIQUE (catalog_id, certificate_hash, pubkey) + ) + """.trimIndent(), + "CREATE INDEX certificate_proofs_certificate_idx ON certificate_proofs(catalog_id, certificate_hash)", + """ + CREATE TABLE app_variants ( + id BLOB PRIMARY KEY, + app_id BLOB NOT NULL + REFERENCES apps(id) ON DELETE CASCADE, + channel TEXT NOT NULL DEFAULT '', + variant TEXT NOT NULL DEFAULT '', + certificate_hash BLOB NOT NULL, + version TEXT NOT NULL, + version_code INTEGER NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}', + UNIQUE (app_id, channel, variant), + CHECK (channel <> '' OR variant <> '') + ) + """.trimIndent(), + "CREATE INDEX app_variants_version_idx ON app_variants(app_id, channel, version_code DESC)", + """ + CREATE TABLE preferences ( + identifier TEXT PRIMARY KEY, + updated_at INTEGER NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}' + ) + """.trimIndent(), + APPS_SEARCH.trimIndent(), + """ + CREATE TABLE comments ( + event_id BLOB PRIMARY KEY, + pubkey BLOB NOT NULL, + created_at INTEGER NOT NULL, + content TEXT NOT NULL, + app_id TEXT, + stack TEXT, + parent_event_id BLOB, + metadata TEXT NOT NULL DEFAULT '{}' + ) + """.trimIndent(), + "CREATE INDEX comments_app_idx ON comments(app_id, created_at DESC)", + "CREATE INDEX comments_stack_idx ON comments(stack, created_at DESC)", + "CREATE INDEX comments_parent_idx ON comments(parent_event_id)", + """ + CREATE TABLE zaps ( + event_id BLOB PRIMARY KEY, + pubkey BLOB NOT NULL, + amount_sats INTEGER NOT NULL CHECK (amount_sats > 0), + created_at INTEGER NOT NULL, + app_id TEXT NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}' + ) + """.trimIndent(), + "CREATE INDEX zaps_app_idx ON zaps(app_id, created_at DESC)", + "CREATE INDEX zaps_pubkey_idx ON zaps(pubkey, created_at DESC)", + """ + CREATE TABLE stacks ( + pubkey BLOB NOT NULL, + identifier TEXT NOT NULL, + event_id BLOB NOT NULL, + name TEXT NOT NULL, + description TEXT, + public_apps TEXT NOT NULL DEFAULT '[]', + updated_at INTEGER NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}', + PRIMARY KEY (pubkey, identifier) + ) + """.trimIndent(), + "CREATE INDEX stacks_updated_idx ON stacks(updated_at DESC)", + """ + CREATE TABLE profiles ( + pubkey BLOB PRIMARY KEY, + updated_at INTEGER NOT NULL, + name TEXT, + picture_url TEXT, + metadata TEXT NOT NULL DEFAULT '{}' + ) + """.trimIndent(), + """ + CREATE TABLE outbox ( + event_id BLOB PRIMARY KEY, + event_json TEXT NOT NULL + ) + """.trimIndent(), + ) +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Search.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Search.kt new file mode 100644 index 0000000..f02f752 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Search.kt @@ -0,0 +1,119 @@ +package dev.zapstore.iolite + +import java.util.PriorityQueue + +/** leaf-ir-v1 document and query width. */ +const val EMBEDDING_DIMS = 768 + +/** Queries are bounded to the same size as `POST /search`. */ +const val MAX_QUERY_BYTES = 256 + +/** + * int8 dot of two leaf-ir-v1 vectors. Cosine is about `dot * (0.3/127)^2`. + * A maps query is about 100000 against a maps listing and about 33000 against a password manager. + */ +internal const val MIN_VECTOR_DOT = 57_348 + +/** About cosine 0.45, so a close vector can outrank a name that merely shares a prefix. */ +internal const val PREFIX_BOOST = 80_645 + +/** Larger than any int8 dot, so an exact name or app ID stays above semantic hits. */ +internal const val EXACT_BOOST = 20_000_000 + +/** Trimmed, whitespace-collapsed, lowercased, and cut to [MAX_QUERY_BYTES]. */ +fun normalizeSearchQuery(query: String): String { + val folded = foldSearchText(query) + val bytes = folded.encodeToByteArray() + if (bytes.size <= MAX_QUERY_BYTES) return folded + var end = MAX_QUERY_BYTES + while (end > 0 && bytes[end].toInt() and 0xC0 == 0x80) end-- + return bytes.decodeToString(0, end).trimEnd() +} + +internal fun foldSearchText(value: String): String { + val out = StringBuilder(value.length) + var pendingSpace = false + for (ch in value.trim()) { + if (ch.isWhitespace()) { + if (out.isNotEmpty()) pendingSpace = true + continue + } + if (pendingSpace) { + out.append(' ') + pendingSpace = false + } + out.append(ch.lowercaseChar()) + } + return out.toString() +} + +/** + * Exact and prefix name or app-ID hits stay in the list without an embedding. + * Other listings are included only when [queryVector] clears [MIN_VECTOR_DOT]. + * Keeps ids only, and at most [limit] of them, so the caller can load full rows for the survivors. + */ +internal class SearchRanker(private val query: String, queryVector: ByteArray?, private val limit: Int?) { + private val vector = queryVector?.takeIf { it.size == EMBEDDING_DIMS } + private val kept = ArrayList() + private val worstFirst = limit?.takeIf { it > 0 }?.let { PriorityQueue(it, dropFirst) } + + fun consider(id: ByteArray, appId: String, name: String, embedding: ByteArray?) { + if (limit == 0) return + val score = score(name, appId, embedding) ?: return + val heap = worstFirst + if (heap == null) { + kept += Ranked(id, appId, score) + return + } + if (heap.size < limit!!) { + heap += Ranked(id, appId, score) + return + } + val worst = checkNotNull(heap.peek()) + if (score > worst.score || (score == worst.score && appId < worst.appId)) { + heap.poll() + heap += Ranked(id, appId, score) + } + } + + fun ids(): List { + val ranked = worstFirst?.toMutableList() ?: kept + ranked.sortWith(bestFirst) + return ranked.map { it.id } + } + + private fun score(name: String, appId: String, embedding: ByteArray?): Int? { + val boost = textBoost(query, name, appId) + val dot = vector?.let { left -> + embedding?.takeIf { it.size == EMBEDDING_DIMS }?.let { embeddingDot(left, it) } + } + if (boost == 0 && (dot == null || dot < MIN_VECTOR_DOT)) return null + return boost + (dot ?: 0) + } +} + +private class Ranked(val id: ByteArray, val appId: String, val score: Int) + +private val bestFirst = compareByDescending { it.score }.thenBy { it.appId } + +/** Lowest score first, and the larger app id first when scores tie, so the queue head is the first to drop. */ +private val dropFirst = compareBy { it.score }.thenByDescending { it.appId } + +private fun textBoost(query: String, name: String, appId: String): Int { + if (query.isEmpty()) return 0 + val foldedName = foldSearchText(name) + val foldedId = appId.lowercase() + return when { + query == foldedName || query == foldedId -> EXACT_BOOST + foldedName.startsWith(query) || foldedId.startsWith(query) -> PREFIX_BOOST + else -> 0 + } +} + +private fun embeddingDot(left: ByteArray, right: ByteArray): Int { + var sum = 0 + for (i in 0 until EMBEDDING_DIMS) { + sum += left[i].toInt() * right[i].toInt() + } + return sum +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/Signer.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/Signer.kt new file mode 100644 index 0000000..421cb1a --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/Signer.kt @@ -0,0 +1,57 @@ +package dev.zapstore.iolite + +import org.json.JSONObject + +interface Signer { + val publicKey: String + suspend fun sign(createdAt: Long, kind: Int, tags: List>, content: String): Event +} + +class LocalSigner(secretKey: ByteArray) : Signer { + private val secret = secretKey.copyOf() + private val selfConversationKey: ByteArray = + Nip44.conversationKey(secret, Crypto.xOnlyPubkey(secret)) + + override val publicKey: String = Hex.encode(Crypto.xOnlyPubkey(secret)) + + override suspend fun sign( + createdAt: Long, + kind: Int, + tags: List>, + content: String, + ): Event = Event.sign(secret, createdAt, kind, tags, content) + + fun encryptToSelf(plaintext: String): String = Nip44.encrypt(plaintext, selfConversationKey) + + fun decryptFromSelf(payload: String): String = Nip44.decrypt(payload, selfConversationKey) +} + +/** + * NIP-55 signer. Android supplies [requestSignedEvent] by talking to a signer + * app such as Amber. Iolite never holds the user's nsec for user content. + */ +class Nip55Signer( + override val publicKey: String, + private val requestSignedEvent: suspend (unsignedJson: String) -> String, +) : Signer { + override suspend fun sign( + createdAt: Long, + kind: Int, + tags: List>, + content: String, + ): Event { + val unsigned = JSONObject() + .put("created_at", createdAt) + .put("kind", kind) + .put("tags", tags.toJsonArray()) + .put("content", content) + .put("pubkey", publicKey) + val signed = Event.parse(requestSignedEvent(unsigned.toString())) + require(signed.verify()) { "NIP-55 returned an invalid signature" } + require(signed.pubkey == publicKey) { "NIP-55 pubkey does not match the requested signer" } + require(signed.kind == kind && signed.createdAt == createdAt && signed.content == content) { + "NIP-55 returned a different event template" + } + return signed + } +} diff --git a/iolite/src/main/kotlin/dev/zapstore/iolite/TarZstd.kt b/iolite/src/main/kotlin/dev/zapstore/iolite/TarZstd.kt new file mode 100644 index 0000000..de279f4 --- /dev/null +++ b/iolite/src/main/kotlin/dev/zapstore/iolite/TarZstd.kt @@ -0,0 +1,162 @@ +package dev.zapstore.iolite + +import com.github.luben.zstd.ZstdInputStream +import java.io.ByteArrayOutputStream +import java.io.EOFException +import java.io.InputStream +import java.util.Locale + +internal data class TarMember(val name: String, val data: ByteArray) + +internal object TarZstd { + fun read( + compressed: InputStream, + maxCompressedBytes: Long, + maxUncompressedBytes: Long, + maxMembers: Int, + maxMemberBytes: Long, + ): List { + val limited = compressed.bounded(maxCompressedBytes, "compressed catalog bundle exceeds limit") + val zstd = ZstdInputStream(limited) + val tar = CountingInputStream(zstd) + val members = ArrayList() + val names = HashSet() + while (true) { + if (tar.bytesRead >= maxUncompressedBytes) { + error("uncompressed catalog bundle exceeds limit") + } + val header = ByteArray(512) + if (!tar.readFully(header)) break + if (header.all { it == 0.toByte() }) { + val next = ByteArray(512) + tar.readFully(next) + break + } + val name = header.cString(0, 100) + val size = header.octal(124, 12) + val type = header[156].toInt().toChar() + val magic = header.cString(257, 6) + if (magic.startsWith("ustar")) { + // ustar + } + val prefix = header.cString(345, 155) + val fullName = if (prefix.isEmpty()) name else "$prefix/$name" + rejectName(fullName) + when (type) { + '0', '\u0000' -> Unit + 'x', 'g', 'L', 'K' -> error("PAX or GNU tar extensions are not allowed") + '1', '2', '3', '4', '5', '6' -> error("directories, links, and devices are not allowed") + else -> error("unsupported tar type $type") + } + if (size > maxMemberBytes) error("catalog member $fullName exceeds size limit") + if (members.size >= maxMembers) error("catalog bundle has too many members") + if (!names.add(fullName)) error("duplicate tar path $fullName") + val data = tar.readExact(size.toInt()) + val padding = ((512 - (size % 512)) % 512).toInt() + if (padding > 0) tar.skipFully(padding.toLong()) + if (tar.bytesRead > maxUncompressedBytes) error("uncompressed catalog bundle exceeds limit") + members += TarMember(fullName, data) + } + if (members.isEmpty() || members.first().name != "manifest.json") { + error("manifest.json must be the first tar member") + } + return members + } + + private fun rejectName(name: String) { + if (name.isEmpty() || name.startsWith("/") || name.contains('\u0000') || name.contains("..") || name.contains('\\')) { + error("illegal tar path $name") + } + } +} + +private fun ByteArray.cString(offset: Int, length: Int): String { + var end = offset + val limit = offset + length + while (end < limit && this[end] != 0.toByte()) end++ + return copyOfRange(offset, end).toString(Charsets.UTF_8) +} + +private fun ByteArray.octal(offset: Int, length: Int): Long { + val raw = cString(offset, length).trim() + if (raw.isEmpty()) return 0 + return raw.trimEnd { it == ' ' || it == '\u0000' }.toLong(8) +} + +private class CountingInputStream(private val input: InputStream) : InputStream() { + var bytesRead = 0L + private set + + override fun read(): Int { + val value = input.read() + if (value >= 0) bytesRead++ + return value + } + + override fun read(b: ByteArray, off: Int, len: Int): Int { + val n = input.read(b, off, len) + if (n > 0) bytesRead += n + return n + } + + fun readFully(buffer: ByteArray): Boolean { + var offset = 0 + while (offset < buffer.size) { + val n = read(buffer, offset, buffer.size - offset) + if (n < 0) { + if (offset == 0) return false + throw EOFException("truncated tar") + } + offset += n + } + return true + } + + fun readExact(size: Int): ByteArray { + val out = ByteArrayOutputStream(size) + var remaining = size + val buf = ByteArray(minOf(size, 16_384)) + while (remaining > 0) { + val n = read(buf, 0, minOf(remaining, buf.size)) + if (n < 0) throw EOFException("truncated tar member") + out.write(buf, 0, n) + remaining -= n + } + return out.toByteArray() + } + + fun skipFully(n: Long) { + var remaining = n + val buf = ByteArray(512) + while (remaining > 0) { + val read = read(buf, 0, minOf(remaining, buf.size.toLong()).toInt()) + if (read < 0) throw EOFException("truncated tar padding") + remaining -= read + } + } +} + +private fun InputStream.bounded(limit: Long, message: String): InputStream = object : InputStream() { + var readCount = 0L + override fun read(): Int { + val value = this@bounded.read() + if (value >= 0) { + readCount++ + if (readCount > limit) error(message) + } + return value + } + + override fun read(b: ByteArray, off: Int, len: Int): Int { + val n = this@bounded.read(b, off, len) + if (n > 0) { + readCount += n + if (readCount > limit) error(message) + } + return n + } +} + +internal fun ByteArray.sha256Hex(): String = Hex.encode(Crypto.sha256(this)) + +internal fun String.asciiLower(): String = lowercase(Locale.ROOT) diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/CatalogArtifactsTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/CatalogArtifactsTest.kt new file mode 100644 index 0000000..7eae947 --- /dev/null +++ b/iolite/src/test/kotlin/dev/zapstore/iolite/CatalogArtifactsTest.kt @@ -0,0 +1,180 @@ +package dev.zapstore.iolite + +import java.io.File +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class CatalogArtifactsTest { + @Test + fun parsesFactCsvAndKeepsReason() { + val rows = parseFactRows( + "fact,value,reason\n" + + "gms,no,\"no play services, com.google\"\n" + + "gms,yes,\n" + + "fcm,yes,\"uses, cloud\"\n", + ) + assertEquals(2, rows.size) + assertEquals("gms", rows[0].key) + assertTrue(rows[0].yes) + assertEquals("no play services, com.google", rows[0].reason) + assertEquals("fcm", rows[1].key) + assertEquals("uses, cloud", rows[1].reason) + } + + @Test + fun parsesAboutSecurityFactsWarningsVectorIconAndHexAvatar() { + val pubkey = "ab".repeat(32) + val parsed = CatalogArtifacts.fromMembers( + mapOf( + "com.example.app/about" to TarMember("com.example.app/about", "Maps.".toByteArray()), + "com.example.app/security" to TarMember("com.example.app/security", "It asks for location.\n⚠️ It runs a shell.".toByteArray()), + "com.example.app/facts" to TarMember("com.example.app/facts", "gms: no".toByteArray()), + "com.example.app/vector" to TarMember("com.example.app/vector", ByteArray(EMBEDDING_DIMS) { 7 }), + "com.example.app/icon.webp" to TarMember("com.example.app/icon.webp", "icon".toByteArray()), + "$pubkey.webp" to TarMember("$pubkey.webp", "avatar".toByteArray()), + ), + maxIconBytes = 256L * 1024, + ) + val app = parsed.apps.single() + assertEquals("com.example.app", app.appId) + assertEquals("Maps.", app.about) + assertEquals("It asks for location.\n⚠️ It runs a shell.", app.security) + assertEquals("gms: no", app.facts) + assertEquals(EMBEDDING_DIMS, app.embedding!!.size) + assertEquals(pubkey, parsed.avatars.single().pubkey) + assertArrayEquals("avatar".toByteArray(), parsed.avatars.single().webp) + } + + @Test + fun rejectsPrefixedPathsAndNpubAvatars() { + assertFalse(CatalogArtifacts.allowedMember("artifacts/com.example.app/about")) + assertFalse(CatalogArtifacts.allowedMember("com.example.app/analysis")) + assertFalse(CatalogArtifacts.allowedMember("com.example.app/warnings")) + assertFalse(CatalogArtifacts.allowedMember("artifacts.bin")) + assertFalse(CatalogArtifacts.allowedMember("npub1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqkxqqu.webp")) + assertTrue(CatalogArtifacts.allowedMember("diff.jsonl")) + assertTrue(CatalogArtifacts.allowedMember("index")) + assertTrue(CatalogArtifacts.allowedMember("${"cd".repeat(32)}.webp")) + val failure = runCatching { + CatalogArtifacts.fromMembers( + mapOf("com.example.app/vector" to TarMember("com.example.app/vector", ByteArray(32 + EMBEDDING_DIMS))), + maxIconBytes = 256L * 1024, + ) + }.exceptionOrNull() + assertTrue(failure is CatalogImportException) + assertEquals("vector com.example.app", failure!!.message) + } + + @Test + fun keepsVectorAndAboutWhenALaterRecordOmitsThem() { + val dir = File("build/tmp/catalog-artifacts-${System.nanoTime()}").apply { mkdirs() } + val store = IoliteStore(File(dir, "iolite.db").absolutePath) + store.insertFixtureCatalog() + val catalog = store.catalog(1)!! + val device = DeviceProfile(abis = listOf("arm64-v8a"), sdk = 34) + val pubkey = "cd".repeat(32) + try { + store.write { tx -> + tx.db.prepare( + """ + INSERT INTO apps ( + id, catalog_id, app_id, certificate_hash, app_event_created_at, pubkey, event_pubkey, + name, summary, repository, version, version_code, channel, metadata + ) VALUES (?, 1, 'com.example.app', ?, 1, NULL, ?, 'Example', '', NULL, '1.0.0', 1, 'main', '{}') + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, ByteArray(32) { 1 }) + statement.bindBlob(2, ByteArray(32) { 2 }) + statement.bindBlob(3, ByteArray(32) { 3 }) + statement.step() + } + } + store.importCatalog( + catalog = catalog, + events = emptyList(), + deletes = emptyList(), + replaceAll = false, + toEpoch = 1, + device = device, + now = 0, + artifacts = CatalogArtifact( + apps = listOf( + AppArtifact( + appId = "com.example.app", + webp = "one".toByteArray(), + embedding = ByteArray(EMBEDDING_DIMS) { 7 }, + facts = "maps", + about = "first note", + security = "Asks for location.\n⚠️ first", + ), + ), + avatars = listOf(AvatarArtifact(pubkey, "face".toByteArray())), + ), + ) + val imported = store.apps().single() + assertEquals("first note", imported.about) + assertEquals("Asks for location.\n⚠️ first", imported.security) + assertEquals("⚠️ first", imported.securityWarnings) + assertEquals("one", imported.iconFile!!.readText()) + assertEquals("face", store.avatarFile(pubkey).readText()) + val embedding = searchEmbedding(store) + + store.importCatalog( + catalog = store.catalog(1)!!, + events = emptyList(), + deletes = emptyList(), + replaceAll = false, + toEpoch = 2, + device = device, + now = 0, + artifacts = CatalogArtifact( + apps = listOf( + AppArtifact( + appId = "com.example.app", + webp = "two".toByteArray(), + embedding = null, + facts = null, + about = null, + security = null, + ), + ), + ), + ) + val kept = store.apps().single() + assertEquals("first note", kept.about) + assertEquals("Asks for location.\n⚠️ first", kept.security) + assertEquals("two", kept.iconFile!!.readText()) + assertArrayEquals(embedding, searchEmbedding(store)) + assertEquals("maps", searchFeatures(store)) + } finally { + store.close() + } + } + + @Test + fun npubRoundTrip() { + val pubkey = "cd".repeat(32) + assertEquals(pubkey, pubkey.toNpub().decodeNpub()) + } + + private fun searchEmbedding(store: IoliteStore): ByteArray = store.read { db -> + db.prepare( + "SELECT embedding FROM apps_search WHERE id = (SELECT id FROM apps WHERE app_id = 'com.example.app')", + ).use { statement -> + assertTrue(statement.step()) + statement.getBlob(0) + } + } + + private fun searchFeatures(store: IoliteStore): String = store.read { db -> + db.prepare( + "SELECT facts FROM apps WHERE app_id = 'com.example.app'", + ).use { statement -> + assertTrue(statement.step()) + statement.getText(0) + } + } +} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/CatalogImporterTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/CatalogImporterTest.kt new file mode 100644 index 0000000..4f8ad37 --- /dev/null +++ b/iolite/src/test/kotlin/dev/zapstore/iolite/CatalogImporterTest.kt @@ -0,0 +1,492 @@ +package dev.zapstore.iolite + +import java.io.ByteArrayInputStream +import java.io.File +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.job +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class CatalogImporterTest { + @Test + fun importsRelayGoldenFromZero() { + withStore { store, importer -> + val bundle = golden("bundle-0-1.tar.zst") + val result = importer.importBundle(1, bundle, localRelay) + assertEquals(0, result.from) + assertEquals(1, result.to) + val app = store.apps().single() + assertEquals("com.example.app", app.appId) + assertEquals("1.0.0", app.version) + assertEquals(100, app.versionCode) + assertEquals(1, store.catalog(1)!!.epoch) + assertEquals(listOf("ws://127.0.0.1:3334"), store.catalog(1)!!.relays.map { it.url }) + assertEquals(manifestPubkey(bundle), store.catalog(1)!!.manifestPubkey) + } + } + + @Test + fun importsCoalescedFromZeroToTwo() { + withStore { store, importer -> + val result = importer.importBundle(1, golden("bundle-0-2.tar.zst"), localRelay) + assertEquals(0, result.from) + assertEquals(2, result.to) + val app = store.apps().single() + assertEquals("com.example.app", app.appId) + assertEquals("1.1.0", app.version) + assertEquals(110, app.versionCode) + assertEquals(2, store.catalog(1)!!.epoch) + } + } + + @Test + fun appliesAdjacentDeltaFromOneToTwo() { + withStore { store, importer -> + importer.importBundle(1, golden("bundle-0-1.tar.zst"), localRelay) + val result = importer.importBundle(1, golden("bundle-1-2.tar.zst")) + assertEquals(1, result.from) + assertEquals(2, result.to) + val app = store.apps().single() + assertEquals("1.1.0", app.version) + assertEquals(110, app.versionCode) + assertEquals(2, store.catalog(1)!!.epoch) + } + } + + @Test + fun wipeClearsAppsAndEpoch() { + withStore { store, importer -> + importer.importBundle(1, golden("bundle-0-1.tar.zst"), localRelay) + assertEquals(1, store.catalog(1)!!.epoch) + assertEquals(1, store.apps().size) + store.wipe() + assertEquals(0, store.catalog(1)!!.epoch) + assertEquals(0, store.apps().size) + assertNull(store.catalog(1)!!.manifestPubkey) + assertEquals(listOf("ws://127.0.0.1:3334"), store.catalog(1)!!.relays.map { it.url }) + } + } + + @Test + fun fromZeroReplacesLeftoverListings() { + withStore { store, importer -> + store.insertFixtureCatalog() + store.write { tx -> + tx.db.prepare( + """ + INSERT INTO apps ( + id, catalog_id, app_id, certificate_hash, app_event_created_at, pubkey, event_pubkey, + name, summary, repository, version, version_code, channel, metadata + ) VALUES (?, 1, 'com.example.leftover', ?, 1, NULL, ?, 'Leftover', '', NULL, '9.0.0', 9, 'main', '{}') + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, ByteArray(32) { 1 }) + statement.bindBlob(2, ByteArray(32) { 2 }) + statement.bindBlob(3, ByteArray(32) { 3 }) + statement.step() + } + } + importer.importBundle(1, golden("bundle-0-1.tar.zst")) + assertEquals(listOf("com.example.app"), store.apps().map { it.appId }) + } + } + + @Test + fun observeAppsEmitsAfterImport() = runBlocking { + val dir = File("build/tmp/iolite-observe-${System.nanoTime()}").apply { mkdirs() } + val scope = CoroutineScope(SupervisorJob()) + val iolite = Iolite.createForTesting( + databasePath = File(dir, "iolite.db").absolutePath, + parentScope = scope, + ) + iolite.importCatalogUpdate(1, golden("bundle-0-1.tar.zst"), localRelay) + val apps = iolite.observeApps().first { it.isNotEmpty() } + assertEquals("com.example.app", apps.single().appId) + iolite.close() + scope.coroutineContext.job.cancel() + } + + @Test + fun observeAppAndSearchReadImportedListing() = runBlocking { + val dir = File("build/tmp/iolite-query-${System.nanoTime()}").apply { mkdirs() } + val scope = CoroutineScope(SupervisorJob()) + val iolite = Iolite.createForTesting( + databasePath = File(dir, "iolite.db").absolutePath, + parentScope = scope, + ) + iolite.importCatalogUpdate(1, golden("bundle-0-1.tar.zst"), localRelay) + val app = iolite.observeApp("com.example.app").first { it != null }!! + assertEquals("com.example.app", app.appId) + assertEquals(listOf("com.example.app"), iolite.apps(AppFilter(search = app.name)).map { it.appId }) + assertEquals(emptyList(), iolite.apps(AppFilter(search = "nomatch"))) + iolite.close() + scope.coroutineContext.job.cancel() + } + + @Test + fun bundledImportThenSyncUsesStoredEndpoint() = runBlocking { + val dir = File("build/tmp/iolite-sync-${System.nanoTime()}").apply { mkdirs() } + val scope = CoroutineScope(SupervisorJob()) + val requested = mutableListOf() + val http = HttpTransport { url, _ -> + requested += url + when (url) { + "http://127.0.0.1:3334/deltas?from=1" -> HttpResponse(304, emptyMap(), null) + else -> error("unexpected catalog URL $url") + } + } + val iolite = Iolite.createForTesting( + databasePath = File(dir, "iolite.db").absolutePath, + parentScope = scope, + http = http, + ) + val bundle = golden("bundle-0-1.tar.zst") + val first = iolite.syncCatalog(catalogId = 1, bundled = bundle, endpoint = localRelay) + val imported = first.imported!! + assertEquals(0, imported.from) + assertEquals(1, imported.to) + assertEquals(bundle.size.toLong(), first.bytes) + assertEquals(manifestPubkey(bundle), iolite.defaultCatalog().manifestPubkey) + assertEquals(listOf("ws://127.0.0.1:3334"), iolite.defaultCatalog().relays.map { it.url }) + val unchanged = iolite.syncCatalog() + assertNull(unchanged.imported) + assertTrue(unchanged.notModified) + assertEquals( + listOf( + "http://127.0.0.1:3334/deltas?from=1", + "http://127.0.0.1:3334/deltas?from=1", + ), + requested, + ) + iolite.close() + scope.coroutineContext.job.cancel() + } + + @Test + fun emptyCatalogFetchesFirstBundleFromEndpoint() = runBlocking { + val dir = File("build/tmp/iolite-bootstrap-${System.nanoTime()}").apply { mkdirs() } + val scope = CoroutineScope(SupervisorJob()) + val bundle = golden("bundle-0-1.tar.zst") + val requested = mutableListOf() + val http = HttpTransport { url, _ -> + requested += url + when (url) { + "http://127.0.0.1:3334/deltas?from=0" -> HttpResponse(200, emptyMap(), bundle) + else -> error("unexpected catalog URL $url") + } + } + val iolite = Iolite.createForTesting( + databasePath = File(dir, "iolite.db").absolutePath, + parentScope = scope, + http = http, + ) + val first = iolite.syncCatalog(catalogId = 1, bundled = ByteArray(0), endpoint = localRelay) + val imported = first.imported!! + assertEquals(0, imported.from) + assertEquals(1, imported.to) + assertEquals(bundle.size.toLong(), first.bytes) + assertEquals(manifestPubkey(bundle), iolite.defaultCatalog().manifestPubkey) + assertEquals(listOf("ws://127.0.0.1:3334"), iolite.defaultCatalog().relays.map { it.url }) + assertEquals(listOf("http://127.0.0.1:3334/deltas?from=0"), requested) + iolite.close() + scope.coroutineContext.job.cancel() + } + + @Test + fun syncRelayUsesOnionOnlyWhenAsked() { + val clearnet = "wss://relay.zapstore.dev".normalizeRelayUrl() + val onion = "ws://abcdefghijklmnopqrstuvwxyz234567.onion".normalizeRelayUrl() + val catalog = CatalogRecord( + id = 1, + relayUrl = clearnet, + relays = listOf(clearnet, onion), + position = 0, + isPrivate = false, + manifestPubkey = null, + epoch = 1, + ) + assertEquals(onion.url, catalog.syncRelay(useOnion = true).url) + assertEquals(clearnet.url, catalog.syncRelay(useOnion = false).url) + } + + @Test + fun pinnedKeyRejectsADifferentSigner() { + withStore { store, importer -> + store.insertFixtureCatalog() + store.pinManifestPubkeyIfAbsent(1, "aa".repeat(32)) + val failure = runCatching { importer.importBundle(1, golden("bundle-0-1.tar.zst")) }.exceptionOrNull() + assertTrue(failure is CatalogImportException) + assertEquals("manifest pubkey does not match the pinned key", failure!!.message) + assertEquals("aa".repeat(32), store.catalog(1)!!.manifestPubkey) + assertEquals(0, store.catalog(1)!!.epoch) + } + } + + @Test + fun importsCatalogStackForCuratedStacks() { + withStore { store, _ -> + store.insertFixtureCatalog() + val secret = Hex.decode("00".repeat(31) + "01") + val otherSecret = Hex.decode("00".repeat(31) + "03") + val device = DeviceProfile(abis = listOf("arm64-v8a"), sdk = 34) + val catalogStack = stackEvent(secret, "editors") + val otherStack = stackEvent(otherSecret, "mine") + store.pinManifestPubkeyIfAbsent(1, catalogStack.pubkey) + store.ingest(listOf(otherStack), IngestContext(device, null)) + val catalog = store.catalog(1)!! + val rejected = runCatching { + store.importCatalog( + catalog = catalog, + events = listOf(otherStack), + deletes = emptyList(), + replaceAll = true, + toEpoch = 1, + device = device, + now = 1_700_000_000, + stackPubkey = catalogStack.pubkey, + ) + }.exceptionOrNull() + assertTrue(rejected is CatalogImportException) + store.importCatalog( + catalog = catalog, + events = listOf(catalogStack, stackEvent(secret, "desktop", platform = "android-x86_64")), + deletes = emptyList(), + replaceAll = true, + toEpoch = 1, + device = device, + now = 1_700_000_000, + stackPubkey = catalogStack.pubkey, + ) + val curated = store.stacks(catalogStack.pubkey) + assertEquals(listOf("editors"), curated.map { it.identifier }) + assertEquals(listOf("com.example.app"), curated.single().appIds) + assertEquals(listOf("mine"), store.stacks(otherStack.pubkey).map { it.identifier }) + store.importCatalog( + catalog = store.catalog(1)!!, + events = emptyList(), + deletes = listOf(CatalogDelete.Coordinate(Kinds.AppStack, catalogStack.pubkey, "editors")), + replaceAll = false, + toEpoch = 2, + device = device, + now = 1_700_000_000, + stackPubkey = catalogStack.pubkey, + ) + assertEquals(emptyList(), store.stacks(catalogStack.pubkey)) + assertEquals(listOf("mine"), store.stacks(otherStack.pubkey).map { it.identifier }) + } + } + + @Test + fun partialDeltaPatchesStoredListing() { + withStore { store, _ -> + store.insertFixtureCatalog() + val secret = Hex.decode("11".repeat(32)) + val ditto = fixtureListing(secret, "pub.ditto.app", "Ditto", "1.0.0", 10) + import(store, ditto.events, 1) + val renamed = appEvent(secret, "pub.ditto.app", "Ditto Nightly", 1_700_000_100, "renamed description") + import(store, listOf(renamed), 2) + val app = store.apps().single() + assertEquals("Ditto Nightly", app.name) + assertEquals("renamed description", app.description) + assertEquals("1.0.0", app.version) + assertEquals(10, app.versionCode) + assertEquals(2, store.catalog(1)!!.epoch) + } + } + + @Test + fun assetOnlyDeltaUpdatesVersionCode() { + withStore { store, _ -> + store.insertFixtureCatalog() + val secret = Hex.decode("11".repeat(32)) + val ditto = fixtureListing(secret, "pub.ditto.app", "Ditto", "1.0.0", 10) + import(store, ditto.events, 1) + val asset = assetEvent(secret, "pub.ditto.app", "9.9.9", 42, 1_700_000_100, hash = "ef".repeat(32)) + import(store, listOf(asset), 2) + val app = store.apps().single() + assertEquals("Ditto", app.name) + assertEquals("9.9.9", app.version) + assertEquals(42, app.versionCode) + assertEquals(asset.tagValue("x"), app.apkHash) + } + } + + @Test + fun malformedListingDoesNotAbortSync() { + withStore { store, _ -> + store.insertFixtureCatalog() + val secret = Hex.decode("11".repeat(32)) + val ditto = fixtureListing(secret, "pub.ditto.app", "Ditto", "1.0.0", 10) + import(store, ditto.events, 1) + val broken = Event.sign( + secret, + 1_700_000_050, + Kinds.Release, + listOf( + listOf("a", "32267:${"ab".repeat(32)}:other.app"), + listOf("i", "pub.ditto.app"), + listOf("version", "3.0.0"), + listOf("d", "pub.ditto.app@3.0.0"), + listOf("c", "main"), + listOf("e", ditto.asset.id), + ), + "notes", + ) + val renamed = appEvent(secret, "pub.ditto.app", "Ditto Nightly", 1_700_000_050, "renamed description") + val other = fixtureListing(secret, "com.example.keep", "Keep", "2.0.0", 20, 1_700_000_200) + val stray = assetEvent(secret, "com.example.unknown", "0.0.1", 1, 1_700_000_300) + import(store, listOf(broken, renamed, stray) + other.events, 2) + val apps = store.apps().associateBy { it.appId } + assertEquals(setOf("pub.ditto.app", "com.example.keep", "com.example.unknown"), apps.keys) + assertEquals("Ditto Nightly", apps.getValue("pub.ditto.app").name) + assertEquals("notes", apps.getValue("pub.ditto.app").releaseNotes) + assertEquals("1.0.0", apps.getValue("pub.ditto.app").version) + assertEquals(10, apps.getValue("pub.ditto.app").versionCode) + assertEquals("Keep", apps.getValue("com.example.keep").name) + assertEquals("0.0.1", apps.getValue("com.example.unknown").version) + assertEquals(1, apps.getValue("com.example.unknown").versionCode) + assertEquals(2, store.catalog(1)!!.epoch) + } + } + + private fun import(store: IoliteStore, events: List, epoch: Long) { + store.importCatalog( + catalog = store.catalog(1)!!, + events = events, + deletes = emptyList(), + replaceAll = false, + toEpoch = epoch, + device = DeviceProfile(abis = listOf("arm64-v8a"), sdk = 34), + now = 1_700_000_000, + ) + } + + private data class FixtureListing(val app: Event, val release: Event, val asset: Event) { + val events: List get() = listOf(app, release, asset) + } + + private fun fixtureListing( + secret: ByteArray, + appId: String, + name: String, + version: String, + versionCode: Long, + createdAt: Long = 1_700_000_000, + ): FixtureListing { + val asset = assetEvent(secret, appId, version, versionCode, createdAt) + val pubkey = asset.pubkey + val release = Event.sign( + secret, + createdAt, + Kinds.Release, + listOf( + listOf("a", "32267:$pubkey:$appId"), + listOf("i", appId), + listOf("version", version), + listOf("d", "$appId@$version"), + listOf("c", "main"), + listOf("e", asset.id), + ), + "notes", + ) + return FixtureListing(appEvent(secret, appId, name, createdAt), release, asset) + } + + private fun appEvent( + secret: ByteArray, + appId: String, + name: String, + createdAt: Long, + content: String = "nightly build", + ): Event = Event.sign( + secret, + createdAt, + Kinds.App, + listOf(listOf("d", appId), listOf("name", name), listOf("summary", "short")), + content, + ) + + private fun assetEvent( + secret: ByteArray, + appId: String, + version: String, + versionCode: Long, + createdAt: Long, + hash: String = "ab".repeat(32), + ): Event = Event.sign( + secret, + createdAt, + Kinds.Asset, + listOf( + listOf("i", appId), + listOf("x", hash), + listOf("version", version), + listOf("m", "application/vnd.android.package-archive"), + listOf("f", "android-arm64-v8a"), + listOf("version_code", versionCode.toString()), + listOf("apk_certificate_hash", "cd".repeat(32)), + listOf("url", "https://example.invalid/$appId.apk"), + listOf("filename", "$appId.apk"), + ), + "", + ) + + private fun stackEvent(secret: ByteArray, identifier: String, platform: String = "android-arm64-v8a"): Event { + val pubkey = Hex.encode(Crypto.xOnlyPubkey(secret)) + return Event.sign( + secret, + 1_700_000_000, + Kinds.AppStack, + listOf( + listOf("d", identifier), + listOf("name", identifier), + listOf("a", "32267:$pubkey:com.example.app"), + listOf("f", platform), + ), + "", + ) + } + + private fun withStore(block: (IoliteStore, CatalogImporter) -> Unit) { + val dir = File("build/tmp/iolite-import-${System.nanoTime()}").apply { mkdirs() } + val store = IoliteStore(File(dir, "iolite.db").absolutePath) + val importer = CatalogImporter( + store = store, + iconsDir = File(dir, "icons"), + config = IoliteConfig(), + device = DeviceProfile(abis = listOf("arm64-v8a"), sdk = 34), + ) + try { + block(store, importer) + } finally { + store.close() + } + } + + private fun manifestPubkey(bundle: ByteArray): String { + val config = IoliteConfig() + val members = TarZstd.read( + ByteArrayInputStream(bundle), + maxCompressedBytes = config.maxCompressedBytes, + maxUncompressedBytes = config.maxUncompressedBytes, + maxMembers = config.maxMembers, + maxMemberBytes = config.maxUncompressedBytes, + ) + return Event.parse(members.first().data.toString(Charsets.UTF_8)).pubkey + } + + private fun golden(name: String): ByteArray { + val file = File("src/test/resources/golden", name) + check(file.isFile) { "missing golden $name" } + return file.readBytes() + } + + companion object { + private val localRelay = "ws://127.0.0.1:3334".normalizeRelayUrl() + } +} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/EventCryptoTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/EventCryptoTest.kt new file mode 100644 index 0000000..c32d985 --- /dev/null +++ b/iolite/src/test/kotlin/dev/zapstore/iolite/EventCryptoTest.kt @@ -0,0 +1,58 @@ +package dev.zapstore.iolite + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class EventCryptoTest { + @Test + fun signAndVerifyRoundTrip() { + val secret = Hex.decode("0000000000000000000000000000000000000000000000000000000000000001") + val event = Event.sign( + secretKey = secret, + createdAt = 1_700_000_000, + kind = 1, + tags = listOf(listOf("t", "zapstore")), + content = "hello", + ) + assertEquals("79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", event.pubkey) + assertTrue(event.verify()) + assertEquals(event.id, Event.computeId(event.pubkey, event.createdAt, event.kind, event.tags, event.content)) + } + + @Test + fun rejectsTamperedContent() { + val secret = Hex.decode("0000000000000000000000000000000000000000000000000000000000000001") + val event = Event.sign(secret, 1, 1, emptyList(), "hello") + val tampered = event.copy(content = "bye") + assertTrue(!tampered.verify()) + } + + @Test + fun replacementOrderingPrefersNewerThenLowestId() { + val older = Event("b".repeat(64), "a".repeat(64), 10, 1, emptyList(), "", "c".repeat(128)) + val newer = older.copy(createdAt = 11, id = "c".repeat(64)) + val sameTimeLowerId = older.copy(id = "a".repeat(64)) + assertTrue(supersedes(newer.createdAt, newer.id, older.createdAt, older.id)) + assertTrue(supersedes(sameTimeLowerId.createdAt, sameTimeLowerId.id, older.createdAt, older.id)) + assertTrue(!supersedes(older.createdAt, older.id, newer.createdAt, newer.id)) + } + + @Test + fun filterSerializesTagKeysWithHash() { + val json = Filter(kinds = listOf(Kinds.App), tags = mapOf("d" to listOf("com.example.app"))).toJsonObject() + assertEquals("com.example.app", json.getJSONArray("#d").getString(0)) + assertEquals(Kinds.App, json.getJSONArray("kinds").getInt(0)) + } + + @Test + fun verifiesRelayAssetWhoseUrlContainsSlash() { + val event = Event.parse( + """{"id":"fb5b64912c37e33f0da48745032f0c7be9e571bfb82da10a09b31c085fccff85","pubkey":"c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5","created_at":1789657776,"kind":3063,"tags":[["i","com.example.app"],["x","a000000000000000000000000000000000000000000000000000000000000000"],["version","1.0.0"],["m","application/vnd.android.package-archive"],["version_code","100"],["apk_certificate_hash","b000000000000000000000000000000000000000000000000000000000000000"],["url","https://cdn.example.com/com.example.app.apk"]],"content":"","sig":"de1a0c2353acc9ceb8fc9122ff14d205a62b8bef0f93742b4286e988e0a3de6612d285424ae8bec5b56ca7b1b6f26164346685703930d8bcec34c2cadd37956d"}""", + ) + assertTrue(event.verify()) + val canonical = String(Event.canonicalBytes(event.pubkey, event.createdAt, event.kind, event.tags, event.content), Charsets.UTF_8) + assertTrue(canonical.contains("https://cdn.example.com/com.example.app.apk")) + assertTrue(!canonical.contains("\\/")) + } +} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/IoliteContractsTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/IoliteContractsTest.kt new file mode 100644 index 0000000..afe2bd9 --- /dev/null +++ b/iolite/src/test/kotlin/dev/zapstore/iolite/IoliteContractsTest.kt @@ -0,0 +1,441 @@ +package dev.zapstore.iolite + +import androidx.sqlite.execSQL +import java.io.File +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.job +import kotlinx.coroutines.runBlocking +import org.json.JSONArray +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class IoliteContractsTest { + @Test + fun catalogRelayListDoesNotPinFromTheNetwork() = runBlocking { + val signer = LocalSigner(SECRET) + val extra = "wss://relay.example.com" + val privateRelay = "wss://private.example" + val http = HttpTransport { _, _ -> error("catalog relay list must not fetch NIP-11") } + withIolite(http = http, signer = signer) { iolite -> + val content = signer.encryptToSelf(JSONArray().put(privateRelay).toString()) + val event = signer.sign( + 1_700_000_100, + Kinds.CatalogRelayList, + listOf(listOf("r", "wss://relay.zapstore.dev"), listOf("r", extra)), + content, + ) + iolite.ingest(listOf(event)) + val selected = iolite.catalogs() + assertEquals( + listOf("wss://relay.zapstore.dev", extra, privateRelay), + selected.map { it.relayUrl.url }, + ) + assertEquals(listOf(false, false, true), selected.map { it.isPrivate }) + assertTrue(selected.all { it.manifestPubkey == null }) + delay(80) + assertTrue(iolite.allCatalogs().all { it.manifestPubkey == null }) + } + } + + @Test + fun rehydrateCatalogPreferencesAndStack() = runBlocking { + val signer = LocalSigner(SECRET) + withIolite(signer = signer) { iolite -> + val privateRelay = "wss://private.example" + val list = signer.sign( + 1_700_000_100, + Kinds.CatalogRelayList, + listOf(listOf("r", "wss://relay.zapstore.dev")), + signer.encryptToSelf(JSONArray().put(privateRelay).toString()), + ) + iolite.ingest(listOf(list)) + iolite.ingest( + listOf(signer.sign( + 1_700_000_101, + Kinds.Preferences, + listOf(listOf("d", "ui"), listOf("client", "zapstore")), + """{"theme":"dark"}""", + )), + ) + iolite.ingest( + listOf(signer.sign( + 1_700_000_102, + Kinds.AppStack, + listOf( + listOf("d", "privacy"), + listOf("name", "Privacy"), + listOf("a", "32267:${signer.publicKey}:com.example.app"), + listOf("client", "zapstore"), + ), + signer.encryptToSelf(JSONArray().put("32267:${signer.publicKey}:com.hidden").toString()), + )), + ) + + val restoredList = iolite.rehydrateCatalogRelayList() + assertTrue(restoredList.id != list.id) + assertEquals(listOf("wss://relay.zapstore.dev"), restoredList.tagValues("r")) + assertEquals( + listOf(privateRelay), + JSONArray(signer.decryptFromSelf(restoredList.content)).let { array -> + List(array.length()) { array.getString(it) } + }, + ) + + val restoredPrefs = iolite.rehydratePreferences("ui", signer) + assertEquals("ui", restoredPrefs.dTag()) + assertEquals("""{"theme":"dark"}""", restoredPrefs.content) + assertEquals("zapstore", restoredPrefs.tagValue("client")) + + val restoredStack = iolite.rehydrateStack("privacy", signer) + assertEquals("privacy", restoredStack.dTag()) + assertEquals("Privacy", restoredStack.tagValue("name")) + assertEquals("android-arm64-v8a", restoredStack.tagValue("f")) + assertEquals("zapstore", restoredStack.tagValue("client")) + assertEquals( + listOf("32267:${signer.publicKey}:com.hidden"), + JSONArray(signer.decryptFromSelf(restoredStack.content)).let { array -> + List(array.length()) { array.getString(it) } + }, + ) + } + } + + @Test + fun catalogManifestsAreNotStoredAsPreferences() = runBlocking { + val signer = LocalSigner(SECRET) + withIolite(signer = signer) { iolite -> + val manifest = signer.sign( + 1_700_000_100, + Kinds.Preferences, + listOf(listOf("d", "catalog"), listOf("from", "0"), listOf("to", "1"), listOf("v", "1")), + "{}", + ) + assertEquals(0, iolite.ingest(listOf(manifest))) + assertTrue(runCatching { iolite.rehydratePreferences("catalog", signer) }.isFailure) + } + } + + @Test + fun pruneRemovesOldZaps() = runBlocking { + val signer = LocalSigner(SECRET) + withIolite( + signer = signer, + config = IoliteConfig( + expirationSweepInterval = 20.milliseconds, + pruneRules = mapOf(Kinds.Zap to 1.seconds), + ), + nowMillis = { 10_000L }, + ) { iolite -> + iolite.ingest( + listOf(signer.sign( + 1, + Kinds.Zap, + listOf(listOf("i", "com.example.app"), listOf("amount", "1000000")), + "", + )), + ) + val newZap = signer.sign( + 10, + Kinds.Zap, + listOf(listOf("i", "com.example.app"), listOf("amount", "2000000")), + "", + ) + iolite.ingest(listOf(newZap)) + delay(80) + val zaps = iolite.query(Query.zaps(AppCoordinate(signer.publicKey, "com.example.app"))).first { true } + assertEquals(QueryPhase.LocalOnly, zaps.phase) + assertEquals(listOf(newZap.id), zaps.items.map { it.eventId }) + assertEquals(2_000L, zaps.items.single().amountSats) + } + } + + @Test + fun commentsAreIndexedFromAddressTag() = runBlocking { + val signer = LocalSigner(SECRET) + withIolite(signer = signer) { iolite -> + val coordinate = AppCoordinate(signer.publicKey, "com.example.app") + val event = signer.sign( + 1_700_000_000, + Kinds.Comment, + listOf(listOf("A", coordinate.toString())), + "great app", + ) + iolite.ingest(listOf(event)) + val comments = iolite.query(Query.comments(coordinate)).first { it.items.isNotEmpty() } + assertEquals(listOf(event.id), comments.items.map { it.eventId }) + assertEquals(listOf("great app"), comments.items.map { it.content }) + assertEquals(listOf("com.example.app"), comments.items.map { it.appId }) + } + } + + @Test + fun commentsIndexParentEventTag() = runBlocking { + val signer = LocalSigner(SECRET) + withIolite(signer = signer) { iolite -> + val coordinate = AppCoordinate(signer.publicKey, "com.example.app") + val root = signer.sign( + 1_700_000_000, + Kinds.Comment, + listOf(listOf("A", coordinate.toString())), + "root", + ) + val reply = signer.sign( + 1_700_000_100, + Kinds.Comment, + listOf(listOf("A", coordinate.toString()), listOf("e", root.id)), + "reply", + ) + iolite.ingest(listOf(root, reply)) + val comments = iolite.query(Query.comments(coordinate)).first { it.items.size == 2 } + assertEquals(root.id, comments.items.single { it.eventId == reply.id }.parentEventId) + assertNull(comments.items.single { it.eventId == root.id }.parentEventId) + } + } + + @Test + fun proofExpiryClearsAppPubkey() { + val dir = File("build/tmp/iolite-c1-${System.nanoTime()}").apply { mkdirs() } + val store = IoliteStore(File(dir, "iolite.db").absolutePath) + store.insertFixtureCatalog() + val appId = ByteArray(32) { 1 } + val cert = ByteArray(32) { 2 } + val pubkey = Hex.decode(LocalSigner(SECRET).publicKey) + store.write { tx -> + tx.db.prepare( + """ + INSERT INTO apps ( + id, catalog_id, app_id, certificate_hash, app_event_created_at, pubkey, event_pubkey, + name, summary, repository, version, version_code, channel, metadata + ) VALUES (?, 1, 'com.example.app', ?, 1, ?, ?, 'Example', '', NULL, '1.0', 1, 'main', '{}') + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, appId) + statement.bindBlob(2, cert) + statement.bindBlob(3, pubkey) + statement.bindBlob(4, pubkey) + statement.step() + } + tx.db.prepare( + """ + INSERT INTO certificate_proofs ( + id, catalog_id, certificate_hash, pubkey, event_id, created_at, expiry, revoked, delegations + ) VALUES (?, 1, ?, ?, ?, 50, 150, 0, '[]') + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, ByteArray(32) { 3 }) + statement.bindBlob(2, cert) + statement.bindBlob(3, pubkey) + statement.bindBlob(4, ByteArray(32) { 4 }) + statement.step() + } + } + store.recomputeProofs(now = 100L) + assertEquals(Hex.encode(pubkey), store.app("com.example.app")!!.proofPubkey) + assertTrue(store.app("com.example.app")!!.isVerified) + store.recomputeProofs(now = 200L) + assertNull(store.app("com.example.app")!!.proofPubkey) + assertEquals(150L, store.nextProofExpiry(100L)) + store.close() + } + + @Test + fun appSearchMatchesNameAndAppIdPrefix() { + val dir = File("build/tmp/iolite-search-${System.nanoTime()}").apply { mkdirs() } + val store = IoliteStore(File(dir, "iolite.db").absolutePath) + store.insertFixtureCatalog() + insertApp(store, 1, "com.example.app", "Zapstore", embedding = null) + assertEquals(listOf("com.example.app"), store.apps(AppFilter(search = "zap")).map { it.appId }) + assertEquals(emptyList(), store.apps(AppFilter(search = "example"))) + assertEquals(listOf("com.example.app"), store.apps(AppFilter(search = "com.example")).map { it.appId }) + assertEquals(listOf("com.example.app"), store.apps(AppFilter(search = "COM.EXAMPLE")).map { it.appId }) + store.close() + } + + @Test + fun appSearchRanksEmbeddings() { + val dir = File("build/tmp/iolite-search-vec-${System.nanoTime()}").apply { mkdirs() } + val store = IoliteStore(File(dir, "iolite.db").absolutePath) + store.insertFixtureCatalog() + val close = ByteArray(EMBEDDING_DIMS) { 20 } + val far = ByteArray(EMBEDDING_DIMS) { 1 } + val opposite = ByteArray(EMBEDDING_DIMS) { -20 } + insertApp(store, 1, "app.map", "Map", opposite) + insertApp(store, 2, "app.maple", "Maple", embedding = null) + insertApp(store, 3, "app.atlas", "Atlas", close) + insertApp(store, 4, "app.noise", "Noise", far) + val vector = ByteArray(EMBEDDING_DIMS) { 20 } + assertEquals( + listOf("app.map", "app.atlas", "app.maple"), + store.apps(AppFilter(search = "map", queryVector = vector)).map { it.appId }, + ) + assertEquals(listOf("app.atlas"), store.apps(AppFilter(search = "nope", queryVector = vector)).map { it.appId }) + assertEquals(listOf("app.map", "app.maple"), store.apps(AppFilter(search = "map")).map { it.appId }) + assertEquals(listOf("app.map"), store.apps(AppFilter(search = "map", queryVector = vector, limit = 1)).map { it.appId }) + store.close() + } + + @Test + fun appLimitReturnsNewestRows() { + val dir = File("build/tmp/iolite-limit-${System.nanoTime()}").apply { mkdirs() } + val store = IoliteStore(File(dir, "iolite.db").absolutePath) + store.insertFixtureCatalog() + val pubkey = Hex.decode(LocalSigner(SECRET).publicKey) + store.write { tx -> + repeat(25) { index -> + tx.db.prepare( + """ + INSERT INTO apps ( + id, catalog_id, app_id, certificate_hash, app_event_created_at, pubkey, event_pubkey, + name, summary, repository, version, version_code, channel, metadata + ) VALUES (?, 1, ?, ?, ?, ?, ?, ?, '', NULL, '1.0', 1, 'main', '{}') + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, ByteArray(32) { index.toByte() }) + statement.bindText(2, "app.$index") + statement.bindBlob(3, ByteArray(32) { 2 }) + statement.bindLong(4, index + 1L) + statement.bindBlob(5, pubkey) + statement.bindBlob(6, pubkey) + statement.bindText(7, "App $index") + statement.step() + } + } + } + val page = store.apps(AppFilter(limit = 20)) + assertEquals(20, page.size) + assertEquals((24 downTo 5).map { "app.$it" }, page.map { it.appId }) + store.close() + } + + @Test + fun wipeAndCloseTruncateWal() { + val dir = File("build/tmp/iolite-wal-${System.nanoTime()}").apply { mkdirs() } + val db = File(dir, "iolite.db") + val wal = File("${db.path}-wal") + val store = IoliteStore(db.absolutePath) + store.insertFixtureCatalog() + val pubkey = Hex.decode(LocalSigner(SECRET).publicKey) + store.write { tx -> + repeat(40) { index -> + tx.db.prepare( + """ + INSERT INTO apps ( + id, catalog_id, app_id, certificate_hash, app_event_created_at, pubkey, event_pubkey, + name, summary, repository, version, version_code, channel, metadata + ) VALUES (?, 1, ?, ?, ?, ?, ?, ?, '', NULL, '1.0', 1, 'main', '{}') + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, ByteArray(32) { index.toByte() }) + statement.bindText(2, "app.$index") + statement.bindBlob(3, ByteArray(32) { 2 }) + statement.bindLong(4, index + 1L) + statement.bindBlob(5, pubkey) + statement.bindBlob(6, pubkey) + statement.bindText(7, "App $index") + statement.step() + } + } + } + store.wipe() + assertTrue(!wal.exists() || wal.length() == 0L) + store.write { tx -> + tx.db.execSQL("UPDATE catalogs SET epoch = 1") + tx.touch(Table.Catalogs) + } + store.close() + assertTrue(!wal.exists() || wal.length() == 0L) + } + + @Test + fun ingestBatchCommitsTogether() { + val dir = File("build/tmp/iolite-batch-${System.nanoTime()}").apply { mkdirs() } + val store = IoliteStore(File(dir, "iolite.db").absolutePath) + val signer = LocalSigner(SECRET) + val device = DeviceProfile(listOf("arm64-v8a"), 34) + val first = runBlocking { signer.sign(1, Kinds.Profile, emptyList(), """{"name":"Ada"}""") } + val second = runBlocking { signer.sign(2, Kinds.Profile, emptyList(), """{"name":"Bob"}""") } + assertEquals(2, store.ingest(listOf(first, second), IngestContext(device, signer))) + val profile = store.profiles(listOf(signer.publicKey)).getValue(signer.publicKey) + assertEquals("Bob", profile.name) + assertEquals(second.id, profile.eventId) + store.close() + } + + private suspend fun withIolite( + signer: LocalSigner, + http: HttpTransport? = null, + config: IoliteConfig = IoliteConfig(eoseGrace = 5.milliseconds, oneShotTimeout = 2.seconds), + nowMillis: () -> Long = { System.currentTimeMillis() }, + block: suspend (Iolite) -> Unit, + ) { + val dir = File("build/tmp/iolite-contracts-${System.nanoTime()}").apply { mkdirs() } + val scope = CoroutineScope(SupervisorJob()) + val iolite = Iolite.createForTesting( + databasePath = File(dir, "iolite.db").absolutePath, + parentScope = scope, + config = config, + http = http, + signer = signer, + nowMillis = nowMillis, + ) + try { + block(iolite) + } finally { + iolite.close() + scope.coroutineContext.job.cancel() + } + } + + private fun insertApp(store: IoliteStore, index: Int, appId: String, name: String, embedding: ByteArray?) { + val pubkey = Hex.decode(LocalSigner(SECRET).publicKey) + val id = ByteArray(32) { index.toByte() } + store.write { tx -> + tx.db.prepare( + """ + INSERT INTO apps ( + id, catalog_id, app_id, certificate_hash, app_event_created_at, pubkey, event_pubkey, + name, summary, repository, version, version_code, channel, metadata + ) VALUES (?, 1, ?, ?, ?, ?, ?, ?, 'example summary', NULL, '1.0', 1, 'main', '{}') + """.trimIndent(), + ).use { statement -> + statement.bindBlob(1, id) + statement.bindText(2, appId) + statement.bindBlob(3, ByteArray(32) { 2 }) + statement.bindLong(4, index.toLong()) + statement.bindBlob(5, pubkey) + statement.bindBlob(6, pubkey) + statement.bindText(7, name) + statement.step() + } + if (embedding != null) { + tx.db.prepare("INSERT INTO apps_search (id, embedding) VALUES (?, ?)").use { statement -> + statement.bindBlob(1, id) + statement.bindBlob(2, embedding) + statement.step() + } + } + } + } + + companion object { + private val SECRET = Hex.decode("0000000000000000000000000000000000000000000000000000000000000001") + } +} + +fun IoliteStore.insertFixtureCatalog() { + write { tx -> + tx.db.prepare( + """ + INSERT INTO catalogs (id, relay_url, position, is_private, epoch, endpoints) + VALUES (1, 'ws://127.0.0.1:3334', 0, 0, 0, '["ws://127.0.0.1:3334"]') + """.trimIndent(), + ).use { it.step() } + } +} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/LegacyQueryTestAdapter.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/LegacyQueryTestAdapter.kt deleted file mode 100644 index ba7a0ce..0000000 --- a/iolite/src/test/kotlin/dev/zapstore/iolite/LegacyQueryTestAdapter.kt +++ /dev/null @@ -1,53 +0,0 @@ -package dev.zapstore.iolite - -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import kotlinx.coroutines.flow.Flow -import kotlin.time.Duration - -/** - * Keeps the pre-refactor behavioral tests readable while they exercise the new - * QueryOptions API. New contract tests should use QueryOptions directly. - */ -internal sealed interface QuerySource { - val options: QueryOptions - - data object Local : QuerySource { - override val options = QueryOptions.local() - } - - data class LocalAndRemote( - val relays: Set, - val mode: RemoteMode = RemoteMode.Stream, - val cachedFor: Duration? = null, - ) : QuerySource { - override val options = QueryOptions.localAndRemote(relays, mode, cachedFor) - } - - data class Remote( - val relays: Set, - val mode: RemoteMode = RemoteMode.Stream, - ) : QuerySource { - override val options = QueryOptions.remote(relays, mode) - } -} - -internal object QuerySync { - val LocalOnly: QueryPhase = QueryPhase.LocalOnly - val Cached: QueryPhase = QueryPhase.Cached - val Connecting: QueryPhase = QueryPhase.Connecting - val CatchingUp: QueryPhase = QueryPhase.CatchingUp - val Live: QueryPhase = QueryPhase.Live - val Complete: QueryPhase = QueryPhase.Complete - val TimedOut: QueryPhase = QueryPhase.TimedOut - val Failed: QueryPhase = QueryPhase.Failed -} - -internal val QueryState.sync: QueryPhase - get() = phase - -internal fun Iolite.query(filter: Filter, source: QuerySource): Flow = - query(filter, source.options) - -internal fun Iolite.query(filters: List, source: QuerySource): Flow = - query(filters, source.options) diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/Nip44Test.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/Nip44Test.kt new file mode 100644 index 0000000..73bdf02 --- /dev/null +++ b/iolite/src/test/kotlin/dev/zapstore/iolite/Nip44Test.kt @@ -0,0 +1,89 @@ +package dev.zapstore.iolite + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class Nip44Test { + private val vectors = JSONObject( + javaClass.classLoader!!.getResourceAsStream("nip44.vectors.json")!!.reader().readText(), + ).getJSONObject("v2") + + @Test + fun conversationKeysMatchOfficialVectors() { + val cases = vectors.getJSONObject("valid").getJSONArray("get_conversation_key") + for (i in 0 until cases.length()) { + val c = cases.getJSONObject(i) + val key = Nip44.conversationKey(c.getString("sec1").hex(), c.getString("pub2").hex()) + assertEquals("case $i", c.getString("conversation_key"), Hex.encode(key)) + } + } + + @Test + fun messageKeysMatchOfficialVectors() { + val block = vectors.getJSONObject("valid").getJSONObject("get_message_keys") + val conversationKey = block.getString("conversation_key").hex() + val keys = block.getJSONArray("keys") + for (i in 0 until keys.length()) { + val c = keys.getJSONObject(i) + val (chachaKey, chachaNonce, hmacKey) = Nip44.messageKeys(conversationKey, c.getString("nonce").hex()) + assertEquals(c.getString("chacha_key"), Hex.encode(chachaKey)) + assertEquals(c.getString("chacha_nonce"), Hex.encode(chachaNonce)) + assertEquals(c.getString("hmac_key"), Hex.encode(hmacKey)) + } + } + + @Test + fun paddedLengthsMatchOfficialVectors() { + val cases = vectors.getJSONObject("valid").getJSONArray("calc_padded_len") + for (i in 0 until cases.length()) { + val pair = cases.getJSONArray(i) + assertEquals(pair.getInt(1), Nip44.paddedLength(pair.getInt(0))) + } + } + + @Test + fun encryptDecryptMatchesOfficialVectors() { + val cases = vectors.getJSONObject("valid").getJSONArray("encrypt_decrypt") + for (i in 0 until cases.length()) { + val c = cases.getJSONObject(i) + val sec1 = c.getString("sec1").hex() + val sec2 = c.getString("sec2").hex() + val pub2 = Crypto.xOnlyPubkey(sec2) + val conversationKey = Nip44.conversationKey(sec1, pub2) + assertEquals(c.getString("conversation_key"), Hex.encode(conversationKey)) + val payload = Nip44.encrypt(c.getString("plaintext"), conversationKey, c.getString("nonce").hex()) + assertEquals("encrypt $i", c.getString("payload"), payload) + val pub1 = Crypto.xOnlyPubkey(sec1) + val roundTripKey = Nip44.conversationKey(sec2, pub1) + assertEquals(c.getString("plaintext"), Nip44.decrypt(payload, roundTripKey)) + } + } + + @Test + fun invalidConversationKeysAreRejected() { + val cases = vectors.getJSONObject("invalid").getJSONArray("get_conversation_key") + for (i in 0 until cases.length()) { + val c = cases.getJSONObject(i) + val failed = runCatching { + Nip44.conversationKey(c.getString("sec1").hex(), c.getString("pub2").hex()) + }.isFailure + assertTrue("case $i should fail", failed) + } + } + + @Test + fun invalidPayloadsAreRejected() { + val cases = vectors.getJSONObject("invalid").getJSONArray("decrypt") + for (i in 0 until cases.length()) { + val c = cases.getJSONObject(i) + val failed = runCatching { + Nip44.decrypt(c.getString("payload"), c.getString("conversation_key").hex()) + }.isFailure + assertTrue("case $i ${c.optString("note")} should fail", failed) + } + } + + private fun String.hex(): ByteArray = Hex.decode(this) +} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/ObservationTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/ObservationTest.kt deleted file mode 100644 index baf0224..0000000 --- a/iolite/src/test/kotlin/dev/zapstore/iolite/ObservationTest.kt +++ /dev/null @@ -1,397 +0,0 @@ -package dev.zapstore.iolite - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.client.EmptyNostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress -import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import java.util.concurrent.CopyOnWriteArrayList -import java.util.concurrent.atomic.AtomicInteger -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeout -import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue -import org.junit.Test -import kotlin.time.Duration.Companion.hours -import kotlin.time.Duration.Companion.seconds - -class ObservationTest { - private val relay = "wss://relay.example".normalizeRelayUrl() - private val eventSequence = AtomicInteger() - - @Test - fun `sessions do not re-emit on other sessions inserts`() = runBlocking { - val client = MultiSubClient() - val purpleQuartz = Iolite.createForTesting( - RecordingEventStore(), - client, - this, - eventVerifier = { true }, - ) - val statesA = CopyOnWriteArrayList() - val statesB = CopyOnWriteArrayList() - val collectionA = collect(purpleQuartz, Filter(kinds = listOf(1)), statesA) - val collectionB = collect(purpleQuartz, Filter(kinds = listOf(7)), statesB) - - client.awaitSubscriptions(2) - client.started(relay) - client.eose(relay) - awaitState(statesA) { it.sync == QuerySync.Live } - awaitState(statesB) { it.sync == QuerySync.Live } - val emittedA = statesA.size - - client.event(relay, signedEvent(kind = 7, content = "for-b")) - awaitState(statesB) { state -> state.items.any { it.content == "for-b" } } - delay(150) - - assertEquals(emittedA, statesA.size) - - collectionA.cancel() - collectionB.cancel() - collectionA.join() - collectionB.join() - purpleQuartz.close() - } - - @Test - fun `replaceable update re-emits with the new version`() = runBlocking { - val client = MultiSubClient() - val purpleQuartz = Iolite.createForTesting( - RecordingEventStore(), - client, - this, - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val collection = collect(purpleQuartz, Filter(kinds = listOf(0)), states) - - client.awaitSubscriptions(1) - client.started(relay) - client.eose(relay) - awaitState(states) { it.sync == QuerySync.Live } - - val v1 = signedEvent(kind = 0, content = "v1", createdAt = 100) - client.event(relay, v1) - awaitState(states) { state -> state.items.any { it.id == v1.id } } - - val v2 = signedEvent(kind = 0, content = "v2", createdAt = 200) - client.event(relay, v2) - val updated = awaitState(states) { it.items.singleOrNull()?.id == v2.id } - assertEquals("v2", updated.items.single().content) - - collection.cancel() - collection.join() - purpleQuartz.close() - } - - @Test - fun `events are persisted via a single batch at eose`() = runBlocking { - val client = MultiSubClient() - val store = RecordingEventStore() - val purpleQuartz = Iolite.createForTesting( - store, - client, - this, - config = IoliteConfig(ingestFlushInterval = 10.seconds), - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val source = QuerySource.LocalAndRemote( - relays = setOf(relay), - mode = RemoteMode.OneShot(5.seconds), - cachedFor = 6.hours, - ) - val collection = launch { - purpleQuartz.query(Filter(kinds = listOf(1)), source).collect(states::add) - } - - client.awaitSubscriptions(1) - client.started(relay) - val events = listOf( - signedEvent(content = "one"), - signedEvent(content = "two"), - signedEvent(content = "three"), - ) - events.forEach { client.event(relay, it) } - client.eose(relay) - - val complete = awaitState(states) { it.sync == QuerySync.Complete } - assertEquals(events.map { it.id }.toSet(), complete.items.map { it.id }.toSet()) - assertEquals(listOf(events.map { it.id }), store.batches) - - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `remote burst is coalesced and complete state contains every event`() = runBlocking { - val client = MultiSubClient() - val purpleQuartz = Iolite.createForTesting( - RecordingEventStore(), - client, - this, - config = IoliteConfig(ingestBatchSize = 1_000, ingestFlushInterval = 10.seconds), - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val collection = launch { - purpleQuartz.query( - Filter(kinds = listOf(1)), - QueryOptions.remote(setOf(relay), RemoteMode.OneShot(5.seconds)), - ).collect(states::add) - } - - client.awaitSubscriptions(1) - client.started(relay) - val events = (0 until 500).map { signedEvent(content = "burst-$it") } - events.forEach { client.event(relay, it) } - client.eose(relay) - - val complete = awaitState(states) { it.phase == QueryPhase.Complete } - assertEquals(events.map(Event::id), complete.items.map(Event::id)) - assertTrue("burst snapshots were not coalesced", states.size < events.size / 2) - - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `duplicate batch rejection is verified with one id query`() = runBlocking { - val client = MultiSubClient() - val store = RecordingEventStore(rejectBatchInserts = true) - val events = (0 until 100).map { signedEvent(content = "duplicate-$it") } - events.forEach { store.insert(it) } - val purpleQuartz = Iolite.createForTesting( - store, - client, - this, - config = IoliteConfig(ingestBatchSize = 100, ingestFlushInterval = 10.seconds), - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val collection = launch { - purpleQuartz.query( - Filter(kinds = listOf(1)), - QueryOptions.remote(setOf(relay), RemoteMode.OneShot(5.seconds)), - ).collect(states::add) - } - - client.awaitSubscriptions(1) - client.started(relay) - events.forEach { client.event(relay, it) } - client.eose(relay) - - awaitState(states) { it.phase == QueryPhase.Complete } - assertEquals(1, store.queryCalls.get()) - - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `slow local collector cannot block remote persistence`() = runBlocking { - val client = MultiSubClient() - val purpleQuartz = Iolite.createForTesting( - RecordingEventStore(), - client, - this, - config = IoliteConfig(ingestBatchSize = 500, ingestFlushInterval = 10.seconds), - eventVerifier = { true }, - ) - val slowCollection = launch { - purpleQuartz.query(Filter(kinds = listOf(1))).collect { - delay(100) - } - } - val remoteStates = CopyOnWriteArrayList() - val remoteCollection = launch { - purpleQuartz.query( - Filter(kinds = listOf(1)), - QueryOptions.remote(setOf(relay), RemoteMode.OneShot(5.seconds)), - ).collect(remoteStates::add) - } - - client.awaitSubscriptions(1) - client.started(relay) - repeat(400) { client.event(relay, signedEvent(content = "write-$it")) } - client.eose(relay) - - val complete = awaitState(remoteStates) { it.phase == QueryPhase.Complete } - assertEquals(400, complete.items.size) - - withTimeout(2.seconds) { remoteCollection.join() } - slowCollection.cancel() - slowCollection.join() - purpleQuartz.close() - } - - private fun kotlinx.coroutines.CoroutineScope.collect( - purpleQuartz: Iolite, - filter: Filter, - states: MutableList, - ): Job = launch { - purpleQuartz.query( - filter, - QuerySource.LocalAndRemote(setOf(relay), RemoteMode.Stream, cachedFor = 6.hours), - ).collect(states::add) - } - - private suspend fun awaitState( - states: List, - predicate: (QueryState) -> Boolean, - ): QueryState = withTimeout(5.seconds) { - while (true) { - states.lastOrNull(predicate)?.let { return@withTimeout it } - delay(10) - } - error("unreachable") - } - - private fun signedEvent( - kind: Int = 1, - content: String, - createdAt: Long = 1_750_000_000, - ): Event { - val id = eventSequence.incrementAndGet().toString(16).padStart(64, '0') - return Event( - id = id, - pubKey = "1".repeat(64), - createdAt = createdAt, - kind = kind, - tags = emptyArray(), - content = content, - sig = "2".repeat(128), - ) - } -} - -private class MultiSubClient : INostrClient by EmptyNostrClient() { - private data class Sub( - val listener: SubscriptionListener, - val filters: Map>, - ) - - private val subs = CopyOnWriteArrayList() - - override fun subscribe( - subId: String, - filters: Map>, - listener: SubscriptionListener?, - ) { - if (listener != null) subs += Sub(listener, filters) - } - - override fun unsubscribe(subId: String) = Unit - - suspend fun awaitSubscriptions(count: Int) { - withTimeout(2.seconds) { - while (subs.size < count) delay(10) - } - } - - fun started(relay: NormalizedRelayUrl) { - subs.forEach { it.listener.onSubscriptionStarted(relay.url, it.filters[relay].orEmpty()) } - } - - fun eose(relay: NormalizedRelayUrl) { - subs.forEach { it.listener.onEose(relay, it.filters[relay]) } - } - - fun event(relay: NormalizedRelayUrl, event: Event) { - subs.forEach { sub -> - val filters = sub.filters[relay].orEmpty() - if (filters.any { it.match(event) }) { - sub.listener.onEvent(event, true, relay, filters) - } - } - } -} - -private class RecordingEventStore( - private val rejectBatchInserts: Boolean = false, -) : IEventStore { - override val relay: NormalizedRelayUrl? = null - val batches = CopyOnWriteArrayList>() - val queryCalls = AtomicInteger() - private val events = mutableListOf() - - override suspend fun insert(event: Event) { - synchronized(events) { - if (events.none { it.id == event.id }) events += event - } - } - - override suspend fun batchInsert(events: List): List { - batches += events.map { it.id } - if (rejectBatchInserts) return events.map { IEventStore.InsertOutcome.Rejected("duplicate") } - return events.map { event -> - synchronized(this.events) { - if (this.events.none { it.id == event.id }) this.events += event - } - IEventStore.InsertOutcome.Accepted - } - } - - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { - body(object : IEventStore.ITransaction { - override fun insert(event: Event) { - synchronized(events) { - if (events.none { it.id == event.id }) events += event - } - } - }) - } - - @Suppress("UNCHECKED_CAST") - override suspend fun query(filter: Filter): List { - queryCalls.incrementAndGet() - return synchronized(events) { events.filter(filter::match).map { it as T } } - } - - @Suppress("UNCHECKED_CAST") - override suspend fun query(filters: List): List { - queryCalls.incrementAndGet() - return synchronized(events) { - events.filter { event -> filters.any { it.match(event) } }.map { it as T } - } - } - - override suspend fun query(filter: Filter, onEach: (T) -> Unit) { - query(filter).forEach(onEach) - } - - override suspend fun query(filters: List, onEach: (T) -> Unit) { - query(filters).forEach(onEach) - } - - override suspend fun count(filter: Filter): Int = query(filter).size - override suspend fun count(filters: List): Int = query(filters).size - - override suspend fun delete(filter: Filter) { - synchronized(events) { events.removeAll(filter::match) } - } - - override suspend fun delete(filters: List) { - synchronized(events) { - events.removeAll { event -> filters.any { it.match(event) } } - } - } - - override suspend fun deleteExpiredEvents() = Unit - override suspend fun reindexFullTextSearch() = Unit - - override suspend fun reindexFullTextSearch( - resumeFrom: String?, - batchSize: Int, - ): FtsReindexProgress = throw UnsupportedOperationException() - - override fun close() = Unit -} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/OutboxRouterTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/OutboxRouterTest.kt deleted file mode 100644 index d3970c2..0000000 --- a/iolite/src/test/kotlin/dev/zapstore/iolite/OutboxRouterTest.kt +++ /dev/null @@ -1,276 +0,0 @@ -package dev.zapstore.iolite - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.client.EmptyNostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress -import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import java.util.concurrent.CopyOnWriteArrayList -import kotlinx.coroutines.cancelAndJoin -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeout -import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue -import org.junit.Test -import kotlin.time.Duration.Companion.seconds - -class OutboxRouterTest { - private val bootstrap = setOf("wss://bootstrap.example".normalizeRelayUrl()) - private val fallback = setOf("wss://fallback.example".normalizeRelayUrl()) - private val outbox = "wss://outbox.example".normalizeRelayUrl() - private val author = "2".repeat(64) - - @Test - fun `resolveReadRelays returns read and unmarked relays from the local store`() = runBlocking { - val store = RouterTestStore() - store.insert(relayListEvent(arrayOf( - arrayOf("r", "wss://read.example"), - arrayOf("r", "wss://read2.example", "read"), - arrayOf("r", "wss://write.example", "write"), - arrayOf("r", "http://insecure.example"), - ))) - val client = RouterTestClient() - val purpleQuartz = Iolite.createForTesting(store, client, this) - val router = OutboxRouter(purpleQuartz, bootstrap) - - var resolved: Set? = null - val resolveJob = launch { resolved = router.resolveReadRelays(author) } - client.eose(client.awaitSub(kinds = listOf(OutboxRouter.RELAY_LIST_KIND))) - withTimeout(2.seconds) { resolveJob.join() } - - assertEquals( - setOf("wss://read.example".normalizeRelayUrl(), "wss://read2.example".normalizeRelayUrl()), - resolved, - ) - purpleQuartz.close() - } - - @Test - fun `resolveReadRelays uses the freshness cache without touching relays`() = runBlocking { - val store = RouterTestStore() - store.insert(relayListEvent(arrayOf(arrayOf("r", "wss://read.example")))) - val refreshCache = InMemoryQueryRefreshCache() - val filter = Filter(authors = listOf(author), kinds = listOf(OutboxRouter.RELAY_LIST_KIND), limit = 1) - refreshCache.recordRefresh(QueryFingerprint.create(listOf(filter), bootstrap), System.currentTimeMillis()) - val client = RouterTestClient() - val purpleQuartz = Iolite.createForTesting(store, client, this, refreshCache = refreshCache) - val router = OutboxRouter(purpleQuartz, bootstrap) - - val resolved = withTimeout(2.seconds) { router.resolveReadRelays(author) } - - assertEquals(setOf("wss://read.example".normalizeRelayUrl()), resolved) - assertTrue(client.subs.isEmpty()) - purpleQuartz.close() - } - - @Test - fun `queryWithOutbox restarts the remote side on the expanded relay set`() = runBlocking { - val store = RouterTestStore() - store.insert(relayListEvent(arrayOf(arrayOf("r", outbox.url)))) - val client = RouterTestClient() - val purpleQuartz = Iolite.createForTesting(store, client, this) - val router = OutboxRouter(purpleQuartz, bootstrap) - - val collection = launch { - router.queryWithOutbox( - listOf(Filter(kinds = listOf(9735), limit = 10)), - authors = listOf(author), - fallbackRelays = fallback, - ).collect { } - } - - val fallbackSub = client.awaitSub(kinds = listOf(9735), relays = fallback) - client.eose(client.awaitSub(kinds = listOf(OutboxRouter.RELAY_LIST_KIND))) - val expandedSub = client.awaitSub(kinds = listOf(9735), relays = fallback + outbox, skip = fallbackSub) - - assertEquals(fallback + outbox, expandedSub.relays) - collection.cancelAndJoin() - purpleQuartz.close() - } - - @Test - fun `queryWithOutbox can replace the fallback set with resolved relays`() = runBlocking { - val store = RouterTestStore() - store.insert(relayListEvent(arrayOf(arrayOf("r", outbox.url)))) - val client = RouterTestClient() - val purpleQuartz = Iolite.createForTesting(store, client, this) - val router = OutboxRouter(purpleQuartz, bootstrap) - - val collection = launch { - router.queryWithOutbox( - listOf(Filter(kinds = listOf(0), limit = 1)), - authors = listOf(author), - fallbackRelays = fallback, - unionWithFallback = false, - ).collect { } - } - - client.awaitSub(kinds = listOf(0), relays = fallback) - client.eose(client.awaitSub(kinds = listOf(OutboxRouter.RELAY_LIST_KIND))) - val expandedSub = client.awaitSub(kinds = listOf(0), relays = setOf(outbox)) - - assertEquals(setOf(outbox), expandedSub.relays) - collection.cancelAndJoin() - purpleQuartz.close() - } - - @Test - fun `queryWithOutbox keeps the fallback relays when nothing resolves`() = runBlocking { - val store = RouterTestStore() - val client = RouterTestClient() - val purpleQuartz = Iolite.createForTesting(store, client, this) - val router = OutboxRouter(purpleQuartz, bootstrap) - - val collection = launch { - router.queryWithOutbox( - listOf(Filter(kinds = listOf(9735), limit = 10)), - authors = listOf(author), - fallbackRelays = fallback, - ).collect { } - } - - client.awaitSub(kinds = listOf(9735), relays = fallback) - client.eose(client.awaitSub(kinds = listOf(OutboxRouter.RELAY_LIST_KIND))) - delay(300) - - assertEquals(1, client.subs.count { it.kinds == listOf(9735) }) - collection.cancelAndJoin() - purpleQuartz.close() - } - - @Test - fun `resolveReadRelays returns empty when the fetch cannot complete`() = runBlocking { - val store = RouterTestStore() - val client = RouterTestClient() - val purpleQuartz = Iolite.createForTesting(store, client, this) - val router = OutboxRouter(purpleQuartz, bootstrap, resolveTimeout = 1.seconds) - - var resolved: Set? = null - val resolveJob = launch { resolved = router.resolveReadRelays(author) } - client.fail(client.awaitSub(kinds = listOf(OutboxRouter.RELAY_LIST_KIND))) - withTimeout(4.seconds) { resolveJob.join() } - - assertEquals(emptySet(), resolved) - purpleQuartz.close() - } - - private fun relayListEvent(tags: Array>): Event = Event( - id = "3".repeat(64), - pubKey = author, - createdAt = 1_750_000_000, - kind = OutboxRouter.RELAY_LIST_KIND, - tags = tags, - content = "", - sig = "4".repeat(128), - ) -} - -private class RouterTestClient : INostrClient by EmptyNostrClient() { - class Sub( - val filters: Map>, - val listener: SubscriptionListener, - ) { - val relays: Set get() = filters.keys - val kinds: List get() = filters.values.flatten().flatMap { it.kinds.orEmpty() }.distinct().sorted() - } - - val subs = CopyOnWriteArrayList() - - override fun subscribe( - subId: String, - filters: Map>, - listener: SubscriptionListener?, - ) { - if (listener != null) subs += Sub(filters, listener) - } - - override fun unsubscribe(subId: String) = Unit - - suspend fun awaitSub( - kinds: List, - relays: Set? = null, - skip: Sub? = null, - ): Sub = withTimeout(2.seconds) { - while (true) { - subs.firstOrNull { - it !== skip && it.kinds == kinds && (relays == null || it.relays == relays) - }?.let { return@withTimeout it } - delay(10) - } - @Suppress("UNREACHABLE_CODE") error("unreachable") - } - - fun eose(sub: Sub) { - sub.filters.forEach { (relay, filters) -> - sub.listener.onSubscriptionStarted(relay.url, filters) - sub.listener.onEose(relay, filters) - } - } - - fun fail(sub: Sub) { - sub.filters.forEach { (relay, filters) -> - sub.listener.onCannotConnect(relay, "relay unreachable", filters) - } - } -} - -private class RouterTestStore : IEventStore { - override val relay: NormalizedRelayUrl? = null - private val events = mutableListOf() - - override suspend fun insert(event: Event) { - events += event - } - - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { - body(object : IEventStore.ITransaction { - override fun insert(event: Event) { - events += event - } - }) - } - - @Suppress("UNCHECKED_CAST") - override suspend fun query(filter: Filter): List = - events.filter(filter::match).map { it as T } - - @Suppress("UNCHECKED_CAST") - override suspend fun query(filters: List): List = - events.filter { event -> filters.any { it.match(event) } }.map { it as T } - - override suspend fun query(filter: Filter, onEach: (T) -> Unit) { - query(filter).forEach(onEach) - } - - override suspend fun query(filters: List, onEach: (T) -> Unit) { - query(filters).forEach(onEach) - } - - override suspend fun count(filter: Filter): Int = query(filter).size - override suspend fun count(filters: List): Int = query(filters).size - - override suspend fun delete(filter: Filter) { - events.removeAll(filter::match) - } - - override suspend fun delete(filters: List) { - events.removeAll { event -> filters.any { it.match(event) } } - } - - override suspend fun deleteExpiredEvents() = Unit - override suspend fun reindexFullTextSearch() = Unit - - override suspend fun reindexFullTextSearch( - resumeFrom: String?, - batchSize: Int, - ): FtsReindexProgress = throw UnsupportedOperationException() - - override fun close() = Unit -} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/PublicContractsTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/PublicContractsTest.kt index dfb095a..fcea170 100644 --- a/iolite/src/test/kotlin/dev/zapstore/iolite/PublicContractsTest.kt +++ b/iolite/src/test/kotlin/dev/zapstore/iolite/PublicContractsTest.kt @@ -1,28 +1,18 @@ package dev.zapstore.iolite -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.sockets.WebsocketBuilder -import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore -import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore -import com.vitorpamplona.quartz.nip40Expiration.isExpired -import java.lang.reflect.Modifier import kotlin.time.Duration -import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.milliseconds +import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test class PublicContractsTest { @Test - fun `config rejects invalid lifecycle values`() { + fun configRejectsInvalidLifecycleValues() { assertFails { IoliteConfig(databaseName = "../store.db").validate() } assertFails { IoliteConfig(oneShotTimeout = 0.milliseconds).validate() } + assertFails { IoliteConfig(eoseGrace = 0.milliseconds).validate() } assertFails { IoliteConfig(ingestionCapacity = 0).validate() } assertFails { IoliteConfig(expirationSweepInterval = 0.milliseconds).validate() } assertFails { IoliteConfig(ingestBatchSize = 0).validate() } @@ -32,7 +22,7 @@ class PublicContractsTest { } @Test - fun `query options enforce complementary source and remote modes`() { + fun queryOptionsEnforceModes() { assertFails { QueryOptions(SourceMode.Remote, RemoteMode.Stream) } assertFails { QueryOptions(SourceMode.LocalAndRemote, RemoteMode.Stream) } assertFails { QueryOptions(SourceMode.Remote) } @@ -51,41 +41,10 @@ class PublicContractsTest { QueryOptions(SourceMode.Remote, RemoteMode.Stream, setOf(relay), cachedFor = 6.hours) } assertTrue(QueryOptions.remote(setOf(relay), RemoteMode.Stream).relays.contains(relay)) - assertTrue( - QueryOptions.localAndRemote(setOf(relay), RemoteMode.Stream, cachedFor = 6.hours) - .relays.contains(relay), - ) } @Test - fun `published Quartz APIs used by the facade remain callable`() { - // This is deliberately a compile-level compatibility gate for the public APIs - // Iolite relies on. Behavioral compatibility is covered by Android tests. - val invalidEvent = Event( - id = "0".repeat(64), - pubKey = "0".repeat(64), - createdAt = 0, - kind = 1, - tags = emptyArray(), - content = "", - sig = "0".repeat(128), - ) - assertTrue(invalidEvent.isExpired().not()) - - val filter = Filter(kinds = listOf(1)) - assertTrue(filter.match(invalidEvent)) - - assertPublicType() - assertPublicType() - assertPublicType() - assertPublicType() - assertPublicType() - assertPublicType() - assertPublicType() - } - - @Test - fun `query phases reject transitions out of terminal states`() { + fun queryPhasesRejectTransitionsOutOfTerminalStates() { assertTrue(isLegalPhaseTransition(null, QueryPhase.Connecting)) assertTrue(isLegalPhaseTransition(QueryPhase.Connecting, QueryPhase.CatchingUp)) assertTrue(isLegalPhaseTransition(QueryPhase.CatchingUp, QueryPhase.Live)) @@ -96,8 +55,21 @@ class PublicContractsTest { assertTrue(!isLegalPhaseTransition(QueryPhase.Failed, QueryPhase.Connecting)) } - private inline fun assertPublicType() { - assertTrue(Modifier.isPublic(T::class.java.modifiers)) + @Test + fun deltasUrlUsesRelayOrigin() { + assertEquals( + "https://relay.example.com/deltas?from=0", + "wss://relay.example.com/nostr?x=1#frag".normalizeRelayUrl().deltasUrl(0), + ) + assertEquals( + "http://127.0.0.1:3334/deltas?from=7", + "ws://127.0.0.1:3334".normalizeRelayUrl().deltasUrl(7), + ) + } + + @Test + fun bolt11ParsesMilliAmount() { + assertEquals(100_000, Bolt11.amountSats("lnbc1m1p...")) } private fun assertFails(block: () -> Unit) { diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/QueryInvariantTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/QueryInvariantTest.kt deleted file mode 100644 index 0544ecd..0000000 --- a/iolite/src/test/kotlin/dev/zapstore/iolite/QueryInvariantTest.kt +++ /dev/null @@ -1,819 +0,0 @@ -package dev.zapstore.iolite - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.client.EmptyNostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener -import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener -import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient -import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.Command -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress -import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import java.util.concurrent.CopyOnWriteArrayList -import java.util.concurrent.atomic.AtomicInteger -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.toList -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeout -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test -import kotlin.time.Duration.Companion.milliseconds -import kotlin.time.Duration.Companion.hours -import kotlin.time.Duration.Companion.seconds - -class QueryInvariantTest { - private val relay = "wss://relay.example".normalizeRelayUrl() - private val eventSequence = AtomicInteger() - - @Test - fun `successful empty one-shot is not cached across facades`() = runBlocking { - val cache = InMemoryQueryRefreshCache() - val clock = MutableEpochClock(1_000) - val filter = Filter(kinds = listOf(1)) - val source = QuerySource.LocalAndRemote( - relays = setOf(relay), - mode = RemoteMode.OneShot(5.seconds), - cachedFor = 6.hours, - ) - val firstClient = ControlledClient() - val first = Iolite.createForTesting( - ControlledEventStore(), - firstClient, - this, - databasePath = "persistent-cache-test", - eventVerifier = { true }, - refreshCache = cache, - clock = clock, - ) - val firstStates = CopyOnWriteArrayList() - val firstCollection = launch { first.query(filter, source).collect(firstStates::add) } - - firstClient.awaitSubscription() - firstClient.started(relay) - firstClient.eose(relay) - awaitState(firstStates) { it.sync == QuerySync.Complete } - withTimeout(2.seconds) { firstCollection.join() } - first.close() - - val secondClient = ControlledClient() - val second = Iolite.createForTesting( - ControlledEventStore(), - secondClient, - this, - databasePath = "persistent-cache-test", - eventVerifier = { true }, - refreshCache = cache, - clock = clock, - ) - val staleCollection = launch { second.query(filter, source).collect { } } - secondClient.awaitSubscription() - assertEquals(1, secondClient.requests.size) - - staleCollection.cancel() - staleCollection.join() - second.close() - } - - @Test - fun `fresh one-shot completes from a non-empty local projection`() = runBlocking { - val cache = InMemoryQueryRefreshCache() - val clock = MutableEpochClock(10_000) - val filter = Filter(kinds = listOf(1)) - val options = QueryOptions.localAndRemote( - relays = setOf(relay), - remoteMode = RemoteMode.OneShot(5.seconds), - cachedFor = 1.hours, - ) - cache.recordRefresh(QueryFingerprint.create(listOf(filter), setOf(relay)), clock.now()) - val store = ControlledEventStore() - val event = signedEvent(content = "persisted") - store.insert(event) - val client = ControlledClient() - val purpleQuartz = Iolite.createForTesting( - store, - client, - this, - eventVerifier = { true }, - refreshCache = cache, - clock = clock, - ) - - val states = purpleQuartz.query(filter, options).toList() - - assertEquals(listOf(QueryPhase.Cached), states.map(QueryState::phase)) - assertEquals(listOf(event.id), states.single().items.map(Event::id)) - assertTrue(client.requests.isEmpty()) - purpleQuartz.close() - } - - @Test - fun `freshness marker is ignored when local projection is empty`() = runBlocking { - val cache = InMemoryQueryRefreshCache() - val clock = MutableEpochClock(10_000) - val filter = Filter(kinds = listOf(1)) - val options = QueryOptions.localAndRemote( - relays = setOf(relay), - remoteMode = RemoteMode.Stream, - cachedFor = 1.hours, - ) - cache.recordRefresh(QueryFingerprint.create(listOf(filter), setOf(relay)), clock.now()) - val client = ControlledClient() - val purpleQuartz = Iolite.createForTesting( - ControlledEventStore(), - client, - this, - eventVerifier = { true }, - refreshCache = cache, - clock = clock, - ) - val states = CopyOnWriteArrayList() - val collection = launch { purpleQuartz.query(filter, options).collect(states::add) } - - client.awaitSubscription() - assertTrue(states.none { it.phase == QueryPhase.Cached }) - - collection.cancel() - collection.join() - purpleQuartz.close() - } - - @Test - fun `fresh stream defers remote and observes an extended refresh window`() = runBlocking { - val cache = InMemoryQueryRefreshCache() - val filter = Filter(kinds = listOf(1)) - val source = QuerySource.LocalAndRemote( - relays = setOf(relay), - mode = RemoteMode.Stream, - cachedFor = 1.seconds, - ) - val fingerprint = QueryFingerprint.create(listOf(filter), setOf(relay)) - cache.recordRefresh(fingerprint, System.currentTimeMillis()) - val client = ControlledClient() - val store = ControlledEventStore() - store.insert(signedEvent(content = "cached")) - val purpleQuartz = Iolite.createForTesting( - store, - client, - this, - eventVerifier = { true }, - refreshCache = cache, - ) - val states = CopyOnWriteArrayList() - val collection = collect(purpleQuartz, source, states) - - val cached = awaitState(states) { it.sync == QuerySync.Cached } - assertTrue(cached.relays.isEmpty()) - assertTrue(client.requests.isEmpty()) - - cache.recordRefresh(fingerprint, System.currentTimeMillis()) - delay(700) - cache.recordRefresh(fingerprint, System.currentTimeMillis()) - delay(500) - assertTrue(client.requests.isEmpty()) - - client.awaitSubscription() - assertEquals(1, client.requests.size) - - collection.cancel() - collection.join() - purpleQuartz.close() - } - - @Test - fun `timeout and persistence failure do not populate freshness`() = runBlocking { - val filter = Filter(kinds = listOf(1)) - val source = QuerySource.LocalAndRemote( - relays = setOf(relay), - mode = RemoteMode.OneShot(50.milliseconds), - cachedFor = 1.hours, - ) - val fingerprint = QueryFingerprint.create(listOf(filter), setOf(relay)) - - val timeoutCache = InMemoryQueryRefreshCache() - val timeoutClient = ControlledClient() - val timeoutQuartz = Iolite.createForTesting( - ControlledEventStore(), - timeoutClient, - this, - config = IoliteConfig(oneShotTimeout = 50.milliseconds), - eventVerifier = { true }, - refreshCache = timeoutCache, - ) - val timeoutStates = CopyOnWriteArrayList() - val timeoutCollection = collect(timeoutQuartz, source, timeoutStates) - timeoutClient.awaitSubscription() - awaitState(timeoutStates) { it.sync == QuerySync.TimedOut } - withTimeout(2.seconds) { timeoutCollection.join() } - assertEquals(null, timeoutCache.lastRefresh(fingerprint)) - timeoutQuartz.close() - - val failureCache = InMemoryQueryRefreshCache() - val failureClient = ControlledClient() - val failureQuartz = Iolite.createForTesting( - ControlledEventStore(failInserts = true), - failureClient, - this, - eventVerifier = { true }, - refreshCache = failureCache, - ) - val failureStates = CopyOnWriteArrayList() - val failureCollection = collect(failureQuartz, source, failureStates) - failureClient.awaitSubscription() - failureClient.started(relay) - failureClient.event(relay, signedEvent(content = "failed-refresh")) - awaitState(failureStates) { it.sync == QuerySync.Failed } - withTimeout(2.seconds) { failureCollection.join() } - assertEquals(null, failureCache.lastRefresh(fingerprint)) - failureQuartz.close() - } - - @Test - fun `failed final local projection fails query without caching`() = runBlocking { - val cache = InMemoryQueryRefreshCache() - val client = ControlledClient() - val filter = Filter(kinds = listOf(1)) - val source = QuerySource.LocalAndRemote( - relays = setOf(relay), - mode = RemoteMode.OneShot(5.seconds), - cachedFor = 1.hours, - ) - val fingerprint = QueryFingerprint.create(listOf(filter), setOf(relay)) - val purpleQuartz = Iolite.createForTesting( - ControlledEventStore(failQueriesAfter = 1), - client, - this, - eventVerifier = { true }, - refreshCache = cache, - ) - val states = CopyOnWriteArrayList() - val collection = collect(purpleQuartz, source, states) - - client.awaitSubscription() - client.started(relay) - client.eose(relay) - - val failed = awaitState(states) { it.sync == QuerySync.Failed } - assertTrue(failed.error is QueryError.UnsupportedLocalProjection) - assertEquals(null, cache.lastRefresh(fingerprint)) - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `reconnect advances generation and accepts replacement EOSE`() = runBlocking { - val client = ControlledClient() - val purpleQuartz = Iolite.createForTesting( - ControlledEventStore(), - client, - this, - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val collection = collect(purpleQuartz, QuerySource.Remote(setOf(relay)), states) - - client.awaitSubscription() - client.started(relay) - client.eose(relay) - awaitState(states) { it.sync == QuerySync.Live && it.relays[relay]?.generation == 1L } - - client.disconnected(relay) - client.connecting(relay) - client.started(relay) - client.eose(relay) - - val reconnected = awaitState(states) { - it.sync == QuerySync.Live && it.relays[relay]?.generation == 2L - } - assertEquals(RelayConnectionState.Connected, reconnected.relays.getValue(relay).connection) - assertTrue(reconnected.relays.getValue(relay).eose) - - collection.cancel() - collection.join() - purpleQuartz.close() - } - - @Test - fun `local and remote stream preserves items through disconnect and reconnect`() = runBlocking { - val client = ControlledClient() - val purpleQuartz = Iolite.createForTesting( - ControlledEventStore(), - client, - this, - eventVerifier = { true }, - ) - val options = QueryOptions.localAndRemote(setOf(relay), RemoteMode.Stream) - val states = CopyOnWriteArrayList() - val collection = launch { - purpleQuartz.query(Filter(kinds = listOf(1)), options).collect(states::add) - } - - client.awaitSubscription() - client.started(relay) - val event = signedEvent(content = "offline-survivor") - client.event(relay, event) - client.eose(relay) - awaitState(states) { it.phase == QueryPhase.Live && it.items.any { item -> item.id == event.id } } - - client.disconnected(relay) - val disconnected = awaitState(states) { - it.phase == QueryPhase.Connecting && it.items.any { item -> item.id == event.id } - } - assertEquals(listOf(event.id), disconnected.items.map(Event::id)) - - client.connecting(relay) - client.started(relay) - client.eose(relay) - val reconnected = awaitState(states) { - it.phase == QueryPhase.Live && it.relays[relay]?.generation == 2L - } - assertEquals(listOf(event.id), reconnected.items.map(Event::id)) - - collection.cancel() - collection.join() - purpleQuartz.close() - } - - @Test - fun `facade close emits lifecycle failure and completes active collectors`() = runBlocking { - val client = ControlledClient() - val store = ControlledEventStore() - val purpleQuartz = Iolite.createForTesting(store, client, this, eventVerifier = { true }) - val states = CopyOnWriteArrayList() - val collection = collect(purpleQuartz, QuerySource.Remote(setOf(relay)), states) - - client.awaitSubscription() - awaitState(states) { it.sync == QuerySync.Connecting } - purpleQuartz.close() - - withTimeout(2.seconds) { collection.join() } - assertTrue(states.last().error is QueryError.Lifecycle) - assertEquals(QuerySync.Failed, states.last().sync) - assertEquals(1, client.unsubscribeCount.get()) - assertEquals(1, client.closeCount.get()) - assertEquals(1, store.closeCount.get()) - - purpleQuartz.close() - } - - @Test - fun `local first completion waits for commit and final projection`() = runBlocking { - val client = ControlledClient() - val store = ControlledEventStore(blockInserts = true) - val purpleQuartz = Iolite.createForTesting(store, client, this, eventVerifier = { true }) - val states = CopyOnWriteArrayList() - val source = QuerySource.LocalAndRemote(setOf(relay), RemoteMode.OneShot(5.seconds)) - val collection = collect(purpleQuartz, source, states) - val event = signedEvent(content = "local-first") - - client.awaitSubscription() - client.started(relay) - client.event(relay, event) - client.eose(relay) - withTimeout(2.seconds) { store.insertionStarted.await() } - - assertFalse(states.any { state -> state.items.any { it.id == event.id } }) - assertFalse(states.any { it.sync == QuerySync.Complete }) - - store.releaseInsert.complete(Unit) - val complete = awaitState(states) { it.sync == QuerySync.Complete } - assertTrue(complete.items.any { it.id == event.id }) - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `timeout unsubscribes immediately then drains accepted persistence`() = runBlocking { - val client = ControlledClient() - val store = ControlledEventStore(blockInserts = true) - val config = IoliteConfig(oneShotTimeout = 100.milliseconds) - val purpleQuartz = Iolite.createForTesting(store, client, this, config, eventVerifier = { true }) - val states = CopyOnWriteArrayList() - val source = QuerySource.Remote(setOf(relay), RemoteMode.OneShot()) - val collection = collect(purpleQuartz, source, states) - val event = signedEvent(content = "timeout") - - client.awaitSubscription() - client.started(relay) - client.event(relay, event) - withTimeout(2.seconds) { store.insertionStarted.await() } - client.eose(relay) - withTimeout(2.seconds) { - while (client.unsubscribeCount.get() == 0) delay(10) - } - - assertFalse(states.any { it.sync == QuerySync.TimedOut }) - assertFalse(states.any { it.sync == QuerySync.Complete }) - assertTrue(states.any { state -> state.items.any { it.id == event.id } }) - - store.releaseInsert.complete(Unit) - val timedOut = awaitState(states) { it.sync == QuerySync.TimedOut } - assertTrue(timedOut.items.any { it.id == event.id }) - assertFalse(states.any { it.sync == QuerySync.Complete }) - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `bounded ingestion fails closed instead of dropping EOSE`() = runBlocking { - val client = ControlledClient() - val store = ControlledEventStore(blockInserts = true) - val config = IoliteConfig(ingestionCapacity = 1, ingestBatchSize = 1) - val purpleQuartz = Iolite.createForTesting(store, client, this, config, eventVerifier = { true }) - val states = CopyOnWriteArrayList() - val source = QuerySource.Remote(setOf(relay), RemoteMode.OneShot(5.seconds)) - val collection = collect(purpleQuartz, source, states) - - client.awaitSubscription() - client.started(relay) - client.event(relay, signedEvent(content = "first")) - withTimeout(2.seconds) { store.insertionStarted.await() } - client.event(relay, signedEvent(content = "queued")) - client.eose(relay) - - val failed = awaitState(states) { it.sync == QuerySync.Failed } - assertTrue(failed.error is QueryError.IngestionSaturated) - assertFalse(states.any { it.sync == QuerySync.Complete || it.sync == QuerySync.Live }) - assertEquals(1, client.unsubscribeCount.get()) - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `remote persistence failure keeps directly emitted item visible`() = runBlocking { - val client = ControlledClient() - val store = ControlledEventStore(failInserts = true) - val purpleQuartz = Iolite.createForTesting(store, client, this, eventVerifier = { true }) - val states = CopyOnWriteArrayList() - val collection = collect(purpleQuartz, QuerySource.Remote(setOf(relay)), states) - val event = signedEvent(content = "visible-before-save") - - client.awaitSubscription() - client.started(relay) - client.event(relay, event) - - val failed = awaitState(states) { it.sync == QuerySync.Failed } - assertTrue(failed.error is QueryError.PersistenceFailure) - assertTrue(failed.items.any { it.id == event.id }) - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `prune rules delete events beyond max age and keep the rest`() = runBlocking { - val store = ControlledEventStore() - val nowMillis = 1_800_000_000_000L - val clock = MutableEpochClock(nowMillis) - val purpleQuartz = Iolite.createForTesting( - store, - ControlledClient(), - this, - config = IoliteConfig( - expirationSweepInterval = 50.milliseconds, - pruneRules = mapOf(9735 to 1.hours), - ), - eventVerifier = { true }, - clock = clock, - ) - val nowSeconds = nowMillis / 1_000 - val old = signedEvent(content = "old", kind = 9735, createdAt = nowSeconds - 2.hours.inWholeSeconds) - val recent = signedEvent(content = "recent", kind = 9735, createdAt = nowSeconds) - val otherKind = signedEvent(content = "other", kind = 1, createdAt = nowSeconds - 2.hours.inWholeSeconds) - store.insert(old) - store.insert(recent) - store.insert(otherKind) - - withTimeout(2.seconds) { - while (store.query(Filter(kinds = listOf(9735))).size > 1) delay(10) - } - - val remaining = store.query(Filter()) - assertEquals(setOf(recent.id, otherKind.id), remaining.map { it.id }.toSet()) - purpleQuartz.close() - } - - @Test - fun `relay CLOSED exposes closed state and fails the query`() = runBlocking { - val client = ControlledClient() - val purpleQuartz = Iolite.createForTesting( - ControlledEventStore(), - client, - this, - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val collection = collect(purpleQuartz, QuerySource.Remote(setOf(relay)), states) - - client.awaitSubscription() - client.started(relay) - client.closed(relay) - - val failed = awaitState(states) { it.sync == QuerySync.Failed } - assertTrue(failed.error is QueryError.RelayFailure) - assertEquals(RelayConnectionState.Closed, failed.relays.getValue(relay).connection) - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `one closed relay does not terminate a healthy multi-relay stream`() = runBlocking { - val otherRelay = "wss://other.example".normalizeRelayUrl() - val client = ControlledClient() - val purpleQuartz = Iolite.createForTesting( - ControlledEventStore(), - client, - this, - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val options = QueryOptions.remote(setOf(relay, otherRelay), RemoteMode.Stream) - val collection = launch { - purpleQuartz.query(Filter(kinds = listOf(1)), options).collect(states::add) - } - - client.awaitSubscription() - client.started(relay) - client.started(otherRelay) - client.closed(relay) - client.eose(otherRelay) - - val live = awaitState(states) { it.phase == QueryPhase.Live } - assertEquals(RelayConnectionState.Closed, live.relays.getValue(relay).connection) - assertEquals(RelayConnectionState.Connected, live.relays.getValue(otherRelay).connection) - assertTrue(collection.isActive) - - collection.cancel() - collection.join() - purpleQuartz.close() - } - - @Test - fun `subscription startup failure is classified as incompatible dependency`() = runBlocking { - val client = ControlledClient(failSubscribe = true) - val purpleQuartz = Iolite.createForTesting( - ControlledEventStore(), - client, - this, - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val collection = collect(purpleQuartz, QuerySource.Remote(setOf(relay)), states) - - val failed = awaitState(states) { it.sync == QuerySync.Failed } - assertTrue(failed.error is QueryError.IncompatibleDependency) - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - @Test - fun `stream query emits every ingested event at eose`() = runBlocking { - val client = ControlledClient() - val purpleQuartz = Iolite.createForTesting( - ControlledEventStore(), - client, - this, - eventVerifier = { true }, - ) - val states = CopyOnWriteArrayList() - val collection = collect( - purpleQuartz, - QuerySource.LocalAndRemote(setOf(relay), RemoteMode.Stream), - states, - ) - - client.awaitSubscription() - client.started(relay) - val events = (1..5).map { signedEvent(content = "note-$it", createdAt = 1_750_000_000L + it) } - events.forEach { client.event(relay, it) } - client.eose(relay) - - val live = awaitState(states) { it.sync == QuerySync.Live } - assertEquals(events.map { it.id }.toSet(), live.items.map { it.id }.toSet()) - - collection.cancel() - withTimeout(2.seconds) { collection.join() } - purpleQuartz.close() - } - - private fun kotlinx.coroutines.CoroutineScope.collect( - purpleQuartz: Iolite, - source: QuerySource, - states: MutableList, - ): Job = launch { - purpleQuartz.query(Filter(kinds = listOf(1)), source).collect(states::add) - } - - private suspend fun awaitState( - states: List, - predicate: (QueryState) -> Boolean, - ): QueryState = withTimeout(5.seconds) { - while (true) { - states.lastOrNull(predicate)?.let { return@withTimeout it } - delay(10) - } - error("unreachable") - } - - private fun signedEvent( - content: String, - kind: Int = 1, - createdAt: Long = 1_750_000_000, - ): Event { - val id = eventSequence.incrementAndGet().toString(16).padStart(64, '0') - return Event( - id = id, - pubKey = "1".repeat(64), - createdAt = createdAt, - kind = kind, - tags = emptyArray(), - content = content, - sig = "2".repeat(128), - ) - } -} - -private class ControlledClient( - private val failSubscribe: Boolean = false, -) : INostrClient by EmptyNostrClient() { - val requests = CopyOnWriteArrayList>>() - val unsubscribeCount = AtomicInteger() - val closeCount = AtomicInteger() - private val connectionListeners = CopyOnWriteArrayList() - - @Volatile - private var subscriptionListener: SubscriptionListener? = null - - override fun subscribe( - subId: String, - filters: Map>, - listener: SubscriptionListener?, - ) { - if (failSubscribe) throw IllegalStateException("controlled subscription failure") - requests += filters - subscriptionListener = listener - } - - override fun unsubscribe(subId: String) { - unsubscribeCount.incrementAndGet() - } - - override fun addConnectionListener(listener: RelayConnectionListener) { - connectionListeners += listener - } - - override fun removeConnectionListener(listener: RelayConnectionListener) { - connectionListeners -= listener - } - - override fun close() { - closeCount.incrementAndGet() - } - - suspend fun awaitSubscription(expectedCount: Int = 1) { - withTimeout(2.seconds) { - while (subscriptionListener == null || requests.size < expectedCount) delay(10) - } - } - - fun started(relay: NormalizedRelayUrl) { - subscriptionListener?.onSubscriptionStarted(relay.url, requests.last().getValue(relay)) - } - - fun event(relay: NormalizedRelayUrl, event: Event) { - subscriptionListener?.onEvent(event, false, relay, requests.last().getValue(relay)) - } - - fun eose(relay: NormalizedRelayUrl) { - subscriptionListener?.onEose(relay, requests.last().getValue(relay)) - } - - fun closed(relay: NormalizedRelayUrl) { - subscriptionListener?.onClosed("controlled close", relay, requests.last().getValue(relay)) - } - - fun connecting(relay: NormalizedRelayUrl) { - val client = TestRelayClient(relay) - connectionListeners.forEach { it.onConnecting(client) } - } - - fun disconnected(relay: NormalizedRelayUrl) { - val client = TestRelayClient(relay) - connectionListeners.forEach { it.onDisconnected(client) } - } -} - -private class MutableEpochClock( - private var current: Long, -) : EpochMillisClock { - override fun now(): Long = current - - fun advance(milliseconds: Long) { - current += milliseconds - } -} - -private class TestRelayClient( - override val url: NormalizedRelayUrl, -) : IRelayClient { - override fun connect() = Unit - override fun needsToReconnect(): Boolean = false - override fun connectAndSyncFiltersIfDisconnected(ignoreRetryDelays: Boolean) = Unit - override fun isConnected(): Boolean = true - override fun sendOrConnectAndSync(cmd: Command) = Unit - override fun sendIfConnected(cmd: Command) = Unit - override fun disconnect() = Unit -} - -private class ControlledEventStore( - private val blockInserts: Boolean = false, - private val failInserts: Boolean = false, - private val failQueriesAfter: Int? = null, -) : IEventStore { - override val relay: NormalizedRelayUrl? = null - val insertionStarted = CompletableDeferred() - val releaseInsert = CompletableDeferred() - val closeCount = AtomicInteger() - private val queryCount = AtomicInteger() - private val events = mutableListOf() - - override suspend fun insert(event: Event) { - insertionStarted.complete(Unit) - if (blockInserts) releaseInsert.await() - if (failInserts) throw IllegalStateException("controlled insert failure") - synchronized(events) { - if (events.none { it.id == event.id }) events += event - } - } - - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { - body(object : IEventStore.ITransaction { - override fun insert(event: Event) { - synchronized(events) { - if (events.none { it.id == event.id }) events += event - } - } - }) - } - - @Suppress("UNCHECKED_CAST") - override suspend fun query(filter: Filter): List { - beforeQuery() - return synchronized(events) { events.filter(filter::match).map { it as T } } - } - - @Suppress("UNCHECKED_CAST") - override suspend fun query(filters: List): List { - beforeQuery() - return synchronized(events) { - events.filter { event -> filters.any { it.match(event) } }.map { it as T } - } - } - - override suspend fun query(filter: Filter, onEach: (T) -> Unit) { - query(filter).forEach(onEach) - } - - override suspend fun query(filters: List, onEach: (T) -> Unit) { - query(filters).forEach(onEach) - } - - override suspend fun count(filter: Filter): Int = query(filter).size - override suspend fun count(filters: List): Int = query(filters).size - - override suspend fun delete(filter: Filter) { - synchronized(events) { events.removeAll(filter::match) } - } - - override suspend fun delete(filters: List) { - synchronized(events) { - events.removeAll { event -> filters.any { it.match(event) } } - } - } - - override suspend fun deleteExpiredEvents() = Unit - override suspend fun reindexFullTextSearch() = Unit - - override suspend fun reindexFullTextSearch( - resumeFrom: String?, - batchSize: Int, - ): FtsReindexProgress = throw UnsupportedOperationException() - - override fun close() { - closeCount.incrementAndGet() - } - - private fun beforeQuery() { - val allowed = failQueriesAfter ?: return - if (queryCount.incrementAndGet() > allowed) { - throw IllegalStateException("controlled query failure") - } - } -} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/QueryRefreshCacheTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/QueryRefreshCacheTest.kt deleted file mode 100644 index 7e9c41a..0000000 --- a/iolite/src/test/kotlin/dev/zapstore/iolite/QueryRefreshCacheTest.kt +++ /dev/null @@ -1,72 +0,0 @@ -package dev.zapstore.iolite - -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import kotlin.time.Duration.Companion.seconds -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertNotEquals -import org.junit.Assert.assertTrue -import org.junit.Test - -class QueryRefreshCacheTest { - @Test - fun `fingerprint canonicalizes filter and relay ordering`() { - val relayOne = "wss://one.example".normalizeRelayUrl() - val relayTwo = "wss://two.example".normalizeRelayUrl() - val idOne = "a".repeat(64) - val idTwo = "b".repeat(64) - val first = listOf( - Filter( - ids = listOf(idOne, idTwo), - kinds = listOf(1, 3), - tags = linkedMapOf("t" to listOf("one", "two"), "x" to listOf("value")), - ), - Filter(search = "profile", limit = 1), - ) - val reordered = listOf( - Filter(search = "profile", limit = 1), - Filter( - ids = listOf(idOne, idTwo), - kinds = listOf(1, 3), - tags = linkedMapOf("x" to listOf("value"), "t" to listOf("one", "two")), - ), - ) - - assertEquals( - QueryFingerprint.create(first, linkedSetOf(relayOne, relayTwo)), - QueryFingerprint.create(reordered, linkedSetOf(relayTwo, relayOne)), - ) - } - - @Test - fun `fingerprint isolates filters and relay sets`() { - val relayOne = "wss://one.example".normalizeRelayUrl() - val relayTwo = "wss://two.example".normalizeRelayUrl() - val authorOne = "a".repeat(64) - val authorTwo = "b".repeat(64) - val profile = listOf(Filter(authors = listOf(authorOne), kinds = listOf(0))) - - val baseline = QueryFingerprint.create(profile, setOf(relayOne)) - - assertNotEquals( - baseline, - QueryFingerprint.create(listOf(Filter(authors = listOf(authorTwo), kinds = listOf(0))), setOf(relayOne)), - ) - assertNotEquals(baseline, QueryFingerprint.create(profile, setOf(relayTwo))) - } - - @Test - fun `freshness expires and treats a backwards clock as stale`() { - val cache = InMemoryQueryRefreshCache() - assertTrue(cache.recordRefresh("profile", 1_000)) - - val fresh = cache.freshness("profile", 60.seconds, 31_000) - assertTrue(fresh.isFresh) - assertEquals(30_000, fresh.remainingMillis) - - assertFalse(cache.freshness("profile", 60.seconds, 61_000).isFresh) - assertFalse(cache.freshness("profile", 60.seconds, 999).isFresh) - assertFalse(cache.freshness("missing", 60.seconds, 1_000).isFresh) - } -} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/QueryWiringTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/QueryWiringTest.kt deleted file mode 100644 index 98175c0..0000000 --- a/iolite/src/test/kotlin/dev/zapstore/iolite/QueryWiringTest.kt +++ /dev/null @@ -1,228 +0,0 @@ -package dev.zapstore.iolite - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.client.EmptyNostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient -import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl -import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress -import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import java.util.concurrent.CopyOnWriteArrayList -import kotlinx.coroutines.cancelAndJoin -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.take -import kotlinx.coroutines.flow.toList -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeout -import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue -import org.junit.Test - -class QueryWiringTest { - @Test - fun `local query emits an empty local seed without subscribing`() = runBlocking { - val client = RecordingClient() - val purpleQuartz = Iolite.createForTesting(MemoryEventStore(), client, this) - - val states = purpleQuartz.query(Filter(kinds = listOf(1))).take(1).toList() - - assertEquals(listOf(QueryPhase.LocalOnly), states.map { it.phase }) - assertTrue(states.single().items.isEmpty()) - assertTrue(client.requests.isEmpty()) - purpleQuartz.close() - } - - @Test - fun `remote request uses exactly the supplied relay set`() = runBlocking { - val client = RecordingClient() - val purpleQuartz = Iolite.createForTesting(MemoryEventStore(), client, this) - val relays = setOf( - "wss://one.example".normalizeRelayUrl(), - "wss://two.example".normalizeRelayUrl(), - ) - - val collection = launch { - purpleQuartz.query( - Filter(kinds = listOf(1)), - QueryOptions.remote(relays, RemoteMode.Stream), - ).collect { } - } - withTimeout(1_000) { - while (client.requests.isEmpty()) delay(10) - } - - assertEquals(1, client.requests.size) - assertEquals(relays, client.requests.single().keys) - assertTrue(client.requests.single().values.all { it.single().kinds == listOf(1) }) - - collection.cancelAndJoin() - purpleQuartz.close() - } - - @Test - fun `foreground refresh bypasses relay retry delays`() = runBlocking { - val client = RecordingClient() - val purpleQuartz = Iolite.createForTesting(MemoryEventStore(), client, this) - - purpleQuartz.refreshConnections() - - assertEquals(listOf(ReconnectCall(true, true)), client.reconnectCalls) - purpleQuartz.close() - } - - @Test - fun `relay traffic toggle disconnects and reconnects the client`() = runBlocking { - val client = RecordingClient() - val purpleQuartz = Iolite.createForTesting(MemoryEventStore(), client, this) - - purpleQuartz.setRelayTrafficEnabled(false) - purpleQuartz.setRelayTrafficEnabled(true) - - assertEquals(listOf(false, true), client.traffic) - assertEquals(listOf(ReconnectCall(true, true)), client.reconnectCalls) - purpleQuartz.close() - } - - @Test - fun `local and remote query seeds persisted rows while relay traffic is suspended`() = runBlocking { - val client = RecordingClient() - val store = MemoryEventStore() - val event = testEvent("offline") - store.insert(event) - val purpleQuartz = Iolite.createForTesting(store, client, this) - val relay = "wss://offline.example".normalizeRelayUrl() - purpleQuartz.setRelayTrafficEnabled(false) - val states = CopyOnWriteArrayList() - val collection = launch { - purpleQuartz.query( - Filter(kinds = listOf(1)), - QueryOptions.localAndRemote(setOf(relay), RemoteMode.Stream), - ).collect(states::add) - } - val state = withTimeout(1_000) { - while (states.isEmpty()) delay(10) - states.first() - } - - assertEquals(QueryPhase.Connecting, state.phase) - assertEquals(listOf(event.id), state.items.map(Event::id)) - assertEquals(false, client.isActive()) - assertTrue(client.requests.isEmpty()) - - purpleQuartz.setRelayTrafficEnabled(true) - withTimeout(1_000) { - while (client.requests.isEmpty()) delay(10) - } - assertEquals(1, client.requests.size) - - collection.cancelAndJoin() - purpleQuartz.close() - } - -} - -private class RecordingClient : INostrClient by EmptyNostrClient() { - val requests = mutableListOf>>() - val reconnectCalls = mutableListOf() - val traffic = mutableListOf() - private var active = true - - override fun subscribe( - subId: String, - filters: Map>, - listener: SubscriptionListener?, - ) { - requests += filters - } - - override fun reconnect(onlyIfChanged: Boolean, ignoreRetryDelays: Boolean) { - reconnectCalls += ReconnectCall(onlyIfChanged, ignoreRetryDelays) - } - - override fun connect() { - active = true - traffic += true - } - - override fun disconnect() { - active = false - traffic += false - } - - override fun isActive() = active -} - -private data class ReconnectCall( - val onlyIfChanged: Boolean, - val ignoreRetryDelays: Boolean, -) - -private fun testEvent(content: String): Event = Event( - id = content.padEnd(64, '0'), - pubKey = "1".repeat(64), - createdAt = 1_750_000_000, - kind = 1, - tags = emptyArray(), - content = content, - sig = "2".repeat(128), -) - -private class MemoryEventStore : IEventStore { - override val relay: NormalizedRelayUrl? = null - private val events = mutableListOf() - - override suspend fun insert(event: Event) { - events += event - } - - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { - body(object : IEventStore.ITransaction { - override fun insert(event: Event) { - events += event - } - }) - } - - @Suppress("UNCHECKED_CAST") - override suspend fun query(filter: Filter): List = - events.filter(filter::match).map { it as T } - - @Suppress("UNCHECKED_CAST") - override suspend fun query(filters: List): List = - events.filter { event -> filters.any { it.match(event) } }.map { it as T } - - override suspend fun query(filter: Filter, onEach: (T) -> Unit) { - query(filter).forEach(onEach) - } - - override suspend fun query(filters: List, onEach: (T) -> Unit) { - query(filters).forEach(onEach) - } - - override suspend fun count(filter: Filter): Int = query(filter).size - - override suspend fun count(filters: List): Int = query(filters).size - - override suspend fun delete(filter: Filter) { - events.removeAll(filter::match) - } - - override suspend fun delete(filters: List) { - events.removeAll { event -> filters.any { it.match(event) } } - } - - override suspend fun deleteExpiredEvents() = Unit - - override suspend fun reindexFullTextSearch() = Unit - - override suspend fun reindexFullTextSearch( - resumeFrom: String?, - batchSize: Int, - ): FtsReindexProgress = throw UnsupportedOperationException() - - override fun close() = Unit -} diff --git a/iolite/src/test/kotlin/dev/zapstore/iolite/RelayOutboxTest.kt b/iolite/src/test/kotlin/dev/zapstore/iolite/RelayOutboxTest.kt new file mode 100644 index 0000000..9ca365d --- /dev/null +++ b/iolite/src/test/kotlin/dev/zapstore/iolite/RelayOutboxTest.kt @@ -0,0 +1,252 @@ +package dev.zapstore.iolite + +import java.io.File +import java.util.concurrent.CopyOnWriteArrayList +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.take +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.job +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.json.JSONArray +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class RelayOutboxTest { + @Test + fun localSignerRoundTripAndNip42Auth() = runBlocking { + val signer = LocalSigner(SECRET) + val event = signer.sign(1_700_000_000, 1, emptyList(), "hello") + assertTrue(event.verify()) + assertEquals(signer.publicKey, event.pubkey) + + val auth = Nip42.auth(signer, "wss://relay.example".normalizeRelayUrl(), "challenge-1", 1_700_000_001) + assertEquals(Kinds.Auth, auth.kind) + assertEquals("challenge-1", auth.tagValue("challenge")) + assertEquals("wss://relay.example", auth.tagValue("relay")) + assertTrue(auth.verify()) + assertTrue(Nip42.wire(auth).startsWith("""["AUTH",""")) + } + + @Test + fun nip55SignerAcceptsVerifiedCallback() = runBlocking { + val local = LocalSigner(SECRET) + val nip55 = Nip55Signer(local.publicKey) { unsigned -> + val json = org.json.JSONObject(unsigned) + local.sign( + json.getLong("created_at"), + json.getInt("kind"), + json.getJSONArray("tags").toStringLists(), + json.getString("content"), + ).toJson() + } + val signed = nip55.sign(10, 1, listOf(listOf("t", "zap")), "hi") + assertTrue(signed.verify()) + assertEquals(local.publicKey, signed.pubkey) + } + + @Test + fun queryLocalAndRemoteIngestsProfileAfterEoseGrace() = runBlocking { + val network = MemoryNetwork() + val signer = LocalSigner(SECRET) + val profile = signer.sign(1_700_000_000, Kinds.Profile, emptyList(), """{"name":"Ada"}""") + val relay = "ws://relay.test".normalizeRelayUrl() + withIolite(webSocket = network.factory(), writeRelays = setOf(relay), signer = signer) { iolite -> + val states = async { + withTimeout(5_000) { + iolite.query( + Query.profile(signer.publicKey), + QueryOptions.localAndRemote(setOf(relay), RemoteMode.OneShot(2.seconds)), + ).first { state -> state.phase == QueryPhase.Complete && state.items?.name == "Ada" } + } + } + network.awaitOpen(relay.url) + val req = network.relays.getValue(relay.url).sent.first { it.startsWith("""["REQ"""") } + val subId = JSONArray(req).getString(1) + network.push(relay.url, JSONArray().put("EVENT").put(subId).put(profile.toJsonObject()).toString()) + network.push(relay.url, JSONArray().put("EOSE").put(subId).toString()) + val complete = states.await() + assertEquals(QueryPhase.Complete, complete.phase) + assertEquals("Ada", complete.items!!.name) + } + } + + @Test + fun durableOutboxDeletesAfterOkThreshold() = runBlocking { + val network = MemoryNetwork() + val signer = LocalSigner(SECRET) + val one = "ws://one.test".normalizeRelayUrl() + val two = "ws://two.test".normalizeRelayUrl() + val three = "ws://three.test".normalizeRelayUrl() + withIolite(webSocket = network.factory(), writeRelays = setOf(one, two, three), signer = signer) { iolite -> + val event = iolite.publish(1, emptyList(), "note", signer, relays = setOf(one, two, three)) + assertEquals(1, iolite.pendingOutbox().size) + network.awaitOpen(one.url) + network.awaitOpen(two.url) + fun ok(url: RelayUrl) { + network.push(url.url, JSONArray().put("OK").put(event.id).put(true).put("").toString()) + } + ok(one) + assertEquals(1, iolite.pendingOutbox().size) + ok(two) + assertTrue(iolite.pendingOutbox().isEmpty()) + } + } + + @Test + fun oneOrTwoRelaysNeedSingleOk() = runBlocking { + val network = MemoryNetwork() + val signer = LocalSigner(SECRET) + val relay = "ws://one.test".normalizeRelayUrl() + withIolite(webSocket = network.factory(), writeRelays = setOf(relay), signer = signer) { iolite -> + val event = iolite.publish(1, emptyList(), "note", signer) + network.awaitOpen(relay.url) + network.push(relay.url, JSONArray().put("OK").put(event.id).put(true).put("").toString()) + assertTrue(iolite.pendingOutbox().isEmpty()) + } + } + + @Test + fun authChallengeIsNotOutboxed() = runBlocking { + val network = MemoryNetwork() + val signer = LocalSigner(SECRET) + val relay = "ws://auth.test".normalizeRelayUrl() + withIolite(webSocket = network.factory(), writeRelays = setOf(relay), signer = signer) { iolite -> + val collected = async { + iolite.query( + Query.profile(signer.publicKey), + QueryOptions.localAndRemote(setOf(relay), RemoteMode.OneShot(2.seconds)), + ).take(3).toList() + } + network.awaitOpen(relay.url) + network.push(relay.url, JSONArray().put("AUTH").put("abc").toString()) + val auth = withTimeout(2_000) { + while (true) { + val found = network.relays.getValue(relay.url).sent.firstOrNull { it.startsWith("""["AUTH"""") } + if (found != null) return@withTimeout found + kotlinx.coroutines.delay(5) + } + error("unreachable") + } + val event = Event.parse(JSONArray(auth).getJSONObject(1)) + assertEquals(Kinds.Auth, event.kind) + assertTrue(iolite.pendingOutbox().none { it.kind == Kinds.Auth }) + collected.cancel() + } + } + + @Test + fun cachedQuerySkipsRelay() = runBlocking { + val network = MemoryNetwork() + val signer = LocalSigner(SECRET) + val profile = signer.sign(1_700_000_000, Kinds.Profile, emptyList(), """{"name":"Ada"}""") + val relay = "ws://cache.test".normalizeRelayUrl() + var now = 1_000L + withIolite( + webSocket = network.factory(), + writeRelays = setOf(relay), + signer = signer, + nowMillis = { now }, + config = IoliteConfig(eoseGrace = 5.milliseconds, oneShotTimeout = 2.seconds), + ) { iolite -> + val first = async { + iolite.query( + Query.profile(signer.publicKey), + QueryOptions.localAndRemote(setOf(relay), RemoteMode.OneShot(2.seconds), cachedFor = 30.seconds), + ).first { it.phase == QueryPhase.Complete } + } + network.awaitOpen(relay.url) + val req = network.relays.getValue(relay.url).sent.first { it.startsWith("""["REQ"""") } + val subId = JSONArray(req).getString(1) + network.push(relay.url, JSONArray().put("EVENT").put(subId).put(profile.toJsonObject()).toString()) + network.push(relay.url, JSONArray().put("EOSE").put(subId).toString()) + first.await() + val sentAfter = network.relays.getValue(relay.url).sent.size + now = 2_000L + val cached = iolite.query( + Query.profile(signer.publicKey), + QueryOptions.localAndRemote(setOf(relay), RemoteMode.OneShot(2.seconds), cachedFor = 30.seconds), + ).first { it.phase == QueryPhase.Cached } + assertEquals("Ada", cached.items!!.name) + assertEquals(sentAfter, network.relays.getValue(relay.url).sent.size) + } + } + + private suspend fun withIolite( + webSocket: WebSocketFactory, + writeRelays: Set, + signer: Signer, + nowMillis: () -> Long = { System.currentTimeMillis() }, + config: IoliteConfig = IoliteConfig(eoseGrace = 5.milliseconds, oneShotTimeout = 2.seconds), + block: suspend (Iolite) -> Unit, + ) { + val dir = File("build/tmp/iolite-relay-${System.nanoTime()}").apply { mkdirs() } + val scope = CoroutineScope(SupervisorJob()) + val iolite = Iolite.createForTesting( + databasePath = File(dir, "iolite.db").absolutePath, + parentScope = scope, + config = config, + webSocket = webSocket, + signer = signer, + writeRelays = writeRelays, + nowMillis = nowMillis, + ) + try { + block(iolite) + } finally { + iolite.close() + scope.coroutineContext.job.cancel() + } + } + + companion object { + private val SECRET = Hex.decode("0000000000000000000000000000000000000000000000000000000000000001") + } +} + +internal class MemoryNetwork { + val relays = mutableMapOf() + + fun factory(): WebSocketFactory = WebSocketFactory { url, listener -> + relays.getOrPut(url) { MemoryRelay(url) }.connect(listener) + } + + fun push(url: String, text: String) { + relays.getValue(url).listener?.onMessage(text) + } + + suspend fun awaitOpen(url: String) { + withTimeout(2_000) { + while (relays[url]?.listener == null) { + kotlinx.coroutines.delay(5) + } + } + } +} + +internal class MemoryRelay(val url: String) { + val sent = CopyOnWriteArrayList() + @Volatile var listener: WebSocketListener? = null + + fun connect(listener: WebSocketListener): WebSocketSession { + this.listener = listener + listener.onOpen() + return object : WebSocketSession { + override fun send(text: String) { + sent += text + } + + override fun close() { + listener.onClosing(1000, "") + } + } + } +} diff --git a/iolite/src/test/resources/golden/from-0-to-1.tar.zst b/iolite/src/test/resources/golden/from-0-to-1.tar.zst new file mode 100644 index 0000000000000000000000000000000000000000..07a570f3a07503d037d198b9f4ce3de2e1158d8a GIT binary patch literal 1049 zcmV+!1m^oFwJ-f-01;gv07_I?4I&_I!w$22xY1LyW`kSF7%vzA$iigJWo4TiyWqee z00O`@*#N=-!2qttsPN~}Q+yKF-Lh*!M$x1}h<~vZIY5q)AE)=QLU)X_8>>!;(YRi;A*Pb8Gtz2FyiX%%6hBBs*+YJktwmRMtyMBta4=9LPaPf;1N1STL9z2M&-cV_FQan}fF{x>YEDEwpH7$RcVv&6T*XXvb!k?s(|k(5raHf={0utDw@V@i z@r7a+$U&GuAqrvSAPa`Vp}`*%i3B;w-l@{V>{^Ff9=W}HT{4Ik*;!6S&oR1l4(;5x zyx+?k`L$y*3owBw>fS8)`+D8>D78VA*6fcK>b707FDcFGL59wWziqzo&+4SgZ#P>0 zv`mwtcl~4Mx-A*i+^L%XD*tL{7u0F4ju*}9!vH)B4LhI}jD}t;IRXJDkh{Y*=5OXQ z(DktF${=l1@G+HpMC8X3rh|H;cUUfPnpl?jkVqQN=fDA zO2OSu^>qGD-CNX>et6WRlPDP>Ze-?Iy5z=;)7F*hyyl~RP&y_x+=?rdT>hfEb>OtAZ{8 z;G74?Rs*U7NU8yH3V?x^ZCJu6+W%8@OSwBo8|Ez}4uU|4j=#1paBY$ TfnYC&tRR3Jj$v|2?uvgMSpMn7 literal 0 HcmV?d00001 diff --git a/iolite/src/test/resources/golden/from-0-to-2.tar.zst b/iolite/src/test/resources/golden/from-0-to-2.tar.zst new file mode 100644 index 0000000000000000000000000000000000000000..933ccddc209efcc790aabc50548cd6aad2234a00 GIT binary patch literal 1056 zcmV+*1mF88wJ-f-01=fS0IF1E4I&_I!w$22xY1LyW`kR4E`JC>2;qe6W)mqQtKdLL z03ZNr+W^A=!T@DXT5pz5t=T2%RcBH)3E%8!IVcr5NCSt)!XKO*#2YC77-JHKC`2K! zn1VvW;Q={F13Ab;AqQbJiX3Et(eT8;ae&x>GN8yo5;0&PK^zingB*!fCPC6d!GIiS z3DQ^u$s6kpO%9TZmVd^`vb-WqS-ex>)X!Q9iwn~mX}Ma2dYL=vJKi}fk_3t14A=n! zImlz3r-R23GA=qanzYT)#Hvz7H5J7C2laC5D|F?Psp}d|N)jX~h=eKt4cl;VPyzrT zJhQR_aL!d#0a!2`79bi9{))ve1O*m$!N9NxZ@{py5FD| zd^`~wWooQO>Q@JyGU*Z~w{!II)HR|n=FO)J-pO1y$DWPh@~E{J*G@LtVK0gj_PK(- zThm>&%^|~LCJ^PUqv#ww*uSMUO{bsZaVfOJWwlgFE6tdWVLz4poRyMD)l90(zbvVb zu{G-6Eckeg|NV+_OuTRA=3b>vG20L$KVOO@NCJfeIS5IR#=;v5Mw8>f0fO9q&2RFk zot3e(u3JX|Wz#MCuGz6iV=j?<;B=#j-D#j_GIBTC7cp z+W%O|vp-tM)o90LDLaLIYse(Wbm@N_@oM5>RPHw)2Ao@H*a1nwXh@4C2l)Xe5dO#a z_V!|{=lELfe9(ESnKY_iN-{d*=uv(aqk`y_^^%`)u_w0IMTh3dG+DC!D=LJQ_B&lI zktt}({E zv!xZUrhY{4_M)2FS9WX1YtlUSHKH2(G>8o*bA?RpUwciuWKP;4<7?}O%BDd8GUEzGW+Tj9q6L2y=@E9G2EubUcii+%B5wWav$4Z&7@>4Ljhyxa+;Ifeb z05&1G2slCzF{n*ihy+NO2Mr+gr7z9uhX~u%2*mWFYXSgpGY~CjXlMb>rdT>hfEb>O ztAd_jV*scF0Nhanssl)>0dfj}ftPJq!YJCGlxRf_ak*lct&li~7a=;zj?w`(2qY%} aSloaHC{PB1y%@5B0B$&j$teLj^@)4_F5KV% literal 0 HcmV?d00001 diff --git a/iolite/src/test/resources/golden/from-1-to-2.tar.zst b/iolite/src/test/resources/golden/from-1-to-2.tar.zst new file mode 100644 index 0000000000000000000000000000000000000000..ccd080d0de5d7784ab6b29dbf43cc9391f0dfff7 GIT binary patch literal 1052 zcmV+%1mpWCwJ-f-01yA%P|Q-zhs2^2jk)BmmYtNKuK=L=06JQ z;n_L7HVn(`!cE`3L-)xbwN#a%S~YzPKboV2tSAjNEgDwI&Kh%Uq|VuCJcX9p9P=-g zTot6aBH2ViN(->V9j+inu}B;ZZ%7b`f|NaIhR~x*obK+34>2oFKF!3Q-uN=d^S8`P zl_gBZrOgiW44BpVTYkWHP<7;&MDT_Qm(Q3G+Oiu2X#fnk=QQ{Z_80 z(lL4L{?)9rOBsXo=q@n!ppaO!Vzq(-!ohev5EP^^G!$?_3IfL>FyN2v2 z?+Q|UE!sdsf}k6l;0Jc0mvJ~OMT3H3kvz5&1_ombQW9H`!XTlf3H|M}O2*vUwaur_ zcy;w>dbb-@rHyvDhrCCf&&JNwbFWU~lk1?R)BD%nvv3`x4%6|ybxdXhK>#x8elT!0 zdy<#{2~z3g&(_-E0(BE`GC%Mb9fmESBj1XO>|PPEtaQgpnXvLxFt&&T7Np>^kpKWT zA-D)|oDwmpO4Fbst02Vi( W0Sc6XU@wNOAb=Z=VRA|}XL6)*O7Bnr literal 0 HcmV?d00001 diff --git a/iolite/src/test/resources/nip44.vectors.json b/iolite/src/test/resources/nip44.vectors.json new file mode 100644 index 0000000..18559c1 --- /dev/null +++ b/iolite/src/test/resources/nip44.vectors.json @@ -0,0 +1,648 @@ +{ + "v2": { + "valid": { + "get_conversation_key": [ + { + "sec1": "315e59ff51cb9209768cf7da80791ddcaae56ac9775eb25b6dee1234bc5d2268", + "pub2": "c2f9d9948dc8c7c38321e4b85c8558872eafa0641cd269db76848a6073e69133", + "conversation_key": "3dfef0ce2a4d80a25e7a328accf73448ef67096f65f79588e358d9a0eb9013f1" + }, + { + "sec1": "a1e37752c9fdc1273be53f68c5f74be7c8905728e8de75800b94262f9497c86e", + "pub2": "03bb7947065dde12ba991ea045132581d0954f042c84e06d8c00066e23c1a800", + "conversation_key": "4d14f36e81b8452128da64fe6f1eae873baae2f444b02c950b90e43553f2178b" + }, + { + "sec1": "98a5902fd67518a0c900f0fb62158f278f94a21d6f9d33d30cd3091195500311", + "pub2": "aae65c15f98e5e677b5050de82e3aba47a6fe49b3dab7863cf35d9478ba9f7d1", + "conversation_key": "9c00b769d5f54d02bf175b7284a1cbd28b6911b06cda6666b2243561ac96bad7" + }, + { + "sec1": "86ae5ac8034eb2542ce23ec2f84375655dab7f836836bbd3c54cefe9fdc9c19f", + "pub2": "59f90272378089d73f1339710c02e2be6db584e9cdbe86eed3578f0c67c23585", + "conversation_key": "19f934aafd3324e8415299b64df42049afaa051c71c98d0aa10e1081f2e3e2ba" + }, + { + "sec1": "2528c287fe822421bc0dc4c3615878eb98e8a8c31657616d08b29c00ce209e34", + "pub2": "f66ea16104c01a1c532e03f166c5370a22a5505753005a566366097150c6df60", + "conversation_key": "c833bbb292956c43366145326d53b955ffb5da4e4998a2d853611841903f5442" + }, + { + "sec1": "49808637b2d21129478041813aceb6f2c9d4929cd1303cdaf4fbdbd690905ff2", + "pub2": "74d2aab13e97827ea21baf253ad7e39b974bb2498cc747cdb168582a11847b65", + "conversation_key": "4bf304d3c8c4608864c0fe03890b90279328cd24a018ffa9eb8f8ccec06b505d" + }, + { + "sec1": "af67c382106242c5baabf856efdc0629cc1c5b4061f85b8ceaba52aa7e4b4082", + "pub2": "bdaf0001d63e7ec994fad736eab178ee3c2d7cfc925ae29f37d19224486db57b", + "conversation_key": "a3a575dd66d45e9379904047ebfb9a7873c471687d0535db00ef2daa24b391db" + }, + { + "sec1": "0e44e2d1db3c1717b05ffa0f08d102a09c554a1cbbf678ab158b259a44e682f1", + "pub2": "1ffa76c5cc7a836af6914b840483726207cb750889753d7499fb8b76aa8fe0de", + "conversation_key": "a39970a667b7f861f100e3827f4adbf6f464e2697686fe1a81aeda817d6b8bdf" + }, + { + "sec1": "5fc0070dbd0666dbddc21d788db04050b86ed8b456b080794c2a0c8e33287bb6", + "pub2": "31990752f296dd22e146c9e6f152a269d84b241cc95bb3ff8ec341628a54caf0", + "conversation_key": "72c21075f4b2349ce01a3e604e02a9ab9f07e35dd07eff746de348b4f3c6365e" + }, + { + "sec1": "1b7de0d64d9b12ddbb52ef217a3a7c47c4362ce7ea837d760dad58ab313cba64", + "pub2": "24383541dd8083b93d144b431679d70ef4eec10c98fceef1eff08b1d81d4b065", + "conversation_key": "dd152a76b44e63d1afd4dfff0785fa07b3e494a9e8401aba31ff925caeb8f5b1" + }, + { + "sec1": "df2f560e213ca5fb33b9ecde771c7c0cbd30f1cf43c2c24de54480069d9ab0af", + "pub2": "eeea26e552fc8b5e377acaa03e47daa2d7b0c787fac1e0774c9504d9094c430e", + "conversation_key": "770519e803b80f411c34aef59c3ca018608842ebf53909c48d35250bd9323af6" + }, + { + "sec1": "cffff919fcc07b8003fdc63bc8a00c0f5dc81022c1c927c62c597352190d95b9", + "pub2": "eb5c3cca1a968e26684e5b0eb733aecfc844f95a09ac4e126a9e58a4e4902f92", + "conversation_key": "46a14ee7e80e439ec75c66f04ad824b53a632b8409a29bbb7c192e43c00bb795" + }, + { + "sec1": "64ba5a685e443e881e9094647ddd32db14444bb21aa7986beeba3d1c4673ba0a", + "pub2": "50e6a4339fac1f3bf86f2401dd797af43ad45bbf58e0801a7877a3984c77c3c4", + "conversation_key": "968b9dbbfcede1664a4ca35a5d3379c064736e87aafbf0b5d114dff710b8a946" + }, + { + "sec1": "dd0c31ccce4ec8083f9b75dbf23cc2878e6d1b6baa17713841a2428f69dee91a", + "pub2": "b483e84c1339812bed25be55cff959778dfc6edde97ccd9e3649f442472c091b", + "conversation_key": "09024503c7bde07eb7865505891c1ea672bf2d9e25e18dd7a7cea6c69bf44b5d" + }, + { + "sec1": "af71313b0d95c41e968a172b33ba5ebd19d06cdf8a7a98df80ecf7af4f6f0358", + "pub2": "2a5c25266695b461ee2af927a6c44a3c598b8095b0557e9bd7f787067435bc7c", + "conversation_key": "fe5155b27c1c4b4e92a933edae23726a04802a7cc354a77ac273c85aa3c97a92" + }, + { + "sec1": "6636e8a389f75fe068a03b3edb3ea4a785e2768e3f73f48ffb1fc5e7cb7289dc", + "pub2": "514eb2064224b6a5829ea21b6e8f7d3ea15ff8e70e8555010f649eb6e09aec70", + "conversation_key": "ff7afacd4d1a6856d37ca5b546890e46e922b508639214991cf8048ddbe9745c" + }, + { + "sec1": "94b212f02a3cfb8ad147d52941d3f1dbe1753804458e6645af92c7b2ea791caa", + "pub2": "f0cac333231367a04b652a77ab4f8d658b94e86b5a8a0c472c5c7b0d4c6a40cc", + "conversation_key": "e292eaf873addfed0a457c6bd16c8effde33d6664265697f69f420ab16f6669b" + }, + { + "sec1": "aa61f9734e69ae88e5d4ced5aae881c96f0d7f16cca603d3bed9eec391136da6", + "pub2": "4303e5360a884c360221de8606b72dd316da49a37fe51e17ada4f35f671620a6", + "conversation_key": "8e7d44fd4767456df1fb61f134092a52fcd6836ebab3b00766e16732683ed848" + }, + { + "sec1": "5e914bdac54f3f8e2cba94ee898b33240019297b69e96e70c8a495943a72fc98", + "pub2": "5bd097924f606695c59f18ff8fd53c174adbafaaa71b3c0b4144a3e0a474b198", + "conversation_key": "f5a0aecf2984bf923c8cd5e7bb8be262d1a8353cb93959434b943a07cf5644bc" + }, + { + "sec1": "8b275067add6312ddee064bcdbeb9d17e88aa1df36f430b2cea5cc0413d8278a", + "pub2": "65bbbfca819c90c7579f7a82b750a18c858db1afbec8f35b3c1e0e7b5588e9b8", + "conversation_key": "2c565e7027eb46038c2263563d7af681697107e975e9914b799d425effd248d6" + }, + { + "sec1": "1ac848de312285f85e0f7ec208aac20142a1f453402af9b34ec2ec7a1f9c96fc", + "pub2": "45f7318fe96034d23ee3ddc25b77f275cc1dd329664dd51b89f89c4963868e41", + "conversation_key": "b56e970e5057a8fd929f8aad9248176b9af87819a708d9ddd56e41d1aec74088" + }, + { + "sec1": "295a1cf621de401783d29d0e89036aa1c62d13d9ad307161b4ceb535ba1b40e6", + "pub2": "840115ddc7f1034d3b21d8e2103f6cb5ab0b63cf613f4ea6e61ae3d016715cdd", + "conversation_key": "b4ee9c0b9b9fef88975773394f0a6f981ca016076143a1bb575b9ff46e804753" + }, + { + "sec1": "a28eed0fe977893856ab9667e06ace39f03abbcdb845c329a1981be438ba565d", + "pub2": "b0f38b950a5013eba5ab4237f9ed29204a59f3625c71b7e210fec565edfa288c", + "conversation_key": "9d3a802b45bc5aeeb3b303e8e18a92ddd353375710a31600d7f5fff8f3a7285b" + }, + { + "sec1": "7ab65af72a478c05f5c651bdc4876c74b63d20d04cdbf71741e46978797cd5a4", + "pub2": "f1112159161b568a9cb8c9dd6430b526c4204bcc8ce07464b0845b04c041beda", + "conversation_key": "943884cddaca5a3fef355e9e7f08a3019b0b66aa63ec90278b0f9fdb64821e79" + }, + { + "sec1": "95c79a7b75ba40f2229e85756884c138916f9d103fc8f18acc0877a7cceac9fe", + "pub2": "cad76bcbd31ca7bbda184d20cc42f725ed0bb105b13580c41330e03023f0ffb3", + "conversation_key": "81c0832a669eea13b4247c40be51ccfd15bb63fcd1bba5b4530ce0e2632f301b" + }, + { + "sec1": "baf55cc2febd4d980b4b393972dfc1acf49541e336b56d33d429bce44fa12ec9", + "pub2": "0c31cf87fe565766089b64b39460ebbfdedd4a2bc8379be73ad3c0718c912e18", + "conversation_key": "37e2344da9ecdf60ae2205d81e89d34b280b0a3f111171af7e4391ded93b8ea6" + }, + { + "sec1": "6eeec45acd2ed31693c5256026abf9f072f01c4abb61f51cf64e6956b6dc8907", + "pub2": "e501b34ed11f13d816748c0369b0c728e540df3755bab59ed3327339e16ff828", + "conversation_key": "afaa141b522ddb27bb880d768903a7f618bb8b6357728cae7fb03af639b946e6" + }, + { + "sec1": "261a076a9702af1647fb343c55b3f9a4f1096273002287df0015ba81ce5294df", + "pub2": "b2777c863878893ae100fb740c8fab4bebd2bf7be78c761a75593670380a6112", + "conversation_key": "76f8d2853de0734e51189ced523c09427c3e46338b9522cd6f74ef5e5b475c74" + }, + { + "sec1": "ed3ec71ca406552ea41faec53e19f44b8f90575eda4b7e96380f9cc73c26d6f3", + "pub2": "86425951e61f94b62e20cae24184b42e8e17afcf55bafa58645efd0172624fae", + "conversation_key": "f7ffc520a3a0e9e9b3c0967325c9bf12707f8e7a03f28b6cd69ae92cf33f7036" + }, + { + "sec1": "5a788fc43378d1303ac78639c59a58cb88b08b3859df33193e63a5a3801c722e", + "pub2": "a8cba2f87657d229db69bee07850fd6f7a2ed070171a06d006ec3a8ac562cf70", + "conversation_key": "7d705a27feeedf78b5c07283362f8e361760d3e9f78adab83e3ae5ce7aeb6409" + }, + { + "sec1": "63bffa986e382b0ac8ccc1aa93d18a7aa445116478be6f2453bad1f2d3af2344", + "pub2": "b895c70a83e782c1cf84af558d1038e6b211c6f84ede60408f519a293201031d", + "conversation_key": "3a3b8f00d4987fc6711d9be64d9c59cf9a709c6c6481c2cde404bcc7a28f174e" + }, + { + "sec1": "e4a8bcacbf445fd3721792b939ff58e691cdcba6a8ba67ac3467b45567a03e5c", + "pub2": "b54053189e8c9252c6950059c783edb10675d06d20c7b342f73ec9fa6ed39c9d", + "conversation_key": "7b3933b4ef8189d347169c7955589fc1cfc01da5239591a08a183ff6694c44ad" + }, + { + "sec1": "fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364139", + "pub2": "0000000000000000000000000000000000000000000000000000000000000002", + "conversation_key": "8b6392dbf2ec6a2b2d5b1477fc2be84d63ef254b667cadd31bd3f444c44ae6ba", + "note": "sec1 = n-2, pub2: random, 0x02" + }, + { + "sec1": "0000000000000000000000000000000000000000000000000000000000000002", + "pub2": "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdeb", + "conversation_key": "be234f46f60a250bef52a5ee34c758800c4ca8e5030bf4cc1a31d37ba2104d43", + "note": "sec1 = 2, pub2: rand" + }, + { + "sec1": "0000000000000000000000000000000000000000000000000000000000000001", + "pub2": "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", + "conversation_key": "3b4610cb7189beb9cc29eb3716ecc6102f1247e8f3101a03a1787d8908aeb54e", + "note": "sec1 == pub2" + } + ], + "get_message_keys": { + "conversation_key": "a1a3d60f3470a8612633924e91febf96dc5366ce130f658b1f0fc652c20b3b54", + "keys": [ + { + "nonce": "e1e6f880560d6d149ed83dcc7e5861ee62a5ee051f7fde9975fe5d25d2a02d72", + "chacha_key": "f145f3bed47cb70dbeaac07f3a3fe683e822b3715edb7c4fe310829014ce7d76", + "chacha_nonce": "c4ad129bb01180c0933a160c", + "hmac_key": "027c1db445f05e2eee864a0975b0ddef5b7110583c8c192de3732571ca5838c4" + }, + { + "nonce": "e1d6d28c46de60168b43d79dacc519698512ec35e8ccb12640fc8e9f26121101", + "chacha_key": "e35b88f8d4a8f1606c5082f7a64b100e5d85fcdb2e62aeafbec03fb9e860ad92", + "chacha_nonce": "22925e920cee4a50a478be90", + "hmac_key": "46a7c55d4283cb0df1d5e29540be67abfe709e3b2e14b7bf9976e6df994ded30" + }, + { + "nonce": "cfc13bef512ac9c15951ab00030dfaf2626fdca638dedb35f2993a9eeb85d650", + "chacha_key": "020783eb35fdf5b80ef8c75377f4e937efb26bcbad0e61b4190e39939860c4bf", + "chacha_nonce": "d3594987af769a52904656ac", + "hmac_key": "237ec0ccb6ebd53d179fa8fd319e092acff599ef174c1fdafd499ef2b8dee745" + }, + { + "nonce": "ea6eb84cac23c5c1607c334e8bdf66f7977a7e374052327ec28c6906cbe25967", + "chacha_key": "ff68db24b34fa62c78ac5ffeeaf19533afaedf651fb6a08384e46787f6ce94be", + "chacha_nonce": "50bb859aa2dde938cc49ec7a", + "hmac_key": "06ff32e1f7b29753a727d7927b25c2dd175aca47751462d37a2039023ec6b5a6" + }, + { + "nonce": "8c2e1dd3792802f1f9f7842e0323e5d52ad7472daf360f26e15f97290173605d", + "chacha_key": "2f9daeda8683fdeede81adac247c63cc7671fa817a1fd47352e95d9487989d8b", + "chacha_nonce": "400224ba67fc2f1b76736916", + "hmac_key": "465c05302aeeb514e41c13ed6405297e261048cfb75a6f851ffa5b445b746e4b" + }, + { + "nonce": "05c28bf3d834fa4af8143bf5201a856fa5fac1a3aee58f4c93a764fc2f722367", + "chacha_key": "1e3d45777025a035be566d80fd580def73ed6f7c043faec2c8c1c690ad31c110", + "chacha_nonce": "021905b1ea3afc17cb9bf96f", + "hmac_key": "74a6e481a89dcd130aaeb21060d7ec97ad30f0007d2cae7b1b11256cc70dfb81" + }, + { + "nonce": "5e043fb153227866e75a06d60185851bc90273bfb93342f6632a728e18a07a17", + "chacha_key": "1ea72c9293841e7737c71567d8120145a58991aaa1c436ef77bf7adb83f882f1", + "chacha_nonce": "72f69a5a5f795465cee59da8", + "hmac_key": "e9daa1a1e9a266ecaa14e970a84bce3fbbf329079bbccda626582b4e66a0d4c9" + }, + { + "nonce": "7be7338eaf06a87e274244847fe7a97f5c6a91f44adc18fcc3e411ad6f786dbf", + "chacha_key": "881e7968a1f0c2c80742ee03cd49ea587e13f22699730f1075ade01931582bf6", + "chacha_nonce": "6e69be92d61c04a276021565", + "hmac_key": "901afe79e74b19967c8829af23617d7d0ffbf1b57190c096855c6a03523a971b" + }, + { + "nonce": "94571c8d590905bad7becd892832b472f2aa5212894b6ce96e5ba719c178d976", + "chacha_key": "f80873dd48466cb12d46364a97b8705c01b9b4230cb3ec3415a6b9551dc42eef", + "chacha_nonce": "3dda53569cfcb7fac1805c35", + "hmac_key": "e9fc264345e2839a181affebc27d2f528756e66a5f87b04bf6c5f1997047051e" + }, + { + "nonce": "13a6ee974b1fd759135a2c2010e3cdda47081c78e771125e4f0c382f0284a8cb", + "chacha_key": "bc5fb403b0bed0d84cf1db872b6522072aece00363178c98ad52178d805fca85", + "chacha_nonce": "65064239186e50304cc0f156", + "hmac_key": "e872d320dde4ed3487958a8e43b48aabd3ced92bc24bb8ff1ccb57b590d9701a" + }, + { + "nonce": "082fecdb85f358367b049b08be0e82627ae1d8edb0f27327ccb593aa2613b814", + "chacha_key": "1fbdb1cf6f6ea816349baf697932b36107803de98fcd805ebe9849b8ad0e6a45", + "chacha_nonce": "2e605e1d825a3eaeb613db9c", + "hmac_key": "fae910f591cf3c7eb538c598583abad33bc0a03085a96ca4ea3a08baf17c0eec" + }, + { + "nonce": "4c19020c74932c30ec6b2d8cd0d5bb80bd0fc87da3d8b4859d2fb003810afd03", + "chacha_key": "1ab9905a0189e01cda82f843d226a82a03c4f5b6dbea9b22eb9bc953ba1370d4", + "chacha_nonce": "cbb2530ea653766e5a37a83a", + "hmac_key": "267f68acac01ac7b34b675e36c2cef5e7b7a6b697214add62a491bedd6efc178" + }, + { + "nonce": "67723a3381497b149ce24814eddd10c4c41a1e37e75af161930e6b9601afd0ff", + "chacha_key": "9ecbd25e7e2e6c97b8c27d376dcc8c5679da96578557e4e21dba3a7ef4e4ac07", + "chacha_nonce": "ef649fcf335583e8d45e3c2e", + "hmac_key": "04dbbd812fa8226fdb45924c521a62e3d40a9e2b5806c1501efdeba75b006bf1" + }, + { + "nonce": "42063fe80b093e8619b1610972b4c3ab9e76c14fd908e642cd4997cafb30f36c", + "chacha_key": "211c66531bbcc0efcdd0130f9f1ebc12a769105eb39608994bcb188fa6a73a4a", + "chacha_nonce": "67803605a7e5010d0f63f8c8", + "hmac_key": "e840e4e8921b57647369d121c5a19310648105dbdd008200ebf0d3b668704ff8" + }, + { + "nonce": "b5ac382a4be7ac03b554fe5f3043577b47ea2cd7cfc7e9ca010b1ffbb5cf1a58", + "chacha_key": "b3b5f14f10074244ee42a3837a54309f33981c7232a8b16921e815e1f7d1bb77", + "chacha_nonce": "4e62a0073087ed808be62469", + "hmac_key": "c8efa10230b5ea11633816c1230ca05fa602ace80a7598916d83bae3d3d2ccd7" + }, + { + "nonce": "e9d1eba47dd7e6c1532dc782ff63125db83042bb32841db7eeafd528f3ea7af9", + "chacha_key": "54241f68dc2e50e1db79e892c7c7a471856beeb8d51b7f4d16f16ab0645d2f1a", + "chacha_nonce": "a963ed7dc29b7b1046820a1d", + "hmac_key": "aba215c8634530dc21c70ddb3b3ee4291e0fa5fa79be0f85863747bde281c8b2" + }, + { + "nonce": "a94ecf8efeee9d7068de730fad8daf96694acb70901d762de39fa8a5039c3c49", + "chacha_key": "c0565e9e201d2381a2368d7ffe60f555223874610d3d91fbbdf3076f7b1374dd", + "chacha_nonce": "329bb3024461e84b2e1c489b", + "hmac_key": "ac42445491f092481ce4fa33b1f2274700032db64e3a15014fbe8c28550f2fec" + }, + { + "nonce": "533605ea214e70c25e9a22f792f4b78b9f83a18ab2103687c8a0075919eaaa53", + "chacha_key": "ab35a5e1e54d693ff023db8500d8d4e79ad8878c744e0eaec691e96e141d2325", + "chacha_nonce": "653d759042b85194d4d8c0a7", + "hmac_key": "b43628e37ba3c31ce80576f0a1f26d3a7c9361d29bb227433b66f49d44f167ba" + }, + { + "nonce": "7f38df30ceea1577cb60b355b4f5567ff4130c49e84fed34d779b764a9cc184c", + "chacha_key": "a37d7f211b84a551a127ff40908974eb78415395d4f6f40324428e850e8c42a3", + "chacha_nonce": "b822e2c959df32b3cb772a7c", + "hmac_key": "1ba31764f01f69b5c89ded2d7c95828e8052c55f5d36f1cd535510d61ba77420" + }, + { + "nonce": "11b37f9dbc4d0185d1c26d5f4ed98637d7c9701fffa65a65839fa4126573a4e5", + "chacha_key": "964f38d3a31158a5bfd28481247b18dd6e44d69f30ba2a40f6120c6d21d8a6ba", + "chacha_nonce": "5f72c5b87c590bcd0f93b305", + "hmac_key": "2fc4553e7cedc47f29690439890f9f19c1077ef3e9eaeef473d0711e04448918" + }, + { + "nonce": "8be790aa483d4cdd843189f71f135b3ec7e31f381312c8fe9f177aab2a48eafa", + "chacha_key": "95c8c74d633721a131316309cf6daf0804d59eaa90ea998fc35bac3d2fbb7a94", + "chacha_nonce": "409a7654c0e4bf8c2c6489be", + "hmac_key": "21bb0b06eb2b460f8ab075f497efa9a01c9cf9146f1e3986c3bf9da5689b6dc4" + }, + { + "nonce": "19fd2a718ea084827d6bd73f509229ddf856732108b59fc01819f611419fd140", + "chacha_key": "cc6714b9f5616c66143424e1413d520dae03b1a4bd202b82b0a89b0727f5cdc8", + "chacha_nonce": "1b7fd2534f015a8f795d8f32", + "hmac_key": "2bef39c4ce5c3c59b817e86351373d1554c98bc131c7e461ed19d96cfd6399a0" + }, + { + "nonce": "3c2acd893952b2f6d07d8aea76f545ca45961a93fe5757f6a5a80811d5e0255d", + "chacha_key": "c8de6c878cb469278d0af894bc181deb6194053f73da5014c2b5d2c8db6f2056", + "chacha_nonce": "6ffe4f1971b904a1b1a81b99", + "hmac_key": "df1cd69dd3646fca15594284744d4211d70e7d8472e545d276421fbb79559fd4" + }, + { + "nonce": "7dbea4cead9ac91d4137f1c0a6eebb6ba0d1fb2cc46d829fbc75f8d86aca6301", + "chacha_key": "c8e030f6aa680c3d0b597da9c92bb77c21c4285dd620c5889f9beba7446446b0", + "chacha_nonce": "a9b5a67d081d3b42e737d16f", + "hmac_key": "355a85f551bc3cce9a14461aa60994742c9bbb1c81a59ca102dc64e61726ab8e" + }, + { + "nonce": "45422e676cdae5f1071d3647d7a5f1f5adafb832668a578228aa1155a491f2f3", + "chacha_key": "758437245f03a88e2c6a32807edfabff51a91c81ca2f389b0b46f2c97119ea90", + "chacha_nonce": "263830a065af33d9c6c5aa1f", + "hmac_key": "7c581cf3489e2de203a95106bfc0de3d4032e9d5b92b2b61fb444acd99037e17" + }, + { + "nonce": "babc0c03fad24107ad60678751f5db2678041ff0d28671ede8d65bdf7aa407e9", + "chacha_key": "bd68a28bd48d9ffa3602db72c75662ac2848a0047a313d2ae2d6bc1ac153d7e9", + "chacha_nonce": "d0f9d2a1ace6c758f594ffdd", + "hmac_key": "eb435e3a642adfc9d59813051606fc21f81641afd58ea6641e2f5a9f123bb50a" + }, + { + "nonce": "7a1b8aac37d0d20b160291fad124ab697cfca53f82e326d78fef89b4b0ea8f83", + "chacha_key": "9e97875b651a1d30d17d086d1e846778b7faad6fcbc12e08b3365d700f62e4fe", + "chacha_nonce": "ccdaad5b3b7645be430992eb", + "hmac_key": "6f2f55cf35174d75752f63c06cc7cbc8441759b142999ed2d5a6d09d263e1fc4" + }, + { + "nonce": "8370e4e32d7e680a83862cab0da6136ef607014d043e64cdf5ecc0c4e20b3d9a", + "chacha_key": "1472bed5d19db9c546106de946e0649cd83cc9d4a66b087a65906e348dcf92e2", + "chacha_nonce": "ed02dece5fc3a186f123420b", + "hmac_key": "7b3f7739f49d30c6205a46b174f984bb6a9fc38e5ccfacef2dac04fcbd3b184e" + }, + { + "nonce": "9f1c5e8a29cd5677513c2e3a816551d6833ee54991eb3f00d5b68096fc8f0183", + "chacha_key": "5e1a7544e4d4dafe55941fcbdf326f19b0ca37fc49c4d47e9eec7fb68cde4975", + "chacha_nonce": "7d9acb0fdc174e3c220f40de", + "hmac_key": "e265ab116fbbb86b2aefc089a0986a0f5b77eda50c7410404ad3b4f3f385c7a7" + }, + { + "nonce": "c385aa1c37c2bfd5cc35fcdbdf601034d39195e1cabff664ceb2b787c15d0225", + "chacha_key": "06bf4e60677a13e54c4a38ab824d2ef79da22b690da2b82d0aa3e39a14ca7bdd", + "chacha_nonce": "26b450612ca5e905b937e147", + "hmac_key": "22208152be2b1f5f75e6bfcc1f87763d48bb7a74da1be3d102096f257207f8b3" + }, + { + "nonce": "3ff73528f88a50f9d35c0ddba4560bacee5b0462d0f4cb6e91caf41847040ce4", + "chacha_key": "850c8a17a23aa761d279d9901015b2bbdfdff00adbf6bc5cf22bd44d24ecabc9", + "chacha_nonce": "4a296a1fb0048e5020d3b129", + "hmac_key": "b1bf49a533c4da9b1d629b7ff30882e12d37d49c19abd7b01b7807d75ee13806" + }, + { + "nonce": "2dcf39b9d4c52f1cb9db2d516c43a7c6c3b8c401f6a4ac8f131a9e1059957036", + "chacha_key": "17f8057e6156ba7cc5310d01eda8c40f9aa388f9fd1712deb9511f13ecc37d27", + "chacha_nonce": "a8188daff807a1182200b39d", + "hmac_key": "47b89da97f68d389867b5d8a2d7ba55715a30e3d88a3cc11f3646bc2af5580ef" + } + ] + }, + "calc_padded_len": [ + [16, 32], + [32, 32], + [33, 64], + [37, 64], + [45, 64], + [49, 64], + [64, 64], + [65, 96], + [100, 128], + [111, 128], + [200, 224], + [250, 256], + [320, 320], + [383, 384], + [384, 384], + [400, 448], + [500, 512], + [512, 512], + [515, 640], + [700, 768], + [800, 896], + [900, 1024], + [1020, 1024], + [65536, 65536] + ], + "encrypt_decrypt": [ + { + "sec1": "0000000000000000000000000000000000000000000000000000000000000001", + "sec2": "0000000000000000000000000000000000000000000000000000000000000002", + "conversation_key": "c41c775356fd92eadc63ff5a0dc1da211b268cbea22316767095b2871ea1412d", + "nonce": "0000000000000000000000000000000000000000000000000000000000000001", + "plaintext": "a", + "payload": "AgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABee0G5VSK0/9YypIObAtDKfYEAjD35uVkHyB0F4DwrcNaCXlCWZKaArsGrY6M9wnuTMxWfp1RTN9Xga8no+kF5Vsb" + }, + { + "sec1": "0000000000000000000000000000000000000000000000000000000000000002", + "sec2": "0000000000000000000000000000000000000000000000000000000000000001", + "conversation_key": "c41c775356fd92eadc63ff5a0dc1da211b268cbea22316767095b2871ea1412d", + "nonce": "f00000000000000000000000000000f00000000000000000000000000000000f", + "plaintext": "🍕🫃", + "payload": "AvAAAAAAAAAAAAAAAAAAAPAAAAAAAAAAAAAAAAAAAAAPSKSK6is9ngkX2+cSq85Th16oRTISAOfhStnixqZziKMDvB0QQzgFZdjLTPicCJaV8nDITO+QfaQ61+KbWQIOO2Yj" + }, + { + "sec1": "5c0c523f52a5b6fad39ed2403092df8cebc36318b39383bca6c00808626fab3a", + "sec2": "4b22aa260e4acb7021e32f38a6cdf4b673c6a277755bfce287e370c924dc936d", + "conversation_key": "3e2b52a63be47d34fe0a80e34e73d436d6963bc8f39827f327057a9986c20a45", + "nonce": "b635236c42db20f021bb8d1cdff5ca75dd1a0cc72ea742ad750f33010b24f73b", + "plaintext": "表ポあA鷗ŒéＢ逍Üߪąñ丂㐀𠀀", + "payload": "ArY1I2xC2yDwIbuNHN/1ynXdGgzHLqdCrXUPMwELJPc7s7JqlCMJBAIIjfkpHReBPXeoMCyuClwgbT419jUWU1PwaNl4FEQYKCDKVJz+97Mp3K+Q2YGa77B6gpxB/lr1QgoqpDf7wDVrDmOqGoiPjWDqy8KzLueKDcm9BVP8xeTJIxs=" + }, + { + "sec1": "8f40e50a84a7462e2b8d24c28898ef1f23359fff50d8c509e6fb7ce06e142f9c", + "sec2": "b9b0a1e9cc20100c5faa3bbe2777303d25950616c4c6a3fa2e3e046f936ec2ba", + "conversation_key": "d5a2f879123145a4b291d767428870f5a8d9e5007193321795b40183d4ab8c2b", + "nonce": "b20989adc3ddc41cd2c435952c0d59a91315d8c5218d5040573fc3749543acaf", + "plaintext": "ability🤝的 ȺȾ", + "payload": "ArIJia3D3cQc0sQ1lSwNWakTFdjFIY1QQFc/w3SVQ6yvbG2S0x4Yu86QGwPTy7mP3961I1XqB6SFFTzqDZZavhxoWMj7mEVGMQIsh2RLWI5EYQaQDIePSnXPlzf7CIt+voTD" + }, + { + "sec1": "875adb475056aec0b4809bd2db9aa00cff53a649e7b59d8edcbf4e6330b0995c", + "sec2": "9c05781112d5b0a2a7148a222e50e0bd891d6b60c5483f03456e982185944aae", + "conversation_key": "3b15c977e20bfe4b8482991274635edd94f366595b1a3d2993515705ca3cedb8", + "nonce": "8d4442713eb9d4791175cb040d98d6fc5be8864d6ec2f89cf0895a2b2b72d1b1", + "plaintext": "pepper👀їжак", + "payload": "Ao1EQnE+udR5EXXLBA2Y1vxb6IZNbsL4nPCJWisrctGxY3AduCS+jTUgAAnfvKafkmpy15+i9YMwCdccisRa8SvzW671T2JO4LFSPX31K4kYUKelSAdSPwe9NwO6LhOsnoJ+" + }, + { + "sec1": "eba1687cab6a3101bfc68fd70f214aa4cc059e9ec1b79fdb9ad0a0a4e259829f", + "sec2": "dff20d262bef9dfd94666548f556393085e6ea421c8af86e9d333fa8747e94b3", + "conversation_key": "4f1538411098cf11c8af216836444787c462d47f97287f46cf7edb2c4915b8a5", + "nonce": "2180b52ae645fcf9f5080d81b1f0b5d6f2cd77ff3c986882bb549158462f3407", + "plaintext": "( ͡° ͜ʖ ͡°)", + "payload": "AiGAtSrmRfz59QgNgbHwtdbyzXf/PJhogrtUkVhGLzQHv4qhKQwnFQ54OjVMgqCea/Vj0YqBSdhqNR777TJ4zIUk7R0fnizp6l1zwgzWv7+ee6u+0/89KIjY5q1wu6inyuiv" + }, + { + "sec1": "d5633530f5bcfebceb5584cfbbf718a30df0751b729dd9a789b9f30c0587d74e", + "sec2": "b74e6a341fb134127272b795a08b59250e5fa45a82a2eb4095e4ce9ed5f5e214", + "conversation_key": "75fe686d21a035f0c7cd70da64ba307936e5ca0b20710496a6b6b5f573377bdd", + "nonce": "e4cd5f7ce4eea024bc71b17ad456a986a74ac426c2c62b0a15eb5c5c8f888b68", + "plaintext": "مُنَاقَشَةُ سُبُلِ اِسْتِخْدَامِ اللُّغَةِ فِي النُّظُمِ الْقَائِمَةِ وَفِيم يَخُصَّ التَّطْبِيقَاتُ الْحاسُوبِيَّةُ،", + "payload": "AuTNX3zk7qAkvHGxetRWqYanSsQmwsYrChXrXFyPiItoIBsWu1CB+sStla2M4VeANASHxM78i1CfHQQH1YbBy24Tng7emYW44ol6QkFD6D8Zq7QPl+8L1c47lx8RoODEQMvNCbOk5ffUV3/AhONHBXnffrI+0025c+uRGzfqpYki4lBqm9iYU+k3Tvjczq9wU0mkVDEaM34WiQi30MfkJdRbeeYaq6kNvGPunLb3xdjjs5DL720d61Flc5ZfoZm+CBhADy9D9XiVZYLKAlkijALJur9dATYKci6OBOoc2SJS2Clai5hOVzR0yVeyHRgRfH9aLSlWW5dXcUxTo7qqRjNf8W5+J4jF4gNQp5f5d0YA4vPAzjBwSP/5bGzNDslKfcAH" + }, + { + "sec1": "d5633530f5bcfebceb5584cfbbf718a30df0751b729dd9a789b9f30c0587d74e", + "sec2": "b74e6a341fb134127272b795a08b59250e5fa45a82a2eb4095e4ce9ed5f5e214", + "conversation_key": "75fe686d21a035f0c7cd70da64ba307936e5ca0b20710496a6b6b5f573377bdd", + "nonce": "38d1ca0abef9e5f564e89761a86cee04574b6825d3ef2063b10ad75899e4b023", + "plaintext": "الكل في المجمو عة (5)", + "payload": "AjjRygq++eX1ZOiXYahs7gRXS2gl0+8gY7EK11iZ5LAjbOTrlfrxak5Lki42v2jMPpLSicy8eHjsWkkMtF0i925vOaKG/ZkMHh9ccQBdfTvgEGKzztedqDCAWb5TP1YwU1PsWaiiqG3+WgVvJiO4lUdMHXL7+zKKx8bgDtowzz4QAwI=" + }, + { + "sec1": "d5633530f5bcfebceb5584cfbbf718a30df0751b729dd9a789b9f30c0587d74e", + "sec2": "b74e6a341fb134127272b795a08b59250e5fa45a82a2eb4095e4ce9ed5f5e214", + "conversation_key": "75fe686d21a035f0c7cd70da64ba307936e5ca0b20710496a6b6b5f573377bdd", + "nonce": "4f1a31909f3483a9e69c8549a55bbc9af25fa5bbecf7bd32d9896f83ef2e12e0", + "plaintext": "𝖑𝖆𝖟𝖞 社會科學院語學研究所", + "payload": "Ak8aMZCfNIOp5pyFSaVbvJryX6W77Pe9MtmJb4PvLhLgh/TsxPLFSANcT67EC1t/qxjru5ZoADjKVEt2ejdx+xGvH49mcdfbc+l+L7gJtkH7GLKpE9pQNQWNHMAmj043PAXJZ++fiJObMRR2mye5VHEANzZWkZXMrXF7YjuG10S1pOU=" + }, + { + "sec1": "d5633530f5bcfebceb5584cfbbf718a30df0751b729dd9a789b9f30c0587d74e", + "sec2": "b74e6a341fb134127272b795a08b59250e5fa45a82a2eb4095e4ce9ed5f5e214", + "conversation_key": "75fe686d21a035f0c7cd70da64ba307936e5ca0b20710496a6b6b5f573377bdd", + "nonce": "a3e219242d85465e70adcd640b564b3feff57d2ef8745d5e7a0663b2dccceb54", + "plaintext": "🙈 🙉 🙊 0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟 Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗", + "payload": "AqPiGSQthUZecK3NZAtWSz/v9X0u+HRdXnoGY7LczOtUf05aMF89q1FLwJvaFJYICZoMYgRJHFLwPiOHce7fuAc40kX0wXJvipyBJ9HzCOj7CgtnC1/cmPCHR3s5AIORmroBWglm1LiFMohv1FSPEbaBD51VXxJa4JyWpYhreSOEjn1wd0lMKC9b+osV2N2tpbs+rbpQem2tRen3sWflmCqjkG5VOVwRErCuXuPb5+hYwd8BoZbfCrsiAVLd7YT44dRtKNBx6rkabWfddKSLtreHLDysOhQUVOp/XkE7OzSkWl6sky0Hva6qJJ/V726hMlomvcLHjE41iKmW2CpcZfOedg==" + } + ], + "encrypt_decrypt_long_msg": [ + { + "conversation_key": "8fc262099ce0d0bb9b89bac05bb9e04f9bc0090acc181fef6840ccee470371ed", + "nonce": "326bcb2c943cd6bb717588c9e5a7e738edf6ed14ec5f5344caa6ef56f0b9cff7", + "pattern": "x", + "repeat": 65535, + "plaintext_sha256": "09ab7495d3e61a76f0deb12cb0306f0696cbb17ffc12131368c7a939f12f56d3", + "payload_sha256": "90714492225faba06310bff2f249ebdc2a5e609d65a629f1c87f2d4ffc55330a" + }, + { + "conversation_key": "56adbe3720339363ab9c3b8526ffce9fd77600927488bfc4b59f7a68ffe5eae0", + "nonce": "ad68da81833c2a8ff609c3d2c0335fd44fe5954f85bb580c6a8d467aa9fc5dd0", + "pattern": "!", + "repeat": 65535, + "plaintext_sha256": "6af297793b72ae092c422e552c3bb3cbc310da274bd1cf9e31023a7fe4a2d75e", + "payload_sha256": "8013e45a109fad3362133132b460a2d5bce235fe71c8b8f4014793fb52a49844" + }, + { + "conversation_key": "7fc540779979e472bb8d12480b443d1e5eb1098eae546ef2390bee499bbf46be", + "nonce": "34905e82105c20de9a2f6cd385a0d541e6bcc10601d12481ff3a7575dc622033", + "pattern": "🦄", + "repeat": 16383, + "plaintext_sha256": "a249558d161b77297bc0cb311dde7d77190f6571b25c7e4429cd19044634a61f", + "payload_sha256": "b3348422471da1f3c59d79acfe2fe103f3cd24488109e5b18734cdb5953afd15" + } + ] + }, + "invalid": { + "encrypt_msg_lengths": [0, 65536, 100000, 10000000], + "get_conversation_key": [ + { + "sec1": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "pub2": "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "note": "sec1 higher than curve.n" + }, + { + "sec1": "0000000000000000000000000000000000000000000000000000000000000000", + "pub2": "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "note": "sec1 is 0" + }, + { + "sec1": "fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364139", + "pub2": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "note": "pub2 is invalid, no sqrt, all-ff" + }, + { + "sec1": "fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", + "pub2": "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "note": "sec1 == curve.n" + }, + { + "sec1": "0000000000000000000000000000000000000000000000000000000000000002", + "pub2": "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "note": "pub2 is invalid, no sqrt" + }, + { + "sec1": "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + "pub2": "0000000000000000000000000000000000000000000000000000000000000000", + "note": "pub2 is point of order 3 on twist" + }, + { + "sec1": "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + "pub2": "eb1f7200aecaa86682376fb1c13cd12b732221e774f553b0a0857f88fa20f86d", + "note": "pub2 is point of order 13 on twist" + }, + { + "sec1": "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", + "pub2": "709858a4c121e4a84eb59c0ded0261093c71e8ca29efeef21a6161c447bcaf9f", + "note": "pub2 is point of order 3319 on twist" + } + ], + "decrypt": [ + { + "conversation_key": "ca2527a037347b91bea0c8a30fc8d9600ffd81ec00038671e3a0f0cb0fc9f642", + "nonce": "daaea5ca345b268e5b62060ca72c870c48f713bc1e00ff3fc0ddb78e826f10db", + "plaintext": "n o b l e", + "payload": "#Atqupco0WyaOW2IGDKcshwxI9xO8HgD/P8Ddt46CbxDbrhdG8VmJdU0MIDf06CUvEvdnr1cp1fiMtlM/GrE92xAc1K5odTpCzUB+mjXgbaqtntBUbTToSUoT0ovrlPwzGjyp", + "note": "unknown encryption version" + }, + { + "conversation_key": "36f04e558af246352dcf73b692fbd3646a2207bd8abd4b1cd26b234db84d9481", + "nonce": "ad408d4be8616dc84bb0bf046454a2a102edac937c35209c43cd7964c5feb781", + "plaintext": "⚠️", + "payload": "AK1AjUvoYW3IS7C/BGRUoqEC7ayTfDUgnEPNeWTF/reBZFaha6EAIRueE9D1B1RuoiuFScC0Q94yjIuxZD3JStQtE8JMNacWFs9rlYP+ZydtHhRucp+lxfdvFlaGV/sQlqZz", + "note": "unknown encryption version 0" + }, + { + "conversation_key": "ca2527a037347b91bea0c8a30fc8d9600ffd81ec00038671e3a0f0cb0fc9f642", + "nonce": "daaea5ca345b268e5b62060ca72c870c48f713bc1e00ff3fc0ddb78e826f10db", + "plaintext": "n o s t r", + "payload": "Atфupco0WyaOW2IGDKcshwxI9xO8HgD/P8Ddt46CbxDbrhdG8VmJZE0UICD06CUvEvdnr1cp1fiMtlM/GrE92xAc1EwsVCQEgWEu2gsHUVf4JAa3TpgkmFc3TWsax0v6n/Wq", + "note": "invalid base64" + }, + { + "conversation_key": "cff7bd6a3e29a450fd27f6c125d5edeb0987c475fd1e8d97591e0d4d8a89763c", + "nonce": "09ff97750b084012e15ecb84614ce88180d7b8ec0d468508a86b6d70c0361a25", + "plaintext": "¯\\_(ツ)_/¯", + "payload": "Agn/l3ULCEAS4V7LhGFM6IGA17jsDUaFCKhrbXDANholyySBfeh+EN8wNB9gaLlg4j6wdBYh+3oK+mnxWu3NKRbSvQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "note": "invalid MAC" + }, + { + "conversation_key": "cfcc9cf682dfb00b11357f65bdc45e29156b69db424d20b3596919074f5bf957", + "nonce": "65b14b0b949aaa7d52c417eb753b390e8ad6d84b23af4bec6d9bfa3e03a08af4", + "plaintext": "🥎", + "payload": "AmWxSwuUmqp9UsQX63U7OQ6K1thLI69L7G2b+j4DoIr0oRWQ8avl4OLqWZiTJ10vIgKrNqjoaX+fNhE9RqmR5g0f6BtUg1ijFMz71MO1D4lQLQfW7+UHva8PGYgQ1QpHlKgR", + "note": "invalid MAC" + }, + { + "conversation_key": "5254827d29177622d40a7b67cad014fe7137700c3c523903ebbe3e1b74d40214", + "nonce": "7ab65dbb8bbc2b8e35cafb5745314e1f050325a864d11d0475ef75b3660d91c1", + "plaintext": "elliptic-curve cryptography", + "payload": "Anq2XbuLvCuONcr7V0UxTh8FAyWoZNEdBHXvdbNmDZHB573MI7R7rrTYftpqmvUpahmBC2sngmI14/L0HjOZ7lWGJlzdh6luiOnGPc46cGxf08MRC4CIuxx3i2Lm0KqgJ7vA", + "note": "invalid padding" + }, + { + "conversation_key": "fea39aca9aa8340c3a78ae1f0902aa7e726946e4efcd7783379df8096029c496", + "nonce": "7d4283e3b54c885d6afee881f48e62f0a3f5d7a9e1cb71ccab594a7882c39330", + "plaintext": "noble", + "payload": "An1Cg+O1TIhdav7ogfSOYvCj9dep4ctxzKtZSniCw5MwRrrPJFyAQYZh5VpjC2QYzny5LIQ9v9lhqmZR4WBYRNJ0ognHVNMwiFV1SHpvUFT8HHZN/m/QarflbvDHAtO6pY16", + "note": "invalid padding" + }, + { + "conversation_key": "0c4cffb7a6f7e706ec94b2e879f1fc54ff8de38d8db87e11787694d5392d5b3f", + "nonce": "6f9fd72667c273acd23ca6653711a708434474dd9eb15c3edb01ce9a95743e9b", + "plaintext": "censorship-resistant and global social network", + "payload": "Am+f1yZnwnOs0jymZTcRpwhDRHTdnrFcPtsBzpqVdD6b2NZDaNm/TPkZGr75kbB6tCSoq7YRcbPiNfJXNch3Tf+o9+zZTMxwjgX/nm3yDKR2kHQMBhVleCB9uPuljl40AJ8kXRD0gjw+aYRJFUMK9gCETZAjjmrsCM+nGRZ1FfNsHr6Z", + "note": "invalid padding" + }, + { + "conversation_key": "5cd2d13b9e355aeb2452afbd3786870dbeecb9d355b12cb0a3b6e9da5744cd35", + "nonce": "b60036976a1ada277b948fd4caa065304b96964742b89d26f26a25263a5060bd", + "plaintext": "0", + "payload": "", + "note": "invalid payload length: 0" + }, + { + "conversation_key": "d61d3f09c7dfe1c0be91af7109b60a7d9d498920c90cbba1e137320fdd938853", + "nonce": "1a29d02c8b4527745a2ccb38bfa45655deb37bc338ab9289d756354cea1fd07c", + "plaintext": "1", + "payload": "Ag==", + "note": "invalid payload length: 4" + }, + { + "conversation_key": "873bb0fc665eb950a8e7d5971965539f6ebd645c83c08cd6a85aafbad0f0bc47", + "nonce": "c826d3c38e765ab8cc42060116cd1464b2a6ce01d33deba5dedfb48615306d4a", + "plaintext": "2", + "payload": "AqxgToSh3H7iLYRJjoWAM+vSv/Y1mgNlm6OWWjOYUClrFF8=", + "note": "invalid payload length: 48" + }, + { + "conversation_key": "9f2fef8f5401ac33f74641b568a7a30bb19409c76ffdc5eae2db6b39d2617fbe", + "nonce": "9ff6484642545221624eaac7b9ea27133a4cc2356682a6033aceeef043549861", + "plaintext": "3", + "payload": "Ap/2SEZCVFIhYk6qx7nqJxM6TMI1ZoKmAzrO7vBDVJhhuZXWiM20i/tIsbjT0KxkJs2MZjh1oXNYMO9ggfk7i47WQA==", + "note": "invalid payload length: 92" + } + ] + } + } +}