diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index 59603ccd38..1fb620a8ba 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -56,6 +56,7 @@ class FsEventStore( private val layout = FsLayout(root) private val hasher: TagNameValueHasher private val indexer: FsIndexer + private val slots: FsSlots private val planner: FsQueryPlanner init { @@ -63,6 +64,7 @@ class FsEventStore( cleanStaging() hasher = TagNameValueHasher(layout.readOrCreateSeed()) indexer = FsIndexer(layout, hasher, indexingStrategy) + slots = FsSlots(layout, indexer) planner = FsQueryPlanner(layout, hasher) } @@ -73,8 +75,25 @@ class FsEventStore( override fun insert(event: Event) { if (event.kind.isEphemeral()) return + val slot = slots.slotPathFor(event) + val existingSlot = slot?.let { slots.readSlot(it) } + if (existingSlot != null && existingSlot.createdAt >= event.createdAt) { + // Newer or equal-timestamp version already owns this slot. + // Matches ReplaceableModule / AddressableModule blocking + // behaviour in SQLite. + return + } + val canonical = layout.canonical(event.id) - if (canonical.exists()) return + if (canonical.exists()) { + // Same id already written. If this event is a replaceable / + // addressable whose slot points somewhere else, still install + // the slot so the winner is consistent. + if (slot != null && existingSlot?.id != event.id) { + slots.install(slot, canonical, event, existingSlot) + } + return + } Files.createDirectories(canonical.parent) val tmp = Files.createTempFile(layout.staging, event.id, FsLayout.JSON_EXT) @@ -90,6 +109,9 @@ class FsEventStore( } Files.setLastModifiedTime(canonical, FileTime.from(event.createdAt, TimeUnit.SECONDS)) indexer.link(event, canonical) + if (slot != null) { + slots.install(slot, canonical, event, existingSlot) + } } catch (t: Throwable) { Files.deleteIfExists(tmp) throw t @@ -185,7 +207,14 @@ class FsEventStore( fun delete(id: HexKey): Int { val canonical = layout.canonical(id) val event = readEvent(id) // need tags to know which index links to remove - if (event != null) indexer.unlink(event) + if (event != null) { + indexer.unlink(event) + val slot = slots.slotPathFor(event) + if (slot != null) { + val winner = slots.readSlot(slot) + if (winner?.id == id) slots.clear(slot) + } + } return if (canonical.deleteIfExists()) 1 else 0 } diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt index 89ff20ddc2..72d1640a92 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt @@ -26,6 +26,7 @@ import java.nio.ByteBuffer import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardCopyOption +import java.security.MessageDigest import java.security.SecureRandom import kotlin.io.path.exists @@ -44,6 +45,8 @@ internal class FsLayout( val idxAuthor: Path = idx.resolve(IDX_AUTHOR) val idxOwner: Path = idx.resolve(IDX_OWNER) val idxTag: Path = idx.resolve(IDX_TAG) + val replaceable: Path = root.resolve(REPLACEABLE_DIR) + val addressable: Path = root.resolve(ADDRESSABLE_DIR) fun canonical(id: HexKey): Path { require(id.length >= 4) { "event id must be at least 4 hex chars, got '$id'" } @@ -88,6 +91,19 @@ internal class FsLayout( fun authorDir(pubkey: HexKey): Path = idxAuthor.resolve(pubkey) + /** Slot path for a replaceable event (kinds 0, 3, 10000-19999). */ + fun replaceableSlot( + kind: Kind, + pubkey: HexKey, + ): Path = replaceable.resolve(kind.toString()).resolve("$pubkey$JSON_EXT") + + /** Slot path for an addressable event (kinds 30000-39999). */ + fun addressableSlot( + kind: Kind, + pubkey: HexKey, + dTag: String, + ): Path = addressable.resolve(kind.toString()).resolve(pubkey).resolve("${sha256Hex(dTag)}$JSON_EXT") + fun ensureSkeleton() { Files.createDirectories(events) Files.createDirectories(staging) @@ -95,6 +111,8 @@ internal class FsLayout( Files.createDirectories(idxAuthor) Files.createDirectories(idxOwner) Files.createDirectories(idxTag) + Files.createDirectories(replaceable) + Files.createDirectories(addressable) } /** @@ -124,9 +142,26 @@ internal class FsLayout( const val IDX_AUTHOR = "author" const val IDX_OWNER = "owner" const val IDX_TAG = "tag" + const val REPLACEABLE_DIR = "replaceable" + const val ADDRESSABLE_DIR = "addressable" const val SEED_FILE = ".seed" const val JSON_EXT = ".json" + /** Lowercase hex SHA-256 of the given UTF-8 string. Used for d-tag slots. */ + fun sha256Hex(s: String): String { + val md = MessageDigest.getInstance("SHA-256") + val bytes = md.digest(s.encodeToByteArray()) + val sb = StringBuilder(bytes.size * 2) + for (b in bytes) { + val v = b.toInt() and 0xff + sb.append(HEX[v ushr 4]) + sb.append(HEX[v and 0x0f]) + } + return sb.toString() + } + + private val HEX = "0123456789abcdef".toCharArray() + /** zero-padded to 10 digits so lex order == chronological order through year 2286. */ fun tsPad(ts: Long): String = ts.toString().padStart(TS_PAD_WIDTH, '0') diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlots.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlots.kt new file mode 100644 index 0000000000..d0cad6fe97 --- /dev/null +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlots.kt @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.fs + +import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.isAddressable +import com.vitorpamplona.quartz.nip01Core.core.isReplaceable +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.util.UUID +import kotlin.io.path.deleteIfExists +import kotlin.io.path.exists +import kotlin.io.path.readText + +/** + * Replaceable- and addressable-event slot management. + * + * A *slot* is a single hardlink whose parent directory encodes the + * Nostr uniqueness constraint: + * + * replaceable//.json # kinds 0, 3, 10000-19999 + * addressable///.json # kinds 30000-39999 + * + * Because a directory can only hold one entry with a given name, the + * filesystem *is* the `UNIQUE(kind, pubkey[, d])` constraint. Insertion + * rules match SQLite's ReplaceableModule / AddressableModule triggers: + * + * Tnew = incoming event.createdAt + * Told = current slot winner's createdAt (if any) + * + * Tnew > Told → atomically install new slot, evict old + * Tnew <= Told → reject the insert entirely + * no existing slot → install new slot + * not replaceable/addr. → no-op (kinds that don't have slot semantics) + * + * Eviction deletes the old canonical file and all its index hardlinks. + */ +internal class FsSlots( + private val layout: FsLayout, + private val indexer: FsIndexer, +) { + /** Path of the slot that owns this event's identity, or null if none. */ + fun slotPathFor(event: Event): Path? = + when { + event.kind.isReplaceable() -> { + layout.replaceableSlot(event.kind, event.pubKey) + } + + event is AddressableEvent && event.kind.isAddressable() -> { + layout.addressableSlot(event.kind, event.pubKey, event.dTag()) + } + + else -> { + null + } + } + + /** + * Pre-insert check. Returns the *existing* winner if it blocks the + * insert (i.e. its createdAt is >= incoming.createdAt). Returns null + * when insertion may proceed — possibly with an evictable older winner, + * which the caller looks up separately via [readSlot]. + */ + fun shouldBlock( + event: Event, + slot: Path, + ): Boolean { + val existing = readSlot(slot) ?: return false + return existing.createdAt >= event.createdAt + } + + fun readSlot(slot: Path): Event? { + if (!slot.exists()) return null + return try { + Event.fromJson(slot.readText()) + } catch (_: java.nio.file.NoSuchFileException) { + null + } catch (_: Exception) { + null + } + } + + /** + * Install [canonical] at [slot] atomically, evicting [evicting] if + * present and distinct from the new event. The old canonical and all + * its index hardlinks are removed; the slot itself is swapped in via + * `rename(2)` (REPLACE_EXISTING + ATOMIC_MOVE). + */ + fun install( + slot: Path, + canonical: Path, + newEvent: Event, + evicting: Event?, + ) { + Files.createDirectories(slot.parent) + val tmp = slot.resolveSibling("${slot.fileName}.tmp.${UUID.randomUUID()}") + try { + Files.createLink(tmp, canonical) + Files.move(tmp, slot, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING) + } catch (t: Throwable) { + Files.deleteIfExists(tmp) + throw t + } + if (evicting != null && evicting.id != newEvent.id) { + indexer.unlink(evicting) + layout.canonical(evicting.id).deleteIfExists() + } + } + + /** Remove a slot without installing a replacement. Used by NIP-09 cascades. */ + fun clear(slot: Path) { + Files.deleteIfExists(slot) + } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlotsTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlotsTest.kt new file mode 100644 index 0000000000..232424a081 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlotsTest.kt @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.fs + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent +import com.vitorpamplona.quartz.utils.Secp256k1Instance +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.readText +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class FsSlotsTest { + private val signer = NostrSignerSync() + private lateinit var root: Path + private lateinit var store: FsEventStore + + @BeforeTest + fun setup() { + Secp256k1Instance + root = Files.createTempDirectory("fs-slot-") + store = FsEventStore(root) + } + + @AfterTest + fun tearDown() { + store.close() + if (root.exists()) { + Files.walk(root).use { it.sorted(Comparator.reverseOrder()).forEach { p -> Files.deleteIfExists(p) } } + } + } + + // ------------------------------------------------------------------ + // Replaceable (kind 0, 3, 10000-19999) + // ------------------------------------------------------------------ + + private fun metadata( + name: String, + createdAt: Long, + ) = signer.sign( + createdAt = createdAt, + kind = MetadataEvent.KIND, + tags = emptyArray(), + content = "{\"name\":\"$name\"}", + ) + + @Test + fun `newer replaceable evicts older`() { + val v1 = metadata("old", 100) + val v2 = metadata("new", 200) + store.insert(v1) + store.insert(v2) + + // Only the newer survives a query by author. + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) + assertEquals(listOf(v2.id), got.map { it.id }) + + // The older canonical is gone. + assertFalse(store.hasCanonical(v1.id), "older canonical should be removed") + } + + @Test + fun `older replaceable is rejected when newer exists`() { + val newer = metadata("new", 200) + val older = metadata("old", 100) + store.insert(newer) + store.insert(older) + + // Newer still wins. + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) + assertEquals(listOf(newer.id), got.map { it.id }) + + // And the older was never persisted. + assertFalse(store.hasCanonical(older.id), "older should have been rejected") + } + + @Test + fun `equal timestamp replaceable is rejected`() { + val a = metadata("a", 100) + val b = metadata("b", 100) + store.insert(a) + store.insert(b) + + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) + assertEquals(1, got.size) + assertEquals(a.id, got.single().id) + } + + @Test + fun `replaceable slot file contains the current winner`() { + val v = metadata("only", 100) + store.insert(v) + + val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") + assertTrue(slot.exists(), "slot must exist") + val parsed = Event.fromJson(slot.readText()) + assertEquals(v.id, parsed.id) + } + + @Test + fun `replaceable slot survives canonical deletion via hardlink`() { + val v = metadata("x", 100) + store.insert(v) + + // Simulate a user (or bug) removing the canonical file. + val canonical = + root + .resolve("events") + .resolve(v.id.substring(0, 2)) + .resolve(v.id.substring(2, 4)) + .resolve("${v.id}.json") + assertTrue(Files.deleteIfExists(canonical)) + + val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") + assertTrue(slot.exists(), "slot should persist even if canonical is gone (hardlink to same inode)") + val parsed = Event.fromJson(slot.readText()) + assertEquals(v.id, parsed.id) + } + + @Test + fun `eviction unlinks index hardlinks for the old winner`() { + val v1 = metadata("old", 100) + val v2 = metadata("new", 200) + store.insert(v1) + store.insert(v2) + + // Author index should have exactly one entry — the winner. + val authorDir = root.resolve("idx/author/${signer.pubKey}") + val entries = + Files.list(authorDir).use { s -> + s.toList().map { it.fileName.toString() } + } + assertEquals(1, entries.size, "author index should only hold the winner") + assertTrue(entries.single().endsWith("-${v2.id}"), "author index entry must point at winner") + } + + @Test + fun `delete of current replaceable winner clears the slot`() { + val v = metadata("only", 100) + store.insert(v) + + val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") + assertTrue(slot.exists()) + + store.delete(v.id) + assertFalse(slot.exists(), "slot should be cleared when winner is deleted") + } + + // ------------------------------------------------------------------ + // Addressable (kinds 30000-39999) + // ------------------------------------------------------------------ + + private fun article( + slug: String, + body: String, + createdAt: Long, + ): LongTextNoteEvent = + signer.sign( + createdAt = createdAt, + kind = LongTextNoteEvent.KIND, + tags = arrayOf(arrayOf("d", slug)), + content = body, + ) + + @Test + fun `newer addressable evicts older for same d-tag`() { + val v1 = article("intro", "draft 1", 10) + val v2 = article("intro", "draft 2", 20) + store.insert(v1) + store.insert(v2) + + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) + assertEquals(listOf(v2.id), got.map { it.id }) + assertFalse(store.hasCanonical(v1.id), "older draft canonical should be removed") + } + + @Test + fun `addressable with different d-tags coexist`() { + val intro = article("intro", "hello", 10) + val about = article("about", "bio", 15) + store.insert(intro) + store.insert(about) + + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) + assertEquals(setOf(intro.id, about.id), got.map { it.id }.toSet()) + } + + @Test + fun `older addressable is rejected when newer exists`() { + val newer = article("slug", "new", 200) + val older = article("slug", "old", 100) + store.insert(newer) + store.insert(older) + + val got = store.query(Filter(authors = listOf(signer.pubKey))) + assertEquals(listOf(newer.id), got.map { it.id }) + } + + @Test + fun `addressable slot file contains the current winner`() { + val v = article("intro", "hello", 10) + store.insert(v) + + val dHash = FsLayout.sha256Hex("intro") + val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") + assertTrue(slot.exists()) + val parsed = Event.fromJson(slot.readText()) + assertEquals(v.id, parsed.id) + } + + @Test + fun `empty d-tag gets its own slot`() { + val v = article("", "homepage", 1) + store.insert(v) + + val dHash = FsLayout.sha256Hex("") + val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") + assertTrue(slot.exists()) + } + + @Test + fun `delete of current addressable winner clears the slot`() { + val v = article("intro", "hello", 10) + store.insert(v) + val dHash = FsLayout.sha256Hex("intro") + val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") + assertTrue(slot.exists()) + + store.delete(v.id) + assertFalse(slot.exists()) + } + + // ------------------------------------------------------------------ + // Non-replaceable events: no slot involvement + // ------------------------------------------------------------------ + + @Test + fun `regular text note has no slot`() { + val note = + signer.sign( + createdAt = 1, + kind = 1, + tags = emptyArray(), + content = "plain", + ) + store.insert(note) + + // No entries under replaceable/ or addressable/ — only the scaffolded dirs exist. + val replaceableDir = root.resolve("replaceable") + val addressableDir = root.resolve("addressable") + assertEquals( + 0, + Files.walk(replaceableDir).use { s -> s.filter { Files.isRegularFile(it) }.count() }, + ) + assertEquals( + 0, + Files.walk(addressableDir).use { s -> s.filter { Files.isRegularFile(it) }.count() }, + ) + } + + // helper — check canonical existence + private fun FsEventStore.hasCanonical(id: String): Boolean { + val p = + root + .resolve("events") + .resolve(id.substring(0, 2)) + .resolve(id.substring(2, 4)) + .resolve("$id.json") + return p.exists() + } +}