From 3c6d36cde177eb5d023953e2b656fbbf52f68b9f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 16:30:30 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20authors-only=20query=20index=20?= =?UTF-8?q?=E2=80=94=20close=20the=20one=20gap=20vs=20strfry's=20index=20s?= =?UTF-8?q?et?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed strfry's LMDB indices (golpe.yaml) against geode's SQLite set: the two are nearly isomorphic — time, id, kind+time, author+kind+time, tag+time, plus conditional deletion/expiration/replaceable entries (geode's are partial indexes, so ordinary events don't pay for them). Nothing to drop. One real hole: strfry maintains a plain pubkey(+created_at) index and geode had none, so an authors-only filter (no kinds) — archive pulls, account-migration tools, 'everything by these pubkeys' — degraded to a full walk of the time index. EXPLAIN confirmed: SCAN query_by_created_at_id. - quartz: IndexingStrategy.indexEventsByPubkeyAlone (default false — clients query their supported kinds and can skip it) gates a new query_by_pubkey_created index; DATABASE_VERSION 2→3 with an idempotent migration that backfills it for opted-in strategies. - geode: relayIndexingStrategy turns it on. - relayBench: new 'author-archive' scenario — every kind by 3 *quiet* pubkeys. Quiet is the point: prolific authors are dense in the time index and a scan finds them quickly, which is why the suite never caught this; sparse authors force the full walk. Measured (50k corpus): author-archive EOSE p50 42.8 ms -> 3.6 ms (12x, and the old path grows linearly with table size); ingest 5,337 -> 5,156 events/s (~3%, the one extra B-tree per event). strfry reference on the same scenario: 0.52 ms. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NeoCvXnTxsKzqurkmjdC46 --- .../geode/RelayIndexingStrategy.kt | 5 ++++ .../store/sqlite/EventIndexesModule.kt | 23 +++++++++++++++++++ .../store/sqlite/IndexingStrategy.kt | 12 ++++++++++ .../store/sqlite/SQLiteEventStore.kt | 7 +++++- .../com/vitorpamplona/relaybench/Scenarios.kt | 12 ++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/RelayIndexingStrategy.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/RelayIndexingStrategy.kt index dadef48a06..3809672cae 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/RelayIndexingStrategy.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/RelayIndexingStrategy.kt @@ -44,6 +44,11 @@ import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy fun relayIndexingStrategy(fullTextSearch: Boolean = true) = DefaultIndexingStrategy( indexEventsByCreatedAtAlone = true, + // Authors-only filters (no kinds) are relay-common — archives, + // migration tools. strfry maintains the same (pubkey, created_at) + // index unconditionally; without it the filter walks the whole + // time index. + indexEventsByPubkeyAlone = true, indexFullTextSearch = fullTextSearch, ) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt index 75ef7ce718..f0228217b0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt @@ -79,6 +79,13 @@ class EventIndexesModule( db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers ($orderBy)") } + // queries by author only, no kind — "everything by these pubkeys" + // (profile archives, migration/backup tools). Relays need it; + // clients query their supported kinds and can skip it. + if (indexStrategy.indexEventsByPubkeyAlone) { + db.execSQL("CREATE INDEX query_by_pubkey_created ON event_headers (pubkey, $orderBy)") + } + // queries by kind only, mostly used in Global Feeds when author is not important. db.execSQL("CREATE INDEX query_by_kind_created ON event_headers (kind, $orderBy)") @@ -133,6 +140,22 @@ class EventIndexesModule( db.execSQL("DROP TABLE IF EXISTS event_headers") } + /** + * v2 → v3 migration: the authors-only index arrived after v2 schemas + * shipped. Backfills it (idempotently) for strategies that want it; + * everyone else just gets the version bump. + */ + fun migrateV2AddPubkeyIndex(db: SQLiteConnection) { + if (!indexStrategy.indexEventsByPubkeyAlone) return + val orderBy = + if (indexStrategy.useAndIndexIdOnOrderBy) { + "created_at DESC, id ASC" + } else { + "created_at DESC" + } + db.execSQL("CREATE INDEX IF NOT EXISTS query_by_pubkey_created ON event_headers (pubkey, $orderBy)") + } + val sqlInsertHeader = """ INSERT INTO event_headers diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt index bcb36f6140..21fc90f52e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt @@ -41,6 +41,17 @@ interface IndexingStrategy { */ val indexEventsByCreatedAtAlone: Boolean + /** + * Activate this for filters that carry `authors` but no `kinds` — + * "everything by these pubkeys". Clients rarely need it (they query + * their supported kinds), but relays receive it constantly: profile + * archives, account migration/backup tools, and follow-everything + * feeds. Without it those filters degrade to a full scan of the + * time index. strfry maintains the equivalent (`pubkey`) index + * unconditionally. + */ + val indexEventsByPubkeyAlone: Boolean + /** * Activate this if you see too many Tag-centric Filters without * kind, pubkey or id. @@ -107,6 +118,7 @@ interface IndexingStrategy { */ class DefaultIndexingStrategy( override val indexEventsByCreatedAtAlone: Boolean = false, + override val indexEventsByPubkeyAlone: Boolean = false, override val indexTagsByCreatedAtAlone: Boolean = false, override val indexTagsWithKindAndPubkey: Boolean = false, override val useAndIndexIdOnOrderBy: Boolean = false, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 61d7dfc13f..985b7c3786 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -43,7 +43,7 @@ class SQLiteEventStore( val numReaders: Int = 4, ) { companion object { - const val DATABASE_VERSION = 2 + const val DATABASE_VERSION = 3 } val seedModule = SeedModule() @@ -163,6 +163,11 @@ class SQLiteEventStore( modules.reversed().forEach { it.drop(db) } modules.forEach { it.create(db) } } + 2 -> { + // Upgrade from version 2 to 3: authors-only query index + // (created only for strategies that opt in). + eventIndexModule.migrateV2AddPubkeyIndex(db) + } } } } diff --git a/relayBench/src/main/kotlin/com/vitorpamplona/relaybench/Scenarios.kt b/relayBench/src/main/kotlin/com/vitorpamplona/relaybench/Scenarios.kt index 43da3db874..5c73ac5ed5 100644 --- a/relayBench/src/main/kotlin/com/vitorpamplona/relaybench/Scenarios.kt +++ b/relayBench/src/main/kotlin/com/vitorpamplona/relaybench/Scenarios.kt @@ -121,6 +121,18 @@ object Scenarios { Filter(kinds = listOf(0), authors = topAuthors.take(50)), ), ) + // Deliberately *quiet* authors (popularity tail): their rows + // are sparse in the time index, so an authors-only filter + // exposes whether the store has a (pubkey, created_at) path + // or has to walk the whole table. Prolific authors would + // mask the difference — a time-index scan finds them fast. + add( + Scenario( + "author-archive", + "every kind by ${topAuthors.takeLast(3).size} quiet pubkeys (archive/migration pull)", + Filter(authors = topAuthors.takeLast(3), limit = 500), + ), + ) add( Scenario( "follow-feed",