mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-08-10 16:33:27 +00:00
feat: authors-only query index — close the one gap vs strfry's index set
Reviewed strfry's LMDB indices (golpe.yaml) against geode's SQLite set: the two are nearly isomorphic — time, id, kind+time, author+kind+time, tag+time, plus conditional deletion/expiration/replaceable entries (geode's are partial indexes, so ordinary events don't pay for them). Nothing to drop. One real hole: strfry maintains a plain pubkey(+created_at) index and geode had none, so an authors-only filter (no kinds) — archive pulls, account-migration tools, 'everything by these pubkeys' — degraded to a full walk of the time index. EXPLAIN confirmed: SCAN query_by_created_at_id. - quartz: IndexingStrategy.indexEventsByPubkeyAlone (default false — clients query their supported kinds and can skip it) gates a new query_by_pubkey_created index; DATABASE_VERSION 2→3 with an idempotent migration that backfills it for opted-in strategies. - geode: relayIndexingStrategy turns it on. - relayBench: new 'author-archive' scenario — every kind by 3 *quiet* pubkeys. Quiet is the point: prolific authors are dense in the time index and a scan finds them quickly, which is why the suite never caught this; sparse authors force the full walk. Measured (50k corpus): author-archive EOSE p50 42.8 ms -> 3.6 ms (12x, and the old path grows linearly with table size); ingest 5,337 -> 5,156 events/s (~3%, the one extra B-tree per event). strfry reference on the same scenario: 0.52 ms. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NeoCvXnTxsKzqurkmjdC46
This commit is contained in:
@@ -44,6 +44,11 @@ import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
|
||||
fun relayIndexingStrategy(fullTextSearch: Boolean = true) =
|
||||
DefaultIndexingStrategy(
|
||||
indexEventsByCreatedAtAlone = true,
|
||||
// Authors-only filters (no kinds) are relay-common — archives,
|
||||
// migration tools. strfry maintains the same (pubkey, created_at)
|
||||
// index unconditionally; without it the filter walks the whole
|
||||
// time index.
|
||||
indexEventsByPubkeyAlone = true,
|
||||
indexFullTextSearch = fullTextSearch,
|
||||
)
|
||||
|
||||
|
||||
+23
@@ -79,6 +79,13 @@ class EventIndexesModule(
|
||||
db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers ($orderBy)")
|
||||
}
|
||||
|
||||
// queries by author only, no kind — "everything by these pubkeys"
|
||||
// (profile archives, migration/backup tools). Relays need it;
|
||||
// clients query their supported kinds and can skip it.
|
||||
if (indexStrategy.indexEventsByPubkeyAlone) {
|
||||
db.execSQL("CREATE INDEX query_by_pubkey_created ON event_headers (pubkey, $orderBy)")
|
||||
}
|
||||
|
||||
// queries by kind only, mostly used in Global Feeds when author is not important.
|
||||
db.execSQL("CREATE INDEX query_by_kind_created ON event_headers (kind, $orderBy)")
|
||||
|
||||
@@ -133,6 +140,22 @@ class EventIndexesModule(
|
||||
db.execSQL("DROP TABLE IF EXISTS event_headers")
|
||||
}
|
||||
|
||||
/**
|
||||
* v2 → v3 migration: the authors-only index arrived after v2 schemas
|
||||
* shipped. Backfills it (idempotently) for strategies that want it;
|
||||
* everyone else just gets the version bump.
|
||||
*/
|
||||
fun migrateV2AddPubkeyIndex(db: SQLiteConnection) {
|
||||
if (!indexStrategy.indexEventsByPubkeyAlone) return
|
||||
val orderBy =
|
||||
if (indexStrategy.useAndIndexIdOnOrderBy) {
|
||||
"created_at DESC, id ASC"
|
||||
} else {
|
||||
"created_at DESC"
|
||||
}
|
||||
db.execSQL("CREATE INDEX IF NOT EXISTS query_by_pubkey_created ON event_headers (pubkey, $orderBy)")
|
||||
}
|
||||
|
||||
val sqlInsertHeader =
|
||||
"""
|
||||
INSERT INTO event_headers
|
||||
|
||||
+12
@@ -41,6 +41,17 @@ interface IndexingStrategy {
|
||||
*/
|
||||
val indexEventsByCreatedAtAlone: Boolean
|
||||
|
||||
/**
|
||||
* Activate this for filters that carry `authors` but no `kinds` —
|
||||
* "everything by these pubkeys". Clients rarely need it (they query
|
||||
* their supported kinds), but relays receive it constantly: profile
|
||||
* archives, account migration/backup tools, and follow-everything
|
||||
* feeds. Without it those filters degrade to a full scan of the
|
||||
* time index. strfry maintains the equivalent (`pubkey`) index
|
||||
* unconditionally.
|
||||
*/
|
||||
val indexEventsByPubkeyAlone: Boolean
|
||||
|
||||
/**
|
||||
* Activate this if you see too many Tag-centric Filters without
|
||||
* kind, pubkey or id.
|
||||
@@ -107,6 +118,7 @@ interface IndexingStrategy {
|
||||
*/
|
||||
class DefaultIndexingStrategy(
|
||||
override val indexEventsByCreatedAtAlone: Boolean = false,
|
||||
override val indexEventsByPubkeyAlone: Boolean = false,
|
||||
override val indexTagsByCreatedAtAlone: Boolean = false,
|
||||
override val indexTagsWithKindAndPubkey: Boolean = false,
|
||||
override val useAndIndexIdOnOrderBy: Boolean = false,
|
||||
|
||||
+6
-1
@@ -43,7 +43,7 @@ class SQLiteEventStore(
|
||||
val numReaders: Int = 4,
|
||||
) {
|
||||
companion object {
|
||||
const val DATABASE_VERSION = 2
|
||||
const val DATABASE_VERSION = 3
|
||||
}
|
||||
|
||||
val seedModule = SeedModule()
|
||||
@@ -163,6 +163,11 @@ class SQLiteEventStore(
|
||||
modules.reversed().forEach { it.drop(db) }
|
||||
modules.forEach { it.create(db) }
|
||||
}
|
||||
2 -> {
|
||||
// Upgrade from version 2 to 3: authors-only query index
|
||||
// (created only for strategies that opt in).
|
||||
eventIndexModule.migrateV2AddPubkeyIndex(db)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,6 +121,18 @@ object Scenarios {
|
||||
Filter(kinds = listOf(0), authors = topAuthors.take(50)),
|
||||
),
|
||||
)
|
||||
// Deliberately *quiet* authors (popularity tail): their rows
|
||||
// are sparse in the time index, so an authors-only filter
|
||||
// exposes whether the store has a (pubkey, created_at) path
|
||||
// or has to walk the whole table. Prolific authors would
|
||||
// mask the difference — a time-index scan finds them fast.
|
||||
add(
|
||||
Scenario(
|
||||
"author-archive",
|
||||
"every kind by ${topAuthors.takeLast(3).size} quiet pubkeys (archive/migration pull)",
|
||||
Filter(authors = topAuthors.takeLast(3), limit = 500),
|
||||
),
|
||||
)
|
||||
add(
|
||||
Scenario(
|
||||
"follow-feed",
|
||||
|
||||
Reference in New Issue
Block a user