From f29196f0fc54922839e76d3f1b581b314cb6dc2b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 16:04:20 +0000 Subject: [PATCH] =?UTF-8?q?test(store):=20measure=20follow-feed=20read/wri?= =?UTF-8?q?te/size=20tradeoff=20=E2=80=94=20keep=20current=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit follow-feed (kinds=[1,6] × 150 authors, ORDER BY created_at DESC LIMIT 500) was geode's 5.5× loss (97.7ms vs strfry 17.6ms). Investigated whether any change is worth it, across read + write + size. Read (FollowFeedReadBenchmark, in-memory, scale 5 ≈ 1.05M events): prolific-recent sparse-old current 5.7 ms 1.9 ms scan (strfry) 1.0 ms 1601.9 ms union 316.9 ms 20.0 ms - scan (created_at index + early LIMIT) wins for active follows but is catastrophic for sparse/inactive follows AND grows with corpus size (234ms→1601ms from scale 1→5) — following rarely-posting accounts is common, so it'd be a severe regression. - union (300 per-branch subqueries) is dominated by branch overhead. - current is the only robust option — flat across scale, bounded by the followed set, never catastrophic. The 97.7ms is a worst case (the 150 MOST prolific authors, disk-bound reading all their matching rows). No safe SQL-level swap exists; each alternative trades geode's worst case for a worse one on a common workload. The only universal improvement is strfry's app-level k-way merge (O(LIMIT+streams)) — a real new executor, not a SQL tweak. Write & size: neutral for every candidate — all reuse existing indexes (query_by_kind_pubkey_created / query_by_created_at_id), none adds a CREATE INDEX, so ingest throughput and storage are untouched regardless of choice. A new index was considered and rejected (taxes every write, helps one shape, reverts under ANALYZE). Decision: keep the current composite plan. Full write-up in quartz/plans/2026-07-04-follow-feed-read-tradeoff.md. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012EZeWww5TJnzBZKPoc6mvU --- quartz/build.gradle.kts | 1 + .../2026-07-04-follow-feed-read-tradeoff.md | 73 +++++++ .../prodbench/FollowFeedReadBenchmark.kt | 201 ++++++++++++++++++ 3 files changed, 275 insertions(+) create mode 100644 quartz/plans/2026-07-04-follow-feed-read-tradeoff.md create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FollowFeedReadBenchmark.kt diff --git a/quartz/build.gradle.kts b/quartz/build.gradle.kts index efc1db5eb7..10c0817ee8 100644 --- a/quartz/build.gradle.kts +++ b/quartz/build.gradle.kts @@ -93,6 +93,7 @@ kotlin { } // Forward the negentropy-benchmark corpus size to the test JVM. System.getProperty("negBenchN")?.let { systemProperty("negBenchN", it) } + System.getProperty("followBenchScale")?.let { systemProperty("followBenchScale", it) } // Opt-in JFR profiling of a benchmark run (-PnegProfile=/tmp/neg.jfr). (project.findProperty("negProfile") as? String)?.let { jvmArgs("-XX:+FlightRecorder", "-XX:StartFlightRecording=filename=$it,settings=profile,dumponexit=true") diff --git a/quartz/plans/2026-07-04-follow-feed-read-tradeoff.md b/quartz/plans/2026-07-04-follow-feed-read-tradeoff.md new file mode 100644 index 0000000000..d33a790b33 --- /dev/null +++ b/quartz/plans/2026-07-04-follow-feed-read-tradeoff.md @@ -0,0 +1,73 @@ +# follow-feed: measured read/write/size tradeoff — keep the current plan + +The 1M run showed geode's `follow-feed` (`kinds=[1,6] AND authors=[150] +ORDER BY created_at DESC LIMIT 500`) at 97.7 ms vs strfry 17.6 ms. This +measured whether any change is worth it, deliberately across **read**, +**write**, and **size** so a read win doesn't quietly cost elsewhere. + +## What the current plan actually does + +Composite seek `query_by_kind_pubkey_created (kind, pubkey, created_at)` for +the 300 `(kind, pubkey)` combos, feeding every matching row into a +LIMIT-bounded top-500 sorter (`USE TEMP B-TREE FOR ORDER BY`). So it reads +**O(followed authors' matching history)** rows — cheap in memory, but on a +cold on-disk 1M DB, following prolific accounts means reading hundreds of +thousands of index rows: that's the 97.7 ms (and it's a worst case — the +relayBench follow set is the 150 *most prolific* authors). + +## Read — measured (`FollowFeedReadBenchmark`, in-memory) + +Three strategies, all on **existing** indexes, across two opposite follow +profiles, at two corpus sizes: + +| | prolific-recent | sparse-old | +|---|---:|---:| +| **current** (composite + bounded sort) | 5.7 ms | 1.9 ms | +| **scan** (`created_at` index, early-LIMIT — strfry's shape) | **1.0 ms** | **1601.9 ms** | +| **union** (per-(author,kind) LIMIT 500, merged) | 316.9 ms | 20.0 ms | + +*(scale 5 ≈ 1.05M events; scale 1 numbers: scan sparse-old 234 ms, so it +grows 234 → 1601 ms as the corpus grows.)* + +- **scan** is the strfry approach and wins big for active follows (1.0 ms), + but is **catastrophic for sparse/inactive follows** — it walks newest-first + through the whole corpus to reach their old events, so it's slow **and + scales with total corpus size** (234 → 1601 ms). Following people who rarely + post is the common case; this would be a severe regression. +- **union** (300 branches) is dominated by per-branch overhead — non-viable. +- **current** is the only *robust* option: never catastrophic, flat across + scale, bounded by the followed set (not the corpus). + +**No safe SQL-level swap exists.** Each alternative trades geode's worst case +for a worse one on a common workload. The only universally-better plan is +strfry's algorithm — an **app-level k-way merge**: open a `(kind, pubkey, +created_at DESC)` cursor per `(author, kind)`, heap-merge them, stop at the +LIMIT. That reads **O(LIMIT + streams)** regardless of follow activity or +corpus size. It's a real new query-execution path (not a SQL tweak), worth +building only if prolific-follow feeds become a measured production priority. + +## Write & size — neutral for every candidate + +All four strategies (current, scan, union, k-way merge) run on indexes that +already exist: + +| strategy | index | new index? | +|---|---|---| +| current / union / k-way | `query_by_kind_pubkey_created` (unconditional) | no | +| scan | `query_by_created_at_id` (`indexEventsByCreatedAtAlone`, on for geode) | no | + +So **no read-strategy choice here changes the index set** — write throughput +(geode's 1.3× ingest lead) and on-disk footprint (geode's storage win) are +untouched no matter which we pick. Adding a *new* index to help this one +shape was considered and rejected: it would tax every insert and grow the DB +for a single query pattern, and (as the profiles investigation already +showed) SQLite reverts to the scan under `ANALYZE` anyway. + +## Decision + +Keep the current composite plan. It's the robust default; the tempting SQL +fixes each break a common workload and get worse at scale, which is exactly +the "don't break something else" risk. If follow-feed latency on prolific +follows becomes a priority, the app-level k-way merge is the only approach +that improves it universally, and it's write/size-neutral. Benchmark kept as +the evidence and as a guard for anyone tempted by the `scan` shortcut. diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FollowFeedReadBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FollowFeedReadBenchmark.kt new file mode 100644 index 0000000000..4a54e5242e --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/FollowFeedReadBenchmark.kt @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.prodbench + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.nip01Core.store.sqlite.explainQuery +import com.vitorpamplona.quartz.utils.EventFactory +import kotlinx.coroutines.runBlocking +import kotlin.test.Test + +/** + * Investigates the `follow-feed` query — `kinds=[1,6] AND authors=[150] + * ORDER BY created_at DESC LIMIT 500` — which the 1M run showed geode losing + * 5.5× on (97.7 ms vs strfry 17.6 ms). The current plan seeks the + * (kind,pubkey) index for the 150 authors, **collects every matching event**, + * then TEMP B-TREE sorts to 500 — no early termination, so it explodes when + * followed authors are prolific. + * + * Compares three read strategies (all on **existing** indexes, so no + * write/size cost) across two opposite author profiles: + * - **current**: composite seek + collect-all + sort. + * - **scan**: `INDEXED BY query_by_created_at_id` — walk newest-first, + * filter kind+author, stop at LIMIT (strfry's early-terminating shape). + * - **union**: per-(author,kind) `ORDER BY created_at DESC LIMIT 500` + * branches merged — bounds each branch's read to the LIMIT. + * + * Scenarios (same store, two disjoint follow sets + background): + * - **prolific-recent**: followed authors post densely & recently. + * - **sparse-old**: followed authors post rarely & long ago (scan must skip + * a lot of newer background to reach them). + * + * Size the seed with `-DfollowBenchScale=N` (default 1 ≈ ~130k events). + */ +class FollowFeedReadBenchmark { + companion object { + val SCALE = System.getProperty("followBenchScale")?.toInt() ?: 1 + } + + private val hex = "0123456789abcdef" + + private fun mix(seed: Long): Long { + var z = seed + -0x61c8864680b583ebL + z = (z xor (z ushr 30)) * -0x40a7b892e31b1a47L + z = (z xor (z ushr 27)) * -0x6b2fb644ecceee15L + return z xor (z ushr 31) + } + + private fun hex64( + salt: Long, + index: Int, + ): String { + val out = CharArray(64) + for (w in 0 until 4) { + val v = mix(salt * 1_000_003 + index.toLong() * 4 + w) + for (b in 0 until 8) { + val byte = ((v ushr (b * 8)) and 0xFF).toInt() + out[(w * 8 + b) * 2] = hex[byte ushr 4] + out[(w * 8 + b) * 2 + 1] = hex[byte and 0xF] + } + } + return String(out) + } + + private val sig = "0".repeat(128) + private var idSeq = 0 + + private fun ev( + pubkey: String, + createdAt: Long, + kind: Int, + ): Event = EventFactory.create(hex64(7, idSeq++), pubkey, createdAt, kind, emptyArray(), "", sig) + + private val cols = "id, pubkey, created_at, kind, tags, content, sig" + + @Test + fun compareFollowFeedStrategies() = + runBlocking { + val store = EventStore(dbName = null, indexStrategy = DefaultIndexingStrategy(indexEventsByCreatedAtAlone = true, indexEventsByPubkeyAlone = true, indexFullTextSearch = false)) + + val base = 1_700_000_000L + val span = 3_000_000L // ~35 days + val batch = ArrayList(10_000) + + fun add(e: Event) { + batch.add(e) + if (batch.size >= 10_000) { + runBlocking { store.batchInsert(batch) } + batch.clear() + } + } + + // Background: many authors posting throughout the whole window. + val bgAuthors = 3_000 * SCALE + for (a in 0 until bgAuthors) { + val pk = hex64(1, a) + repeat(20) { add(ev(pk, base + (mix(a * 31L + it) and 0x7fffffff) % span, 1)) } + } + + // prolific-recent: 150 authors, dense in the newest 10% of time. + val prolific = (0 until 150).map { hex64(2, it) } + val recentStart = base + span * 9 / 10 + for ((i, pk) in prolific.withIndex()) { + repeat(1_000 * SCALE) { + val kind = if (it % 8 == 0) 6 else 1 + add(ev(pk, recentStart + (mix(i * 131L + it) and 0x7fffffff) % (span / 10), kind)) + } + } + + // sparse-old: 150 authors, a few events each, in the oldest 10%. + val sparse = (0 until 150).map { hex64(3, it) } + for ((i, pk) in sparse.withIndex()) { + repeat(6) { + val kind = if (it % 3 == 0) 6 else 1 + add(ev(pk, base + (mix(i * 17L + it) and 0x7fffffff) % (span / 10), kind)) + } + } + if (batch.isNotEmpty()) store.batchInsert(batch) + + val total = runBlocking { store.count(Filter()) } + println("─ FollowFeedReadBenchmark: $total events (scale=$SCALE) ─") + + for ((scenario, authors) in listOf("prolific-recent" to prolific, "sparse-old" to sparse)) { + val inList = authors.joinToString(",") { "'$it'" } + val whereTail = "WHERE kind IN (1,6) AND pubkey IN ($inList) ORDER BY created_at DESC LIMIT 500" + + val current = "SELECT $cols FROM event_headers $whereTail" + val scan = "SELECT $cols FROM event_headers INDEXED BY query_by_created_at_id $whereTail" + // Per-(author,kind) branch, each wrapped in a subquery so its + // ORDER BY/LIMIT is legal inside the UNION ALL; the outer merge + // sorts the ≤ 300×500 collected rows down to 500. + val union = + "SELECT $cols FROM (\n" + + authors.joinToString("\nUNION ALL\n") { pk -> + listOf(1, 6).joinToString("\nUNION ALL\n") { k -> + "SELECT $cols FROM (SELECT $cols FROM event_headers WHERE kind = $k AND pubkey = '$pk' ORDER BY created_at DESC LIMIT 500)" + } + } + + "\n) ORDER BY created_at DESC LIMIT 500" + + println(" ═ $scenario ═") + for ((name, sql) in listOf("current" to current, "scan" to scan, "union" to union)) { + try { + val plan = + runBlocking { store.store.explainQuery(sql) } + .lineSequence() + .filter { it.contains("SEARCH") || it.contains("SCAN ") || (it.contains("USE ") && !it.contains("USING")) } + .joinToString(" | ") { it.trimStart('│', '├', '└', '─', ' ') } + .let { if (it.length > 90) it.take(90) + "…" else it } + val (n, ms) = timeRaw(store, sql) + println(" %-8s %6.1f ms (%d rows) %s".format(name, ms, n, plan)) + } catch (e: Exception) { + println(" %-8s ERROR: %s".format(name, e.message?.take(80))) + } + } + } + store.close() + } + + private fun timeRaw( + store: EventStore, + sql: String, + ): Pair { + fun run() = + runBlocking { + store.store.pool.useReader { c -> + c.prepare(sql).use { s -> + var n = 0 + while (s.step()) n++ + n + } + } + } + repeat(3) { run() } + val runs = 10 + val start = System.nanoTime() + var got = 0 + repeat(runs) { got = run() } + return got to (System.nanoTime() - start) / 1e6 / runs + } +}