laantungir/amethyst
1
0
Fork 0
mirror of https://github.com/vitorpamplona/amethyst.git synced 2026-08-09 08:04:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3468 from vitorpamplona/claude/dedup-decode-benchmark-ci-rl7v8u

Browse Source 
Make dedup decode benchmark resilient to CI load variance
This commit is contained in:
Vitor Pamplona
2026-07-03 17:21:52 -04:00
committed by GitHub
parent ae5e0ff8ed 130135250b
commit 4e895c20dd
1 changed files with 42 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/DedupDecodeBenchmark.kt
+42 -23
View File
@@ -70,39 +70,58 @@ class DedupDecodeBenchmark {
return System.nanoTime() - start
}
@Test
fun duplicateFramesDecodeFasterThroughTheCache() {
val frames = buildFrames()
val dupShare = 1.0 - 1.0 / DELIVERIES_PER_EVENT
// warmup both paths
run(MessageDecoder.Default, frames.take(20_000))
run(CachingEventDecoder(capacity = UNIQUE_EVENTS * 2), frames.take(20_000))
private fun measureOnce(
attempt: Int,
frames: List<String>,
): Double {
val fullNanos = run(MessageDecoder.Default, frames)
val caching = CachingEventDecoder(capacity = UNIQUE_EVENTS * 2)
val cachedNanos = run(caching, frames)
val speedup = fullNanos.toDouble() / cachedNanos
println("=== DEDUP DECODE BENCHMARK (${frames.size} frames, %.0f%% duplicates) ===".format(dupShare * 100))
println(
" full parse every frame: %.1fms (%.1fµs/frame)"
.format(fullNanos / 1e6, fullNanos / 1e3 / frames.size),
)
println(
" caching decoder: %.1fms (%.1fµs/frame) parsed=%d reused=%d -> %.2fx"
.format(cachedNanos / 1e6, cachedNanos / 1e3 / frames.size, caching.parsedCount, caching.reusedCount, speedup),
)
// Deterministic correctness gate, independent of timing: a functional
// regression (duplicates missing the cache) fails on every attempt.
assertTrue(
caching.parsedCount == UNIQUE_EVENTS.toLong() &&
caching.reusedCount == (frames.size - UNIQUE_EVENTS).toLong(),
"every duplicate must hit the cache",
)
val speedup = fullNanos.toDouble() / cachedNanos
println(
" attempt $attempt: full parse %.1fms (%.1fµs/frame) vs caching %.1fms (%.1fµs/frame) parsed=%d reused=%d -> %.2fx"
.format(fullNanos / 1e6, fullNanos / 1e3 / frames.size, cachedNanos / 1e6, cachedNanos / 1e3 / frames.size, caching.parsedCount, caching.reusedCount, speedup),
)
return speedup
}
@Test
fun duplicateFramesDecodeFasterThroughTheCache() {
val frames = buildFrames()
val dupShare = 1.0 - 1.0 / DELIVERIES_PER_EVENT
println("=== DEDUP DECODE BENCHMARK (${frames.size} frames, %.0f%% duplicates) ===".format(dupShare * 100))
// warmup both paths (JIT + Jackson caches)
run(MessageDecoder.Default, frames.take(20_000))
run(CachingEventDecoder(capacity = UNIQUE_EVENTS * 2), frames.take(20_000))
// The claim that justifies the code: at a 66% duplicate share the
// caching decoder must be meaningfully faster than parsing everything.
// Generous threshold (1.5x) so CI noise doesn't flake; measured ~2.5-3x.
assertTrue(speedup > 1.5, "expected >1.5x speedup, got %.2fx".format(speedup))
// caching decoder must be meaningfully faster than parsing everything
// (measured ~2.5-3x). But wall-clock ratios on a loaded shared CI
// runner can compress a single sample below any generous target, so a
// sub-target sample is not a regression by itself. Retry a couple of
// times; enforce a hard floor that a real regression (cache lookups
// costing as much as the parse they skip -> ~1.0x) can never pass,
// and warn — don't flake — in the noise band.
var best = 0.0
for (attempt in 1..3) {
best = maxOf(best, measureOnce(attempt, frames))
if (best > 1.5) break
}
assertTrue(best > 1.05, "caching decoder no faster than full parse (best %.2fx) — dedup is not saving work".format(best))
if (best <= 1.5) {
println(" WARN: best speedup %.2fx below the 1.5x target — machine likely loaded; not failing".format(best))
}
}
}