Merge pull request #4216 from davotoula/fix/audio-visualizer-frame-drops

fix(audio): make the audio visualizer update in real time
This commit is contained in:
Vitor Pamplona
2026-09-27 09:48:02 -04:00
committed by GitHub
8 changed files with 423 additions and 22 deletions
@@ -22,9 +22,13 @@ package com.vitorpamplona.amethyst.service.playback
/**
* Shared logcat tag for the playback diagnostic trace (source routing, player lifecycle, error
* recovery, HLS liveness learning). Emitted with `Log.d`, so it appears only in a debug build
* (`Log.minLevel = DEBUG`) and is silent in benchmark/release (`ERROR`). To capture a playback
* investigation, install a debug build and run:
* recovery, HLS liveness learning).
*
* Emitted with `Log.d`, which fires only while `Log.minLevel <= DEBUG`. No shipped variant is
* there by default: [com.vitorpamplona.amethyst.Amethyst.DEFAULT_LOG_LEVEL] gives debug AND
* benchmark builds `INFO` (the `benchmark` build type counts as `isDebug`) and release `WARN`, so
* this trace is silent everywhere until `Amethyst.VERBOSE_LOGS` is flipped to true. To capture a
* playback investigation, set that flag, install a debug build and run:
*
* ```
* adb logcat -s PlaybackDiag
@@ -148,9 +148,11 @@ const val VIDEO_QUALITY_TAG = "VideoQuality"
* Traces which rendition adaptive selection landed on, against the full ladder the manifest
* offered.
*
* The listener is registered only when the trace can actually be emitted — debug builds set
* `Log.minLevel = DEBUG` while benchmark/release set `ERROR` (see [PLAYBACK_DIAG_TAG]) — so the
* release path keeps the "no listener per player" property that dropping the old selector bought.
* The listener is registered only when the trace can actually be emitted. `Log.minLevel` is above
* `DEBUG` in every variant by default — `INFO` for debug and benchmark builds, `WARN` for release
* (see [PLAYBACK_DIAG_TAG]) — so this costs nothing until `Amethyst.VERBOSE_LOGS` is turned on,
* and every build keeps the "no listener per player" property that dropping the old selector
* bought.
*/
@Composable
fun LogVideoQualitySelection(player: Player) {
@@ -27,9 +27,11 @@ import androidx.media3.exoplayer.audio.TeeAudioProcessor
import com.vitorpamplona.amethyst.commons.audio.AudioWindow
import com.vitorpamplona.amethyst.commons.audio.Fft
import com.vitorpamplona.amethyst.commons.audio.Spectrum
import com.vitorpamplona.amethyst.commons.audio.SpectrumTrail
import com.vitorpamplona.amethyst.commons.audio.normalizeToPeakInPlace
import com.vitorpamplona.amethyst.commons.audio.toLogBins
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableSharedFlow
import java.nio.ByteBuffer
@@ -65,6 +67,10 @@ class SpectrumAudioBufferSink(
private var channels = 1
private var encoding = C.ENCODING_PCM_16BIT
// Audio time one fft frame covers: fftSize samples per channel at the stream's sample rate.
// Zero until the first flush reports a rate, which leaves the frame unpaced rather than wrong.
private var frameDurationNanos = 0L
@kotlin.OptIn(ExperimentalCoroutinesApi::class)
override fun flush(
sampleRateHz: Int,
@@ -73,6 +79,7 @@ class SpectrumAudioBufferSink(
) {
this.channels = channelCount.coerceAtLeast(1)
this.encoding = encoding
this.frameDurationNanos = if (sampleRateHz > 0) fftSize * 1_000_000_000L / sampleRateHz else 0L
filled = 0
output?.resetReplayCache()
}
@@ -98,7 +105,7 @@ class SpectrumAudioBufferSink(
// Skip the DC bin (index 0): toLogBins ignores it, so letting a DC/offset component be the
// peak would scale every audible bin toward zero and wash the spectrum out.
mags.normalizeToPeakInPlace(fromIndex = 1)
output?.tryEmit(Spectrum(mags.toLogBins(binCount)))
output?.tryEmit(Spectrum(mags.toLogBins(binCount), frameDurationNanos))
}
}
@@ -167,7 +174,15 @@ object PcmTapRegistry {
if (!fedByLiveSink && !stillCollected) iter.remove()
}
}
MutableSharedFlow(replay = 1, extraBufferCapacity = 1)
// Frames arrive in clusters (see SpectrumTrail) emitted within a few ms on the audio
// thread, while the collector sits on the main dispatcher and cannot run in between;
// a 2-slot buffer kept ~2 frames of each cluster and dropped the rest. Hold as much as
// the visualizer's own backlog, which is the real lag bound, dropping the stalest.
MutableSharedFlow(
replay = 1,
extraBufferCapacity = SpectrumTrail.MAX_BACKLOG_FRAMES,
onBufferOverflow = BufferOverflow.DROP_OLDEST,
)
}
}
}
@@ -24,7 +24,11 @@ import androidx.annotation.OptIn
import androidx.media3.common.C
import androidx.media3.common.util.UnstableApi
import com.vitorpamplona.amethyst.commons.audio.Spectrum
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.test.advanceUntilIdle
import kotlinx.coroutines.test.runTest
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
@@ -33,6 +37,7 @@ import java.nio.ByteOrder
import kotlin.math.PI
import kotlin.math.sin
@kotlin.OptIn(ExperimentalCoroutinesApi::class)
@OptIn(UnstableApi::class)
class SpectrumAudioBufferSinkTest {
private val fftSize = 64
@@ -136,4 +141,62 @@ class SpectrumAudioBufferSinkTest {
assertTrue("non-16-bit PCM must not emit a spectrum", out.replayCache.isEmpty())
}
// Pacing draws each frame for the audio time it covers, so the sink has to say how long that is.
// It is fftSize samples PER CHANNEL: a stereo stream must not report half (or double) the time.
@Test
fun eachFrameCarriesTheAudioTimeItCovers() {
val sink = SpectrumAudioBufferSink(fftSize = fftSize, binCount = binCount)
val out = MutableSharedFlow<Spectrum>(replay = 1, extraBufferCapacity = 1)
sink.output = out
sink.flush(48000, 1, C.ENCODING_PCM_16BIT)
sink.handleBuffer(monoPcm(sineShorts(k = 2, n = fftSize)))
assertEquals(fftSize * 1_000_000_000L / 48000, out.replayCache.last().durationNanos)
}
@Test
fun stereoFramesCoverTheSameTimeAsMono() {
val sink = SpectrumAudioBufferSink(fftSize = fftSize, binCount = binCount)
val out = MutableSharedFlow<Spectrum>(replay = 1, extraBufferCapacity = 1)
sink.output = out
sink.flush(44100, 2, C.ENCODING_PCM_16BIT)
val tone = sineShorts(k = 2, n = fftSize)
sink.handleBuffer(interleavedStereoPcm(tone, tone))
assertEquals(fftSize * 1_000_000_000L / 44100, out.replayCache.last().durationNanos)
}
/**
* The audio thread emits a cluster of fft frames within a few ms, with no suspension point in
* between, while the UI collector sits on the main dispatcher and cannot interleave. Anything the
* registry's flow cannot hold at that instant is dropped; a 2-slot buffer kept 2 of every cluster.
*/
private fun deliveredFromOneCluster(framesInCluster: Int): Int {
var count = -1
runTest {
val mediaId = "test://cluster-$framesInCluster"
val sink = SpectrumAudioBufferSink(fftSize = fftSize, binCount = binCount)
PcmTapRegistry.bind(mediaId, sink)
sink.flush(48000, 1, C.ENCODING_PCM_16BIT)
val received = mutableListOf<Spectrum>()
val collector = launch { PcmTapRegistry.spectrumFor(mediaId).collect { received.add(it) } }
advanceUntilIdle() // let the collector subscribe
sink.handleBuffer(monoPcm(sineShorts(k = 2, n = fftSize * framesInCluster)))
advanceUntilIdle() // only now does the UI collector get to run
collector.cancel()
PcmTapRegistry.bind(null, sink)
count = received.size
}
return count
}
@Test
fun everyFrameOfAClusterReachesTheVisualizer() {
assertEquals("cluster of 8", 8, deliveredFromOneCluster(8))
assertEquals("cluster of 20", 20, deliveredFromOneCluster(20))
}
}
@@ -26,9 +26,16 @@ import kotlin.math.exp
import kotlin.math.ln
import kotlin.math.log10
/** One frame of frequency-domain magnitudes, ordered low→high Hz and normalized 0f..1f. */
/**
* One frame of frequency-domain magnitudes, ordered low→high Hz and normalized 0f..1f.
*
* [durationNanos] is how much audio the frame describes (fft size / sample rate), which is how long
* it should stay on screen. Zero — the default — means the producer is not pacing to audio (the
* synthetic preview emits one per display frame), so the frame is shown as soon as it arrives.
*/
class Spectrum(
val bins: FloatArray,
val durationNanos: Long = 0L,
)
/**
@@ -0,0 +1,110 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.audio
/**
* The per-displayed-frame step of the spectrum visualizer: queue incoming frames, release each as
* its audio time comes due, apply the decay trail against what is already on screen, and return a
* fresh array to draw.
*
* Why pacing is needed at all: the audio pipeline fills its output buffer in chunks, so frames reach
* the UI in clusters — ~15 frames roughly every 330 ms, measured on a Pixel 9a. Drawn on arrival a
* cluster collapses into one frame; released one per display refresh it plays out in ~110 ms at
* 120 Hz and then freezes (161 stalls averaging 211 ms over 52.8 s). Each frame instead stays up for
* the [Spectrum.durationNanos] of audio it describes, so a cluster spreads across the gap to the
* next. Time lost while starved is not owed back: after a gap, frames resume from the current frame
* time rather than being dumped at once to catch up.
*
* The backlog is capped at [capacity], evicting the stalest frame, so faster-than-real-time playback
* or a stalled UI cannot leave the picture drifting ever further behind the sound.
*
* The decay gives each bin an instant attack and a gradual release, so bars snap up to a transient
* and fall back smoothly instead of flickering.
*
* This lives here, rather than inline in the Compose collector, so the pacing, starvation and decay
* rules are unit-testable without a Compose harness; the composable is left as plumbing.
*
* Not thread-safe by design: both ends run on the UI dispatcher.
*/
class SpectrumTrail(
private val decay: Float,
private val capacity: Int = MAX_BACKLOG_FRAMES,
) {
private val queue = ArrayDeque<Spectrum>(capacity)
private var drawn = FloatArray(0)
// When the frame at the head of the queue may be drawn, in the caller's frame-clock nanos.
private var dueNanos = Long.MIN_VALUE
private var starved = true
/** Queues a freshly decoded frame, evicting the stalest if the backlog is at [capacity]. */
fun offer(frame: Spectrum) {
while (queue.size >= capacity) queue.removeFirst()
queue.addLast(frame)
}
/** True when nothing is queued, so the caller can stop polling until the next [offer]. */
fun isEmpty(): Boolean = queue.isEmpty()
/**
* The array to draw at [frameTimeNanos], or null when nothing new is due and the current one
* should persist.
*
* A fresh array each time is intentional: `mutableStateOf` compares by reference, so a new
* instance is what signals Compose to redraw. Do NOT switch to in-place mutation.
*/
fun nextOrNull(frameTimeNanos: Long): FloatArray? {
if (queue.isEmpty()) {
starved = true
return null
}
if (starved) {
// Resume from now, but never earlier than the last drawn frame's audio time runs out.
dueNanos = maxOf(dueNanos, frameTimeNanos)
starved = false
}
var result: FloatArray? = null
while (dueNanos <= frameTimeNanos) {
val frame = queue.removeFirstOrNull() ?: break
result = decayedFrom(frame)
dueNanos += frame.durationNanos
}
return result
}
private fun decayedFrom(frame: Spectrum): FloatArray {
val prev = drawn
val next =
FloatArray(frame.bins.size) { i ->
val prior = if (i < prev.size) prev[i] * decay else 0f
if (frame.bins[i] > prior) frame.bins[i] else prior
}
drawn = next
return next
}
companion object {
// A cluster is ~15 frames, so 24 (~0.5 s of audio) never trims ordinary playback, while
// bounding how far the picture can lag the sound (e.g. at 2x playback speed).
const val MAX_BACKLOG_FRAMES = 24
}
}
@@ -0,0 +1,183 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.audio
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotSame
import kotlin.test.assertNull
import kotlin.test.assertTrue
/** Pacing, starvation, backlog and decay rules of [SpectrumTrail]; see its KDoc for why each exists. */
class SpectrumTrailTest {
private val ms = 1_000_000L
private fun frame(
value: Float,
durationMs: Long = 20,
) = Spectrum(floatArrayOf(value), durationNanos = durationMs * ms)
@Test
fun yieldsNothingWhenStarvedSoTheDrawnFrameIsHeld() {
val trail = SpectrumTrail(decay = 0.5f)
assertNull(trail.nextOrNull(0))
}
@Test
fun theFirstFrameIsDrawnOnArrivalWithNothingToDecayFrom() {
val trail = SpectrumTrail(decay = 0.5f)
trail.offer(frame(1f))
assertContentEquals(floatArrayOf(1f), trail.nextOrNull(0))
}
@Test
fun aClusterIsSpreadOverTheAudioTimeItCoversNotTheScreenRefresh() {
val trail = SpectrumTrail(decay = 0f)
trail.offer(frame(1f))
trail.offer(frame(2f))
trail.offer(frame(3f))
// 120 Hz refreshes (~8 ms) must not drain a frame that covers 20 ms of audio.
assertContentEquals(floatArrayOf(1f), trail.nextOrNull(0))
assertNull(trail.nextOrNull(8 * ms))
assertNull(trail.nextOrNull(16 * ms))
assertContentEquals(floatArrayOf(2f), trail.nextOrNull(20 * ms))
assertNull(trail.nextOrNull(33 * ms))
assertContentEquals(floatArrayOf(3f), trail.nextOrNull(40 * ms))
assertNull(trail.nextOrNull(48 * ms))
}
@Test
fun afterStarvingTheNextClusterStartsWhenItArrivesInsteadOfBeingDumpedToCatchUp() {
val trail = SpectrumTrail(decay = 0f)
trail.offer(frame(1f))
assertContentEquals(floatArrayOf(1f), trail.nextOrNull(0))
assertNull(trail.nextOrNull(100 * ms)) // starved for a while
trail.offer(frame(2f))
trail.offer(frame(3f))
// The idle time is not owed: frame 2 shows now and frame 3 a full frame later.
assertContentEquals(floatArrayOf(2f), trail.nextOrNull(300 * ms))
assertNull(trail.nextOrNull(308 * ms))
assertContentEquals(floatArrayOf(3f), trail.nextOrNull(320 * ms))
}
@Test
fun aFrameArrivingBeforeThePreviousOneElapsedStillWaitsItsTurn() {
val trail = SpectrumTrail(decay = 0f)
trail.offer(frame(1f))
assertContentEquals(floatArrayOf(1f), trail.nextOrNull(0))
assertNull(trail.nextOrNull(8 * ms)) // queue momentarily empty
trail.offer(frame(2f))
assertNull(trail.nextOrNull(16 * ms)) // frame 1 still covers until 20 ms
assertContentEquals(floatArrayOf(2f), trail.nextOrNull(24 * ms))
}
@Test
fun aSlowFrameClockJumpsToTheLatestDueFrame() {
val trail = SpectrumTrail(decay = 0f)
trail.offer(frame(1f))
trail.offer(frame(2f))
trail.offer(frame(3f))
assertContentEquals(floatArrayOf(1f), trail.nextOrNull(0))
// A janky 45 ms frame: frames 2 (due 20) and 3 (due 40) are both due; show the newest.
assertContentEquals(floatArrayOf(3f), trail.nextOrNull(45 * ms))
}
@Test
fun aFrameWithNoDurationIsShownOnArrival() {
// Producers that do not declare a duration (the synthetic preview emits one per display
// frame) keep the old behaviour: whatever is queued is current.
val trail = SpectrumTrail(decay = 0f)
trail.offer(Spectrum(floatArrayOf(1f)))
trail.offer(Spectrum(floatArrayOf(2f)))
assertContentEquals(floatArrayOf(2f), trail.nextOrNull(0))
}
@Test
fun aBacklogIsCappedByDroppingTheStalestFrame() {
// Faster-than-real-time playback (2x speed) produces frames faster than they come due;
// the cap bounds how far the picture can lag the sound.
val trail = SpectrumTrail(decay = 0f, capacity = 2)
trail.offer(frame(1f))
trail.offer(frame(2f))
trail.offer(frame(3f)) // evicts frame 1
assertContentEquals(floatArrayOf(2f), trail.nextOrNull(0))
assertContentEquals(floatArrayOf(3f), trail.nextOrNull(20 * ms))
assertNull(trail.nextOrNull(40 * ms))
}
@Test
fun aRisingBinTakesItsNewValueImmediately() {
val trail = SpectrumTrail(decay = 0.5f)
trail.offer(frame(0.2f))
trail.nextOrNull(0)
trail.offer(frame(0.9f))
assertContentEquals(floatArrayOf(0.9f), trail.nextOrNull(20 * ms))
}
@Test
fun aFallingBinDecaysFromWhatWasDrawnRatherThanSnapping() {
val trail = SpectrumTrail(decay = 0.5f)
trail.offer(frame(1f))
trail.nextOrNull(0)
trail.offer(frame(0f))
assertContentEquals(floatArrayOf(0.5f), trail.nextOrNull(20 * ms))
}
@Test
fun eachDrawGetsAFreshArraySoComposeSeesTheChange() {
val trail = SpectrumTrail(decay = 0.5f)
trail.offer(frame(1f))
trail.offer(frame(1f))
val first = trail.nextOrNull(0)
val second = trail.nextOrNull(20 * ms)
// mutableStateOf compares by reference; reusing one array would never trigger a redraw.
assertNotSame(first, second)
}
@Test
fun reportsEmptyOnlyOnceEveryQueuedFrameIsReleasedSoTheCallerKnowsWhenToPark() {
val trail = SpectrumTrail(decay = 0f)
assertTrue(trail.isEmpty())
trail.offer(frame(1f))
trail.offer(frame(2f))
assertFalse(trail.isEmpty())
trail.nextOrNull(0)
assertFalse(trail.isEmpty()) // frame 2 is queued but not yet due
trail.nextOrNull(20 * ms)
assertTrue(trail.isEmpty())
}
}
@@ -26,15 +26,23 @@ import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.withFrameMillis
import androidx.compose.runtime.withFrameNanos
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.drawscope.DrawScope
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.flow.Flow
/**
* Collects [spectrum] into a decayed [FloatArray] and optionally runs a monotonic time clock,
* then calls [draw] inside the Canvas draw lambda. The fast-changing state is read
* ONLY in the draw lambda, so new frames trigger the draw phase, never recomposition.
* Pass [animated] = false for non-time-varying styles (bars, radial) to avoid 60fps redraws.
*
* Frames are released through [SpectrumTrail] as the audio they describe comes due. The pacing loop
* only writes state when a frame is due, and parks entirely while nothing is queued (paused, idle),
* so on its own it redraws at the ~43-47 Hz the fft produces and costs nothing when silent.
*
* Pass [animated] = false for non-time-varying styles (bars, radial): that drops the monotonic clock,
* whose whole purpose is to redraw every frame even when the spectrum has not moved.
*/
@Composable
fun SpectrumCanvas(
@@ -46,18 +54,27 @@ fun SpectrumCanvas(
draw: DrawScope.(bins: FloatArray, timeSec: Float, palette: VisualizerPalette) -> Unit,
) {
val smoothed = remember { mutableStateOf(FloatArray(0)) }
LaunchedEffect(spectrum, decay) {
var prev = FloatArray(0)
spectrum.collect { frame ->
// A fresh array each frame is intentional: mutableStateOf compares by reference, so a new
// instance is what signals Compose to redraw. Do NOT switch to in-place mutation.
val next =
FloatArray(frame.bins.size) { i ->
val prior = if (i < prev.size) prev[i] * decay else 0f
if (frame.bins[i] > prior) frame.bins[i] else prior
}
smoothed.value = next
prev = next
val trail = remember(spectrum, decay) { SpectrumTrail(decay) }
val arrivals = remember(trail) { Channel<Unit>(Channel.CONFLATED) }
LaunchedEffect(spectrum, trail) {
spectrum.collect {
trail.offer(it)
arrivals.trySend(Unit)
}
}
LaunchedEffect(trail) {
while (true) {
val next = trail.nextOrNull(withFrameNanos { it })
if (next != null) {
smoothed.value = next
} else if (trail.isEmpty()) {
// Park off the frame clock until the next offer. This must follow a nextOrNull that
// saw the empty queue: that call records the starvation which stops the next cluster
// from being dumped at once to "catch up".
arrivals.receive()
}
}
}