feat(audio): render audio in posts as waveform or cover cards

Inline media in a post now goes through playableLayout(): known audio
renders as a waveform scrubber (the imeta waveform when present, else the
synthetic one), known audio with an imeta image as a cover card with the
same scrubber, and files that could be audio or video (HLS playlists) as
a neutral card until the player's probe settles it. Known video and live
streams keep the video player, and a declared-audio file whose tracks turn
out to include video moves to it. Probe results are cached per URL so a
post scrolled back into view keeps its shape.

The cards are stateless in commonsUI (PlayableMediaCards.kt) and driven on
Android by PlayableAudioView over the pooled player: no autoplay, no loop,
nothing fetched before play when automatic media loading is off; tap or
drag on the waveform seeks, and tapping the card shows the share, save,
picture-in-picture and mute buttons the voice player uses.

MediaUrlVideo now carries the imeta size and waveform, so the card shows
the file size and real waveform before anything loads.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017KKAj1seHQLCcVEiMeePEE
This commit is contained in:
Claude
2026-10-01 14:25:06 +00:00
parent 6fdd258394
commit 7c4aa18b34
12 changed files with 888 additions and 480 deletions
@@ -0,0 +1,280 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.ui.components
import androidx.annotation.OptIn
import androidx.collection.LruCache
import androidx.compose.foundation.layout.Box
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.State
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.layout.boundsInWindow
import androidx.compose.ui.layout.onGloballyPositioned
import androidx.media3.common.C
import androidx.media3.common.Player
import androidx.media3.common.Tracks
import androidx.media3.common.util.UnstableApi
import androidx.media3.common.util.Util
import com.vitorpamplona.amethyst.commons.audio.PlayableLayout
import com.vitorpamplona.amethyst.commons.audio.mediaFormatLabel
import com.vitorpamplona.amethyst.commons.audio.player.AudioCardInfo
import com.vitorpamplona.amethyst.commons.audio.player.AudioPlaybackUi
import com.vitorpamplona.amethyst.commons.audio.player.PlayableMediaCard
import com.vitorpamplona.amethyst.commons.resources.Res
import com.vitorpamplona.amethyst.commons.resources.audio_card_untitled
import com.vitorpamplona.amethyst.commons.resources.audio_card_untitled_no_format
import com.vitorpamplona.amethyst.commons.resources.playable_media_untitled
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo
import com.vitorpamplona.amethyst.commons.ui.stringRes
import com.vitorpamplona.amethyst.commons.viewmodels.AccountViewModel
import com.vitorpamplona.amethyst.service.playback.composable.GetVideoController
import com.vitorpamplona.amethyst.service.playback.composable.MediaControllerState
import com.vitorpamplona.amethyst.service.playback.composable.PauseControllerWhenInBackground
import com.vitorpamplona.amethyst.service.playback.composable.mediaitem.GetMediaItem
import com.vitorpamplona.amethyst.service.playback.composable.mediaitem.LoadedMediaItem
import com.vitorpamplona.amethyst.ui.note.types.RenderTopButtonsForVoice
import kotlinx.coroutines.delay
/**
* What the player found when it probed a URL: true when every track is audio, false when there is a
* video track. Remembered per URL so a post scrolled back into view takes its probed shape at once
* instead of starting from its declaration and reflowing again.
*/
object PlayableMediaProbeCache {
private val cache = LruCache<String, Boolean>(500)
fun get(url: String): Boolean? = cache[url]
fun put(
url: String,
audioOnly: Boolean,
) {
cache.put(url, audioOnly)
}
}
/**
* A playable file inside a post that is (or may be) audio: the shared [PlayableMediaCard] for [layout],
* driven by the same pooled player as every other inline media.
*
* Unlike a feed video it never autoplays — audio playing muted is pointless and playing it aloud
* unasked is worse — and it does not loop. With automatic media loading off (data saver), nothing is
* fetched until play is pressed. Once the player knows the tracks, [onProbed] reports whether they are
* all audio, which is how an undecided file finds its real card and how a mislabelled video leaves for
* the video player.
*/
@Composable
fun PlayableAudioView(
content: MediaUrlVideo,
layout: PlayableLayout,
onProbed: (audioOnly: Boolean) -> Unit,
accountViewModel: AccountViewModel,
) {
val authorName =
remember(content.authorName, content.authorPubKey) {
content.authorName ?: content.authorPubKey?.let { accountViewModel.getUserIfExists(it)?.toBestDisplayName() }
}
val info =
remember(content, authorName) {
AudioCardInfo(
title = null,
artist = authorName,
format = mediaFormatLabel(content.mimeType, content.url),
sizeBytes = content.sizeBytes,
seed = content.hash ?: content.url,
waveform = content.waveform,
artworkUrl = content.artworkUri,
)
}
// The name the system media notification shows.
val mediaTitle =
when {
layout == PlayableLayout.UNDECIDED -> stringRes(Res.string.playable_media_untitled)
info.format != null -> stringRes(Res.string.audio_card_untitled, info.format)
else -> stringRes(Res.string.audio_card_untitled_no_format)
}
var load by remember(content.url) { mutableStateOf(accountViewModel.settings.startVideoPlayback()) }
var playWhenConnected by remember(content.url) { mutableStateOf(false) }
var active by remember(content.url) { mutableStateOf<Pair<MediaControllerState, LoadedMediaItem>?>(null) }
val controllerVisible = remember(content.url) { mutableStateOf(false) }
if (load) {
val proxyPort = remember(content.url) { accountViewModel.httpClientBuilder.proxyPortForVideo(content.url) }
GetMediaItem(
videoUri = content.url,
title = mediaTitle,
artworkUri = content.artworkUri,
authorName = authorName,
callbackUri = content.uri,
mimeType = content.mimeType,
proxyPort = proxyPort,
keepPlaying = false,
isLiveStream = content.isLiveStream,
hash = content.hash,
) { mediaItem ->
GetVideoController(mediaItem = mediaItem, muted = false) { controller ->
PauseControllerWhenInBackground(controller)
ReportProbe(controller.controller, onProbed)
DisposableEffect(controller, mediaItem) {
active = controller to mediaItem
onDispose {
if (active?.first === controller) active = null
}
}
// Play was pressed before the controller connected.
LaunchedEffect(controller) {
if (playWhenConnected) {
playWhenConnected = false
controller.controller.play()
}
}
}
}
}
Box {
val current = active
if (current != null) {
val (controller, mediaItem) = current
val playback by rememberAudioPlaybackUi(controller.controller)
PlayableMediaCard(
layout = layout,
info = info,
playback = playback,
onPlayPause = { playOrPause(controller.controller) },
onSeek = { fraction -> playback.durationMs?.let { controller.controller.seekTo((it * fraction).toLong()) } },
onClick = { controllerVisible.value = !controllerVisible.value },
modifier =
Modifier.onGloballyPositioned { coordinates ->
// Where picture-in-picture animates from.
val bounds = coordinates.boundsInWindow()
controller.visibility.setBounds(bounds.left.toInt(), bounds.top.toInt(), bounds.right.toInt(), bounds.bottom.toInt())
},
overlay = {
RenderTopButtonsForVoice(
mediaData = mediaItem.src,
controllerState = controller,
controllerVisible = controllerVisible,
modifier = Modifier.align(Alignment.TopEnd),
accountViewModel = accountViewModel,
)
},
)
} else {
PlayableMediaCard(
layout = layout,
info = info,
playback = AudioPlaybackUi.Idle,
onPlayPause = {
playWhenConnected = true
load = true
},
onSeek = {},
)
}
}
}
@OptIn(UnstableApi::class)
private fun playOrPause(player: Player) {
// Handles the idle (re-prepare) and ended (restart) states, not just play/pause.
Util.handlePlayPauseButtonAction(player)
}
/** True when the tracks are all audio, false when one is video, null while the player hasn't found any. */
internal fun Tracks.audioOnlyOrNull(): Boolean? {
if (groups.isEmpty()) return null
if (groups.any { it.type == C.TRACK_TYPE_VIDEO }) return false
return if (groups.any { it.type == C.TRACK_TYPE_AUDIO }) true else null
}
@Composable
private fun ReportProbe(
player: Player,
onProbed: (Boolean) -> Unit,
) {
val currentOnProbed by rememberUpdatedState(onProbed)
DisposableEffect(player) {
val listener =
object : Player.Listener {
override fun onTracksChanged(tracks: Tracks) {
tracks.audioOnlyOrNull()?.let { currentOnProbed(it) }
}
}
player.addListener(listener)
player.currentTracks.audioOnlyOrNull()?.let { currentOnProbed(it) }
onDispose { player.removeListener(listener) }
}
}
/** The player's state as the card draws it: event-driven, plus a position poll while it plays. */
@Composable
private fun rememberAudioPlaybackUi(player: Player): State<AudioPlaybackUi> {
val state = remember(player) { mutableStateOf(player.toPlaybackUi()) }
DisposableEffect(player) {
val listener =
object : Player.Listener {
override fun onEvents(
player: Player,
events: Player.Events,
) {
state.value = player.toPlaybackUi()
}
}
player.addListener(listener)
state.value = player.toPlaybackUi()
onDispose { player.removeListener(listener) }
}
val isPlaying = state.value.isPlaying
LaunchedEffect(player, isPlaying) {
while (isPlaying) {
delay(POSITION_POLL_MS)
state.value = player.toPlaybackUi()
}
}
return state
}
@OptIn(UnstableApi::class)
private fun Player.toPlaybackUi() =
AudioPlaybackUi(
// Matches what the button should offer: pause while playing or about to (buffering with
// playWhenReady), play when paused, ended or idle.
isPlaying = !Util.shouldShowPlayButton(this),
positionMs = currentPosition.coerceAtLeast(0),
durationMs = duration.takeIf { it != C.TIME_UNSET && it > 0 },
)
private const val POSITION_POLL_MS = 250L
@@ -63,6 +63,8 @@ import coil3.request.ImageRequest
import coil3.size.Size
import com.google.accompanist.permissions.ExperimentalPermissionsApi
import com.vitorpamplona.amethyst.Amethyst
import com.vitorpamplona.amethyst.commons.audio.PlayableLayout
import com.vitorpamplona.amethyst.commons.audio.playableLayout
import com.vitorpamplona.amethyst.commons.icons.symbols.Icon
import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols
import com.vitorpamplona.amethyst.commons.model.MediaAspectRatioCache
@@ -231,53 +233,86 @@ fun ZoomableContentView(
}
is MediaUrlVideo -> {
// The fallback classifies the URL string, so compute it once per content — for audio
// the cache miss is permanent and this branch re-runs on every recomposition.
val fallbackRatio =
remember(content.url, content.mimeType) {
unknownMediaAspectRatio(content.mimeType, content.url)
// Audio, or media that may be audio, gets an audio card: B (waveform) or C (cover) when the
// declaration says audio, A while it can't tell. The player's probe, once in, overrides the
// declaration — undecided files find their real card, mislabelled videos come back here.
var probedAudioOnly by remember(content.url) { mutableStateOf(PlayableMediaProbeCache.get(content.url)) }
val layout =
remember(content.url, content.mimeType, content.artworkUri, content.isLiveStream, probedAudioOnly) {
// A live stream is the one stream we know is video-first; it keeps the video player.
if (content.isLiveStream) {
PlayableLayout.VIDEO
} else {
playableLayout(content.mimeType, content.url, hasArtwork = content.artworkUri != null, probedAudioOnly = probedAudioOnly)
}
}
val ratio =
content.dim?.aspectRatioOrNull()
?: MediaAspectRatioCache.get(content.url)
?: fallbackRatio
ContentWarningGate(
isSensitive = content.contentWarning != null,
reasons = setOfNotNull(content.contentWarning),
preloadUrls = emptyList(),
accountViewModel = accountViewModel,
modifier = mediaSizingModifier(ratio, contentScale),
backdrop = (content.thumbhash ?: content.blurhash)?.let { { BlurhashBackdrop(content.blurhash, content.description, content.thumbhash) } },
) {
Box(
// The sizing modifier is repeated here because ContentWarningGate only applies
// the one it is handed when the content is actually sensitive — the common
// non-sensitive path emits content() bare. Without a height constraint of its
// own this box stretches to whatever ceiling encloses it and the player
// letterboxes the frame inside, which is what put black bars above and below
// live streams (their enclosure is StreamingHeaderModifier's 300.dp cap).
modifier = mediaSizingModifier(ratio, contentScale).then(boundsTrackingModifier),
contentAlignment = Alignment.Center,
if (layout != PlayableLayout.VIDEO) {
ContentWarningGate(
isSensitive = content.contentWarning != null,
reasons = setOfNotNull(content.contentWarning),
preloadUrls = emptyList(),
accountViewModel = accountViewModel,
) {
VideoView(
videoUri = content.url,
mimeType = content.mimeType,
title = content.description,
artworkUri = content.artworkUri,
authorName = content.authorName,
dimensions = content.dim,
blurhash = content.blurhash,
roundedCorner = roundedCorner,
contentScale = contentScale,
nostrUriCallback = content.uri,
onDialog = { dialogOpen = true },
PlayableAudioView(
content = content,
layout = layout,
onProbed = { audioOnly ->
PlayableMediaProbeCache.put(content.url, audioOnly)
probedAudioOnly = audioOnly
},
accountViewModel = accountViewModel,
thumbhash = content.thumbhash,
isLiveStream = content.isLiveStream,
hash = content.hash,
captions = content.captions,
)
}
} else {
// The fallback classifies the URL string, so compute it once per content — for audio
// the cache miss is permanent and this branch re-runs on every recomposition.
val fallbackRatio =
remember(content.url, content.mimeType) {
unknownMediaAspectRatio(content.mimeType, content.url)
}
val ratio =
content.dim?.aspectRatioOrNull()
?: MediaAspectRatioCache.get(content.url)
?: fallbackRatio
ContentWarningGate(
isSensitive = content.contentWarning != null,
reasons = setOfNotNull(content.contentWarning),
preloadUrls = emptyList(),
accountViewModel = accountViewModel,
modifier = mediaSizingModifier(ratio, contentScale),
backdrop = (content.thumbhash ?: content.blurhash)?.let { { BlurhashBackdrop(content.blurhash, content.description, content.thumbhash) } },
) {
Box(
// The sizing modifier is repeated here because ContentWarningGate only applies
// the one it is handed when the content is actually sensitive — the common
// non-sensitive path emits content() bare. Without a height constraint of its
// own this box stretches to whatever ceiling encloses it and the player
// letterboxes the frame inside, which is what put black bars above and below
// live streams (their enclosure is StreamingHeaderModifier's 300.dp cap).
modifier = mediaSizingModifier(ratio, contentScale).then(boundsTrackingModifier),
contentAlignment = Alignment.Center,
) {
VideoView(
videoUri = content.url,
mimeType = content.mimeType,
title = content.description,
artworkUri = content.artworkUri,
authorName = content.authorName,
dimensions = content.dim,
blurhash = content.blurhash,
roundedCorner = roundedCorner,
contentScale = contentScale,
nostrUriCallback = content.uri,
onDialog = { dialogOpen = true },
accountViewModel = accountViewModel,
thumbhash = content.thumbhash,
isLiveStream = content.isLiveStream,
hash = content.hash,
captions = content.captions,
)
}
}
}
}
@@ -86,3 +86,47 @@ fun declaredAsAudio(
}
private val HLS_EXTENSIONS = listOf("m3u8")
/**
* A short container label for a card — "MP3", "FLAC", "HLS" — from the declared MIME type, else the URL's
* extension. Null when neither names one.
*/
fun mediaFormatLabel(
mimeType: String?,
url: String,
): String? {
val mime = normalizeMimeType(mimeType)
if (RichTextParser.isHlsMimeType(mime) || RichTextParser.hasExtensionIn(url, HLS_EXTENSIONS)) return "HLS"
if (mime != null) {
MIME_LABELS[mime.substringBefore(';').trim().lowercase()]?.let { return it }
}
return urlExtension(url)?.takeIf { RichTextParser.isAudioUrl(url) || RichTextParser.isVideoUrl(url) }?.uppercase()
}
private val MIME_LABELS =
mapOf(
"audio/mpeg" to "MP3",
"audio/mp3" to "MP3",
"audio/mp4" to "M4A",
"audio/x-m4a" to "M4A",
"audio/aac" to "AAC",
"audio/ogg" to "OGG",
"audio/opus" to "OPUS",
"audio/flac" to "FLAC",
"audio/x-flac" to "FLAC",
"audio/wav" to "WAV",
"audio/x-wav" to "WAV",
"audio/wave" to "WAV",
"audio/webm" to "WEBM",
"video/mp4" to "MP4",
"video/webm" to "WEBM",
"video/quicktime" to "MOV",
)
/** The last path segment's extension, ignoring the query and fragment. */
private fun urlExtension(url: String): String? {
val path = url.substringBefore('#').substringBefore('?')
val segment = path.substringAfterLast('/')
val dot = segment.lastIndexOf('.')
return if (dot in 0 until segment.length - 1) segment.substring(dot + 1) else null
}
@@ -118,6 +118,10 @@ open class MediaUrlVideo(
authorPubKey: String? = null,
// NIP-71 `text-track` captions resolved for this video, for the player to side-load.
val captions: ImmutableList<CaptionTrack> = persistentListOf(),
// The imeta `size`, in bytes: an audio card shows it before the player has loaded anything.
val sizeBytes: Long? = null,
// The imeta `waveform` (NIP-A0 amplitudes, not normalised): an audio card draws it as its seek bar.
val waveform: List<Float>? = null,
) : MediaUrlContent(url, description, hash, dim, blurhash, uri, mimeType, thumbhash, authorPubKey)
@Immutable
@@ -38,7 +38,9 @@ import com.vitorpamplona.quartz.nip94FileMetadata.tags.DimensionTag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.HashSha256Tag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.ImageTag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.MimeTypeTag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.SizeTag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.ThumbhashTag
import com.vitorpamplona.quartz.nipA0VoiceMessages.tags.WaveformTag
import com.vitorpamplona.quartz.utils.Log
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.persistentListOf
@@ -95,6 +97,8 @@ class RichTextParser {
mimeType = contentType,
thumbhash = frags[ThumbhashTag.TAG_NAME] ?: tags[ThumbhashTag.TAG_NAME]?.firstOrNull(),
authorPubKey = authorPubKey,
sizeBytes = (frags[SizeTag.TAG_NAME] ?: tags[SizeTag.TAG_NAME]?.firstOrNull())?.toLongOrNull(),
waveform = (frags[WaveformTag.TAG_NAME] ?: tags[WaveformTag.TAG_NAME]?.firstOrNull())?.let { WaveformTag.parseWave(it) },
)
} else if (kind == MediaContentKind.PDF) {
MediaUrlPdf(
@@ -82,4 +82,16 @@ class PlayableLayoutTest {
// An MPEG video mislabelled `audio/mpeg` moves to the video player once its tracks are known.
assertEquals(PlayableLayout.VIDEO, playableLayout("audio/mpeg", "https://x.com/a.mpg", hasArtwork = false, probedAudioOnly = false))
}
@Test
fun formatLabels() {
assertEquals("MP3", mediaFormatLabel("audio/mpeg", blossomMp3))
assertEquals("FLAC", mediaFormatLabel(null, "https://x.com/a.flac?dl=1"))
assertEquals("M4A", mediaFormatLabel("audio/mp4", "https://x.com/abc"))
assertEquals("HLS", mediaFormatLabel("audio/mpegurl", "https://x.com/live"))
assertEquals("HLS", mediaFormatLabel(null, "https://x.com/live.m3u8"))
assertEquals(null, mediaFormatLabel(null, "https://blossom.example/b21c6e2a4d38f2ab"))
// A dot in the host is not an extension.
assertEquals(null, mediaFormatLabel(null, "https://e.nostr.build/a_x_mp3"))
}
}
@@ -0,0 +1,60 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.richtext
import com.vitorpamplona.amethyst.commons.model.ImmutableListOfLists
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertIs
import kotlin.test.assertNull
/** The imeta facts an audio card shows before the player loads: `size` and `waveform` reach [MediaUrlVideo]. */
class RichTextParserAudioImetaTest {
private val url =
"https://npub17u5dneh8qjp43ecfxr6u5e9sjamsmxyuekrg2nlxrrk6nj9rsyrqywt4tp.blossom.band/" +
"b21c6e2a4d38f2abac617ac6643aba919f271b37c1c539b5643c17753716c506.mp3"
private fun parse(vararg imeta: String) =
RichTextParser()
.parseText(
"I can simply generate the esoteric beats I want to hear!\n$url",
ImmutableListOfLists(arrayOf(arrayOf("imeta", "url $url", *imeta))),
null,
).mediaList
.single()
@Test
fun sizeAndMimeReachTheMedia() {
// The post from the original report.
val media = parse("x b21c6e2a4d38f2abac617ac6643aba919f271b37c1c539b5643c17753716c506", "size 4992768", "m audio/mpeg")
assertIs<MediaUrlVideo>(media)
assertEquals("audio/mpeg", media.mimeType)
assertEquals(4_992_768L, media.sizeBytes)
assertNull(media.waveform)
}
@Test
fun waveformReachesTheMedia() {
val media = parse("m audio/mpeg", "waveform 0 7 35 100 42")
assertIs<MediaUrlVideo>(media)
assertEquals(listOf(0f, 7f, 35f, 100f, 42f), media.waveform)
}
}
@@ -6361,4 +6361,7 @@
<string name="browser_pill_download_risky">This file type can run code. Check that the name matches what you meant to get.</string>
<string name="browser_pill_download_save">Save</string>
<string name="browser_pill_download_from">From %1$s</string>
<string name="audio_card_untitled">%1$s audio</string>
<string name="audio_card_untitled_no_format">Audio</string>
<string name="playable_media_untitled">Media</string>
</resources>
@@ -1,288 +0,0 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.audio.player
import androidx.compose.foundation.Image
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.aspectRatio
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import com.vitorpamplona.amethyst.commons.icons.symbols.Icon
import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols
import com.vitorpamplona.amethyst.commons.robohash.CachedRobohash
import com.vitorpamplona.amethyst.commons.ui.theme.ThemeComparisonRow
import com.vitorpamplona.amethyst.commons.ui.theme.isLight
/** The post that prompted the redesign (an MP3 with `m audio/mpeg`, no ID3 tags), plus richer variants. */
object AudioPlayerSamples {
const val AUTHOR_HEX = "f728d9e6e7048358e70930f5ca64b097770d989ccd86854fe618eda9c8a38106"
const val AUTHOR_NAME = "npub17u5d…t4tp"
const val POST_TEXT =
"There aren't many nerdcore hip hop artists making music but I just realized that's no longer a problem: " +
"I can simply generate the esoteric beats I want to hear!"
private const val HASH = "b21c6e2a4d38f2abac617ac6643aba919f271b37c1c539b5643c17753716c506"
private const val URL = "https://npub17u5dneh8qjp43ecfxr6u5e9sjamsmxyuekrg2nlxrrk6nj9rsyrqywt4tp.blossom.band/$HASH.mp3"
/** Straight from the imeta, before ExoPlayer has probed anything: no title, no duration, no waveform. */
val notLoaded =
AudioCardUi(
url = URL,
mimeType = "audio/mpeg",
title = "MP3 audio",
artist = AUTHOR_NAME,
format = "MP3",
sizeBytes = 4_992_768,
durationSeconds = null,
positionSeconds = 0,
isPlaying = false,
seed = HASH,
)
/** Same file, playing: the player now knows it runs 3:28. */
val playing =
AudioCardUi(
url = URL,
mimeType = "audio/mpeg",
title = "MP3 audio",
artist = AUTHOR_NAME,
format = "MP3",
sizeBytes = 4_992_768,
durationSeconds = 208,
positionSeconds = 72,
isPlaying = true,
seed = HASH,
)
/** A track with artwork (imeta `image` or an embedded cover) and ID3 tags naming it. */
val withArtwork =
AudioCardUi(
url = "https://blossom.example/3fa1c27be0d94e1a6c55b8f02d7e4a9c1b6f8e03d2a7c5b941e60f8d3c2b1a07.mp3",
mimeType = "audio/mpeg",
title = "Segfault in the Cipher",
artist = "Esoteric Beats Vol. 1",
format = "MP3",
sizeBytes = 4_992_768,
durationSeconds = 208,
positionSeconds = 72,
isPlaying = true,
seed = "3fa1c27be0d94e1a6c55b8f02d7e4a9c1b6f8e03d2a7c5b941e60f8d3c2b1a07",
hasArtwork = true,
)
/** An HLS playlist: it may carry audio only or video, and nothing tells until the player probes it. */
val undecided =
AudioCardUi(
url = "https://stream.example/live/index.m3u8",
mimeType = null,
title = "Stream",
artist = AUTHOR_NAME,
format = "HLS",
sizeBytes = null,
durationSeconds = null,
positionSeconds = 0,
isPlaying = false,
seed = "9c41e07d2b5a8f36e1d0c4b7a29f58e3d6c1b0a4f7e2d9c8b5a3f1e0d7c6b4a2",
)
}
/** Dark and light side by side, each filling its half, so screenshots have no transparent gaps. */
@Composable
private fun AudioPreviewFrame(content: @Composable () -> Unit) {
ThemeComparisonRow {
Box(Modifier.fillMaxSize().background(MaterialTheme.colorScheme.background)) { content() }
}
}
/** A stripped-down note: avatar, name, the post text, then the audio card where the media goes. */
@Composable
fun MockAudioPost(
text: String = AudioPlayerSamples.POST_TEXT,
media: @Composable () -> Unit,
) {
Row(Modifier.fillMaxWidth().padding(12.dp)) {
Image(
imageVector = CachedRobohash.get(AudioPlayerSamples.AUTHOR_HEX, MaterialTheme.colorScheme.isLight),
contentDescription = null,
modifier = Modifier.size(48.dp).clip(CircleShape),
)
Spacer(Modifier.width(10.dp))
Column(Modifier.weight(1f)) {
Row(verticalAlignment = Alignment.CenterVertically) {
Text(AudioPlayerSamples.AUTHOR_NAME, style = MaterialTheme.typography.bodyMedium, fontWeight = FontWeight.Bold)
Text(" · 2h", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
Spacer(Modifier.height(2.dp))
Text(text, style = MaterialTheme.typography.bodyMedium)
Spacer(Modifier.height(10.dp))
media()
}
}
}
@Composable
private fun Caption(text: String) {
Text(
text,
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.primary,
modifier = Modifier.padding(start = 12.dp, top = 14.dp),
)
}
/** Roughly what ships today: the video player's black square, controls and all, until tracks resolve. */
@Composable
private fun TodaysPlayerSketch() {
Box(
Modifier
.fillMaxWidth()
.aspectRatio(1f)
.clip(RoundedCornerShape(12.dp))
.background(Color.Black),
contentAlignment = Alignment.Center,
) {
Icon(MaterialSymbols.PlayArrow, null, Modifier.size(64.dp), tint = Color.White, filled = true)
Row(
Modifier.align(Alignment.BottomCenter).fillMaxWidth().padding(12.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Text("0:00", style = MaterialTheme.typography.labelSmall, color = Color.White)
Box(Modifier.weight(1f).height(3.dp).background(Color.White.copy(alpha = 0.4f)))
Text("--:--", style = MaterialTheme.typography.labelSmall, color = Color.White)
Icon(MaterialSymbols.Fullscreen, null, Modifier.size(20.dp), tint = Color.White)
}
}
}
@Preview(widthDp = 820, heightDp = 700)
@Composable
fun AudioTodayPreview() =
AudioPreviewFrame {
Column {
Caption("Today · the video player's black square, until ExoPlayer reports tracks")
MockAudioPost { TodaysPlayerSketch() }
}
}
@Preview(widthDp = 820, heightDp = 560)
@Composable
fun AudioKnownAudioPreview() =
AudioPreviewFrame {
Column {
Caption("Known audio, no waveform tag → B with the synthetic waveform · before load")
MockAudioPost { PlayablePostMediaPrototype(AudioPlayerSamples.notLoaded) }
Caption("Playing")
MockAudioPost { PlayablePostMediaPrototype(AudioPlayerSamples.playing) }
}
}
@Preview(widthDp = 820, heightDp = 620)
@Composable
fun AudioWithArtworkPreview() =
AudioPreviewFrame {
Column {
Caption("Known audio with artwork → C, the cover with B's scrubber")
MockAudioPost { PlayablePostMediaPrototype(AudioPlayerSamples.withArtwork) }
}
}
@Preview(widthDp = 820, heightDp = 360)
@Composable
fun AudioUndecidedPreview() =
AudioPreviewFrame {
Column {
Caption("Audio or video? (HLS playlist) → A until the player probes it")
MockAudioPost(text = "Live set tonight, tune in") { PlayablePostMediaPrototype(AudioPlayerSamples.undecided) }
}
}
@Preview(widthDp = 820, heightDp = 520)
@Composable
fun AudioMiniPlayerPreview() =
AudioPreviewFrame {
Column(Modifier.fillMaxSize()) {
Caption("E · Mini player — after the post scrolls away")
// Ghost feed rows standing in for the notes that scrolled in.
repeat(3) {
Row(Modifier.fillMaxWidth().padding(12.dp)) {
Box(Modifier.size(48.dp).clip(CircleShape).background(MaterialTheme.colorScheme.surfaceContainerHigh))
Spacer(Modifier.width(10.dp))
Column(Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(8.dp)) {
Box(
Modifier
.width(120.dp)
.height(12.dp)
.clip(RoundedCornerShape(6.dp))
.background(MaterialTheme.colorScheme.surfaceContainerHigh),
)
Box(
Modifier
.fillMaxWidth()
.height(12.dp)
.clip(RoundedCornerShape(6.dp))
.background(MaterialTheme.colorScheme.surfaceContainer),
)
Box(
Modifier
.fillMaxWidth(0.7f)
.height(12.dp)
.clip(RoundedCornerShape(6.dp))
.background(MaterialTheme.colorScheme.surfaceContainer),
)
}
}
}
Spacer(Modifier.weight(1f))
AudioMiniPlayerPrototype(AudioPlayerSamples.withArtwork, Modifier.padding(horizontal = 8.dp))
Spacer(Modifier.height(8.dp))
// Bottom navigation bar placeholder.
Row(
Modifier.fillMaxWidth().height(56.dp).background(MaterialTheme.colorScheme.surfaceContainer),
horizontalArrangement = Arrangement.SpaceEvenly,
verticalAlignment = Alignment.CenterVertically,
) {
listOf(MaterialSymbols.Home, MaterialSymbols.Search, MaterialSymbols.Notifications).forEach {
Icon(it, null, Modifier.size(24.dp), tint = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
}
}
@@ -0,0 +1,177 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.audio.player
import androidx.compose.foundation.Image
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import com.vitorpamplona.amethyst.commons.audio.PlayableLayout
import com.vitorpamplona.amethyst.commons.audio.playableLayout
import com.vitorpamplona.amethyst.commons.robohash.CachedRobohash
import com.vitorpamplona.amethyst.commons.ui.theme.ThemeComparisonRow
import com.vitorpamplona.amethyst.commons.ui.theme.isLight
/** The post that prompted the audio cards (an MP3 with `m audio/mpeg`, no ID3 tags), plus the other cases. */
object AudioPlayerSamples {
const val AUTHOR_HEX = "f728d9e6e7048358e70930f5ca64b097770d989ccd86854fe618eda9c8a38106"
const val AUTHOR_NAME = "npub17u5d\u2026t4tp"
const val POST_TEXT =
"There aren't many nerdcore hip hop artists making music but I just realized that's no longer a problem: " +
"I can simply generate the esoteric beats I want to hear!"
private const val HASH = "b21c6e2a4d38f2abac617ac6643aba919f271b37c1c539b5643c17753716c506"
const val MP3_URL = "https://npub17u5dneh8qjp43ecfxr6u5e9sjamsmxyuekrg2nlxrrk6nj9rsyrqywt4tp.blossom.band/$HASH.mp3"
const val HLS_URL = "https://stream.example/live/index.m3u8"
/** Everything the imeta gives: format, size, hash. No title, no waveform. */
val mp3 = AudioCardInfo(title = null, artist = AUTHOR_NAME, format = "MP3", sizeBytes = 4_992_768, seed = HASH)
/** The same kind of post with a NIP-A0 `waveform` (0..100 amplitudes). */
val mp3WithWaveform =
AudioCardInfo(
title = null,
artist = AUTHOR_NAME,
format = "MP3",
sizeBytes = 1_204_224,
seed = "5e0f",
waveform = listOf(4f, 9f, 22f, 41f, 38f, 12f, 6f, 30f, 72f, 88f, 64f, 20f, 8f, 15f, 46f, 93f, 100f, 77f, 35f, 10f, 5f, 18f, 52f, 61f, 28f, 9f, 3f, 25f, 58f, 80f, 66f, 31f, 12f, 7f, 19f, 44f, 70f, 49f, 16f, 4f),
)
/** A named track with artwork (the cover here is the generated fallback; the imeta `image` would load over it). */
val withArtwork =
AudioCardInfo(
title = "Segfault in the Cipher",
artist = "Esoteric Beats Vol. 1",
format = "MP3",
sizeBytes = 4_992_768,
seed = "3fa1c27be0d94e1a6c55b8f02d7e4a9c1b6f8e03d2a7c5b941e60f8d3c2b1a07",
)
/** An HLS playlist: audio only or video, nothing tells until the player probes it. */
val stream = AudioCardInfo(title = null, artist = AUTHOR_NAME, format = "HLS", sizeBytes = null, seed = HLS_URL)
val playing = AudioPlaybackUi(isPlaying = true, positionMs = 72_000, durationMs = 208_000)
val pausedEarly = AudioPlaybackUi(isPlaying = false, positionMs = 9_000, durationMs = 41_000)
}
/** Dark and light side by side, each filling its half, so screenshots have no transparent gaps. */
@Composable
private fun AudioPreviewFrame(content: @Composable () -> Unit) {
ThemeComparisonRow {
Box(Modifier.fillMaxSize().background(MaterialTheme.colorScheme.background)) { content() }
}
}
/** A stripped-down note: avatar, name, the post text, then the audio card where the media goes. */
@Composable
fun MockAudioPost(
text: String = AudioPlayerSamples.POST_TEXT,
media: @Composable () -> Unit,
) {
Row(Modifier.fillMaxWidth().padding(12.dp)) {
Image(
imageVector = CachedRobohash.get(AudioPlayerSamples.AUTHOR_HEX, MaterialTheme.colorScheme.isLight),
contentDescription = null,
modifier = Modifier.size(48.dp).clip(CircleShape),
)
Spacer(Modifier.width(10.dp))
Column(Modifier.weight(1f)) {
Row(verticalAlignment = Alignment.CenterVertically) {
Text(AudioPlayerSamples.AUTHOR_NAME, style = MaterialTheme.typography.bodyMedium, fontWeight = FontWeight.Bold)
Text(" · 2h", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
Spacer(Modifier.height(2.dp))
Text(text, style = MaterialTheme.typography.bodyMedium)
Spacer(Modifier.height(10.dp))
media()
}
}
}
@Composable
private fun Caption(text: String) {
Text(
text,
style = MaterialTheme.typography.labelMedium,
color = MaterialTheme.colorScheme.primary,
modifier = Modifier.padding(start = 12.dp, top = 14.dp),
)
}
@Composable
private fun Card(
layout: PlayableLayout,
info: AudioCardInfo,
playback: AudioPlaybackUi = AudioPlaybackUi.Idle,
) = PlayableMediaCard(layout, info, playback, onPlayPause = {}, onSeek = {})
@Preview(widthDp = 820, heightDp = 760)
@Composable
fun AudioKnownAudioPreview() =
AudioPreviewFrame {
val layout = playableLayout("audio/mpeg", AudioPlayerSamples.MP3_URL, hasArtwork = false)
Column {
Caption("Known audio, no waveform tag: synthetic waveform, before load")
MockAudioPost { Card(layout, AudioPlayerSamples.mp3) }
Caption("Playing")
MockAudioPost { Card(layout, AudioPlayerSamples.mp3, AudioPlayerSamples.playing) }
Caption("With a waveform tag")
MockAudioPost(text = "Quick voice memo about the relay outage") { Card(layout, AudioPlayerSamples.mp3WithWaveform, AudioPlayerSamples.pausedEarly) }
}
}
@Preview(widthDp = 820, heightDp = 620)
@Composable
fun AudioWithArtworkPreview() =
AudioPreviewFrame {
Column {
Caption("Known audio with artwork: the cover with the same scrubber")
MockAudioPost { Card(playableLayout("audio/mpeg", AudioPlayerSamples.MP3_URL, hasArtwork = true), AudioPlayerSamples.withArtwork, AudioPlayerSamples.playing) }
}
}
@Preview(widthDp = 820, heightDp = 360)
@Composable
fun AudioUndecidedPreview() =
AudioPreviewFrame {
Column {
Caption("Audio or video? (HLS playlist): neutral card until the player probes it")
MockAudioPost(text = "Live set tonight, tune in") { Card(playableLayout(null, AudioPlayerSamples.HLS_URL, hasArtwork = false), AudioPlayerSamples.stream) }
}
}
@@ -22,6 +22,9 @@ package com.vitorpamplona.amethyst.commons.audio.player
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background
import androidx.compose.foundation.clickable
import androidx.compose.foundation.gestures.detectHorizontalDragGestures
import androidx.compose.foundation.gestures.detectTapGestures
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.BoxScope
@@ -40,11 +43,14 @@ import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.Immutable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
@@ -54,160 +60,193 @@ import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.dp
import coil3.compose.AsyncImage
import com.vitorpamplona.amethyst.commons.audio.PlayableLayout
import com.vitorpamplona.amethyst.commons.audio.playableLayout
import com.vitorpamplona.amethyst.commons.audio.syntheticWaveformFor
import com.vitorpamplona.amethyst.commons.icons.symbols.Icon
import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol
import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols
import com.vitorpamplona.amethyst.commons.resources.Res
import com.vitorpamplona.amethyst.commons.resources.close
import com.vitorpamplona.amethyst.commons.resources.audio_card_untitled
import com.vitorpamplona.amethyst.commons.resources.audio_card_untitled_no_format
import com.vitorpamplona.amethyst.commons.resources.pause
import com.vitorpamplona.amethyst.commons.resources.play
import com.vitorpamplona.amethyst.commons.resources.playable_media_untitled
import com.vitorpamplona.amethyst.commons.ui.stringRes
import kotlin.math.abs
import kotlin.math.roundToInt
/*
* PROTOTYPES — how a playable file inside a post should render, in place of today's video player
* with a square box. Stateless and fed by [AudioCardUi]; nothing here is wired to a player yet.
* Rendered offscreen by `AudioPlayerPrototypesRenderTest` into `commonsUI/build/audio-player/`.
* The cards a playable file inside a post renders as, instead of a video player with no picture.
* Which one is [com.vitorpamplona.amethyst.commons.audio.playableLayout]'s call:
* - known audio, no artwork -> [AudioWaveformCard]: the waveform is the seek bar (the file's own
* waveform, else the synthetic one every other audio renderer draws);
* - known audio with artwork -> [AudioCoverCard]: the artwork with the same scrubber over its foot;
* - audio or video, can't tell -> [UndecidedMediaCard], until the player's probe settles it.
*
* Which card is shown is [playableLayout]'s call, made from the imeta before the player loads a byte:
* - known audio, no artwork -> B, the waveform scrubber (the file's waveform, else a synthetic one);
* - known audio with artwork -> C, the cover with the same scrubber over its foot;
* - audio or video, can't tell (an HLS playlist, say) -> A, the neutral track card, until the
* player's probe settles it;
* - known video -> the video player.
* Stateless: the platform owns the player and feeds [AudioPlaybackUi] in, taking play/pause and seeks
* back out. Everything in [AudioCardInfo] comes from the imeta, so each card has its final shape before
* the player has loaded a byte.
*/
/** Everything a card shows. [durationSeconds] is null until the player has probed the file. */
/** What a card shows about the file. All of it is known from the event, before the player loads anything. */
@Immutable
class AudioCardUi(
val url: String,
val mimeType: String?,
val title: String,
val artist: String,
class AudioCardInfo(
/** The track's name, when something names it. Null shows "MP3 audio" (or the like). */
val title: String?,
/** Usually the post's author. */
val artist: String?,
/** "MP3", "FLAC", "HLS"… from the declared MIME type or the URL. */
val format: String?,
val sizeBytes: Long?,
val durationSeconds: Int?,
val positionSeconds: Int,
val isPlaying: Boolean,
/** The blob's sha256 (`x`): seeds the synthetic waveform and the generated cover, so both are stable per file. */
/** The blob's sha256 (`x`), or the URL: seeds the synthetic waveform and the generated cover. */
val seed: String,
/** The file's own amplitude envelope (a `waveform` tag, or one decoded from the audio), 0..1. */
/** The file's own amplitude envelope (a `waveform` tag), any scale. Null draws the synthetic one. */
val waveform: List<Float>? = null,
/** True when the imeta `image` or an embedded cover gives the file artwork. */
val hasArtwork: Boolean = false,
/** The imeta `image`: the artwork [AudioCoverCard] shows. */
val artworkUrl: String? = null,
) {
val progress: Float
get() = durationSeconds?.takeIf { it > 0 }?.let { (positionSeconds.toFloat() / it).coerceIn(0f, 1f) } ?: 0f
/** "MP3 · 4.8 MB" — known from the imeta alone. */
val fileFacts: String
get() = listOfNotNull(format, sizeBytes?.let(::formatFileSize)).joinToString(" · ")
/** "1:12 / 3:28" while there is a duration; the file facts before that. */
val timeLabel: String
get() = durationSeconds?.let { "${formatClock(positionSeconds)} / ${formatClock(it)}" } ?: fileFacts
val layout: PlayableLayout
get() = playableLayout(mimeType, url, hasArtwork)
}
/** The one entry point a post would call: picks the card from [AudioCardUi.layout]. */
@Composable
fun PlayablePostMediaPrototype(
media: AudioCardUi,
modifier: Modifier = Modifier,
/** What the player is doing. [durationMs] is null until it has probed the file. */
@Immutable
data class AudioPlaybackUi(
val isPlaying: Boolean = false,
val positionMs: Long = 0,
val durationMs: Long? = null,
) {
when (media.layout) {
PlayableLayout.AUDIO_WAVEFORM -> AudioWaveformCardPrototype(media, modifier)
PlayableLayout.AUDIO_COVER -> AudioCoverCardPrototype(media, modifier)
PlayableLayout.UNDECIDED -> UndecidedMediaCardPrototype(media, modifier)
// Not prototyped: known video keeps the existing video player.
PlayableLayout.VIDEO ->
Box(
modifier
.fillMaxWidth()
.aspectRatio(16f / 9f)
.clip(RoundedCornerShape(12.dp))
.background(Color.Black),
)
val progress: Float
get() = durationMs?.takeIf { it > 0 }?.let { (positionMs.toFloat() / it).coerceIn(0f, 1f) } ?: 0f
companion object {
val Idle = AudioPlaybackUi()
}
}
/** Renders whichever card [layout] names. [PlayableLayout.VIDEO] is the video player's job and draws nothing here. */
@Composable
fun PlayableMediaCard(
layout: PlayableLayout,
info: AudioCardInfo,
playback: AudioPlaybackUi,
onPlayPause: () -> Unit,
onSeek: (Float) -> Unit,
modifier: Modifier = Modifier,
onClick: (() -> Unit)? = null,
overlay: @Composable BoxScope.() -> Unit = {},
) {
when (layout) {
PlayableLayout.AUDIO_WAVEFORM -> AudioWaveformCard(info, playback, onPlayPause, onSeek, modifier, onClick, overlay)
PlayableLayout.AUDIO_COVER -> AudioCoverCard(info, playback, onPlayPause, onSeek, modifier, onClick, overlay)
PlayableLayout.UNDECIDED -> UndecidedMediaCard(info, playback, onPlayPause, modifier, onClick, overlay)
PlayableLayout.VIDEO -> Unit
}
}
// ---------------------------------------------------------------------------------------------
// B. Waveform scrubber — known audio. The waveform IS the seek bar.
// Known audio: the waveform is the seek bar.
// ---------------------------------------------------------------------------------------------
@Composable
fun AudioWaveformCardPrototype(
audio: AudioCardUi,
fun AudioWaveformCard(
info: AudioCardInfo,
playback: AudioPlaybackUi,
onPlayPause: () -> Unit,
onSeek: (Float) -> Unit,
modifier: Modifier = Modifier,
onClick: (() -> Unit)? = null,
overlay: @Composable BoxScope.() -> Unit = {},
) {
val bars = rememberBars(audio)
Surface(
modifier = modifier.fillMaxWidth(),
shape = RoundedCornerShape(16.dp),
color = MaterialTheme.colorScheme.surfaceContainerHigh,
val bars = rememberBars(info)
Box(
modifier
.fillMaxWidth()
.clip(RoundedCornerShape(16.dp))
.background(MaterialTheme.colorScheme.surfaceContainerHigh)
.clickableWithoutRipple(onClick),
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
verticalAlignment = Alignment.CenterVertically,
) {
PlayPauseButton(audio.isPlaying, size = 44.dp)
PlayPauseButton(playback.isPlaying, size = 44.dp, onClick = onPlayPause)
Spacer(Modifier.width(12.dp))
Column(Modifier.weight(1f)) {
WaveformBars(
bars = bars,
progress = audio.progress,
progress = playback.progress,
played = MaterialTheme.colorScheme.primary,
unplayed = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.35f),
modifier = Modifier.fillMaxWidth().height(40.dp),
onSeek = onSeek.takeIf { playback.durationMs != null },
)
Spacer(Modifier.height(4.dp))
Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) {
Text(
audio.durationSeconds?.let { formatClock(audio.positionSeconds) } ?: audio.title,
if (playback.durationMs != null) formatClock(playback.positionMs) else audioTitle(info),
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier.weight(1f, fill = false),
)
Spacer(Modifier.width(8.dp))
Text(
listOfNotNull(audio.fileFacts, audio.durationSeconds?.let(::formatClock)).joinToString(" · "),
trailingFacts(info, playback),
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
)
}
}
}
overlay()
}
}
// ---------------------------------------------------------------------------------------------
// C. Cover — known audio with artwork: the cover, with B's scrubber over its foot. The cover here is
// generated from the hash; real artwork would replace the gradient and the record.
// Known audio with artwork: the artwork, with the same scrubber over its foot.
// ---------------------------------------------------------------------------------------------
@Composable
fun AudioCoverCardPrototype(
audio: AudioCardUi,
fun AudioCoverCard(
info: AudioCardInfo,
playback: AudioPlaybackUi,
onPlayPause: () -> Unit,
onSeek: (Float) -> Unit,
modifier: Modifier = Modifier,
onClick: (() -> Unit)? = null,
overlay: @Composable BoxScope.() -> Unit = {},
) {
val bars = rememberBars(audio)
val bars = rememberBars(info)
Box(
modifier =
modifier
.fillMaxWidth()
.heightIn(max = 400.dp)
.aspectRatio(1f)
.clip(RoundedCornerShape(16.dp)),
.clip(RoundedCornerShape(16.dp))
.clickableWithoutRipple(onClick),
) {
GeneratedCover(audio.seed, Modifier.fillMaxSize(), icon = null, record = true)
// The generated cover sits under the artwork, so a slow or failed load still reads as a cover.
GeneratedCover(info.seed, Modifier.fillMaxSize(), icon = null, record = info.artworkUrl == null)
if (info.artworkUrl != null) {
AsyncImage(
model = info.artworkUrl,
contentDescription = null,
contentScale = ContentScale.Crop,
modifier = Modifier.fillMaxSize(),
)
}
// Bottom scrim so the title and the waveform read on any cover.
Box(
@@ -222,62 +261,68 @@ fun AudioCoverCardPrototype(
Row(verticalAlignment = Alignment.CenterVertically) {
Column(Modifier.weight(1f)) {
Text(
audio.title,
audioTitle(info),
style = MaterialTheme.typography.titleLarge,
fontWeight = FontWeight.Bold,
color = Color.White,
maxLines = 2,
overflow = TextOverflow.Ellipsis,
)
Text(
audio.artist,
style = MaterialTheme.typography.bodySmall,
color = Color.White.copy(alpha = 0.8f),
maxLines = 1,
)
info.artist?.let {
Text(it, style = MaterialTheme.typography.bodySmall, color = Color.White.copy(alpha = 0.8f), maxLines = 1, overflow = TextOverflow.Ellipsis)
}
}
Spacer(Modifier.width(12.dp))
PlayPauseButton(audio.isPlaying, size = 52.dp, container = Color.White, content = Color.Black)
PlayPauseButton(playback.isPlaying, size = 52.dp, onClick = onPlayPause, container = Color.White, content = Color.Black)
}
Spacer(Modifier.height(12.dp))
WaveformBars(
bars = bars,
progress = audio.progress,
progress = playback.progress,
played = Color.White,
unplayed = Color.White.copy(alpha = 0.35f),
modifier = Modifier.fillMaxWidth().height(36.dp),
onSeek = onSeek.takeIf { playback.durationMs != null },
)
Spacer(Modifier.height(4.dp))
Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) {
Text(
audio.durationSeconds?.let { formatClock(audio.positionSeconds) } ?: "",
playback.durationMs?.let { formatClock(playback.positionMs) } ?: "",
style = MaterialTheme.typography.labelSmall,
color = Color.White.copy(alpha = 0.8f),
)
Text(
listOfNotNull(audio.fileFacts, audio.durationSeconds?.let(::formatClock)).joinToString(" · "),
trailingFacts(info, playback),
style = MaterialTheme.typography.labelSmall,
color = Color.White.copy(alpha = 0.8f),
maxLines = 1,
)
}
}
overlay()
}
}
// ---------------------------------------------------------------------------------------------
// A. Neutral track card — only while we can't tell audio from video. It commits to neither shape
// (no waveform, no picture box) and is replaced by B, C or the video player once the probe answers.
// Audio or video, can't tell yet: commits to neither shape (no waveform, no picture box) and gives way
// to the waveform, the cover or the video player once the probe answers.
// ---------------------------------------------------------------------------------------------
@Composable
fun UndecidedMediaCardPrototype(
media: AudioCardUi,
fun UndecidedMediaCard(
info: AudioCardInfo,
playback: AudioPlaybackUi,
onPlayPause: () -> Unit,
modifier: Modifier = Modifier,
onClick: (() -> Unit)? = null,
overlay: @Composable BoxScope.() -> Unit = {},
) {
Surface(
modifier = modifier.fillMaxWidth(),
shape = RoundedCornerShape(16.dp),
color = MaterialTheme.colorScheme.surfaceContainerHigh,
Box(
modifier
.fillMaxWidth()
.clip(RoundedCornerShape(16.dp))
.background(MaterialTheme.colorScheme.surfaceContainerHigh)
.clickableWithoutRipple(onClick),
) {
Column {
Row(
@@ -285,7 +330,10 @@ fun UndecidedMediaCardPrototype(
verticalAlignment = Alignment.CenterVertically,
) {
Box(
Modifier.size(64.dp).clip(RoundedCornerShape(10.dp)).background(MaterialTheme.colorScheme.surfaceContainerHighest),
Modifier
.size(64.dp)
.clip(RoundedCornerShape(10.dp))
.background(MaterialTheme.colorScheme.onSurface.copy(alpha = 0.08f)),
contentAlignment = Alignment.Center,
) {
Icon(MaterialSymbols.Podcasts, null, Modifier.size(28.dp), tint = MaterialTheme.colorScheme.onSurfaceVariant)
@@ -293,66 +341,29 @@ fun UndecidedMediaCardPrototype(
Spacer(Modifier.width(12.dp))
Column(Modifier.weight(1f)) {
Text(
media.title,
info.title ?: stringRes(Res.string.playable_media_untitled),
style = MaterialTheme.typography.titleSmall,
fontWeight = FontWeight.SemiBold,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
Text(
media.artist,
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
info.artist?.let {
Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant, maxLines = 1, overflow = TextOverflow.Ellipsis)
}
Spacer(Modifier.height(2.dp))
Text(
if (media.durationSeconds != null) "${media.timeLabel} · ${media.fileFacts}" else media.fileFacts,
trailingFacts(info, playback),
style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
)
}
Spacer(Modifier.width(8.dp))
PlayPauseButton(media.isPlaying, size = 48.dp)
PlayPauseButton(playback.isPlaying, size = 48.dp, onClick = onPlayPause)
}
ThinProgress(media.progress, Modifier.fillMaxWidth().height(3.dp))
}
}
}
// ---------------------------------------------------------------------------------------------
// E. Mini player — where a playing track goes once its post scrolls away. Complements B and C.
// ---------------------------------------------------------------------------------------------
@Composable
fun AudioMiniPlayerPrototype(
audio: AudioCardUi,
modifier: Modifier = Modifier,
) {
Surface(
modifier = modifier.fillMaxWidth(),
shape = RoundedCornerShape(14.dp),
color = MaterialTheme.colorScheme.surfaceContainerHighest,
shadowElevation = 6.dp,
) {
Column {
Row(
modifier = Modifier.padding(8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
GeneratedCover(audio.seed, Modifier.size(44.dp).clip(RoundedCornerShape(8.dp)), icon = MaterialSymbols.MusicNote, iconSize = 20.dp)
Spacer(Modifier.width(10.dp))
Column(Modifier.weight(1f)) {
Text(audio.title, style = MaterialTheme.typography.bodyMedium, fontWeight = FontWeight.SemiBold, maxLines = 1, overflow = TextOverflow.Ellipsis)
Text(audio.artist, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant, maxLines = 1, overflow = TextOverflow.Ellipsis)
}
PlayPauseButton(audio.isPlaying, size = 40.dp, container = Color.Transparent, content = MaterialTheme.colorScheme.onSurface)
Icon(MaterialSymbols.Close, stringRes(Res.string.close), Modifier.padding(horizontal = 8.dp).size(22.dp), tint = MaterialTheme.colorScheme.onSurfaceVariant)
}
ThinProgress(audio.progress, Modifier.fillMaxWidth().height(2.dp))
ThinProgress(playback.progress, Modifier.fillMaxWidth().height(3.dp))
}
overlay()
}
}
@@ -362,18 +373,51 @@ fun AudioMiniPlayerPrototype(
/** The file's own waveform when it has one, else the synthetic one every other audio renderer already uses. */
@Composable
private fun rememberBars(audio: AudioCardUi): List<Float> = remember(audio.seed, audio.waveform) { audio.waveform ?: syntheticWaveformFor(audio.seed).wave }
private fun rememberBars(info: AudioCardInfo): List<Float> = remember(info.seed, info.waveform) { info.waveform?.let(::normalizeWave) ?: syntheticWaveformFor(info.seed).wave }
/** Scales a raw amplitude list (NIP-A0 sends 0..100, others 0..1 or raw RMS) onto 0..1. Null when it has no range. */
internal fun normalizeWave(wave: List<Float>): List<Float>? {
val max = wave.maxOrNull() ?: return null
if (max <= 0f || !max.isFinite()) return null
return wave.map { (it / max).coerceIn(0f, 1f) }
}
/** The track's name, or "MP3 audio" when nothing names it. */
@Composable
private fun audioTitle(info: AudioCardInfo): String =
info.title
?: info.format?.let { stringRes(Res.string.audio_card_untitled, it) }
?: stringRes(Res.string.audio_card_untitled_no_format)
/** "MP3 · 4.8 MB · 3:28" once the duration is known; the file facts before that. */
private fun trailingFacts(
info: AudioCardInfo,
playback: AudioPlaybackUi,
): String = listOfNotNull(info.fileFacts.ifEmpty { null }, playback.durationMs?.let(::formatClock)).joinToString(" · ")
private fun Modifier.clickableWithoutRipple(onClick: (() -> Unit)?): Modifier =
if (onClick == null) {
this
} else {
this.then(Modifier.clickable(interactionSource = null, indication = null, onClick = onClick))
}
@Composable
fun PlayPauseButton(
isPlaying: Boolean,
size: Dp,
onClick: (() -> Unit)?,
modifier: Modifier = Modifier,
container: Color = MaterialTheme.colorScheme.primary,
content: Color = MaterialTheme.colorScheme.onPrimary,
) {
Box(
modifier = modifier.size(size).clip(CircleShape).background(container),
modifier =
modifier
.size(size)
.clip(CircleShape)
.background(container)
.then(if (onClick != null) Modifier.clickable(onClick = onClick) else Modifier),
contentAlignment = Alignment.Center,
) {
Icon(
@@ -436,7 +480,10 @@ private fun ThinProgress(
}
}
/** Centred bars, coloured [played] up to [progress] and [unplayed] after it. */
/**
* Centred bars, coloured [played] up to [progress] and [unplayed] after it. With [onSeek], a tap or a
* horizontal drag seeks: the drag previews its position and seeks once, on release.
*/
@Composable
fun WaveformBars(
bars: List<Float>,
@@ -446,8 +493,36 @@ fun WaveformBars(
modifier: Modifier = Modifier,
barWidth: Dp = 3.dp,
gap: Dp = 2.dp,
onSeek: ((Float) -> Unit)? = null,
) {
Canvas(modifier) {
var dragFraction by remember { mutableStateOf<Float?>(null) }
val currentOnSeek by rememberUpdatedState(onSeek)
val seekModifier =
if (onSeek == null) {
Modifier
} else {
Modifier
.pointerInput(Unit) {
detectTapGestures { offset -> currentOnSeek?.invoke((offset.x / size.width).coerceIn(0f, 1f)) }
}.pointerInput(Unit) {
detectHorizontalDragGestures(
onDragStart = { offset -> dragFraction = (offset.x / size.width).coerceIn(0f, 1f) },
onDragEnd = {
dragFraction?.let { currentOnSeek?.invoke(it) }
dragFraction = null
},
onDragCancel = { dragFraction = null },
onHorizontalDrag = { change, _ ->
change.consume()
dragFraction = (change.position.x / size.width).coerceIn(0f, 1f)
},
)
}
}
Canvas(modifier.then(seekModifier)) {
if (bars.isEmpty()) return@Canvas
val shown = dragFraction ?: progress
val step = (barWidth + gap).toPx()
val count = (size.width / step).toInt().coerceAtLeast(1)
val w = barWidth.toPx()
@@ -456,7 +531,7 @@ fun WaveformBars(
val h = amp * size.height
val x = i * step
drawRoundRect(
color = if (x / size.width < progress) played else unplayed,
color = if (x / size.width < shown) played else unplayed,
topLeft = Offset(x, (size.height - h) / 2f),
size = Size(w, h),
cornerRadius = CornerRadius(w / 2f, w / 2f),
@@ -472,7 +547,13 @@ fun seedColors(seed: String): Pair<Color, Color> {
return Color.hsv(hue, 0.6f, 0.62f) to Color.hsv((hue + 48f) % 360f, 0.75f, 0.32f)
}
fun formatClock(seconds: Int): String = "${seconds / 60}:${(seconds % 60).toString().padStart(2, '0')}"
fun formatClock(millis: Long): String {
val seconds = millis / 1000
val hours = seconds / 3600
val minutes = (seconds % 3600) / 60
val secs = (seconds % 60).toString().padStart(2, '0')
return if (hours > 0) "$hours:${minutes.toString().padStart(2, '0')}:$secs" else "$minutes:$secs"
}
fun formatFileSize(bytes: Long): String {
val tenthsOfMb = (bytes * 10 / 1_048_576.0).roundToInt()
@@ -29,12 +29,12 @@ import kotlin.test.Test
import kotlin.test.assertTrue
/**
* Renders every audio-player prototype offscreen (no device, no display) through `ImageComposeScene`, in the
* Renders every playable-media card offscreen (no device, no display) through `ImageComposeScene`, in the
* dark and light Amethyst themes side by side, and writes each to `commonsUI/build/audio-player/<name>.png`,
* so the candidate designs for audio inside a post can be compared as screenshots. Fails if a screen throws
* or draws nothing.
*/
class AudioPlayerPrototypesRenderTest {
class PlayableMediaCardsRenderTest {
private val outDir = File("build/audio-player").apply { mkdirs() }
private fun render(
@@ -66,16 +66,12 @@ class AudioPlayerPrototypesRenderTest {
}
}
@Test fun today() = render("00-today", 820, 700) { AudioTodayPreview() }
@Test fun knownAudio() = render("01-known-audio-waveform", 820, 560) { AudioKnownAudioPreview() }
@Test fun knownAudio() = render("01-known-audio-waveform", 820, 760) { AudioKnownAudioPreview() }
@Test fun withArtwork() = render("02-known-audio-cover", 820, 620) { AudioWithArtworkPreview() }
@Test fun undecided() = render("03-undecided-track-card", 820, 360) { AudioUndecidedPreview() }
@Test fun miniPlayer() = render("04-mini-player", 820, 520) { AudioMiniPlayerPreview() }
private companion object {
const val SETTLE_FRAMES = 12
const val FRAME_MILLIS = 60L