From 604f5747fffd4096cd7d9a3adc14a1e1f37c9ce5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 14:46:29 +0000 Subject: [PATCH] feat(audio): picture-in-picture for long audio in posts Audio cards for tracks of ten minutes or more (by the player, else the imeta duration) show a picture-in-picture button; the overlay's PiP button does the same for any audio. Both pause the inline player and hand the same pooled player to the PiP window, which resumes from there. For audio, the PiP window is now square and draws the cover (imeta image, else the generated one), the waveform as progress and the time, instead of an empty video surface. It starts unmuted: the feed's mute default is for silently autoplaying video. MediaItemData gains an isAudio flag, and it and the hash survive the PiP intent. MediaUrlVideo carries the imeta duration, so the card shows the length and offers PiP before the player has probed the file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017KKAj1seHQLCcVEiMeePEE --- .../composable/AudioPlaybackUiState.kt | 76 +++++++++ .../composable/mediaitem/MediaItemData.kt | 3 + .../service/playback/pip/IntentExtras.kt | 4 + .../service/playback/pip/PipAudioView.kt | 58 +++++++ .../service/playback/pip/PipVideoActivity.kt | 12 +- .../ui/components/PlayableAudioView.kt | 150 +++++++++--------- .../commons/richtext/MediaContentModels.kt | 3 + .../commons/richtext/RichTextParser.kt | 2 + .../richtext/RichTextParserAudioImetaTest.kt | 8 + .../audio/player/PlayableMediaCardPreviews.kt | 39 ++++- .../audio/player/PlayableMediaCards.kt | 108 ++++++++++++- .../player/PlayableMediaCardsRenderTest.kt | 2 + 12 files changed, 385 insertions(+), 80 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/AudioPlaybackUiState.kt create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/PipAudioView.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/AudioPlaybackUiState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/AudioPlaybackUiState.kt new file mode 100644 index 0000000000..1623954bc1 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/AudioPlaybackUiState.kt @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.service.playback.composable + +import androidx.annotation.OptIn +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.State +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.media3.common.C +import androidx.media3.common.Player +import androidx.media3.common.util.UnstableApi +import androidx.media3.common.util.Util +import com.vitorpamplona.amethyst.commons.audio.player.AudioPlaybackUi +import kotlinx.coroutines.delay + +/** The player's state as the card draws it: event-driven, plus a position poll while it plays. */ +@Composable +fun rememberAudioPlaybackUi(player: Player): State { + val state = remember(player) { mutableStateOf(player.toPlaybackUi()) } + DisposableEffect(player) { + val listener = + object : Player.Listener { + override fun onEvents( + player: Player, + events: Player.Events, + ) { + state.value = player.toPlaybackUi() + } + } + player.addListener(listener) + state.value = player.toPlaybackUi() + onDispose { player.removeListener(listener) } + } + + val isPlaying = state.value.isPlaying + LaunchedEffect(player, isPlaying) { + while (isPlaying) { + delay(POSITION_POLL_MS) + state.value = player.toPlaybackUi() + } + } + return state +} + +@OptIn(UnstableApi::class) +private fun Player.toPlaybackUi() = + AudioPlaybackUi( + // Matches what the button should offer: pause while playing or about to (buffering with + // playWhenReady), play when paused, ended or idle. + isPlaying = !Util.shouldShowPlayButton(this), + positionMs = currentPosition.coerceAtLeast(0), + durationMs = duration.takeIf { it != C.TIME_UNSET && it > 0 }, + ) + +private const val POSITION_POLL_MS = 250L diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/mediaitem/MediaItemData.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/mediaitem/MediaItemData.kt index 7ff0500dae..6c3604c680 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/mediaitem/MediaItemData.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/composable/mediaitem/MediaItemData.kt @@ -54,6 +54,9 @@ data class MediaItemData( // NIP-71 `text-track` captions to side-load. A data class is the cache key, so this has to be // a value that compares by content — ImmutableList keeps that true and keeps the key @Stable. val captions: ImmutableList = persistentListOf(), + // Set by the in-post audio cards: picture-in-picture then draws the audio view (cover, waveform, + // time) in a square window, unmuted, instead of a video surface with no picture. + val isAudio: Boolean = false, ) @Immutable diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/IntentExtras.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/IntentExtras.kt index 53f1ddd1b9..43728e73e2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/IntentExtras.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/IntentExtras.kt @@ -62,6 +62,8 @@ class IntentExtras { keepPlaying = intent.getBoolean("keepPlaying", true), waveformData = intent.getFloatArray("wavefrontData")?.toList()?.let { WaveformData(it) }, isLiveStream = intent.getBoolean("isLiveStream", false), + hash = intent.getString("hash"), + isAudio = intent.getBoolean("isAudio", false), ) } @@ -81,6 +83,8 @@ class IntentExtras { putBoolean("keepPlaying", data.keepPlaying) data.waveformData?.let { putFloatArray("wavefrontData", it.wave.toFloatArray()) } putBoolean("isLiveStream", data.isLiveStream) + data.hash?.let { putString("hash", it) } + putBoolean("isAudio", data.isAudio) bounds?.let { putInt("boundLeft", it.left) } bounds?.let { putInt("boundRight", it.right) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/PipAudioView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/PipAudioView.kt new file mode 100644 index 0000000000..3899f27e72 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/PipAudioView.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.service.playback.pip + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import com.vitorpamplona.amethyst.commons.audio.mediaFormatLabel +import com.vitorpamplona.amethyst.commons.audio.player.AudioCardInfo +import com.vitorpamplona.amethyst.commons.audio.player.AudioPipContent +import com.vitorpamplona.amethyst.service.playback.composable.MediaControllerState +import com.vitorpamplona.amethyst.service.playback.composable.mediaitem.MediaItemData +import com.vitorpamplona.amethyst.service.playback.composable.rememberAudioPlaybackUi + +/** + * The picture-in-picture window for audio sent there from an in-post audio card: the cover (artwork, + * else the one generated from the file hash), the waveform as progress, and the time. There is no + * picture to show, so unlike [RenderPipVideo] it draws no video surface and doesn't hold the screen on — + * audio keeps playing with the screen off. + */ +@Composable +fun RenderPipAudio( + controller: MediaControllerState, + mediaItemData: MediaItemData, +) { + val info = + remember(mediaItemData) { + AudioCardInfo( + title = null, + artist = mediaItemData.authorName, + format = mediaFormatLabel(mediaItemData.mimeType, mediaItemData.videoUri), + sizeBytes = null, + seed = mediaItemData.hash ?: mediaItemData.videoUri, + waveform = mediaItemData.waveformData?.wave, + artworkUrl = mediaItemData.artworkUri, + ) + } + val playback by rememberAudioPlaybackUi(controller.controller) + AudioPipContent(info, playback) +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/PipVideoActivity.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/PipVideoActivity.kt index 6998148f91..8c9aaa13d1 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/PipVideoActivity.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/playback/pip/PipVideoActivity.kt @@ -46,8 +46,10 @@ class PipVideoActivity : ComponentActivity() { setContent { val videoData by rememberVideoDataFromIntents() videoData?.let { mediaItemData -> - // keeps a copy of the value to avoid recompositions here when the DEFAULT value changes - val muted = remember(mediaItemData) { DEFAULT_MUTED_SETTING.value } + // keeps a copy of the value to avoid recompositions here when the DEFAULT value changes. + // Audio is never started muted: the feed's mute default is about silent autoplaying + // video, and an audio track somebody just sent to PiP is meant to be heard. + val muted = remember(mediaItemData) { !mediaItemData.isAudio && DEFAULT_MUTED_SETTING.value } GetMediaItem(mediaItemData) { mediaItem -> GetVideoController( @@ -60,7 +62,11 @@ class PipVideoActivity : ComponentActivity() { RegisterBackgroundMedia(controllerState) RegisterControllerReceiver(controllerState) WatchControllerForActions(mediaItemData, controllerState) - RenderPipVideo(controllerState, mediaItemData.waveformData) + if (mediaItemData.isAudio) { + RenderPipAudio(controllerState, mediaItemData) + } else { + RenderPipVideo(controllerState, mediaItemData.waveformData) + } } } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/PlayableAudioView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/PlayableAudioView.kt index 14d543062d..c3ffb0559f 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/PlayableAudioView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/PlayableAudioView.kt @@ -20,13 +20,14 @@ */ package com.vitorpamplona.amethyst.ui.components +import android.content.pm.PackageManager +import android.graphics.Rect import androidx.annotation.OptIn import androidx.collection.LruCache import androidx.compose.foundation.layout.Box import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.State import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember @@ -36,16 +37,19 @@ import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.layout.boundsInWindow import androidx.compose.ui.layout.onGloballyPositioned +import androidx.compose.ui.platform.LocalContext import androidx.media3.common.C import androidx.media3.common.Player import androidx.media3.common.Tracks import androidx.media3.common.util.UnstableApi import androidx.media3.common.util.Util import com.vitorpamplona.amethyst.commons.audio.PlayableLayout +import com.vitorpamplona.amethyst.commons.audio.WaveformData import com.vitorpamplona.amethyst.commons.audio.mediaFormatLabel import com.vitorpamplona.amethyst.commons.audio.player.AudioCardInfo import com.vitorpamplona.amethyst.commons.audio.player.AudioPlaybackUi import com.vitorpamplona.amethyst.commons.audio.player.PlayableMediaCard +import com.vitorpamplona.amethyst.commons.audio.syntheticWaveformFor import com.vitorpamplona.amethyst.commons.resources.Res import com.vitorpamplona.amethyst.commons.resources.audio_card_untitled import com.vitorpamplona.amethyst.commons.resources.audio_card_untitled_no_format @@ -53,13 +57,15 @@ import com.vitorpamplona.amethyst.commons.resources.playable_media_untitled import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo import com.vitorpamplona.amethyst.commons.ui.stringRes import com.vitorpamplona.amethyst.commons.viewmodels.AccountViewModel +import com.vitorpamplona.amethyst.service.playback.composable.DEFAULT_MUTED_SETTING import com.vitorpamplona.amethyst.service.playback.composable.GetVideoController import com.vitorpamplona.amethyst.service.playback.composable.MediaControllerState import com.vitorpamplona.amethyst.service.playback.composable.PauseControllerWhenInBackground import com.vitorpamplona.amethyst.service.playback.composable.mediaitem.GetMediaItem -import com.vitorpamplona.amethyst.service.playback.composable.mediaitem.LoadedMediaItem +import com.vitorpamplona.amethyst.service.playback.composable.mediaitem.MediaItemData +import com.vitorpamplona.amethyst.service.playback.composable.rememberAudioPlaybackUi +import com.vitorpamplona.amethyst.service.playback.pip.PipVideoActivity import com.vitorpamplona.amethyst.ui.note.types.RenderTopButtonsForVoice -import kotlinx.coroutines.delay /** * What the player found when it probed a URL: true when every track is audio, false when there is a @@ -79,6 +85,9 @@ object PlayableMediaProbeCache { } } +/** Audio at least this long gets a picture-in-picture button on its card: it will outlast the post on screen. */ +private const val LONG_AUDIO_MS = 10 * 60 * 1000L + /** * A playable file inside a post that is (or may be) audio: the shared [PlayableMediaCard] for [layout], * driven by the same pooled player as every other inline media. @@ -88,6 +97,9 @@ object PlayableMediaProbeCache { * fetched until play is pressed. Once the player knows the tracks, [onProbed] reports whether they are * all audio, which is how an undecided file finds its real card and how a mislabelled video leaves for * the video player. + * + * Known audio that runs [LONG_AUDIO_MS] or more (by the player, else the imeta `duration`) shows a + * picture-in-picture button, so a podcast can keep playing in a floating window while the feed scrolls. */ @Composable fun PlayableAudioView( @@ -96,6 +108,7 @@ fun PlayableAudioView( onProbed: (audioOnly: Boolean) -> Unit, accountViewModel: AccountViewModel, ) { + val context = LocalContext.current val authorName = remember(content.authorName, content.authorPubKey) { content.authorName ?: content.authorPubKey?.let { accountViewModel.getUserIfExists(it)?.toBestDisplayName() } @@ -111,6 +124,7 @@ fun PlayableAudioView( seed = content.hash ?: content.url, waveform = content.waveform, artworkUrl = content.artworkUri, + declaredDurationMs = content.durationSeconds?.let { (it * 1000).toLong() }, ) } @@ -122,33 +136,56 @@ fun PlayableAudioView( else -> stringRes(Res.string.audio_card_untitled_no_format) } + val mediaData = + remember(content, mediaTitle, authorName) { + MediaItemData( + videoUri = content.url, + authorName = authorName, + title = mediaTitle, + artworkUri = content.artworkUri, + callbackUri = content.uri, + mimeType = content.mimeType, + proxyPort = accountViewModel.httpClientBuilder.proxyPortForVideo(content.url), + keepPlaying = false, + isLiveStream = content.isLiveStream, + hash = content.hash, + ) + } + var load by remember(content.url) { mutableStateOf(accountViewModel.settings.startVideoPlayback()) } var playWhenConnected by remember(content.url) { mutableStateOf(false) } - var active by remember(content.url) { mutableStateOf?>(null) } + var active by remember(content.url) { mutableStateOf(null) } val controllerVisible = remember(content.url) { mutableStateOf(false) } + // Where picture-in-picture animates from. + val bounds = remember(content.url) { arrayOf(null) } + val pipSupported = remember { context.packageManager.hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE) } + + // Hands playback to the PiP window: it attaches to the same pooled player (same URL), so pausing + // here first keeps the two from playing over each other, and it resumes from this position. + val startPictureInPicture = { + active?.controller?.pause() + PipVideoActivity.callIn( + mediaData.copy( + // Square window, and the audio view instead of an empty video surface. + aspectRatio = 1f, + isAudio = true, + waveformData = WaveformData(info.waveform ?: syntheticWaveformFor(info.seed).wave), + ), + bounds[0], + context.getActivity(), + ) + } if (load) { - val proxyPort = remember(content.url) { accountViewModel.httpClientBuilder.proxyPortForVideo(content.url) } - GetMediaItem( - videoUri = content.url, - title = mediaTitle, - artworkUri = content.artworkUri, - authorName = authorName, - callbackUri = content.uri, - mimeType = content.mimeType, - proxyPort = proxyPort, - keepPlaying = false, - isLiveStream = content.isLiveStream, - hash = content.hash, - ) { mediaItem -> + GetMediaItem(mediaData) { mediaItem -> GetVideoController(mediaItem = mediaItem, muted = false) { controller -> PauseControllerWhenInBackground(controller) ReportProbe(controller.controller, onProbed) - DisposableEffect(controller, mediaItem) { - active = controller to mediaItem + DisposableEffect(controller) { + active = controller onDispose { - if (active?.first === controller) active = null + if (active === controller) active = null } } @@ -163,11 +200,17 @@ fun PlayableAudioView( } } + val trackBounds = + Modifier.onGloballyPositioned { coordinates -> + val b = coordinates.boundsInWindow() + bounds[0] = Rect(b.left.toInt(), b.top.toInt(), b.right.toInt(), b.bottom.toInt()) + } + Box { - val current = active - if (current != null) { - val (controller, mediaItem) = current + val controller = active + if (controller != null) { val playback by rememberAudioPlaybackUi(controller.controller) + val totalMs = playback.durationMs ?: info.declaredDurationMs PlayableMediaCard( layout = layout, info = info, @@ -175,23 +218,25 @@ fun PlayableAudioView( onPlayPause = { playOrPause(controller.controller) }, onSeek = { fraction -> playback.durationMs?.let { controller.controller.seekTo((it * fraction).toLong()) } }, onClick = { controllerVisible.value = !controllerVisible.value }, - modifier = - Modifier.onGloballyPositioned { coordinates -> - // Where picture-in-picture animates from. - val bounds = coordinates.boundsInWindow() - controller.visibility.setBounds(bounds.left.toInt(), bounds.top.toInt(), bounds.right.toInt(), bounds.bottom.toInt()) - }, + onPictureInPicture = startPictureInPicture.takeIf { pipSupported && layout != PlayableLayout.UNDECIDED && totalMs != null && totalMs >= LONG_AUDIO_MS }, + modifier = trackBounds, overlay = { RenderTopButtonsForVoice( - mediaData = mediaItem.src, - controllerState = controller, + mediaData = mediaData, controllerVisible = controllerVisible, + startingMuteState = controller.controller.volume < 0.001, + onMuteClick = { mute -> + DEFAULT_MUTED_SETTING.value = mute + controller.controller.volume = if (mute) 0f else 1f + }, + onPictureInPictureClick = startPictureInPicture, modifier = Modifier.align(Alignment.TopEnd), accountViewModel = accountViewModel, ) }, ) } else { + val totalMs = info.declaredDurationMs PlayableMediaCard( layout = layout, info = info, @@ -201,6 +246,8 @@ fun PlayableAudioView( load = true }, onSeek = {}, + onPictureInPicture = startPictureInPicture.takeIf { pipSupported && layout != PlayableLayout.UNDECIDED && totalMs != null && totalMs >= LONG_AUDIO_MS }, + modifier = trackBounds, ) } } @@ -237,44 +284,3 @@ private fun ReportProbe( onDispose { player.removeListener(listener) } } } - -/** The player's state as the card draws it: event-driven, plus a position poll while it plays. */ -@Composable -private fun rememberAudioPlaybackUi(player: Player): State { - val state = remember(player) { mutableStateOf(player.toPlaybackUi()) } - DisposableEffect(player) { - val listener = - object : Player.Listener { - override fun onEvents( - player: Player, - events: Player.Events, - ) { - state.value = player.toPlaybackUi() - } - } - player.addListener(listener) - state.value = player.toPlaybackUi() - onDispose { player.removeListener(listener) } - } - - val isPlaying = state.value.isPlaying - LaunchedEffect(player, isPlaying) { - while (isPlaying) { - delay(POSITION_POLL_MS) - state.value = player.toPlaybackUi() - } - } - return state -} - -@OptIn(UnstableApi::class) -private fun Player.toPlaybackUi() = - AudioPlaybackUi( - // Matches what the button should offer: pause while playing or about to (buffering with - // playWhenReady), play when paused, ended or idle. - isPlaying = !Util.shouldShowPlayButton(this), - positionMs = currentPosition.coerceAtLeast(0), - durationMs = duration.takeIf { it != C.TIME_UNSET && it > 0 }, - ) - -private const val POSITION_POLL_MS = 250L diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentModels.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentModels.kt index 6148d22d99..63f68c89a8 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentModels.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentModels.kt @@ -122,6 +122,9 @@ open class MediaUrlVideo( val sizeBytes: Long? = null, // The imeta `waveform` (NIP-A0 amplitudes, not normalised): an audio card draws it as its seek bar. val waveform: List? = null, + // The imeta `duration`, in seconds: an audio card shows it, and offers picture-in-picture for long + // audio, before the player has probed the file. + val durationSeconds: Double? = null, ) : MediaUrlContent(url, description, hash, dim, blurhash, uri, mimeType, thumbhash, authorPubKey) @Immutable diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt index 63d1a009cb..b4731798df 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt @@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.nip94FileMetadata.tags.ImageTag import com.vitorpamplona.quartz.nip94FileMetadata.tags.MimeTypeTag import com.vitorpamplona.quartz.nip94FileMetadata.tags.SizeTag import com.vitorpamplona.quartz.nip94FileMetadata.tags.ThumbhashTag +import com.vitorpamplona.quartz.nipA0VoiceMessages.tags.DurationTag import com.vitorpamplona.quartz.nipA0VoiceMessages.tags.WaveformTag import com.vitorpamplona.quartz.utils.Log import kotlinx.collections.immutable.ImmutableList @@ -99,6 +100,7 @@ class RichTextParser { authorPubKey = authorPubKey, sizeBytes = (frags[SizeTag.TAG_NAME] ?: tags[SizeTag.TAG_NAME]?.firstOrNull())?.toLongOrNull(), waveform = (frags[WaveformTag.TAG_NAME] ?: tags[WaveformTag.TAG_NAME]?.firstOrNull())?.let { WaveformTag.parseWave(it) }, + durationSeconds = (frags[DurationTag.TAG_NAME] ?: tags[DurationTag.TAG_NAME]?.firstOrNull())?.toDoubleOrNull()?.takeIf { it > 0 }, ) } else if (kind == MediaContentKind.PDF) { MediaUrlPdf( diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserAudioImetaTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserAudioImetaTest.kt index 121a6d2dbb..3af55ab51c 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserAudioImetaTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserAudioImetaTest.kt @@ -49,6 +49,14 @@ class RichTextParserAudioImetaTest { assertEquals("audio/mpeg", media.mimeType) assertEquals(4_992_768L, media.sizeBytes) assertNull(media.waveform) + assertNull(media.durationSeconds) + } + + @Test + fun durationReachesTheMedia() { + val media = parse("m audio/mpeg", "duration 3725.4") + assertIs(media) + assertEquals(3725.4, media.durationSeconds) } @Test diff --git a/commonsUI/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCardPreviews.kt b/commonsUI/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCardPreviews.kt index d3e3e38bc4..a85d9edce0 100644 --- a/commonsUI/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCardPreviews.kt +++ b/commonsUI/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCardPreviews.kt @@ -33,6 +33,7 @@ import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable @@ -86,7 +87,19 @@ object AudioPlayerSamples { /** An HLS playlist: audio only or video, nothing tells until the player probes it. */ val stream = AudioCardInfo(title = null, artist = AUTHOR_NAME, format = "HLS", sizeBytes = null, seed = HLS_URL) + /** A 1h12 podcast episode: long enough that the card offers picture-in-picture. */ + val podcast = + AudioCardInfo( + title = null, + artist = AUTHOR_NAME, + format = "MP3", + sizeBytes = 69_206_016, + seed = "a7d93c0e5b2f4816", + declaredDurationMs = 4_325_000, + ) + val playing = AudioPlaybackUi(isPlaying = true, positionMs = 72_000, durationMs = 208_000) + val podcastPlaying = AudioPlaybackUi(isPlaying = true, positionMs = 1_534_000, durationMs = 4_325_000) val pausedEarly = AudioPlaybackUi(isPlaying = false, positionMs = 9_000, durationMs = 41_000) } @@ -139,7 +152,8 @@ private fun Card( layout: PlayableLayout, info: AudioCardInfo, playback: AudioPlaybackUi = AudioPlaybackUi.Idle, -) = PlayableMediaCard(layout, info, playback, onPlayPause = {}, onSeek = {}) + pip: Boolean = false, +) = PlayableMediaCard(layout, info, playback, onPlayPause = {}, onSeek = {}, onPictureInPicture = if (pip) ({}) else null) @Preview(widthDp = 820, heightDp = 760) @Composable @@ -175,3 +189,26 @@ fun AudioUndecidedPreview() = MockAudioPost(text = "Live set tonight, tune in") { Card(playableLayout(null, AudioPlayerSamples.HLS_URL, hasArtwork = false), AudioPlayerSamples.stream) } } } + +@Preview(widthDp = 820, heightDp = 1000) +@Composable +fun AudioPictureInPicturePreview() = + AudioPreviewFrame { + val waveform = playableLayout("audio/mpeg", AudioPlayerSamples.MP3_URL, hasArtwork = false) + val cover = playableLayout("audio/mpeg", AudioPlayerSamples.MP3_URL, hasArtwork = true) + Column { + Caption("Long audio offers picture-in-picture (before load: imeta duration)") + MockAudioPost(text = "New episode is out!") { Card(waveform, AudioPlayerSamples.podcast, pip = true) } + MockAudioPost(text = "New episode is out!") { Card(cover, AudioPlayerSamples.podcast, AudioPlayerSamples.podcastPlaying, pip = true) } + Caption("The PiP window") + Row(Modifier.padding(12.dp)) { + Box(Modifier.size(180.dp).clip(RoundedCornerShape(12.dp))) { + AudioPipContent(AudioPlayerSamples.podcast, AudioPlayerSamples.podcastPlaying) + } + Spacer(Modifier.width(12.dp)) + Box(Modifier.size(180.dp).clip(RoundedCornerShape(12.dp))) { + AudioPipContent(AudioPlayerSamples.withArtwork, AudioPlayerSamples.playing) + } + } + } + } diff --git a/commonsUI/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCards.kt b/commonsUI/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCards.kt index 26b654cf58..dfa5e70e4b 100644 --- a/commonsUI/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCards.kt +++ b/commonsUI/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCards.kt @@ -76,6 +76,7 @@ import com.vitorpamplona.amethyst.commons.resources.Res import com.vitorpamplona.amethyst.commons.resources.audio_card_untitled import com.vitorpamplona.amethyst.commons.resources.audio_card_untitled_no_format import com.vitorpamplona.amethyst.commons.resources.pause +import com.vitorpamplona.amethyst.commons.resources.picture_in_picture import com.vitorpamplona.amethyst.commons.resources.play import com.vitorpamplona.amethyst.commons.resources.playable_media_untitled import com.vitorpamplona.amethyst.commons.ui.stringRes @@ -111,6 +112,8 @@ class AudioCardInfo( val waveform: List? = null, /** The imeta `image`: the artwork [AudioCoverCard] shows. */ val artworkUrl: String? = null, + /** The imeta `duration`: shown until the player knows the real one. */ + val declaredDurationMs: Long? = null, ) { /** "MP3 · 4.8 MB" — known from the imeta alone. */ val fileFacts: String @@ -142,11 +145,12 @@ fun PlayableMediaCard( onSeek: (Float) -> Unit, modifier: Modifier = Modifier, onClick: (() -> Unit)? = null, + onPictureInPicture: (() -> Unit)? = null, overlay: @Composable BoxScope.() -> Unit = {}, ) { when (layout) { - PlayableLayout.AUDIO_WAVEFORM -> AudioWaveformCard(info, playback, onPlayPause, onSeek, modifier, onClick, overlay) - PlayableLayout.AUDIO_COVER -> AudioCoverCard(info, playback, onPlayPause, onSeek, modifier, onClick, overlay) + PlayableLayout.AUDIO_WAVEFORM -> AudioWaveformCard(info, playback, onPlayPause, onSeek, modifier, onClick, onPictureInPicture, overlay) + PlayableLayout.AUDIO_COVER -> AudioCoverCard(info, playback, onPlayPause, onSeek, modifier, onClick, onPictureInPicture, overlay) PlayableLayout.UNDECIDED -> UndecidedMediaCard(info, playback, onPlayPause, modifier, onClick, overlay) PlayableLayout.VIDEO -> Unit } @@ -164,6 +168,8 @@ fun AudioWaveformCard( onSeek: (Float) -> Unit, modifier: Modifier = Modifier, onClick: (() -> Unit)? = null, + /** Non-null shows the picture-in-picture button: meant for long audio, where the post will scroll away. */ + onPictureInPicture: (() -> Unit)? = null, overlay: @Composable BoxScope.() -> Unit = {}, ) { val bars = rememberBars(info) @@ -192,7 +198,7 @@ fun AudioWaveformCard( Spacer(Modifier.height(4.dp)) Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) { Text( - if (playback.durationMs != null) formatClock(playback.positionMs) else audioTitle(info), + if (playback.durationMs != null || info.declaredDurationMs != null) formatClock(playback.positionMs) else audioTitle(info), style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.onSurfaceVariant, maxLines = 1, @@ -208,6 +214,10 @@ fun AudioWaveformCard( ) } } + if (onPictureInPicture != null) { + Spacer(Modifier.width(4.dp)) + PictureInPictureButton(onPictureInPicture, tint = MaterialTheme.colorScheme.onSurfaceVariant) + } } overlay() } @@ -225,6 +235,8 @@ fun AudioCoverCard( onSeek: (Float) -> Unit, modifier: Modifier = Modifier, onClick: (() -> Unit)? = null, + /** Non-null shows the picture-in-picture button: meant for long audio, where the post will scroll away. */ + onPictureInPicture: (() -> Unit)? = null, overlay: @Composable BoxScope.() -> Unit = {}, ) { val bars = rememberBars(info) @@ -299,10 +311,84 @@ fun AudioCoverCard( ) } } + if (onPictureInPicture != null) { + PictureInPictureButton( + onPictureInPicture, + tint = Color.White, + modifier = + Modifier + .align(Alignment.TopEnd) + .padding(8.dp) + .clip(CircleShape) + .background(Color.Black.copy(alpha = 0.4f)), + ) + } overlay() } } +// --------------------------------------------------------------------------------------------- +// Picture-in-picture: what the small floating window shows for audio. Square, display-only (a PiP +// window takes no touches; the system draws play/pause over it), the cover behind the waveform. +// --------------------------------------------------------------------------------------------- + +@Composable +fun AudioPipContent( + info: AudioCardInfo, + playback: AudioPlaybackUi, + modifier: Modifier = Modifier, +) { + val bars = rememberBars(info) + Box(modifier.fillMaxSize()) { + GeneratedCover(info.seed, Modifier.fillMaxSize(), icon = null, record = info.artworkUrl == null) + if (info.artworkUrl != null) { + AsyncImage( + model = info.artworkUrl, + contentDescription = null, + contentScale = ContentScale.Crop, + modifier = Modifier.fillMaxSize(), + ) + } + Box( + Modifier + .fillMaxWidth() + .fillMaxHeight(0.6f) + .align(Alignment.BottomCenter) + .background(Brush.verticalGradient(listOf(Color.Transparent, Color.Black.copy(alpha = 0.85f)))), + ) + Column(Modifier.align(Alignment.BottomStart).fillMaxWidth().padding(10.dp)) { + Text( + audioTitle(info), + style = MaterialTheme.typography.titleSmall, + fontWeight = FontWeight.Bold, + color = Color.White, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + ) + info.artist?.let { + Text(it, style = MaterialTheme.typography.labelSmall, color = Color.White.copy(alpha = 0.8f), maxLines = 1, overflow = TextOverflow.Ellipsis) + } + Spacer(Modifier.height(6.dp)) + WaveformBars( + bars = bars, + progress = playback.progress, + played = Color.White, + unplayed = Color.White.copy(alpha = 0.35f), + modifier = Modifier.fillMaxWidth().height(24.dp), + barWidth = 2.dp, + gap = 2.dp, + ) + Spacer(Modifier.height(2.dp)) + Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) { + Text(formatClock(playback.positionMs), style = MaterialTheme.typography.labelSmall, color = Color.White.copy(alpha = 0.8f)) + (playback.durationMs ?: info.declaredDurationMs)?.let { + Text(formatClock(it), style = MaterialTheme.typography.labelSmall, color = Color.White.copy(alpha = 0.8f)) + } + } + } + } +} + // --------------------------------------------------------------------------------------------- // Audio or video, can't tell yet: commits to neither shape (no waveform, no picture box) and gives way // to the waveform, the cover or the video player once the probe answers. @@ -393,7 +479,7 @@ private fun audioTitle(info: AudioCardInfo): String = private fun trailingFacts( info: AudioCardInfo, playback: AudioPlaybackUi, -): String = listOfNotNull(info.fileFacts.ifEmpty { null }, playback.durationMs?.let(::formatClock)).joinToString(" · ") +): String = listOfNotNull(info.fileFacts.ifEmpty { null }, (playback.durationMs ?: info.declaredDurationMs)?.let(::formatClock)).joinToString(" · ") private fun Modifier.clickableWithoutRipple(onClick: (() -> Unit)?): Modifier = if (onClick == null) { @@ -402,6 +488,20 @@ private fun Modifier.clickableWithoutRipple(onClick: (() -> Unit)?): Modifier = this.then(Modifier.clickable(interactionSource = null, indication = null, onClick = onClick)) } +@Composable +private fun PictureInPictureButton( + onClick: () -> Unit, + tint: Color, + modifier: Modifier = Modifier, +) { + Box( + modifier = modifier.size(36.dp).clip(CircleShape).clickable(onClick = onClick), + contentAlignment = Alignment.Center, + ) { + Icon(MaterialSymbols.PictureInPicture, stringRes(Res.string.picture_in_picture), Modifier.size(22.dp), tint = tint) + } +} + @Composable fun PlayPauseButton( isPlaying: Boolean, diff --git a/commonsUI/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCardsRenderTest.kt b/commonsUI/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCardsRenderTest.kt index b0baa0c130..c9d13b11e4 100644 --- a/commonsUI/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCardsRenderTest.kt +++ b/commonsUI/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/audio/player/PlayableMediaCardsRenderTest.kt @@ -72,6 +72,8 @@ class PlayableMediaCardsRenderTest { @Test fun undecided() = render("03-undecided-track-card", 820, 360) { AudioUndecidedPreview() } + @Test fun pictureInPicture() = render("04-picture-in-picture", 820, 1000) { AudioPictureInPicturePreview() } + private companion object { const val SETTLE_FRAMES = 12 const val FRAME_MILLIS = 60L