Merge pull request #3868 from davotoula/fix/file-header-non-media-fallback

fix(media): stop rendering non-media NIP-94 files as video
This commit is contained in:
Vitor Pamplona
2026-08-05 18:20:41 -04:00
committed by GitHub
12 changed files with 579 additions and 179 deletions
@@ -0,0 +1,142 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.ui.components
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalUriHandler
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.vitorpamplona.amethyst.commons.icons.symbols.Icon
import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol
import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols
import com.vitorpamplona.amethyst.commons.util.countToHumanReadableBytes
import com.vitorpamplona.amethyst.commons.util.prettyMime
import com.vitorpamplona.amethyst.ui.components.pdf.extractFilename
import com.vitorpamplona.amethyst.ui.theme.DoubleVertSpacer
import com.vitorpamplona.amethyst.ui.theme.MaxWidthWithHorzPadding
import com.vitorpamplona.amethyst.ui.theme.Size20Modifier
import com.vitorpamplona.amethyst.ui.theme.innerPostModifier
/**
* The renderer for a declared file that none of the media viewers can display — a webxdc app,
* an archive, an installer, any MIME [com.vitorpamplona.amethyst.commons.richtext.RichTextParser.classifyMedia]
* returns null for.
*
* It exists so those files have somewhere to land other than the video player: an unknown blob
* used to fall through an image-or-else-video branch into ExoPlayer, which buffers forever on a
* zip. Everything shown here comes off the event's own tags (NIP-94 `alt`, `m`, `size`), so the
* card costs no network round-trip — unlike routing the URL through the OpenGraph previewer,
* which would try to download the blob just to rediscover the type the event already declared.
*/
@Composable
fun FileAttachmentCard(
url: String,
description: String?,
mimeType: String?,
sizeInBytes: Long?,
) {
val uriHandler = LocalUriHandler.current
val filename = remember(url) { extractFilename(url) }
val subtitle = remember(mimeType, sizeInBytes) { fileSubtitle(mimeType, sizeInBytes) }
Column(
modifier =
MaterialTheme.colorScheme.innerPostModifier
.fillMaxWidth()
.clickable { uriHandler.openUri(url) },
) {
FileAttachmentRow(
symbol = MaterialSymbols.AttachFile,
// The alt/content text names the file for a human ("Webxdc app: Quake");
// the hashed URL basename is the fallback when the event omits it.
title = description?.ifBlank { null } ?: filename,
subtitle = subtitle,
titleMaxLines = 2,
)
Spacer(modifier = DoubleVertSpacer)
}
}
/**
* The icon + title + subtitle row shared by every card that stands in for a file it can't
* render inline: this one and the PDF placeholder/skeleton in
* [com.vitorpamplona.amethyst.ui.components.pdf.PdfPreviewCard].
*/
@Composable
internal fun FileAttachmentRow(
symbol: MaterialSymbol,
title: String,
subtitle: String?,
titleMaxLines: Int = 1,
) {
Row(
modifier = MaxWidthWithHorzPadding.padding(vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Icon(
symbol = symbol,
contentDescription = null,
modifier = Size20Modifier,
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
Column(modifier = Modifier.weight(1f)) {
Text(
text = title,
style = MaterialTheme.typography.bodyMedium,
maxLines = titleMaxLines,
overflow = TextOverflow.Ellipsis,
)
if (subtitle != null) {
Text(
text = subtitle,
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
)
}
}
}
}
/** "APK · 16 MB", dropping either half when the event doesn't declare it. */
private fun fileSubtitle(
mimeType: String?,
sizeInBytes: Long?,
): String? =
listOfNotNull(
mimeType?.ifBlank { null }?.let(::prettyMime),
sizeInBytes?.takeIf { it > 0 }?.let(::countToHumanReadableBytes),
).joinToString(" · ").ifEmpty { null }
@@ -26,38 +26,29 @@ import android.os.ParcelFileDescriptor
import androidx.compose.foundation.ExperimentalFoundationApi
import androidx.compose.foundation.Image
import androidx.compose.foundation.combinedClickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.aspectRatio
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.produceState
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.FilterQuality
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.platform.LocalWindowInfo
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.core.graphics.createBitmap
import com.vitorpamplona.amethyst.commons.icons.symbols.Icon
import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlPdf
import com.vitorpamplona.amethyst.ui.components.ClickableUrl
import com.vitorpamplona.amethyst.ui.components.FileAttachmentRow
import com.vitorpamplona.amethyst.ui.components.ShareMediaAction
import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel
import com.vitorpamplona.amethyst.ui.theme.DoubleVertSpacer
import com.vitorpamplona.amethyst.ui.theme.MaxWidthWithHorzPadding
import com.vitorpamplona.amethyst.ui.theme.Size20Modifier
import com.vitorpamplona.amethyst.ui.theme.innerPostModifier
import com.vitorpamplona.quartz.utils.Log
import kotlinx.coroutines.CancellationException
@@ -207,35 +198,11 @@ private fun PdfSkeletonCard(filename: String) {
private fun FilenameRow(
filename: String,
subtitle: String,
) {
Row(
modifier = MaxWidthWithHorzPadding.padding(vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Icon(
symbol = MaterialSymbols.PictureAsPdf,
contentDescription = null,
modifier = Size20Modifier,
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
Column(modifier = Modifier.weight(1f)) {
Text(
text = filename,
style = MaterialTheme.typography.bodyMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
Text(
text = subtitle,
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
)
}
}
}
) = FileAttachmentRow(
symbol = MaterialSymbols.PictureAsPdf,
title = filename,
subtitle = subtitle,
)
private fun renderFirstPage(
file: java.io.File,
@@ -24,10 +24,13 @@ import androidx.compose.runtime.Composable
import androidx.compose.runtime.remember
import androidx.compose.ui.layout.ContentScale
import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent
import com.vitorpamplona.amethyst.commons.richtext.MediaContentKind
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlPdf
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo
import com.vitorpamplona.amethyst.commons.richtext.RichTextParser
import com.vitorpamplona.amethyst.model.Note
import com.vitorpamplona.amethyst.ui.components.FileAttachmentCard
import com.vitorpamplona.amethyst.ui.components.SensitivityWarning
import com.vitorpamplona.amethyst.ui.components.ZoomableContentView
import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel
@@ -43,50 +46,103 @@ fun FileHeaderDisplay(
) {
val event = (note.event as? FileHeaderEvent) ?: return
val fullUrl = event.url() ?: return
val mimeType = remember(note) { event.mimeType() }
val content = remember(note) { event.toMediaContent(note, fullUrl, mimeType) }
val content: BaseMediaContent =
remember(note) {
val blurHash = event.blurhash()
val thumbHash = event.thumbhash()
val hash = event.hash()
val dimensions = event.dimensions()
val description = event.content.ifEmpty { null } ?: event.alt()
val isImage = event.mimeType()?.startsWith("image/") == true || RichTextParser.isImageUrl(fullUrl)
val uri = note.toNostrUri()
val mimeType = event.mimeType()
if (isImage) {
MediaUrlImage(
url = fullUrl,
description = description,
hash = hash,
blurhash = blurHash,
dim = dimensions,
uri = uri,
mimeType = mimeType,
thumbhash = thumbHash,
)
} else {
MediaUrlVideo(
url = fullUrl,
description = description,
hash = hash,
blurhash = blurHash,
dim = dimensions,
uri = uri,
authorName = note.author?.toBestDisplayName(),
mimeType = mimeType,
thumbhash = thumbHash,
)
}
}
// The sensitivity gate wraps both branches: a content warning is about the file, not about
// which viewer happens to render it, so an NSFW-tagged archive stays behind the same gate.
SensitivityWarning(note = note, accountViewModel = accountViewModel) {
ZoomableContentView(
content = content,
roundedCorner = roundedCorner,
contentScale = contentScale,
accountViewModel = accountViewModel,
)
if (content == null) {
FileHeaderAttachmentCard(event, fullUrl, mimeType)
} else {
ZoomableContentView(
content = content,
roundedCorner = roundedCorner,
contentScale = contentScale,
accountViewModel = accountViewModel,
)
}
}
}
/**
* Builds the viewer for a kind-1063 header, or **null** when no viewer can show the blob.
*
* Kind 1063 is a *generic* file container — its `m` tag can name any type, so unlike a NIP-71
* video event the kind itself asserts nothing about how to render the payload. A null here means
* the file belongs in [FileHeaderAttachmentCard] rather than being pushed into the video player.
*/
internal fun FileHeaderEvent.toMediaContent(
note: Note,
url: String,
mimeType: String?,
): BaseMediaContent? {
val blurHash = blurhash()
val thumbHash = thumbhash()
val hash = hash()
val dimensions = dimensions()
val description = fileDescription()
val uri = note.toNostrUri()
return when (RichTextParser.classifyMedia(url, mimeType)) {
MediaContentKind.IMAGE ->
MediaUrlImage(
url = url,
description = description,
hash = hash,
blurhash = blurHash,
dim = dimensions,
uri = uri,
mimeType = mimeType,
thumbhash = thumbHash,
)
MediaContentKind.VIDEO ->
MediaUrlVideo(
url = url,
description = description,
hash = hash,
blurhash = blurHash,
dim = dimensions,
uri = uri,
authorName = note.author?.toBestDisplayName(),
mimeType = mimeType,
thumbhash = thumbHash,
)
MediaContentKind.PDF ->
MediaUrlPdf(
url = url,
description = description,
hash = hash,
blurhash = blurHash,
dim = dimensions,
uri = uri,
mimeType = mimeType,
thumbhash = thumbHash,
)
null -> null
}
}
/** The link card a kind-1063 header falls back to when [toMediaContent] returns null. */
@Composable
internal fun FileHeaderAttachmentCard(
event: FileHeaderEvent,
url: String,
mimeType: String?,
) {
val description = remember(event) { event.fileDescription() }
val sizeInBytes = remember(event) { event.size()?.toLong() }
FileAttachmentCard(
url = url,
description = description,
mimeType = mimeType,
sizeInBytes = sizeInBytes,
)
}
/** The human-facing name of the file: NIP-94 `content` when present, else the `alt` tag. */
private fun FileHeaderEvent.fileDescription(): String? = content.ifEmpty { null } ?: alt()
@@ -65,6 +65,7 @@ import coil3.compose.AsyncImage
import com.vitorpamplona.amethyst.R
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage
import com.vitorpamplona.amethyst.commons.ui.components.ClickableTextPrimary
import com.vitorpamplona.amethyst.commons.util.prettyMime
import com.vitorpamplona.amethyst.model.LocalCache
import com.vitorpamplona.amethyst.model.MediaAspectRatioCache
import com.vitorpamplona.amethyst.model.Note
@@ -766,27 +767,6 @@ fun RenderSoftwareAsset(
}
}
internal fun prettyMime(mime: String): String =
when (mime) {
"application/vnd.android.package-archive" -> "APK"
"application/vnd.apple.ipa" -> "IPA"
"application/x-apple-diskimage" -> "DMG"
"application/vnd.apple.installer+xml" -> "PKG"
"application/x-msi" -> "MSI"
"application/vnd.appimage" -> "AppImage"
"application/vnd.flatpak" -> "Flatpak"
"application/vnd.oci.image.manifest.v1+json" -> "OCI"
"application/x-executable" -> "ELF"
"application/x-mach-binary" -> "Mach-O"
"application/vnd.microsoft.portable-executable" -> "EXE"
"application/vsix" -> "VSIX"
"application/x-chrome-extension" -> "CRX"
"application/x-xpinstall" -> "XPI"
"application/wasm" -> "WASM"
"application/webbundle" -> "Web Bundle"
else -> mime
}
internal fun formatBytes(bytes: Long): String {
if (bytes < 1024L) return "$bytes B"
val kb = bytes / 1024.0
@@ -43,6 +43,7 @@ import com.vitorpamplona.amethyst.R
import com.vitorpamplona.amethyst.commons.model.EmptyTagList
import com.vitorpamplona.amethyst.commons.model.toImmutableListOfLists
import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent
import com.vitorpamplona.amethyst.commons.richtext.MediaContentKind
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo
import com.vitorpamplona.amethyst.commons.richtext.RichTextParser
@@ -88,7 +89,9 @@ fun VideoDisplay(
val content: BaseMediaContent =
remember(note) {
val description = videoEvent.content.ifBlank { null } ?: event.alt()
val isImage = imeta.mimeType?.startsWith("image/") == true || RichTextParser.isImageUrl(imeta.url)
// A NIP-71 event asserts its own type, so only an explicit image imeta diverts to the
// viewer; an unclassifiable one still belongs in the player. See classifyMedia.
val isImage = RichTextParser.classifyMedia(imeta.url, imeta.mimeType) == MediaContentKind.IMAGE
val uri = note.toNostrUri()
if (isImage) {
@@ -26,6 +26,7 @@ import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.layout.ContentScale
import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent
import com.vitorpamplona.amethyst.commons.richtext.MediaContentKind
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo
import com.vitorpamplona.amethyst.commons.richtext.RichTextParser
@@ -55,7 +56,9 @@ fun JustVideoDisplay(
val imeta = videoEvent.imetaTags().getOrNull(0) ?: return
val isSensitive = remember(note) { event.isSensitiveOrNSFW() }
val reasons = remember(note) { collectContentWarningReasons(event) }
val isImage = remember(note) { imeta.mimeType?.startsWith("image/") == true || RichTextParser.isImageUrl(imeta.url) }
// A NIP-71 event asserts its own type, so only an explicit image imeta diverts to the
// viewer; an unclassifiable one still belongs in the player. See classifyMedia.
val isImage = remember(note) { RichTextParser.classifyMedia(imeta.url, imeta.mimeType) == MediaContentKind.IMAGE }
val content by
remember(note) {
@@ -39,6 +39,7 @@ import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent
import com.vitorpamplona.amethyst.commons.richtext.MediaContentKind
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo
import com.vitorpamplona.amethyst.commons.richtext.RichTextParser
@@ -104,7 +105,9 @@ private fun VideoCardImage(
val imeta = videoEvent.imetaTags().getOrNull(0) ?: return
val isSensitive = remember(note) { event.isSensitiveOrNSFW() }
val reasons = remember(note) { collectContentWarningReasons(event) }
val isImage = remember(note) { imeta.mimeType?.startsWith("image/") == true || RichTextParser.isImageUrl(imeta.url) }
// A NIP-71 event asserts its own type, so only an explicit image imeta diverts to the
// viewer; an unclassifiable one still belongs in the player. See classifyMedia.
val isImage = remember(note) { RichTextParser.classifyMedia(imeta.url, imeta.mimeType) == MediaContentKind.IMAGE }
val content by
remember(note) {
@@ -29,7 +29,6 @@ import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.MutableState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.Modifier
@@ -38,10 +37,7 @@ import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo
import com.vitorpamplona.amethyst.commons.richtext.RichTextParser
import com.vitorpamplona.amethyst.model.MediaAspectRatioCache
import com.vitorpamplona.amethyst.model.Note
import com.vitorpamplona.amethyst.ui.components.BlurhashBackdrop
@@ -51,9 +47,10 @@ import com.vitorpamplona.amethyst.ui.components.collectContentWarningReasons
import com.vitorpamplona.amethyst.ui.components.mediaSizingModifier
import com.vitorpamplona.amethyst.ui.navigation.navs.INav
import com.vitorpamplona.amethyst.ui.note.ReactionsRow
import com.vitorpamplona.amethyst.ui.note.types.FileHeaderAttachmentCard
import com.vitorpamplona.amethyst.ui.note.types.toMediaContent
import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip31Alts.alt
import com.vitorpamplona.quartz.nip36SensitiveContent.isSensitiveOrNSFW
import com.vitorpamplona.quartz.nip94FileMetadata.FileHeaderEvent
@@ -101,45 +98,33 @@ private fun FileHeaderCardImage(
val isSensitive = remember(note) { event.isSensitiveOrNSFW() }
val reasons = remember(note) { collectContentWarningReasons(event) }
val isImage = remember(note) { event.mimeType()?.startsWith("image/") == true || RichTextParser.isImageUrl(fullUrl) }
val mimeType = remember(note) { event.mimeType() }
val blurHash = remember(note) { event.blurhash() }
val thumbHash = remember(note) { event.thumbhash() }
val dimensions = remember(note) { event.dimensions() }
val content by remember(note) {
val hash = event.hash()
val description = event.content.ifEmpty { null } ?: event.alt()
val uri = note.toNostrUri()
val mimeType = event.mimeType()
val content = remember(note) { event.toMediaContent(note, fullUrl, mimeType) }
mutableStateOf<BaseMediaContent>(
if (isImage) {
MediaUrlImage(
url = fullUrl,
description = description,
hash = hash,
blurhash = blurHash,
dim = dimensions,
uri = uri,
mimeType = mimeType,
thumbhash = thumbHash,
)
} else {
MediaUrlVideo(
url = fullUrl,
description = description,
hash = hash,
blurhash = blurHash,
dim = dimensions,
uri = uri,
authorName = note.author?.toBestDisplayName(),
mimeType = mimeType,
thumbhash = thumbHash,
)
},
)
// Reachable despite VideoFeedFilter admitting only image/video types: the filter accepts on
// `urls().any { … }` while this card renders `url()`, the first tag — so a multi-mirror event
// whose first URL is unrenderable lands here. The gate wraps it for the same reason it wraps
// the viewer in FileHeaderDisplay: a content warning is about the file, and the card still
// spells out its filename, alt text, MIME and size. Sizing stays on the gate's defaults
// (fillMaxWidth, no backdrop) — a link card has no aspect ratio to reserve and no blurhash
// to show behind it.
if (content == null) {
ContentWarningGate(
isSensitive = isSensitive,
reasons = reasons,
preloadUrls = emptyList(),
accountViewModel = accountViewModel,
) {
FileHeaderAttachmentCard(event, fullUrl, mimeType)
}
return
}
val isImage = content is MediaUrlImage
val ratio = dimensions?.aspectRatio() ?: MediaAspectRatioCache.get(fullUrl)
ContentWarningGate(
@@ -0,0 +1,37 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.richtext
/**
* Which player/viewer can render a declared blob, as resolved by
* [RichTextParser.classifyMedia].
*
* The set is deliberately closed: it enumerates the renderers [BaseMediaContent] actually has
* (`MediaUrlImage`, `MediaUrlVideo`, `MediaUrlPdf`), so "no constant fits" — a `null`
* classification — is the honest answer for every other file type rather than a bucket some
* caller has to invent a default for. Audio folds into [VIDEO] because both play through the
* same pipeline; see `RichTextParser.videoExt`.
*/
enum class MediaContentKind {
IMAGE,
VIDEO,
PDF,
}
@@ -61,41 +61,12 @@ class RichTextParser {
val contentType = frags[MimeTypeTag.TAG_NAME] ?: tags[MimeTypeTag.TAG_NAME]?.firstOrNull()
var isImage = false
var isVideo = false
var isPdf = false
// Returning null here drops the URL to a plain link, discarding the imeta's `dim`/blurhash
// and forcing a URL-preview round-trip to rediscover a type the imeta already declared —
// which is why classifyMedia falls back to the extension before giving up.
val kind = classifyMedia(fullUrl, contentType)
if (contentType != null) {
isImage = contentType.startsWith("image/")
// HLS playlists are advertised with a non-`video/*` MIME (`application/vnd.apple.mpegurl`
// and three legacy aliases). Without these, an imeta-described `.m3u8` falls into the
// null bucket below and the renderer drops back to a plain hyperlink — even though
// the matching extension would have routed it to MediaUrlVideo. Mirror the canonical
// list used by MediaItemCache.toExoPlayerMimeType / GalleryThumb.isHlsMimeType.
isVideo = contentType.startsWith("video/") || contentType.startsWith("audio/") || isHlsMimeType(contentType)
isPdf = contentType.startsWith("application/pdf")
} else if (fullUrl.startsWith("data:")) {
isImage = fullUrl.startsWith("data:image/")
isVideo = fullUrl.startsWith("data:video/") || fullUrl.startsWith("data:audio/")
isPdf = fullUrl.startsWith("data:application/pdf")
}
// Fall back to file-extension detection when the type is still unknown. This covers both
// the no-MIME case and a *malformed* imeta MIME — e.g. Primal iOS emits `m jpeg` instead
// of `m image/jpeg`, which matches none of the `startsWith` prefixes above. Without this
// fallback such a URL returns null and drops to a plain link: that discards the imeta
// `dim`/blurhash (so the loading placeholder can't reserve the image's height and the
// feed jumps once the bitmap arrives) and forces a needless URL-preview network
// round-trip just to rediscover the type the imeta already declared. `data:` URIs carry
// their type in the prefix, so a miss there is genuine — don't extension-probe them.
if (!isImage && !isVideo && !isPdf && !fullUrl.startsWith("data:")) {
val removedParamsFromUrl = removeQueryParamsForExtensionComparison(fullUrl)
isImage = imageExtensions.any { removedParamsFromUrl.endsWith(it) }
isVideo = videoExtensions.any { removedParamsFromUrl.endsWith(it) }
isPdf = pdfExtensions.any { removedParamsFromUrl.endsWith(it) }
}
return if (isImage) {
return if (kind == MediaContentKind.IMAGE) {
MediaUrlImage(
url = fullUrl,
description = description ?: frags[AltTag.TAG_NAME] ?: tags[AltTag.TAG_NAME]?.firstOrNull(),
@@ -108,7 +79,7 @@ class RichTextParser {
thumbhash = frags[ThumbhashTag.TAG_NAME] ?: tags[ThumbhashTag.TAG_NAME]?.firstOrNull(),
authorPubKey = authorPubKey,
)
} else if (isVideo) {
} else if (kind == MediaContentKind.VIDEO) {
MediaUrlVideo(
url = fullUrl,
description = description ?: frags[AltTag.TAG_NAME] ?: tags[AltTag.TAG_NAME]?.firstOrNull(),
@@ -125,7 +96,7 @@ class RichTextParser {
thumbhash = frags[ThumbhashTag.TAG_NAME] ?: tags[ThumbhashTag.TAG_NAME]?.firstOrNull(),
authorPubKey = authorPubKey,
)
} else if (isPdf) {
} else if (kind == MediaContentKind.PDF) {
MediaUrlPdf(
url = fullUrl,
description = description ?: frags[AltTag.TAG_NAME] ?: tags[AltTag.TAG_NAME]?.firstOrNull(),
@@ -582,6 +553,46 @@ class RichTextParser {
return pdfExtensions.any { removedParamsFromUrl.endsWith(it) }
}
/**
* Resolves which renderer can display a declared blob — the single decision every media
* renderer must make, from a NIP-94 `m` tag, a NIP-92 imeta, or a bare URL.
*
* A declared MIME type wins; the URL extension is the fallback both for the no-MIME case
* and for a *malformed* MIME (Primal iOS emits `m jpeg` rather than `m image/jpeg`, which
* matches no prefix below). `data:` URIs carry their type in the prefix, so a miss there is
* genuine and the base64 payload is never extension-probed.
*
* Returns **null** when nothing can render the file. Callers must not substitute a media
* kind for that null: handing an arbitrary blob — a webxdc app, a zip, an APK — to the
* video player yields a permanently-buffering ExoPlayer where a plain link belongs. The one
* defensible default is on kinds whose *event* already asserts the type (a NIP-71 video
* event is a video however odd its imeta), and those call sites say so explicitly.
*/
fun classifyMedia(
url: String,
mimeType: String?,
): MediaContentKind? {
if (mimeType != null) {
if (mimeType.startsWith("image/")) return MediaContentKind.IMAGE
// HLS playlists are advertised with a non-`video/*` MIME; see [isHlsMimeType].
if (mimeType.startsWith("video/") || mimeType.startsWith("audio/") || isHlsMimeType(mimeType)) return MediaContentKind.VIDEO
if (mimeType.startsWith("application/pdf")) return MediaContentKind.PDF
} else if (url.startsWith("data:")) {
if (url.startsWith("data:image/")) return MediaContentKind.IMAGE
if (url.startsWith("data:video/") || url.startsWith("data:audio/")) return MediaContentKind.VIDEO
if (url.startsWith("data:application/pdf")) return MediaContentKind.PDF
}
if (url.startsWith("data:")) return null
val removedParamsFromUrl = removeQueryParamsForExtensionComparison(url)
if (imageExtensions.any { removedParamsFromUrl.endsWith(it) }) return MediaContentKind.IMAGE
if (videoExtensions.any { removedParamsFromUrl.endsWith(it) }) return MediaContentKind.VIDEO
if (pdfExtensions.any { removedParamsFromUrl.endsWith(it) }) return MediaContentKind.PDF
return null
}
fun isValidURL(url: String?): Boolean = isValidUrl(url)
fun parseImageOrVideo(fullUrl: String): BaseMediaContent {
@@ -0,0 +1,52 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.util
/**
* The short label a user recognises for a distributable file type — "APK", not
* "application/vnd.android.package-archive".
*
* Unmapped types return the raw MIME unchanged, which is the honest fallback: a bare
* `application/x-webxdc` still tells the reader more than an invented label would.
*
* Used by the NIP-82 software-app chips and by the file-attachment card that stands in for any
* blob no viewer can render.
*/
fun prettyMime(mime: String): String =
when (mime) {
"application/vnd.android.package-archive" -> "APK"
"application/vnd.apple.ipa" -> "IPA"
"application/x-apple-diskimage" -> "DMG"
"application/vnd.apple.installer+xml" -> "PKG"
"application/x-msi" -> "MSI"
"application/vnd.appimage" -> "AppImage"
"application/vnd.flatpak" -> "Flatpak"
"application/vnd.oci.image.manifest.v1+json" -> "OCI"
"application/x-executable" -> "ELF"
"application/x-mach-binary" -> "Mach-O"
"application/vnd.microsoft.portable-executable" -> "EXE"
"application/vsix" -> "VSIX"
"application/x-chrome-extension" -> "CRX"
"application/x-xpinstall" -> "XPI"
"application/wasm" -> "WASM"
"application/webbundle" -> "Web Bundle"
else -> mime
}
@@ -0,0 +1,161 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.commons.richtext
import com.vitorpamplona.quartz.nip92IMeta.IMetaTag
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
class ClassifyMediaTest {
@Test
fun webxdcAppIsNotMedia() {
// Regression: a NIP-94 header for a webxdc app (a zip bundle) used to reach the
// ExoPlayer branch, because the only test was `isImage` and everything else fell
// through to video. https://blossom.ditto.pub/<sha256>.xdc, m=application/x-webxdc
assertNull(
RichTextParser.classifyMedia(
"https://blossom.ditto.pub/d810ba7873d710b197fc402c0573cd95ce7d44fff7f904e8f58e48af3a47c107.xdc",
"application/x-webxdc",
),
)
}
@Test
fun unknownTypesAreNotMedia() {
assertNull(RichTextParser.classifyMedia("https://x.com/app.apk", "application/vnd.android.package-archive"))
assertNull(RichTextParser.classifyMedia("https://x.com/archive.zip", "application/zip"))
assertNull(RichTextParser.classifyMedia("https://x.com/notes.txt", "text/plain"))
// No mime at all and an extension we don't render.
assertNull(RichTextParser.classifyMedia("https://x.com/file.xdc", null))
assertNull(RichTextParser.classifyMedia("https://x.com/no-extension-at-all", null))
}
@Test
fun declaredMimeTypesClassify() {
assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a", "image/png"))
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "video/mp4"))
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "audio/mpeg"))
assertEquals(MediaContentKind.PDF, RichTextParser.classifyMedia("https://x.com/a", "application/pdf"))
}
@Test
fun hlsPlaylistMimesAreVideo() {
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "application/vnd.apple.mpegurl"))
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "application/x-mpegURL"))
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "audio/mpegurl"))
}
@Test
fun extensionIsUsedWhenMimeIsAbsent() {
assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.jpg", null))
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a.mp4", null))
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a.m3u8", null))
assertEquals(MediaContentKind.PDF, RichTextParser.classifyMedia("https://x.com/a.pdf", null))
assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.PNG", null))
}
@Test
fun aDeclaredMimeBeatsAContradictingExtension() {
// The check this replaced was an OR — `mime.startsWith("image/") || isImageUrl(url)` —
// so a poster-named video URL classified as an image. A declared MIME is the publisher
// stating the type; the extension is only a guess for when they didn't. Pins the
// precedence against a future "simplification" back to OR-semantics.
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/thumb.jpg", "video/mp4"))
assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/clip.mp4", "image/png"))
assertEquals(MediaContentKind.PDF, RichTextParser.classifyMedia("https://x.com/scan.png", "application/pdf"))
}
@Test
fun anUnrecognisedMimeDefersToTheExtensionRatherThanVetoingIt() {
// Precedence applies only to MIMEs we recognise. An unrecognised one means "no usable
// declaration", not "declared unrenderable" — the two are indistinguishable here, and
// treating them alike is what lets [extensionRescuesAMalformedMime] work. So a real
// video mislabelled `application/x-webxdc` still plays…
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/bundle.mp4", "application/x-webxdc"))
// …while the webxdc app that motivated this class stays unrenderable, because nothing
// rescues it: `.xdc` is in no extension list either.
assertNull(RichTextParser.classifyMedia("https://x.com/bundle.xdc", "application/x-webxdc"))
}
@Test
fun extensionRescuesAMalformedMime() {
// Primal iOS emits `m jpeg` instead of `m image/jpeg`; the extension must still win
// over "unknown". Preserves the behaviour createMediaContent already documented.
assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.jpg", "jpeg"))
}
@Test
fun queryStringsAndFragmentsAreStripped() {
assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.jpg?token=1", null))
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a.mp4#t=10", null))
assertNull(RichTextParser.classifyMedia("https://x.com/a.xdc?token=1", null))
}
@Test
fun dataUrisAreClassifiedByTheirPrefixOnly() {
assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("data:image/png;base64,AAAA", null))
assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("data:video/mp4;base64,AAAA", null))
assertEquals(MediaContentKind.PDF, RichTextParser.classifyMedia("data:application/pdf;base64,AAAA", null))
// A data: URI carries its type in the prefix, so a miss there is genuine — the
// payload must never be extension-probed (base64 can end in any letters).
assertNull(RichTextParser.classifyMedia("data:application/zip;base64,AAAAmp4", null))
}
@Test
fun classifyMediaAgreesWithCreateMediaContent() {
// createMediaContent is the long-standing reference for this decision; the two must
// not drift, since half the renderers call one and half the other.
val cases =
listOf(
"https://x.com/a.jpg" to null,
"https://x.com/a" to "image/png",
"https://x.com/a" to "video/mp4",
"https://x.com/a" to "audio/mpeg",
"https://x.com/a" to "application/pdf",
"https://x.com/a" to "application/vnd.apple.mpegurl",
"https://x.com/a.xdc" to "application/x-webxdc",
"https://x.com/a.zip" to "application/zip",
"https://x.com/a.jpg" to "jpeg",
"data:image/png;base64,AAAA" to null,
"data:application/zip;base64,AAAAmp4" to null,
)
cases.forEach { (url, mime) ->
val tags = mime?.let { mapOf(url to imeta(url, it)) } ?: emptyMap()
val expected =
when (RichTextParser().createMediaContent(url, tags, null)) {
is MediaUrlImage -> MediaContentKind.IMAGE
is MediaUrlVideo -> MediaContentKind.VIDEO
is MediaUrlPdf -> MediaContentKind.PDF
null -> null
else -> error("unexpected content type for $url / $mime")
}
assertEquals<MediaContentKind?>(expected, RichTextParser.classifyMedia(url, mime), "disagreement on $url / $mime")
}
}
private fun imeta(
url: String,
mimeType: String,
) = IMetaTag(url = url, properties = mapOf("m" to listOf(mimeType)))
}