diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt index e3262b35e5..2ec9c44691 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt @@ -115,6 +115,7 @@ import com.vitorpamplona.quartz.experimental.clink.pointers.NDebit import com.vitorpamplona.quartz.experimental.ephemChat.chat.RoomId import com.vitorpamplona.quartz.experimental.interactiveStories.InteractiveStoryBaseEvent import com.vitorpamplona.quartz.experimental.interactiveStories.InteractiveStoryReadingStateEvent +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaReferenceV2 import com.vitorpamplona.quartz.marmot.appComponents.GroupBlossomImageV1 import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent @@ -2453,6 +2454,29 @@ class AccountViewModel( fun marmotMediaExporterSecret(nostrGroupId: String): ByteArray? = account.marmotManager?.mediaExporterSecret(nostrGroupId) + /** + * True when this group carries the `encrypted-media-v2` policy (`0x800b`) + * and a sender should therefore produce v2 references. + * + * A group without it is not a licence to reinterpret the frozen v1 policy + * at `0x8008` as v2 — they are different components — so this is a plain + * "does the group say v2", and the sender falls back to MIP-04 when it + * does not. + */ + fun marmotUsesEncryptedMediaV2(nostrGroupId: String): Boolean = account.marmotManager?.encryptedMediaPolicy(nostrGroupId) != null + + /** Post the kind:9 carrying an `encrypted-media-v2` attachment. */ + suspend fun sendMarmotGroupEncryptedMediaV2( + nostrGroupId: String, + reference: EncryptedMediaReferenceV2, + caption: String, + ) { + val manager = account.marmotManager ?: return + val bundle = manager.buildMediaMessage(nostrGroupId, reference, caption, persistOwn = false) + val relays = account.marmot.marmotGroupRelays(nostrGroupId) + account.marmot.sendMarmotGroupMessage(nostrGroupId, bundle.innerEvent, relays) + } + suspend fun createMarmotGroup( nostrGroupId: String, name: String = "", diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/MarmotGroupChatView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/MarmotGroupChatView.kt index 6075939f97..eaa69731b0 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/MarmotGroupChatView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/MarmotGroupChatView.kt @@ -346,6 +346,7 @@ private fun MarmotGroupFileUploadDialog( } }, context = context, + useEncryptedMediaV2 = accountViewModel.marmotUsesEncryptedMediaV2(nostrGroupId), onceUploaded = { uploads -> MarmotFileSender(nostrGroupId, accountViewModel).send(uploads) onUpload() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/MarmotGroupIconDisplay.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/MarmotGroupIconDisplay.kt index d26d064494..8132972932 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/MarmotGroupIconDisplay.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/MarmotGroupIconDisplay.kt @@ -29,7 +29,7 @@ import com.vitorpamplona.amethyst.commons.model.marmotGroups.MarmotGroupImage import com.vitorpamplona.amethyst.model.nip11RelayInfo.loadRelayInfo import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.quartz.marmot.appComponents.GroupAvatarUrlV1 -import com.vitorpamplona.quartz.marmot.appComponents.MarmotHttpsUrl +import com.vitorpamplona.quartz.marmot.appComponents.MarmotWebUrl import com.vitorpamplona.quartz.marmot.mip01Groups.MarmotGroupImageCipher import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer @@ -118,7 +118,7 @@ fun rememberMarmotGroupAvatarUrl( ): String? { val link = remember(avatarUrl) { - avatarUrl?.url?.takeIf { it.isNotEmpty() && MarmotHttpsUrl.isSafeToContact(it) } + avatarUrl?.url?.takeIf { it.isNotEmpty() && MarmotWebUrl.isSafeToContact(it) } } // Branch rather than resolving both: the Blossom path registers a // decryption cipher and probes servers as a side effect, and neither is diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/send/MarmotFileSender.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/send/MarmotFileSender.kt index bf512b716d..19bc05ce08 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/send/MarmotFileSender.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/send/MarmotFileSender.kt @@ -25,8 +25,14 @@ import com.vitorpamplona.quartz.marmot.mip04EncryptedMedia.buildMip04IMetaTag import com.vitorpamplona.quartz.nip01Core.core.HexKey /** - * Sends uploaded MIP-04 encrypted media as Marmot group messages. - * Each upload result becomes a separate kind:9 message with an imeta tag. + * Sends uploaded encrypted media as Marmot group messages. Each upload result + * becomes a separate kind:9 with an `imeta` tag. + * + * Which reference format the tag carries is the GROUP's decision, made when the + * upload was encrypted: a group carrying the `encrypted-media-v2` policy + * (`0x800b`) gets a v2 reference, and one that does not gets the MIP-04 shape. + * The frozen v1 policy at `0x8008` is a different component and is never + * reinterpreted as v2, so there is no third case here. */ class MarmotFileSender( val nostrGroupId: HexKey, @@ -34,6 +40,16 @@ class MarmotFileSender( ) { suspend fun send(uploads: List) { for (upload in uploads) { + val v2 = upload.encryptedMediaV2 + if (v2 != null) { + accountViewModel.sendMarmotGroupEncryptedMediaV2( + nostrGroupId = nostrGroupId, + reference = v2, + caption = upload.caption.orEmpty(), + ) + continue + } + val imeta = buildMip04IMetaTag( url = upload.url, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/send/MarmotFileUploader.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/send/MarmotFileUploader.kt index 44dd46fda4..6f3a514e11 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/send/MarmotFileUploader.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/marmotGroup/send/MarmotFileUploader.kt @@ -29,6 +29,11 @@ import com.vitorpamplona.amethyst.service.uploads.UploadOrchestrator import com.vitorpamplona.amethyst.service.uploads.UploadingState import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.utils.ChatFileUploadState import com.vitorpamplona.amethyst.ui.stringRes +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaPolicyV2 +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaReferenceV2 +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaV2Cipher +import com.vitorpamplona.quartz.marmot.appComponents.MarmotMediaType +import com.vitorpamplona.quartz.marmot.appComponents.MediaLocatorV2 import com.vitorpamplona.quartz.marmot.mip04EncryptedMedia.Mip04NostrCipher /** @@ -44,6 +49,16 @@ class Mip04UploadResult( val blurhash: String?, val caption: String?, val thumbhash: String? = null, + /** + * The `encrypted-media-v2` reference, when the group's policy asked for + * one. Null means this upload is a MIP-04 attachment and the fields above + * are what builds its tag. + * + * The two are carried together rather than as two result types because the + * upload pipeline is identical — only the cipher and the tag differ — and + * the choice belongs to the group, not to the uploader. + */ + val encryptedMediaV2: EncryptedMediaReferenceV2? = null, ) /** @@ -60,6 +75,11 @@ class MarmotFileUploader( exporterSecret: ByteArray, onError: (title: String, message: String) -> Unit, context: Context, + /** + * Produce `encrypted-media-v2` references instead of MIP-04 ones. + * Decided by the group's policy component, not by the uploader. + */ + useEncryptedMediaV2: Boolean = false, onceUploaded: suspend (List) -> Unit, ) { val multiOrchestrator = viewState.multiOrchestrator ?: return @@ -75,7 +95,14 @@ class MarmotFileUploader( val mimeType = media.mimeType ?: "application/octet-stream" val filename = resolveFilename(context, media.uri, mimeType) - val cipher = Mip04NostrCipher(exporterSecret, mimeType, filename) + // v2 puts `m` inside both the key derivation and the AEAD + // associated data, so it has to be the canonical form and not + // whatever the content resolver reported. A type that will not + // canonicalize falls back to MIP-04 for this file rather than + // producing a reference no receiver can key. + val canonicalMediaType = if (useEncryptedMediaV2) MarmotMediaType.canonicalize(mimeType) else null + val v2Cipher = canonicalMediaType?.let { EncryptedMediaV2Cipher(exporterSecret, it, filename) } + val cipher = v2Cipher ?: Mip04NostrCipher(exporterSecret, mimeType, filename) item.orchestrator.uploadEncrypted( uri = media.uri, @@ -93,17 +120,38 @@ class MarmotFileUploader( val state = item.orchestrator.progressState.value if (state is UploadingState.Finished && state.result is UploadOrchestrator.OrchestratorResult.ServerResult) { val serverResult = state.result + // The reference is built from what the cipher recorded while + // encrypting the bytes the pipeline actually uploaded — after + // compression and metadata stripping — because that is what the + // key was derived from. + val reference = + v2Cipher?.let { + EncryptedMediaReferenceV2( + locators = + listOf( + MediaLocatorV2(EncryptedMediaPolicyV2.INITIAL_LOCATOR_KIND, serverResult.url), + ), + ciphertextSha256 = it.ciphertextSha256, + plaintextSha256 = it.plaintextSha256, + nonce = it.nonce, + mediaType = it.mediaType, + filename = filename, + dim = serverResult.fileHeader.dim?.toString(), + thumbhash = serverResult.fileHeader.thumbHash?.thumbhash, + ) + } results.add( Mip04UploadResult( url = serverResult.url, mimeType = mimeType, filename = filename, - originalFileHash = cipher.originalFileHash, - nonce = cipher.nonce, + originalFileHash = (cipher as? Mip04NostrCipher)?.originalFileHash ?: ByteArray(0), + nonce = (cipher as? Mip04NostrCipher)?.nonce ?: ByteArray(0), dimensions = serverResult.fileHeader.dim?.toString(), blurhash = serverResult.fileHeader.blurHash?.blurhash, caption = viewState.caption.ifEmpty { null }, thumbhash = serverResult.fileHeader.thumbHash?.thumbhash, + encryptedMediaV2 = reference, ), ) } else { diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 5d5997dfa6..a66a197cde 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -51,6 +51,7 @@ import com.vitorpamplona.amethyst.cli.commands.KeyPackageCommands import com.vitorpamplona.amethyst.cli.commands.KindCommand import com.vitorpamplona.amethyst.cli.commands.LoginCommand import com.vitorpamplona.amethyst.cli.commands.LogoffCommand +import com.vitorpamplona.amethyst.cli.commands.MarmotMediaCommands import com.vitorpamplona.amethyst.cli.commands.MarmotResetCommand import com.vitorpamplona.amethyst.cli.commands.MessageCommands import com.vitorpamplona.amethyst.cli.commands.NamecoinCommand @@ -371,12 +372,13 @@ private suspend fun marmotDispatch( route( name = "marmot", tail = tail, - usage = "marmot ", + usage = "marmot ", routes = mapOf( "key-package" to { rest -> KeyPackageCommands.dispatch(dataDir, rest) }, "group" to { rest -> GroupCommands.dispatch(dataDir, rest) }, "message" to { rest -> MessageCommands.dispatch(dataDir, rest) }, + "media" to { rest -> MarmotMediaCommands.dispatch(dataDir, rest) }, "stream" to { rest -> StreamCommands.dispatch(dataDir, rest) }, "await" to { rest -> AwaitCommands.dispatch(dataDir, rest) }, "reset" to { rest -> MarmotResetCommand.run(dataDir, rest) }, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GroupMetadataCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GroupMetadataCommands.kt index 3b298ffb32..b866836940 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GroupMetadataCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GroupMetadataCommands.kt @@ -31,7 +31,7 @@ import com.vitorpamplona.amethyst.commons.util.deleteOrWarn import com.vitorpamplona.quartz.marmot.OutboundGroupEvent import com.vitorpamplona.quartz.marmot.appComponents.GroupAvatarUrlV1 import com.vitorpamplona.quartz.marmot.appComponents.GroupBlossomImageV1 -import com.vitorpamplona.quartz.marmot.appComponents.MarmotHttpsUrl +import com.vitorpamplona.quartz.marmot.appComponents.MarmotWebUrl import com.vitorpamplona.quartz.marmot.mip01Groups.MarmotGroupImageEncryption import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray @@ -164,7 +164,7 @@ object GroupMetadataCommands { val avatar = try { GroupAvatarUrlV1( - url = MarmotHttpsUrl.normalize(url), + url = MarmotWebUrl.normalize(url, label = "avatar URL"), dim = dim?.encodeToByteArray() ?: ByteArray(0), thumbhash = thumbhash?.encodeToByteArray() ?: ByteArray(0), ) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/MarmotMediaCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/MarmotMediaCommands.kt new file mode 100644 index 0000000000..f7bff6f6dd --- /dev/null +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/MarmotMediaCommands.kt @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.cli.commands + +import com.vitorpamplona.amethyst.cli.Args +import com.vitorpamplona.amethyst.cli.Context +import com.vitorpamplona.amethyst.cli.DataDir +import com.vitorpamplona.amethyst.cli.Output +import com.vitorpamplona.amethyst.commons.service.upload.BlossomAuth +import com.vitorpamplona.amethyst.commons.service.upload.BlossomClient +import com.vitorpamplona.quartz.marmot.appComponents.BlobStoreEndpointV2 +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaPolicyV2 +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaReferenceV2 +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaV2 +import com.vitorpamplona.quartz.marmot.appComponents.MarmotMediaType +import com.vitorpamplona.quartz.marmot.appComponents.MediaLocatorV2 +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.utils.sha256.sha256 +import java.io.File + +/** + * `amy marmot media` — `encrypted-media-v2` attachments (`0x800b`). + * + * The server only ever sees ciphertext and its hash. The key is derived from + * the group's own MLS exporter and never leaves the group, so the blob store + * is storage and not a party to the conversation. + */ +object MarmotMediaCommands { + val USAGE: String = + """ + |amy marmot media — encrypted-media-v2 attachments + | + | marmot media policy GID print the group's media policy + | marmot media set-policy GID URL[,URL…] commit a policy naming these blob stores, + | in upload/fetch fallback order + | marmot media send GID FILE [--caption TXT] encrypt, upload, and post the kind:9 + | [--server URL] [--mime TYPE] (--server overrides the policy's first endpoint) + | marmot media get GID EVENT_ID --out PATH fetch, decrypt and verify an attachment + """.trimMargin() + + suspend fun dispatch( + dataDir: DataDir, + tail: Array, + ): Int = + route( + "media", + tail, + "media …", + mapOf( + "policy" to { rest -> policy(dataDir, rest) }, + "set-policy" to { rest -> setPolicy(dataDir, rest) }, + "send" to { rest -> send(dataDir, rest) }, + "get" to { rest -> get(dataDir, rest) }, + ), + help = USAGE, + ) + + private suspend fun policy( + dataDir: DataDir, + rest: Array, + ): Int { + if (rest.isEmpty()) return Output.error("bad_args", "media policy ") + Context.open(dataDir).use { ctx -> + ctx.prepare() + val gid = ctx.resolveGroupId(rest[0]) + ctx.syncIncoming() + if (!ctx.marmot.isMember(gid)) return Output.error("not_member", "not a member of group $gid") + + val policy = ctx.marmot.encryptedMediaPolicy(gid) + Output.emit( + mapOf( + "group_id" to gid, + "media_format" to policy?.mediaFormat, + "allowed_locator_kinds" to policy?.allowedLocatorKinds, + "default_blob_endpoints" to + policy?.defaultBlobEndpoints?.map { + mapOf("locator_kind" to it.locatorKind, "base_url" to it.baseUrl) + }, + ), + ) + return 0 + } + } + + private suspend fun setPolicy( + dataDir: DataDir, + rest: Array, + ): Int { + if (rest.size < 2) return Output.error("bad_args", "media set-policy [,…]") + val urls = rest[1].split(',').map { it.trim() }.filter { it.isNotEmpty() } + if (urls.isEmpty()) return Output.error("bad_args", "media set-policy needs at least one base URL") + + val policy = + try { + EncryptedMediaPolicyV2( + allowedLocatorKinds = listOf(EncryptedMediaPolicyV2.INITIAL_LOCATOR_KIND), + // Order is preserved deliberately: it IS the upload/fetch + // fallback priority, so sorting it would change where the + // group uploads. + defaultBlobEndpoints = + urls.map { BlobStoreEndpointV2(EncryptedMediaPolicyV2.INITIAL_LOCATOR_KIND, it) }, + ) + } catch (e: IllegalArgumentException) { + return Output.error("bad_args", e.message ?: "invalid media policy") + } + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val gid = ctx.resolveGroupId(rest[0]) + ctx.syncIncoming() + if (!ctx.marmot.isMember(gid)) return Output.error("not_member", "not a member of group $gid") + + val commit = ctx.marmot.setEncryptedMediaPolicy(gid, policy) + val targets = ctx.marmotGroupRelays(gid).ifEmpty { ctx.outboxRelays() } + val ack = ctx.publish(commit.signedEvent, targets) + RawEventSupport.publishGuard(ack, commit.signedEvent.id)?.let { return it } + + Output.emit( + mapOf( + "group_id" to gid, + "default_blob_endpoints" to policy.defaultBlobEndpoints.map { it.baseUrl }, + "epoch" to ctx.marmot.groupEpoch(gid), + "commit_event_id" to commit.signedEvent.id, + ) + RawEventSupport.ackFields(ack), + ) + return 0 + } + } + + private suspend fun send( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val gid = args.positional(0, "gid") + val path = args.positional(1, "file") + val serverFlag = args.flag("server") + val caption = args.flag("caption") ?: "" + val mime = args.flag("mime") + args.rejectUnknown() + + val file = File(path) + if (!file.isFile) return Output.error("bad_args", "no such file: $path") + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val resolved = ctx.resolveGroupId(gid) + ctx.syncIncoming() + if (!ctx.marmot.isMember(resolved)) return Output.error("not_member", "not a member of group $resolved") + + val endpoint = + serverFlag + ?: ctx.marmot + .encryptedMediaPolicy(resolved) + ?.defaultBlobEndpoints + ?.firstOrNull { it.locatorKind == EncryptedMediaPolicyV2.INITIAL_LOCATOR_KIND } + ?.baseUrl + ?: return Output.error( + "no_endpoint", + "group $resolved has no encrypted-media policy; pass --server or commit one with media set-policy", + ) + val store = BlobStoreEndpointV2(EncryptedMediaPolicyV2.INITIAL_LOCATOR_KIND, endpoint) + + // `m` has to be byte-for-byte canonical: it feeds both the key + // derivation and the AEAD associated data, so "image/JPEG" and + // "image/jpeg" would be different keys for the same file. + val rawMediaType = mime ?: guessMediaType(file.name) + // A media type that will not canonicalize is refused rather than + // guessed at: `m` is inside both the key derivation and the AEAD + // associated data, so a sender and a receiver that canonicalized it + // differently would not agree on the key at all. + val mediaType = + MarmotMediaType.canonicalize(rawMediaType) + ?: return Output.error("bad_args", "'$rawMediaType' is not a usable media type") + val encrypted = + try { + ctx.marmot.encryptMedia(resolved, file.readBytes(), mediaType, file.name) + } catch (e: IllegalArgumentException) { + return Output.error("bad_args", e.message ?: "cannot encrypt this attachment") + } + + val ciphertextHash = encrypted.ciphertextSha256.toHexKey() + val uploadedUrl: String + try { + val auth = + BlossomAuth.createUploadAuth( + ciphertextHash, + encrypted.ciphertext.size.toLong(), + "Encrypted attachment", + ctx.signer, + ) + val result = + BlossomClient().upload(encrypted.ciphertext, "application/octet-stream", store.serverRoot, auth) + if (result.sha256 != null && result.sha256 != ciphertextHash) { + return Output.error("hash_mismatch", "blossom returned ${result.sha256}, expected $ciphertextHash") + } + // The locator is the canonical BUD-01 URL for the ciphertext + // hash, not whatever the server echoed: the hash is what a + // receiver verifies, and a server-chosen URL could name + // something else entirely. + uploadedUrl = store.blossomFetchUrl(ciphertextHash) + } catch (e: Exception) { + return Output.error("upload_failed", "${e.message}") + } + + val reference = + EncryptedMediaReferenceV2( + locators = listOf(MediaLocatorV2(EncryptedMediaPolicyV2.INITIAL_LOCATOR_KIND, uploadedUrl)), + ciphertextSha256 = encrypted.ciphertextSha256, + plaintextSha256 = encrypted.plaintextSha256, + nonce = encrypted.nonce, + mediaType = mediaType, + filename = file.name, + ) + + val bundle = ctx.marmot.buildMediaMessage(resolved, reference, caption) + val targets = ctx.marmotGroupRelays(resolved).ifEmpty { ctx.outboxRelays() } + val ack = ctx.publish(bundle.outbound.signedEvent, targets) + RawEventSupport.publishGuard(ack, bundle.outbound.signedEvent.id)?.let { return it } + + Output.emit( + mapOf( + "group_id" to resolved, + "inner_event_id" to bundle.innerEvent.id, + "outer_event_id" to bundle.outbound.signedEvent.id, + "locator" to uploadedUrl, + "ciphertext_sha256" to ciphertextHash, + "plaintext_sha256" to encrypted.plaintextSha256.toHexKey(), + "m" to mediaType, + "filename" to file.name, + ) + RawEventSupport.ackFields(ack), + ) + return 0 + } + } + + private suspend fun get( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val gid = args.positional(0, "gid") + val eventId = args.positional(1, "event-id") + val out = args.flag("out") ?: return Output.error("bad_args", "media get --out PATH") + args.rejectUnknown() + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val resolved = ctx.resolveGroupId(gid) + ctx.syncIncoming() + if (!ctx.marmot.isMember(resolved)) return Output.error("not_member", "not a member of group $resolved") + + val message = + ctx.marmot + .loadStoredMessages(resolved) + .mapNotNull { Event.fromJsonOrNull(it) } + .firstOrNull { it.id == eventId } + ?: return Output.error("not_found", "no stored message $eventId in group $resolved") + + val reference = + message.tags + .firstOrNull { it.isNotEmpty() && it[0] == "imeta" } + ?.let { + try { + EncryptedMediaV2.parseImetaTag(it) + } catch (e: IllegalArgumentException) { + return Output.error("bad_reference", e.message ?: "invalid imeta tag") + } + } + ?: return Output.error("no_media", "message $eventId carries no encrypted-media reference") + + val locator = + reference.locators.firstOrNull { it.kind == EncryptedMediaPolicyV2.INITIAL_LOCATOR_KIND } + ?: return Output.error("no_locator", "no blossom-v1 locator in message $eventId") + + val ciphertext = + try { + BlossomClient().download(locator.value) + } catch (e: Exception) { + return Output.error("download_failed", "${e.message}") + } ?: return Output.error("download_failed", "blob ${locator.value} not available") + + // The ciphertext hash is checked BEFORE decryption: it is what the + // locator names, so a server that served something else is caught + // here rather than as a confusing AEAD failure. + if (!sha256(ciphertext).contentEquals(reference.ciphertextSha256)) { + return Output.error("hash_mismatch", "the blob at ${locator.value} is not the one the message names") + } + + val plaintext = + try { + ctx.marmot.decryptMedia(resolved, reference, ciphertext) + } catch (e: Exception) { + // A failure here is not "the file is corrupt": the media + // secret is per-epoch, so an attachment from an older epoch + // simply does not open under the current one. + return Output.error("decrypt_failed", "${e.message}") + } + + File(out).writeBytes(plaintext) + Output.emit( + mapOf( + "group_id" to resolved, + "event_id" to eventId, + "locator" to locator.value, + "out" to out, + "bytes" to plaintext.size, + "m" to reference.mediaType, + "filename" to reference.filename, + ), + ) + return 0 + } + } + + /** Extension-based guess, only as a default for `--mime`. */ + private fun guessMediaType(name: String): String = + when (name.substringAfterLast('.', "").lowercase()) { + "jpg", "jpeg" -> "image/jpeg" + "png" -> "image/png" + "gif" -> "image/gif" + "webp" -> "image/webp" + "mp4" -> "video/mp4" + "webm" -> "video/webm" + "mp3" -> "audio/mpeg" + "pdf" -> "application/pdf" + "txt" -> "text/plain" + else -> "application/octet-stream" + } +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/marmot/MarmotManager.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/marmot/MarmotManager.kt index 48bb22589c..79ec80fd96 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/marmot/MarmotManager.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/marmot/MarmotManager.kt @@ -33,6 +33,9 @@ import com.vitorpamplona.quartz.marmot.WelcomeDelivery import com.vitorpamplona.quartz.marmot.WelcomeResult import com.vitorpamplona.quartz.marmot.appComponents.AdminPolicyV1 import com.vitorpamplona.quartz.marmot.appComponents.CurrentProfileGroupFactory +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaPolicyV2 +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaReferenceV2 +import com.vitorpamplona.quartz.marmot.appComponents.EncryptedMediaV2 import com.vitorpamplona.quartz.marmot.appComponents.GroupAvatarUrlV1 import com.vitorpamplona.quartz.marmot.appComponents.GroupBlossomImageV1 import com.vitorpamplona.quartz.marmot.appComponents.GroupProfileV1 @@ -1447,6 +1450,105 @@ class MarmotManager( }.event } + /** + * Set or replace the group's encrypted-media policy (`0x800b`). + * + * The state is a FULL replacement, including both ordered lists — and + * `default_blob_endpoints` order is the upload/fetch fallback priority, so + * a caller reordering it is changing where the group uploads, not + * reformatting it. + * + * Current profile only. The frozen v1 policy at `0x8008` is a different + * component and MUST NOT be reinterpreted as v2, so there is no legacy + * carrier to fall back to here. + */ + suspend fun setEncryptedMediaPolicy( + nostrGroupId: HexKey, + policy: EncryptedMediaPolicyV2, + relays: List = groupRelays(nostrGroupId), + ): OutboundGroupEvent { + val view = groupView(nostrGroupId) ?: throw IllegalStateException("Not a member of group $nostrGroupId") + check(view.isCurrentProfile) { + "Group $nostrGroupId is a legacy MIP-01 group and has no carrier for an encrypted-media policy" + } + val encoded = policy.encode() + return commitAndPublish(nostrGroupId, relays) { + groupManager.stageAppDataUpdate(nostrGroupId, EncryptedMediaPolicyV2.COMPONENT_ID, encoded) + }.event + } + + /** The group's media policy, or null when it carries none. */ + fun encryptedMediaPolicy(nostrGroupId: HexKey): EncryptedMediaPolicyV2? = groupState(nostrGroupId)?.encryptedMedia + + /** + * Encrypt an attachment under the group's media secret + * (`MLS-Exporter("marmot", "encrypted-media", 32)`). + * + * The secret is the CURRENT epoch's. `source_epoch` is deliberately not a + * field of the reference: it is the epoch of the application message that + * carries the tag, so a sender must publish the message in the same epoch + * it encrypted under — which is what publishing right after this does. + */ + fun encryptMedia( + nostrGroupId: HexKey, + plaintext: ByteArray, + mediaType: String, + filename: String, + ): EncryptedMediaV2.EncryptionResult = + EncryptedMediaV2.encrypt( + plaintext = plaintext, + mediaSecret = groupManager.mediaExporterSecret(nostrGroupId), + mediaType = mediaType, + filename = filename, + ) + + /** Decrypt an attachment a peer sent, verifying it is the file the reference names. */ + fun decryptMedia( + nostrGroupId: HexKey, + reference: EncryptedMediaReferenceV2, + ciphertext: ByteArray, + /** + * The epoch that delivered the carrying message, when the caller knows + * it. The media secret is per-epoch, so a message from an older epoch + * does not open under the current one. + */ + epochSecret: ByteArray? = null, + ): ByteArray = + EncryptedMediaV2.decrypt( + ciphertext = ciphertext, + mediaSecret = epochSecret ?: groupManager.mediaExporterSecret(nostrGroupId), + nonce = reference.nonce, + plaintextSha256 = reference.plaintextSha256, + mediaType = reference.mediaType, + filename = reference.filename, + ) + + /** + * Build the kind:9 that carries an `encrypted-media-v2` attachment. + * + * The reference rides in an `imeta` tag; [caption] is the message body a + * client without media support still reads. The locator URLs are the only + * thing in the tag a server ever sees, and they name ciphertext. + */ + suspend fun buildMediaMessage( + nostrGroupId: HexKey, + reference: EncryptedMediaReferenceV2, + caption: String = "", + persistOwn: Boolean = true, + ): TextMessageBundle { + val template = + com.vitorpamplona.quartz.nip01Core.signers + .eventTemplate(kind = 9, description = caption) { + addUnique(reference.toImetaTag()) + } + val innerEvent = + com.vitorpamplona.quartz.nip59Giftwrap.rumors.RumorAssembler + .assembleRumor(signer.pubKey, template) + val outbound = buildGroupMessage(nostrGroupId, innerEvent) + if (persistOwn) persistDecryptedMessage(nostrGroupId, innerEvent.toJson()) + return TextMessageBundle(outbound = outbound, innerEvent = innerEvent) + } + // --- KeyPackage Management --- /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/CurrentProfileGroupFactory.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/CurrentProfileGroupFactory.kt index 6f8ec036ef..25263a5c0c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/CurrentProfileGroupFactory.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/CurrentProfileGroupFactory.kt @@ -64,19 +64,17 @@ object CurrentProfileGroupFactory { * later as a group we cannot actually participate in. Add an id here only * when the component is implemented. * - * `0x8006` (agent-text-stream over QUIC) is listed for the RECEIVE role - * only, which is what [MlsGroup.currentProfileLeafCapabilities] advertises: - * we decode the group's policy, derive the per-stream record keys, open - * records and fold the transcript. We do not advertise the `send` - * (`0xF2D2`) or `fanout` (`0xF2D4`) capabilities, because publishing needs - * durable per-stream sequence state to avoid reusing an AEAD nonce across - * a restart, and we have none. + * `0x8006` (agent-text-stream over QUIC) is listed for every role + * [MlsGroup.currentProfileLeafCapabilities] advertises — receive, send and + * fanout. Publishing needs durable per-stream sequence state so a restart + * cannot reuse an AEAD nonce, and that store exists. */ val SUPPORTED_COMPONENTS: List = listOf( ComponentsList.APP_COMPONENTS_ID, AppComponentIds.GROUP_PROFILE_V1, AppComponentIds.GROUP_BLOSSOM_IMAGE_V1, + AppComponentIds.GROUP_AVATAR_URL_V1, AppComponentIds.ADMIN_POLICY_V1, AppComponentIds.NOSTR_ROUTING_V1, AppComponentIds.MESSAGE_RETENTION_V1, @@ -176,6 +174,19 @@ object CurrentProfileGroupFactory { profile: GroupProfileV1? = null, additionalAdmins: List = emptyList(), retention: MessageRetentionV1? = null, + /** + * The media policy (`0x800b`), off by default. + * + * The component is "required for new app groups under a media-capable + * application profile", but that is application-profile policy rather + * than something epoch 0 has to carry — and the reference + * implementation's own epoch-0 GroupContext does not carry it. Putting + * it there unasked would make our group context differ from the + * reference's for the same inputs AND would require every joiner to + * advertise `0x800b` before it could be added. A group that wants a + * media policy commits one, which is also how it gets changed later. + */ + encryptedMedia: EncryptedMediaPolicyV2? = null, agentTextStream: AgentTextStreamQuicPolicyV1? = null, ciphersuite: MlsCiphersuite = MlsCiphersuite.DEFAULT, ): MlsGroup { @@ -188,6 +199,7 @@ object CurrentProfileGroupFactory { routing = NostrRoutingV1.of(nostrGroupId, relays), profile = profile, retention = retention, + encryptedMedia = encryptedMedia, lifecycle = GroupLifecycleV1.ACTIVE, agentTextStream = agentTextStream, ) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaPolicyV2.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaPolicyV2.kt index c0f376d067..8b5cb8d6a6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaPolicyV2.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaPolicyV2.kt @@ -193,38 +193,36 @@ data class EncryptedMediaPolicyV2( /** * A base URL is normalized when it is byte-equal to its own - * parse-and-serialize output. + * parse-and-serialize output — the same WHATWG normalization + * `group-avatar-url-v1` defines, which is why it runs through the same + * serializer rather than a second hand-rolled approximation of it. Two + * approximations of one rule is how two components end up disagreeing + * about the same URL. * - * The checks below are the structural subset that decides validity for - * every member identically: scheme, no userinfo, a present host, and no - * query or fragment. Reachability and whether this client is willing to - * contact the host are LOCAL policy and must not influence whether the - * component bytes — or the Commit carrying them — are valid; otherwise - * one member's blocklist would fork the group. + * `http` is permitted here and not for avatars: the media policy says + * so explicitly, and a self-hosted blob store on a private network is + * a real deployment. + * + * A query or fragment is refused outright rather than serialized away. + * The serializer would happily keep a query, but the component says an + * endpoint carrying one is invalid, and dropping it would change where + * the group uploads. + * + * Reachability and whether this client is willing to contact the host + * are LOCAL policy and must not influence whether the component bytes — + * or the Commit carrying them — are valid; otherwise one member's + * blocklist would fork the group. */ fun requireNormalizedBaseUrl(url: String) { val bytes = url.encodeToByteArray() require(bytes.isNotEmpty() && bytes.size <= MAX_BASE_URL_BYTES) { "base_url must be 1..$MAX_BASE_URL_BYTES bytes, was ${bytes.size}" } - val scheme = - when { - url.startsWith("https://") -> "https://" - url.startsWith("http://") -> "http://" - else -> throw IllegalArgumentException("base_url must be http or https: '$url'") - } require('#' !in url) { "base_url must not carry a fragment: '$url'" } require('?' !in url) { "base_url must not carry a query: '$url'" } - - val afterScheme = url.substring(scheme.length) - val authority = afterScheme.substringBefore('/') - require(authority.isNotEmpty()) { "base_url has no host: '$url'" } - require('@' !in authority) { "base_url must not carry userinfo: '$url'" } - require(authority == authority.lowercase()) { - "base_url host is not normalized (lowercase): '$url'" + require(MarmotWebUrl.normalize(url, allowHttp = true, label = "base_url") == url) { + "base_url is not normalized: '$url'" } - require("//" !in afterScheme) { "base_url path is not normalized: '$url'" } - require(".." !in afterScheme) { "base_url path is not normalized: '$url'" } } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaV2Cipher.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaV2Cipher.kt new file mode 100644 index 0000000000..64894ae98a --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaV2Cipher.kt @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.marmot.appComponents + +import com.vitorpamplona.quartz.utils.ciphers.NostrCipher + +/** + * `encrypted-media-v2` as a [NostrCipher], so the existing upload pipeline can + * encrypt with it without knowing anything about Marmot. + * + * The pipeline compresses and strips metadata before handing bytes over, and + * that matters here: v2 derives its key from the hash of the bytes it actually + * encrypts, so the hash has to be taken at this point in the chain and not from + * the file the user picked. [plaintextSha256] and [ciphertextSha256] are + * therefore populated by [encrypt] and read afterwards to build the reference. + * + * A single-use object. The nonce is fresh per [encrypt] call, which is required + * — the key is deterministic in (plaintext hash, media type, filename, epoch), + * so re-encrypting the same file in the same epoch under a repeated nonce would + * break ChaCha20-Poly1305 outright — but it also means the fields below always + * describe the LAST call. + */ +class EncryptedMediaV2Cipher( + private val mediaSecret: ByteArray, + /** Canonical media type — the `m` field, byte-for-byte. */ + val mediaType: String, + val filename: String, +) : NostrCipher { + var nonce: ByteArray = ByteArray(0) + private set + + var plaintextSha256: ByteArray = ByteArray(0) + private set + + var ciphertextSha256: ByteArray = ByteArray(0) + private set + + override fun name(): String = EncryptedMediaV2.VERSION + + override fun encrypt(bytesToEncrypt: ByteArray): ByteArray { + val result = EncryptedMediaV2.encrypt(bytesToEncrypt, mediaSecret, mediaType, filename) + nonce = result.nonce + plaintextSha256 = result.plaintextSha256 + ciphertextSha256 = result.ciphertextSha256 + return result.ciphertext + } + + override fun decrypt(bytesToDecrypt: ByteArray): ByteArray = + EncryptedMediaV2.decrypt( + ciphertext = bytesToDecrypt, + mediaSecret = mediaSecret, + nonce = nonce, + plaintextSha256 = plaintextSha256, + mediaType = mediaType, + filename = filename, + ) + + override fun decryptOrNull(bytesToDecrypt: ByteArray): ByteArray? = + try { + decrypt(bytesToDecrypt) + } catch (_: Exception) { + null + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/GroupAvatarUrlV1.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/GroupAvatarUrlV1.kt index 3d90ce510d..5466ae18c3 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/GroupAvatarUrlV1.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/GroupAvatarUrlV1.kt @@ -88,7 +88,7 @@ data class GroupAvatarUrlV1( // Normalizing at encode is the producer's job: the stored bytes ARE the // serialized form, and every decoder re-derives them to check. - val stored = if (isAbsent) "" else MarmotHttpsUrl.normalize(url) + val stored = if (isAbsent) "" else MarmotWebUrl.normalize(url, label = "avatar URL") val writer = TlsWriter() writer.putOpaqueVarInt(stored.encodeToByteArray()) @@ -112,7 +112,7 @@ data class GroupAvatarUrlV1( companion object { const val COMPONENT_ID = AppComponentIds.GROUP_AVATAR_URL_V1 - const val URL_MAX_BYTES = MarmotHttpsUrl.MAX_BYTES + const val URL_MAX_BYTES = MarmotWebUrl.MAX_BYTES const val HINT_MAX_BYTES = 256 /** The cleared avatar: every field empty. */ @@ -139,7 +139,7 @@ data class GroupAvatarUrlV1( // differ from the serializer's output." A decoder never repairs a // non-normalized URL into canonical state — two members would then // hold different bytes for the same group. - require(MarmotHttpsUrl.normalize(text) == text) { "group avatar URL is not normalized" } + require(MarmotWebUrl.normalize(text, label = "avatar URL") == text) { "group avatar URL is not normalized" } } return GroupAvatarUrlV1(text, dim, thumbhash) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotGroupState.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotGroupState.kt index 6a5081da53..9bf7ccd3ea 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotGroupState.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotGroupState.kt @@ -58,6 +58,13 @@ data class MarmotGroupState( val image: GroupBlossomImageV1?, val avatarUrl: GroupAvatarUrlV1?, val retention: MessageRetentionV1?, + /** + * The group's media policy (`0x800b`). Null means the group has none, and + * a current-profile sender then has nowhere it is told to upload — it is + * not a licence to fall back to the frozen v1 policy at `0x8008`, which + * MUST NOT be reinterpreted as v2. + */ + val encryptedMedia: EncryptedMediaPolicyV2?, val lifecycle: GroupLifecycleV1?, val agentTextStream: AgentTextStreamQuicPolicyV1?, ) { @@ -116,6 +123,10 @@ data class MarmotGroupState( image = dictionary[GroupBlossomImageV1.COMPONENT_ID]?.let { GroupBlossomImageV1.decode(it) }, avatarUrl = dictionary[GroupAvatarUrlV1.COMPONENT_ID]?.let { GroupAvatarUrlV1.decode(it) }, retention = dictionary[MessageRetentionV1.COMPONENT_ID]?.let { MessageRetentionV1.decode(it) }, + encryptedMedia = + dictionary[EncryptedMediaPolicyV2.COMPONENT_ID]?.let { + EncryptedMediaPolicyV2.decode(it) + }, lifecycle = dictionary[GroupLifecycleV1.COMPONENT_ID]?.let { GroupLifecycleV1.decode(it) }, agentTextStream = dictionary[AgentTextStreamQuicPolicyV1.COMPONENT_ID]?.let { @@ -139,6 +150,7 @@ data class MarmotGroupState( image: GroupBlossomImageV1? = null, avatarUrl: GroupAvatarUrlV1? = null, retention: MessageRetentionV1? = null, + encryptedMedia: EncryptedMediaPolicyV2? = null, lifecycle: GroupLifecycleV1? = GroupLifecycleV1.ACTIVE, agentTextStream: AgentTextStreamQuicPolicyV1? = null, extraRequiredComponents: Collection = emptyList(), @@ -169,6 +181,10 @@ data class MarmotGroupState( required.add(MessageRetentionV1.COMPONENT_ID) dictionary = dictionary.with(MessageRetentionV1.COMPONENT_ID, it.encode()) } + encryptedMedia?.let { + required.add(EncryptedMediaPolicyV2.COMPONENT_ID) + dictionary = dictionary.with(EncryptedMediaPolicyV2.COMPONENT_ID, it.encode()) + } lifecycle?.let { required.add(GroupLifecycleV1.COMPONENT_ID) dictionary = dictionary.with(GroupLifecycleV1.COMPONENT_ID, it.encode()) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotHttpsUrl.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotWebUrl.kt similarity index 80% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotHttpsUrl.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotWebUrl.kt index cc6cab0053..82d8f3d9f1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotHttpsUrl.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/marmot/appComponents/MarmotWebUrl.kt @@ -21,18 +21,20 @@ package com.vitorpamplona.quartz.marmot.appComponents /** - * The `https`-only WHATWG URL normalizer Marmot group state needs. + * The WHATWG URL normalizer Marmot group state needs. * * This exists because normalization is part of the wire format, not a * convenience: `marmot.group.avatar-url.v1` stores the serialized form, and a * decoder "MUST reject state whose stored URL bytes differ from the - * serializer's output". So this has to agree with every other implementation - * byte for byte — too lax and we accept state a peer rejects, too strict and we - * reject a group somebody else made. + * serializer's output". `marmot.group.encrypted-media.v2` says the same about + * its blob-store base URLs, and names this same normalization. So this has to + * agree with every other implementation byte for byte — too lax and we accept + * state a peer rejects, too strict and we reject a group somebody else made. * - * It is not a general URL library. It handles exactly the shape the component - * allows — `https`, a host, no userinfo, no fragment — and refuses everything - * else rather than guessing. + * It is not a general URL library. It handles exactly the shapes the components + * allow — an `https` (or, where the component permits it, `http`) URL with a + * host, no userinfo and no fragment — and refuses everything else rather than + * guessing. * * **Known limit: no IDNA.** A host with non-ASCII characters is refused instead * of punycoded. That costs nothing on the decode side, where it matters: a @@ -41,11 +43,11 @@ package com.vitorpamplona.quartz.marmot.appComponents * and must be rejected anyway. It only stops us from *accepting* a * Unicode-typed host from our own user, who can paste the punycode form. */ -object MarmotHttpsUrl { +object MarmotWebUrl { const val MAX_BYTES = 2048 - private const val SCHEME = "https://" - private const val DEFAULT_PORT = "443" + private const val HTTPS_DEFAULT_PORT = "443" + private const val HTTP_DEFAULT_PORT = "80" /** * Parse [raw] and return its WHATWG serialization. @@ -53,50 +55,67 @@ object MarmotHttpsUrl { * @throws IllegalArgumentException when the URL is not a valid group-avatar * URL, or when normalizing it would need something this does not do. */ - fun normalize(raw: String): String { - require(raw.isNotEmpty()) { "avatar URL must not be empty" } - require(raw.encodeToByteArray().size <= MAX_BYTES) { "avatar URL exceeds $MAX_BYTES bytes" } + fun normalize( + raw: String, + /** + * Whether plain `http` is acceptable. Off by default because the + * avatar component is https-only; the media policy permits both, and + * that is a per-component rule rather than a global one. + */ + allowHttp: Boolean = false, + /** What to call this URL in an error, e.g. "avatar URL". */ + label: String = "URL", + ): String { + require(raw.isNotEmpty()) { "$label must not be empty" } + require(raw.encodeToByteArray().size <= MAX_BYTES) { "$label exceeds $MAX_BYTES bytes" } val schemeEnd = raw.indexOf("://") - require(schemeEnd > 0) { "avatar URL must be an absolute https URL" } - require(raw.substring(0, schemeEnd).lowercase() == "https") { "avatar URL scheme must be https" } + require(schemeEnd > 0) { "$label must be an absolute URL" } + val scheme = raw.substring(0, schemeEnd).lowercase() + require(scheme == "https" || (allowHttp && scheme == "http")) { + if (allowHttp) "$label scheme must be http or https" else "$label scheme must be https" + } + val defaultPort = if (scheme == "http") HTTP_DEFAULT_PORT else HTTPS_DEFAULT_PORT var rest = raw.substring(schemeEnd + 3) - require(!rest.contains('#')) { "avatar URL must not include a fragment" } + require(!rest.contains('#')) { "$label must not include a fragment" } // The authority runs to the first "/" or "?" — everything after is path // and query. val authorityEnd = rest.indexOfFirst { it == '/' || it == '?' }.let { if (it < 0) rest.length else it } val authority = rest.substring(0, authorityEnd) rest = rest.substring(authorityEnd) - require(!authority.contains('@')) { "avatar URL must not include credentials" } - require(authority.isNotEmpty()) { "avatar URL must include a host" } + require(!authority.contains('@')) { "$label must not include credentials" } + require(authority.isNotEmpty()) { "$label must include a host" } val (host, port) = splitHostPort(authority) - require(host.isNotEmpty()) { "avatar URL must include a host" } + require(host.isNotEmpty()) { "$label must include a host" } require(host.all { it.code < 0x80 }) { - "avatar URL host must be ASCII — encode an international host as punycode first" + "$label host must be ASCII — encode an international host as punycode first" } val queryStart = rest.indexOf('?') val rawPath = if (queryStart < 0) rest else rest.substring(0, queryStart) val rawQuery = if (queryStart < 0) null else rest.substring(queryStart + 1) - val out = StringBuilder(SCHEME) + val out = StringBuilder(scheme).append("://") out.append(host.lowercase()) - if (port != null && port != DEFAULT_PORT) out.append(':').append(port) + if (port != null && port != defaultPort) out.append(':').append(port) out.append(normalizePath(rawPath)) if (rawQuery != null) out.append('?').append(percentEncode(rawQuery, QUERY_KEEP)) val normalized = out.toString() - require(normalized.encodeToByteArray().size <= MAX_BYTES) { "avatar URL exceeds $MAX_BYTES bytes" } + require(normalized.encodeToByteArray().size <= MAX_BYTES) { "$label exceeds $MAX_BYTES bytes" } return normalized } /** True when [normalize] accepts [raw] and returns it unchanged. */ - fun isNormalized(raw: String): Boolean = + fun isNormalized( + raw: String, + allowHttp: Boolean = false, + ): Boolean = try { - normalize(raw) == raw + normalize(raw, allowHttp) == raw } catch (_: IllegalArgumentException) { false } @@ -115,7 +134,7 @@ object MarmotHttpsUrl { fun isSafeToContact(raw: String): Boolean { val host = try { - hostOf(normalize(raw)) + hostOf(normalize(raw, allowHttp = true)) } catch (_: IllegalArgumentException) { return false } @@ -127,7 +146,7 @@ object MarmotHttpsUrl { } private fun hostOf(normalized: String): String { - val rest = normalized.substring(SCHEME.length) + val rest = normalized.substringAfter("://") val end = rest.indexOfFirst { it == '/' || it == '?' }.let { if (it < 0) rest.length else it } return splitHostPort(rest.substring(0, end)).first } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaV2Test.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaV2Test.kt index 2cf77ffe71..17acac7379 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaV2Test.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/marmot/appComponents/EncryptedMediaV2Test.kt @@ -105,7 +105,6 @@ class EncryptedMediaV2Test { "https://a.example.com/?q=1", "https://a.example.com/#f", "https://A.EXAMPLE.COM/", - "https://a.example.com//double/", "https://a.example.com/../up/", "https:///", ).forEach { url -> @@ -115,6 +114,39 @@ class EncryptedMediaV2Test { } } + @Test + fun acceptsBaseUrlsTheWhatwgSerializerLeavesAlone() { + // An empty path segment is NOT collapsed by the WHATWG serializer — the + // path is a segment list, and only "." and ".." are special. Rejecting + // a doubled slash therefore refuses group state the reference + // implementation produces and accepts, which is the exact failure mode + // this component's normalization rule exists to prevent. + // + // `http` is likewise valid here and not for avatars: the media policy + // permits it, and a self-hosted blob store on a private network is a + // real deployment. + listOf( + "https://a.example.com//double/", + "http://blobs.internal/", + "https://a.example.com:8443/blobs/", + ).forEach { url -> + EncryptedMediaPolicyV2(listOf("blossom-v1"), listOf(BlobStoreEndpointV2("blossom-v1", url))) + } + } + + @Test + fun dropsNothingButRefusesToRepair() { + // The default port is absent from a normalized URL, so a stored value + // carrying it is non-normalized and refused rather than trimmed. A + // decoder that repaired it would hold different bytes than the peer + // that stored them. + listOf("https://a.example.com:443/", "http://a.example.com:80/").forEach { url -> + assertFailsWith("expected '$url' to be rejected") { + EncryptedMediaPolicyV2(listOf("blossom-v1"), listOf(BlobStoreEndpointV2("blossom-v1", url))) + } + } + } + @Test fun rejectsInvalidLocatorKinds() { listOf("", "Blossom-v1", "blossom_v1", "blossom v1", "a".repeat(65)).forEach { kind -> diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/marmot/appComponents/GroupAvatarUrlV1Test.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/marmot/appComponents/GroupAvatarUrlV1Test.kt index 993105d109..90708fde61 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/marmot/appComponents/GroupAvatarUrlV1Test.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/marmot/appComponents/GroupAvatarUrlV1Test.kt @@ -73,7 +73,7 @@ class GroupAvatarUrlV1Test { ) for ((raw, expected) in vectors) { - assertEquals("normalizing $raw", expected, MarmotHttpsUrl.normalize(raw)) + assertEquals("normalizing $raw", expected, MarmotWebUrl.normalize(raw)) } } @@ -86,8 +86,8 @@ class GroupAvatarUrlV1Test { "https://cdn.example.com/%7euser/a b.png", "https://[2001:DB8::1]:8443/", )) { - val once = MarmotHttpsUrl.normalize(raw) - assertEquals(once, MarmotHttpsUrl.normalize(once)) + val once = MarmotWebUrl.normalize(raw) + assertEquals(once, MarmotWebUrl.normalize(once)) } } @@ -106,7 +106,7 @@ class GroupAvatarUrlV1Test { "", )) { assertThrows("must reject $bad", IllegalArgumentException::class.java) { - MarmotHttpsUrl.normalize(bad) + MarmotWebUrl.normalize(bad) } } } @@ -122,14 +122,14 @@ class GroupAvatarUrlV1Test { "https://10.0.0.1/avatar.png", "https://[::1]/avatar.png", )) { - MarmotHttpsUrl.normalize(raw) + MarmotWebUrl.normalize(raw) } - assertTrue(MarmotHttpsUrl.isSafeToContact("https://cdn.example.com/a.png")) - assertTrue(!MarmotHttpsUrl.isSafeToContact("https://localhost/a.png")) - assertTrue(!MarmotHttpsUrl.isSafeToContact("https://127.0.0.1/a.png")) - assertTrue(!MarmotHttpsUrl.isSafeToContact("https://10.0.0.1/a.png")) - assertTrue(!MarmotHttpsUrl.isSafeToContact("https://192.168.1.1/a.png")) - assertTrue(!MarmotHttpsUrl.isSafeToContact("https://[::1]/a.png")) + assertTrue(MarmotWebUrl.isSafeToContact("https://cdn.example.com/a.png")) + assertTrue(!MarmotWebUrl.isSafeToContact("https://localhost/a.png")) + assertTrue(!MarmotWebUrl.isSafeToContact("https://127.0.0.1/a.png")) + assertTrue(!MarmotWebUrl.isSafeToContact("https://10.0.0.1/a.png")) + assertTrue(!MarmotWebUrl.isSafeToContact("https://192.168.1.1/a.png")) + assertTrue(!MarmotWebUrl.isSafeToContact("https://[::1]/a.png")) } @Test @@ -140,7 +140,7 @@ class GroupAvatarUrlV1Test { // punycode and a raw Unicode host is non-normalized anyway. val failure = assertThrows(IllegalArgumentException::class.java) { - MarmotHttpsUrl.normalize("https://bücher.example/a.png") + MarmotWebUrl.normalize("https://bücher.example/a.png") } assertTrue(failure.message.orEmpty().contains("punycode")) } @@ -197,7 +197,7 @@ class GroupAvatarUrlV1Test { @Test fun theBoundsAreEnforcedOnBothFields() { val longPath = "https://cdn.example.com/" + "a".repeat(2100) - assertThrows(IllegalArgumentException::class.java) { MarmotHttpsUrl.normalize(longPath) } + assertThrows(IllegalArgumentException::class.java) { MarmotWebUrl.normalize(longPath) } assertThrows(IllegalArgumentException::class.java) { GroupAvatarUrlV1(url = "https://cdn.example.com/a.png", dim = ByteArray(257)).encode() }